diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e731ee56356d8b9fe5267d975c4dae018aa2ddcb..a603ea432042dd471bbb4cb0f66b16b4b1612da9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,11 +26,11 @@ variables: fail-exit-code-job-Singularity: # This job runs in the build stage, which runs first. stage: deploy # MUST FAIL variables: - SLURM_CPUS_PER_TASK: "1" + SLURM_CPUS_PER_TASK: "1" CI_MODE: "Singularity" CONTAINER: "tensorflow" tags: - - "custom" + - "custom" script: - cd rewagha @@ -38,11 +38,11 @@ fail-timeout-job-Singularity: # This job runs in the build stage, which ru stage: deploy # MUST FAIL variables: SLURM_CPUS_PER_TASK: "1" - SLURM_TIME: "00:01:00" + SLURM_TIME: "00:01:00" CI_MODE: "Singularity" CONTAINER: "tensorflow" tags: - - "custom" + - "custom" script: - echo "Compiling the code..." - sleep 1200 @@ -50,9 +50,9 @@ fail-timeout-job-Singularity: # This job runs in the build stage, which ru fail-exit-code-job: # This job runs in the build stage, which runs first. stage: deploy # MUST FAIL variables: - SLURM_CPUS_PER_TASK: "1" + SLURM_CPUS_PER_TASK: "1" tags: - - "custom" + - "custom" script: - cd rewagha @@ -60,9 +60,9 @@ fail-timeout-job: # This job runs in the build stage, which runs first. stage: deploy # MUST FAIL variables: SLURM_CPUS_PER_TASK: "1" - SLURM_TIME: "00:01:00" + SLURM_TIME: "00:01:00" tags: - - "custom" + - "custom" script: - echo "Compiling the code..." - sleep 1200 @@ -72,7 +72,7 @@ build-job: # This job runs in the build stage, which runs first. variables: SLURM_CPUS_PER_TASK: "2" tags: - - "custom" + - "custom" script: - echo "Compiling the code..." - echo "Compile complete." @@ -80,32 +80,32 @@ build-job: # This job runs in the build stage, which runs first. build-job-Singularity: # This job runs in the build stage, which runs first. stage: build variables: - SLURM_CPUS_PER_TASK: "2" + SLURM_CPUS_PER_TASK: "2" CI_MODE: "Singularity" CONTAINER: "tensorflow" tags: - - "custom" + - "custom" script: - echo "Compiling the code..." - echo "Compile complete." batch-job: # This job runs in the build stage, which runs first. stage: build - variables: + variables: CI_MODE: "Batch" BATCH_SCRIPT: "batch.sh" tags: - - "custom" + - "custom" script: - echo "I do nothing" fail-batch-job: # This job runs in the build stage, which runs first. stage: deploy - variables: + variables: CI_MODE: "Batch" BATCH_SCRIPT: "doesntexist.sh" tags: - - "custom" + - "custom" script: - echo "I do nothing" @@ -114,7 +114,7 @@ unit-test-job: # This job runs in the test stage. variables: SLURM_CPUS_PER_TASK: "4" tags: - - "custom" + - "custom" script: - echo "Running unit tests... This will take about 60 seconds." - sleep 60 @@ -125,7 +125,7 @@ lint-test-job: # This job also runs in the test stage. variables: SLURM_CPUS_PER_TASK: "8" tags: - - "custom" + - "custom" script: - echo "Linting code... This will take about 10 seconds." - sleep 10 @@ -134,7 +134,7 @@ lint-test-job: # This job also runs in the test stage. deploy-job: # This job runs in the deploy stage. stage: deploy # It only runs when *both* jobs in the test stage complete successfully. tags: - - "custom" + - "custom" script: - echo "Deploying application..." - echo "Application successfully deployed." diff --git a/ConfigManager.py b/ConfigManager.py new file mode 100644 index 0000000000000000000000000000000000000000..8ef27ebe2f852302ac42a58bf4f8bfe2027b6bd0 --- /dev/null +++ b/ConfigManager.py @@ -0,0 +1,51 @@ +import os + + + + +def get_file(): + path = os.path.join(os.path.dirname(__file__), "config.txt") + return path + + +def read_config(): + key_path = os.path.join(os.path.dirname(__file__), "id_rsa") + map_path = os.path.join(os.path.dirname(__file__), "Assignment.txt") + runner_path = os.path.dirname(__file__) + user_path = "Runner" + down_scoping = True + with open(get_file(), mode='r') as config_file: + config = config_file.read() + config = config.splitlines() + for line in config: + path_tuple = line.split(":") + if path_tuple[0].strip() == "Runner Path": + preamble = os.path.dirname(__file__) + if path_tuple[1].strip() == "absolute": + preamble = "" + path_tuple[2] = path_tuple[2].replace("$HOME", os.getenv("HOME")) + runner_path = os.path.join(preamble, path_tuple[2].strip()) + elif path_tuple[0].strip() == "Key Path": + preamble = os.path.dirname(__file__) + if path_tuple[1].strip() == "absolute": + preamble = "" + path_tuple[2] = path_tuple[2].replace("$HOME", os.getenv("HOME")) + key_path = os.path.join(preamble, path_tuple[2].strip()) + elif path_tuple[0].strip() == "Map Path": + preamble = os.path.dirname(__file__) + if path_tuple[1].strip() == "absolute": + preamble = "" + path_tuple[2] = path_tuple[2].replace("$HOME", os.getenv("HOME")) + map_path = os.path.join(preamble, path_tuple[2].strip()) + elif path_tuple[0].strip() == "User Path": + if path_tuple[1].strip() == "absolute": + raise RuntimeError("User Path must be a relative path to the users $HOME repo.") + user_path = path_tuple[2].strip() + elif path_tuple[0].strip() == "Down Scoping": + if path_tuple[1].strip() == "local": + down_scoping = False + #print(key_path) + #print(map_path) + #print(runner_path) + #print(user_path) + return {"key_path": key_path, "map_path": map_path, "runner_path": runner_path, "user_path": user_path, "down_scoping": down_scoping} \ No newline at end of file diff --git a/JSONManager.py b/JSONManager.py new file mode 100644 index 0000000000000000000000000000000000000000..15e3fca3ab35699183c577b6366c53b1d13e1aae --- /dev/null +++ b/JSONManager.py @@ -0,0 +1,41 @@ +import time +import json +import rsa +import os + +def load_priv_key(path): + path = os.path.join(os.path.dirname(__file__), path) + with open(path, mode='rb') as privatefile: + keydata = privatefile.read() + return rsa.PrivateKey.load_pkcs1(keydata) + + +def load_pub_key(path): + path = os.path.join(os.path.dirname(__file__), path) + with open(path, mode='rb') as pubfile: + keydata = pubfile.read() + return rsa.PublicKey.load_pkcs1(keydata) + + +def create_keys(): + (pubkey, privkey) = rsa.newkeys(2048) + with open("/home/ppl/Runner/id_rsa.pub", "w") as text_file: + text_file.write(pubkey.save_pkcs1().decode('ascii')) + with open("/home/ppl/Runner/id_rsa", "w") as text_file: + text_file.write(privkey.save_pkcs1().decode('ascii')) + +def get_account(url, pid, uid, key_path, map_path): + with open(map_path, mode='rb') as file: + data = file.read() + json_file = rsa.decrypt(data, load_priv_key(key_path)) + search_file = json.loads(json_file) + instance = search_file[url] + result = instance["uid"][uid] + if result == None: + result = instance["pid"][pid] + if result == None: + print("Cannot assign GitLab user/project to cluster account. Please register here: TODO") + exit(1) + return result + + diff --git a/JSONTest.py b/JSONTest.py new file mode 100644 index 0000000000000000000000000000000000000000..14ca53adc750a55316ef9ee4d90aa0cd570c5467 --- /dev/null +++ b/JSONTest.py @@ -0,0 +1,11 @@ +import JSONManager as man +import json +import rsa + +def create_testset(): + dict = {"https://git-ce.rwth-aachen.de" : {"pid" : {}, "uid" : {"2076": "tester1"}}} + json_file = json.dumps(dict) + man.create_keys() + encrypted = rsa.encrypt(json_file.encode('ascii'), man.load_pub_key("id_rsa.pub")) + with open("/home/ppl/Runner/Assignments.txt", "wb") as text_file: + text_file.write(encrypted) diff --git a/JWTManager.py b/JWTManager.py new file mode 100644 index 0000000000000000000000000000000000000000..984e6f571ee1a254ec42a1a668e2662969b910a5 --- /dev/null +++ b/JWTManager.py @@ -0,0 +1,13 @@ +import jwt +import time + +def get_UID_PID(JWT, url): + jwks_client = jwt.PyJWKClient(url) + signing_key = jwks_client.get_signing_key_from_jwt(JWT) + # wait for token to be valid + time.sleep(2) + data = jwt.decode(JWT, + signing_key.key, + algorithms=["RS256"], + options={"verify_exp": False}) + return data["user_id"], data["project_id"] diff --git a/authmanager.py b/authmanager.py new file mode 100644 index 0000000000000000000000000000000000000000..de5bda363a3538efff72376c3722249a3c5584e0 --- /dev/null +++ b/authmanager.py @@ -0,0 +1,62 @@ +import os +import subprocess +import pwd +import time +import colorama + + +def demote(user, uid, gid): + def demote_function(): + print("starting") + print('uid, gid = %d, %d' % (os.getuid(), os.getgid())) + os.chdir(f"/home/{user}") + print(os.getgroups()) + # initgroups must be run before we lose the privilege to set it! + os.initgroups(user, gid) + os.setgid(gid) + # this must be run last + os.setuid(uid) + print("finished demotion") + print('uid, gid = %d, %d' % (os.getuid(), os.getgid())) + print(os.getgroups()) + return demote_function + + +def get_user(user): + uid = pwd.getpwnam(user).pw_uid + gid = pwd.getpwnam(user).pw_gid + print('uid, gid = %d, %d' % (os.getuid(), os.getgid())) + print('User: uid, gid = %d, %d' % (uid, gid)) + return uid, gid + + +def run_task(user, cmd): + return_code = 0 + err = "" + out = "" + print("requesting rights for user: " + user) + uid, gid = get_user(user) + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=demote(user, uid, gid), + shell=True + ) + + # process can execute normally, no exceptions at startup + process_running = True + while process_running: + if proc.poll() is not None: + process_running = False + # half a second of resolution + time.sleep(0.5) + return_code = proc.returncode + out = proc.stderr.read() + err = proc.stdout.read() + colorama.init() + clean_print(err) + clean_print(out) + +def clean_print(string): + print(str(string)[2:-1].replace("\\n","\n")) diff --git a/config.txt b/config.txt new file mode 100644 index 0000000000000000000000000000000000000000..50ed4c4e049594195c1638a265da875cd204fac5 --- /dev/null +++ b/config.txt @@ -0,0 +1,5 @@ +Runner Path: absolute: $HOME/Runner +Key Path: relative: id_rsa +Map Path: relative: Assignments.txt +User Path: relative: Runner +Down Scoping: local \ No newline at end of file diff --git a/driver.py b/driver.py index 1e8ce2c10cf131003e63ff258d711d14cce8262c..d5de9138d72f787f1566dc7325a39c4130666514 100644 --- a/driver.py +++ b/driver.py @@ -1,56 +1,104 @@ import os -from re import I import sys -import subprocess +import time + import variableHandle as vh +#import authmanager as auth +import JWTManager as jwt +import JSONManager as man +import JSONTest as test +import ConfigManager as conf import subprocess +import stat import random import string + def get_random_string(length): # choose from all lowercase letter letters = string.ascii_letters result_str = ''.join(random.choice(letters) for i in range(length)) return result_str + argv = sys.argv -name = 'Costum_Driver' -version = '0.0.5' +name = 'Custom_Driver' +version = '0.1.0' + +account = "" +user_path = "" +runner_path = "" +down_scoping = True + # generates the path to the build directory def get_build_path(): CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID = os.getenv("CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID") CUSTOM_ENV_CI_PROJECT_PATH_SLUG = os.getenv("CUSTOM_ENV_CI_PROJECT_PATH_SLUG") - HOME = os.getenv("HOME") - return HOME + "/Runner/builds/" + CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID + "/" + CUSTOM_ENV_CI_PROJECT_PATH_SLUG + HOME = f"/home/{account}" + if not down_scoping: + HOME = os.getenv("HOME") + build_path = f'{HOME}/{user_path}/builds/{CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID}/{CUSTOM_ENV_CI_PROJECT_PATH_SLUG}' + return str(build_path) def get_cache_path(): CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID = os.getenv("CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID") CUSTOM_ENV_CI_PROJECT_PATH_SLUG = os.getenv("CUSTOM_ENV_CI_PROJECT_PATH_SLUG") - HOME = os.getenv("HOME") - return HOME + "/Runner/cache/" + CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID + "/" + CUSTOM_ENV_CI_PROJECT_PATH_SLUG + HOME = f"/home/{account}" + if not down_scoping: + HOME = os.getenv("HOME") + cache_path = f'{HOME}/{user_path}/cache/{CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID}/{CUSTOM_ENV_CI_PROJECT_PATH_SLUG}' + return str(cache_path) + -# generates the path to the cloned repository +# generates the path to the cloned repository with regards to the build directory def get_clone_path(): clone_path = os.getenv('CUSTOM_ENV_CI_PROJECT_DIR') return clone_path + def handle(): + global user_path + global runner_path + global account + global down_scoping + + # read config file + config = conf.read_config() + user_path = config["user_path"] + key_path = config["key_path"] + runner_path = config["runner_path"] + map_path = config["map_path"] + down_scoping = config["down_scoping"] + + if down_scoping: + #test.create_testset() + token = os.getenv('CUSTOM_ENV_CI_JOB_JWT') + url = os.getenv('CUSTOM_ENV_CI_SERVER_URL') + + # get uid and pid from JWT + uid, pid = jwt.get_UID_PID(token, f"{url}/-/jwks") + + # get account from mapping file + account = man.get_account(url, pid, uid, key_path, map_path) + + if account is None: + print(f"Error: no mapping for GitLab project: {os.getenv('CUSTOM_ENV_CI_PROJECT_NAME')}, or GitLab user: {os.getenv('CUSTOM_ENV_GITLAB_USER_NAME')} available. Please register CI support for to acces the Runner") + if len(argv) < 2: print("Error: no argument") exit(1) - if argv[1] == 'config': - HOME = os.getenv("HOME") + if argv[1] == 'config': # Do not use print in this step - os.system('mkdir -p ' + HOME + '/Runner/scripts') - os.system('mkdir -p ' + HOME + '/Runner/errorCodes') - os.system('chmod +x ' + HOME + '/Runner/sshRunstep.sh') - os.system('chmod +x ' + HOME + '/Runner/singularityRunstep.sh') - os.system('dos2unix ' + HOME + '/Runner/sshRunstep.sh') - os.system('dos2unix ' + HOME + '/Runner/singularityRunstep.sh') + os.system(f'mkdir -p {runner_path}/scripts') + os.system(f'mkdir -p {runner_path}/errorCodes') + os.system(f'chmod +x {runner_path}/sshRunstep.sh') + os.system(f'chmod +x {runner_path}/singularityRunstep.sh') + os.system(f'dos2unix {runner_path}/sshRunstep.sh') + os.system(f'dos2unix {runner_path}/singularityRunstep.sh') handle_config(get_build_path(), get_cache_path(), name, version) elif argv[1] == 'prepare': @@ -62,6 +110,7 @@ def handle(): else: print('Error') + def handle_config(build_dir, cache_dir, driver_name, driver_version): builder = "{ \"builds_dir\": \"" builder += build_dir @@ -73,64 +122,101 @@ def handle_config(build_dir, cache_dir, driver_name, driver_version): builder += driver_version builder += "\" }, \"job_env\" : { \"CUSTOM_ENVIRONMENT\": \"example\" }}" + # print(builder, file=sys.stderr) print(builder) + def handle_prepare(): - #os.system('module avail') - #os.system('module list') os.system('hostname') - #os.system('echo CUSTOM_ENV_CI_PROJECT_URL:') - #print(os.getenv('CUSTOM_ENV_CI_PROJECT_URL')) + print(os.getenv('CUSTOM_ENV_CI_PROJECT_PATH')) + print(os.getenv('CUSTOM_ENV_GITLAB_USER_NAME')) def handle_run(): + # set user $HOME HOME = os.getenv("HOME") - #os.system('hostname') - #os.system('module list') - #print(argv[3]) - #sys.stderr.write("Failure\n") - #exit(21) + + #Setup CI scripts script_hash = get_random_string(8) - os.system('cp ' + argv[2] + ' ' + HOME + '/Runner/scripts/script' + script_hash) - os.system('chmod +x ' + HOME + '/Runner/scripts/script' + script_hash) + os.system(f'cp {argv[2]} {runner_path}/scripts/script{script_hash}') + os.system(f'chmod +x {runner_path}/scripts/script{script_hash}') + os.system(f'dos2unix {runner_path}/scripts/script{script_hash}') mode, container, script = vh.get_CI_mode() + command_wrapper_ds = [] + exec_command = [] - if argv[3] == 'build_script' or argv[3] == 'step_script': - Slurm_vars = vh.get_slurm_variables() - command = [] + # Handle different modes + if mode == 'local': #Debugging mode + print("local mode only for development.") + exit(1) + os.system(f"chmod -R 777 {runner_path}") + # auth.run_task(USER, f'{runner_path}/scripts/script{script_hash} {argv[3]}') + command_wrapper_ds = f"sudo su --shell /bin/bash --login {account} -c".split() + exec_command = f"{runner_path}/scripts/script{script_hash} {argv[3]}" + vh.set_slurm_env() - if mode == 'Batch': - command += ['sbatch', '--wait', f'{get_clone_path()}/{script}'] + elif argv[3] == 'build_script' or argv[3] == 'step_script': + Slurm_vars = vh.get_slurm_variables() + exec_command = [] + + if mode == 'Batch': # Handle Batch scripts + # Parse parameters from Batchscript + file = open(f'{get_clone_path()}/{script}', 'r') + batch_parameters = [] + for line in file.readlines(): + if line.startswith('#SBATCH'): + batch_parameters.append(line.split()[1]) + file.close() + #Define Batchscript run + exec_command += ['srun'] + batch_parameters + [f'{get_clone_path()}/{script}'] print('Warning: The contents of the script section in the CI definition ' - 'will be ignored in the batch mode. If you want to work on the results ' + 'will be ignored in the batch mode. If you want to work on the results ' 'please create additional stages and connect them via artifacts.') else: - command += ['srun', '--job-name=CI'] + # Define Slurm parameters + exec_command += ['srun', '--job-name=CI'] for x in Slurm_vars: - command += [f'{x[0]}{x[1]}'] - + exec_command += [f'{x[0]}{x[1]}'] + # Handle Slurm shell and singularity shell environment if mode == "Slurm": - command += [f'{HOME}/Runner/scripts/script{script_hash}', 'step_script'] - elif mode =="Singularity": + exec_command += [f'{runner_path}/scripts/script{script_hash}', 'step_script'] + elif mode == "Singularity": if os.path.exists(container): container = f'{get_clone_path()}/{script}' - command += [f'{HOME}/Runner/singularityLocalRunstep.sh', + exec_command += [f'{HOME}/Runner/singularityLocalRunstep.sh', f'{get_clone_path()}/{container}', script_hash] else: - command += [f'{HOME}/Runner/singularityRunstep.sh', container, script_hash] + exec_command += [f'{HOME}/Runner/singularityRunstep.sh', container, script_hash] - print(command) - cmd_ret = subprocess.run(command) - os.remove(HOME + '/Runner/scripts/script' + script_hash) - if int(cmd_ret.returncode) != 0: - exit(1) - else: - exit(0) + exec_command = ' '.join(exec_command) + #print(exec_command) + command_wrapper_ds = f"sudo su --shell /bin/bash --login {account} -c ".split() + + else: #run small scripts on local machine + command_wrapper_ds = f"sudo su --shell /bin/bash --login {account} -c ".split() + exec_command = f'{runner_path}/scripts/script{script_hash} {argv[3]}' + + command_wrapper_ds.append(f"{exec_command}") + + # check for downscoping + command = command_wrapper_ds + if not down_scoping: + command = exec_command.split() + # Run command + print(command) + cmd_ret = subprocess.run(command) + return_code = cmd_ret.returncode + os.remove(f'{runner_path}/scripts/script{script_hash}') + + if int(return_code) != 0: + exit(1) else: - os.system('. ' + '$HOME/Runner/scripts/script' + script_hash + ' ' + argv[3]) + exit(0) + + def handle_cleanup(): os.system("echo cleanup") diff --git a/variableHandle.py b/variableHandle.py index ba249dc7829fdf116cf6958d8340190e40df1b1b..47ed21a4294087f44e9286702be9b98dc92e1597 100644 --- a/variableHandle.py +++ b/variableHandle.py @@ -1,4 +1,5 @@ import os +import subprocess from posixpath import split # Gather Slurm job parameters @@ -52,9 +53,9 @@ def get_CI_mode(): if mode == None: mode = 'Slurm' if mode != 'Slurm' and mode != 'Singularity' and mode != 'Batch': - print("Error: only modes Slurm and Singularity are supported!") + print("Error: only modes Batch, Slurm and Singularity are supported!") os.system('echo mode: ' + mode) - exit(1) + #exit(1) # get container for singularity container = os.getenv('CUSTOM_ENV_CONTAINER') @@ -86,4 +87,13 @@ def get_num_nodes(): num_nodes = "1" return num_nodes - +def set_slurm_env(): + res = "" + proc = subprocess.run('env', stdout=subprocess.PIPE) + for variable in proc.stdout.decode().splitlines(): + if variable.startswith("CUSTOM_ENV_SLURM_ENV_"): + value = variable.split("=") + value[0] = value[0].replace("CUSTOM_ENV_SLURM_ENV_", "") + os.putenv(value[0], value[1]) + #res = f"{res}export {value[0]}={value[1]}; " + #return res \ No newline at end of file