Merge branch '1-allow-downscoping' into 'main'

Resolve "Allow downscoping"

Closes #1

See merge request ci-playground/customdriver!2

ConfigManager.py

+import os
+
+
+
+
+def get_file():
+    path = os.path.join(os.path.dirname(__file__), "config.txt")
+    return path
+
+
+def read_config():
+    key_path = os.path.join(os.path.dirname(__file__), "id_rsa")
+    map_path = os.path.join(os.path.dirname(__file__), "Assignment.txt")
+    runner_path = os.path.dirname(__file__)
+    user_path = "Runner"
+    down_scoping = True
+    with open(get_file(), mode='r') as config_file:
+        config = config_file.read()
+    config = config.splitlines()
+    for line in config:
+        path_tuple = line.split(":")
+        if path_tuple[0].strip() == "Runner Path":
+            preamble = os.path.dirname(__file__)
+            if path_tuple[1].strip() == "absolute":
+                preamble = ""
+                path_tuple[2] = path_tuple[2].replace("$HOME", os.getenv("HOME"))
+            runner_path = os.path.join(preamble, path_tuple[2].strip())
+        elif path_tuple[0].strip() == "Key Path":
+            preamble = os.path.dirname(__file__)
+            if path_tuple[1].strip() == "absolute":
+                preamble = ""
+                path_tuple[2] = path_tuple[2].replace("$HOME", os.getenv("HOME"))
+            key_path = os.path.join(preamble, path_tuple[2].strip())
+        elif path_tuple[0].strip() == "Map Path":
+            preamble = os.path.dirname(__file__)
+            if path_tuple[1].strip() == "absolute":
+                preamble = ""
+                path_tuple[2] = path_tuple[2].replace("$HOME", os.getenv("HOME"))
+            map_path = os.path.join(preamble, path_tuple[2].strip())
+        elif path_tuple[0].strip() == "User Path":
+            if path_tuple[1].strip() == "absolute":
+                raise RuntimeError("User Path must be a relative path to the users $HOME repo.")
+            user_path = path_tuple[2].strip()
+        elif path_tuple[0].strip() == "Down Scoping":
+            if path_tuple[1].strip() == "local":
+                down_scoping = False
+    #print(key_path)
+    #print(map_path)
+    #print(runner_path)
+    #print(user_path)
+    return {"key_path": key_path, "map_path": map_path, "runner_path": runner_path, "user_path": user_path, "down_scoping": down_scoping}
\ No newline at end of file

JSONManager.py

+import time
+import json
+import rsa
+import os
+
+def load_priv_key(path):
+    path = os.path.join(os.path.dirname(__file__), path)
+    with open(path, mode='rb') as privatefile:
+        keydata = privatefile.read()
+    return rsa.PrivateKey.load_pkcs1(keydata)
+
+
+def load_pub_key(path):
+    path = os.path.join(os.path.dirname(__file__), path)
+    with open(path, mode='rb') as pubfile:
+        keydata = pubfile.read()
+    return rsa.PublicKey.load_pkcs1(keydata)
+
+
+def create_keys():
+    (pubkey, privkey) = rsa.newkeys(2048)
+    with open("/home/ppl/Runner/id_rsa.pub", "w") as text_file:
+        text_file.write(pubkey.save_pkcs1().decode('ascii'))
+    with open("/home/ppl/Runner/id_rsa", "w") as text_file:
+        text_file.write(privkey.save_pkcs1().decode('ascii'))
+
+def get_account(url, pid, uid, key_path, map_path):
+    with open(map_path, mode='rb') as file:
+        data = file.read()
+    json_file = rsa.decrypt(data, load_priv_key(key_path))
+    search_file = json.loads(json_file)
+    instance = search_file[url]
+    result = instance["uid"][uid]
+    if result == None:
+        result = instance["pid"][pid]
+    if result == None:
+        print("Cannot assign GitLab user/project to cluster account. Please register here: TODO")
+        exit(1)
+    return result
+
+

JSONTest.py

+import JSONManager as man
+import json
+import rsa
+
+def create_testset():
+    dict = {"https://git-ce.rwth-aachen.de" : {"pid" : {}, "uid" : {"2076": "tester1"}}}
+    json_file = json.dumps(dict)
+    man.create_keys()
+    encrypted = rsa.encrypt(json_file.encode('ascii'), man.load_pub_key("id_rsa.pub"))
+    with open("/home/ppl/Runner/Assignments.txt", "wb") as text_file:
+        text_file.write(encrypted)

JWTManager.py

+import jwt
+import time
+
+def get_UID_PID(JWT, url):
+    jwks_client = jwt.PyJWKClient(url)
+    signing_key = jwks_client.get_signing_key_from_jwt(JWT)
+    # wait for token to be valid
+    time.sleep(2)
+    data = jwt.decode(JWT,
+        signing_key.key,
+        algorithms=["RS256"],
+        options={"verify_exp": False})
+    return data["user_id"], data["project_id"]

authmanager.py

+import os
+import subprocess
+import pwd
+import time
+import colorama
+
+
+def demote(user, uid, gid):
+    def demote_function():
+        print("starting")
+        print('uid, gid = %d, %d' % (os.getuid(), os.getgid()))
+        os.chdir(f"/home/{user}")
+        print(os.getgroups())
+        # initgroups must be run before we lose the privilege to set it!
+        os.initgroups(user, gid)
+        os.setgid(gid)
+        # this must be run last
+        os.setuid(uid)
+        print("finished demotion")
+        print('uid, gid = %d, %d' % (os.getuid(), os.getgid()))
+        print(os.getgroups())
+    return demote_function
+
+
+def get_user(user):
+    uid = pwd.getpwnam(user).pw_uid
+    gid = pwd.getpwnam(user).pw_gid
+    print('uid, gid = %d, %d' % (os.getuid(), os.getgid()))
+    print('User: uid, gid = %d, %d' % (uid, gid))
+    return uid, gid
+
+
+def run_task(user, cmd):
+    return_code = 0
+    err = ""
+    out = ""
+    print("requesting rights for user: " + user)
+    uid, gid = get_user(user)
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        preexec_fn=demote(user, uid, gid),
+        shell=True
+    )
+
+    # process can execute normally, no exceptions at startup
+    process_running = True
+    while process_running:
+        if proc.poll() is not None:
+            process_running = False
+        # half a second of resolution
+        time.sleep(0.5)
+    return_code = proc.returncode
+    out = proc.stderr.read()
+    err = proc.stdout.read()
+    colorama.init()
+    clean_print(err)
+    clean_print(out)
+
+def clean_print(string):
+    print(str(string)[2:-1].replace("\\n","\n"))

config.txt

+Runner Path: absolute: $HOME/Runner
+Key Path: relative: id_rsa
+Map Path: relative: Assignments.txt
+User Path: relative: Runner
+Down Scoping: local
\ No newline at end of file

driver.py

 import os
-from re import I
 import sys
-import subprocess
+import time
+
 import variableHandle as vh
+#import authmanager as auth
+import JWTManager as jwt
+import JSONManager as man
+import JSONTest as test
+import ConfigManager as conf
 import subprocess
+import stat
 
 import random
 import string
 
+
 def get_random_string(length):
     # choose from all lowercase letter
     letters = string.ascii_letters
     result_str = ''.join(random.choice(letters) for i in range(length))
     return result_str
 
+
 argv = sys.argv
 
-name = 'Costum_Driver'
-version = '0.0.5'
+name = 'Custom_Driver'
+version = '0.1.0'
+
+account = ""
+user_path = ""
+runner_path = ""
+down_scoping = True
+
 
 # generates the path to the build directory
 def get_build_path():
     CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID = os.getenv("CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID")
     CUSTOM_ENV_CI_PROJECT_PATH_SLUG = os.getenv("CUSTOM_ENV_CI_PROJECT_PATH_SLUG")
+    HOME = f"/home/{account}"
+    if not down_scoping:
         HOME = os.getenv("HOME")
-    return HOME + "/Runner/builds/" + CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID + "/" + CUSTOM_ENV_CI_PROJECT_PATH_SLUG
+    build_path = f'{HOME}/{user_path}/builds/{CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID}/{CUSTOM_ENV_CI_PROJECT_PATH_SLUG}'
+    return str(build_path)
 
 
 def get_cache_path():
     CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID = os.getenv("CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID")
     CUSTOM_ENV_CI_PROJECT_PATH_SLUG = os.getenv("CUSTOM_ENV_CI_PROJECT_PATH_SLUG")
+    HOME = f"/home/{account}"
+    if not down_scoping:
         HOME = os.getenv("HOME")
-    return HOME + "/Runner/cache/" + CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID + "/" + CUSTOM_ENV_CI_PROJECT_PATH_SLUG
+    cache_path = f'{HOME}/{user_path}/cache/{CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID}/{CUSTOM_ENV_CI_PROJECT_PATH_SLUG}'
+    return str(cache_path)
+
 
-# generates the path to the cloned repository
+# generates the path to the cloned repository with regards to the build directory
 def get_clone_path():
     clone_path = os.getenv('CUSTOM_ENV_CI_PROJECT_DIR')
     return clone_path
 
+
 def handle():
+    global user_path
+    global runner_path
+    global account
+    global down_scoping
+
+    # read config file
+    config = conf.read_config()
+    user_path = config["user_path"]
+    key_path = config["key_path"]
+    runner_path = config["runner_path"]
+    map_path = config["map_path"]
+    down_scoping = config["down_scoping"]
+
+    if down_scoping:
+        #test.create_testset()
+        token = os.getenv('CUSTOM_ENV_CI_JOB_JWT')
+        url = os.getenv('CUSTOM_ENV_CI_SERVER_URL')
+
+        # get uid and pid from JWT
+        uid, pid = jwt.get_UID_PID(token, f"{url}/-/jwks")
+
+        # get account from mapping file
+        account = man.get_account(url, pid, uid, key_path, map_path)
+
+        if account is None:
+            print(f"Error: no mapping for GitLab project: {os.getenv('CUSTOM_ENV_CI_PROJECT_NAME')}, or GitLab user: {os.getenv('CUSTOM_ENV_GITLAB_USER_NAME')} available. Please register CI support for to acces the Runner")
+
     if len(argv) < 2:
         print("Error: no argument")
         exit(1)
-    if argv[1] == 'config':
-        HOME = os.getenv("HOME")
+    if argv[1] == 'config':  # Do not use print in this step
 
-        os.system('mkdir -p ' + HOME + '/Runner/scripts')
-        os.system('mkdir -p ' + HOME + '/Runner/errorCodes')
-        os.system('chmod +x ' + HOME + '/Runner/sshRunstep.sh')
-        os.system('chmod +x ' + HOME + '/Runner/singularityRunstep.sh')
-        os.system('dos2unix ' + HOME + '/Runner/sshRunstep.sh')
-        os.system('dos2unix ' + HOME + '/Runner/singularityRunstep.sh')
+        os.system(f'mkdir -p {runner_path}/scripts')
+        os.system(f'mkdir -p {runner_path}/errorCodes')
+        os.system(f'chmod +x {runner_path}/sshRunstep.sh')
+        os.system(f'chmod +x {runner_path}/singularityRunstep.sh')
+        os.system(f'dos2unix {runner_path}/sshRunstep.sh')
+        os.system(f'dos2unix {runner_path}/singularityRunstep.sh')
 
         handle_config(get_build_path(), get_cache_path(), name, version)
     elif argv[1] == 'prepare':
@@ -62,6 +110,7 @@ def handle():
     else:
         print('Error')
 
+
 def handle_config(build_dir, cache_dir, driver_name, driver_version):
     builder = "{ \"builds_dir\": \""
     builder += build_dir
@@ -73,64 +122,101 @@ def handle_config(build_dir, cache_dir, driver_name, driver_version):
     builder += driver_version
     builder += "\"  },  \"job_env\" : {    \"CUSTOM_ENVIRONMENT\": \"example\"  }}"
 
+    # print(builder, file=sys.stderr)
     print(builder)
 
+
 def handle_prepare():
-    #os.system('module avail')
-    #os.system('module list')
     os.system('hostname')
-    #os.system('echo CUSTOM_ENV_CI_PROJECT_URL:')
-    #print(os.getenv('CUSTOM_ENV_CI_PROJECT_URL'))
+    print(os.getenv('CUSTOM_ENV_CI_PROJECT_PATH'))
+    print(os.getenv('CUSTOM_ENV_GITLAB_USER_NAME'))
 
 
 def handle_run():
+    # set user $HOME
     HOME = os.getenv("HOME")
-    #os.system('hostname')
-    #os.system('module list')
-    #print(argv[3])
-    #sys.stderr.write("Failure\n")
-    #exit(21)
+
+    #Setup CI scripts
     script_hash = get_random_string(8)
-    os.system('cp ' + argv[2] + ' ' + HOME + '/Runner/scripts/script' + script_hash)
-    os.system('chmod +x ' + HOME + '/Runner/scripts/script' + script_hash)
+    os.system(f'cp {argv[2]} {runner_path}/scripts/script{script_hash}')
+    os.system(f'chmod +x {runner_path}/scripts/script{script_hash}')
+    os.system(f'dos2unix {runner_path}/scripts/script{script_hash}')
 
     mode, container, script = vh.get_CI_mode()
+    command_wrapper_ds = []
+    exec_command = []
 
-    if  argv[3] == 'build_script' or argv[3] == 'step_script':
-        Slurm_vars = vh.get_slurm_variables()
-        command = []
+    # Handle different modes
+    if mode == 'local': #Debugging mode
+        print("local mode only for development.")
+        exit(1)
+        os.system(f"chmod -R 777 {runner_path}")
+        # auth.run_task(USER, f'{runner_path}/scripts/script{script_hash}  {argv[3]}')
+        command_wrapper_ds = f"sudo su --shell /bin/bash --login {account} -c".split()
+        exec_command = f"{runner_path}/scripts/script{script_hash} {argv[3]}"
+        vh.set_slurm_env()
 
-        if mode == 'Batch':
-            command += ['sbatch', '--wait', f'{get_clone_path()}/{script}']
+    elif argv[3] == 'build_script' or argv[3] == 'step_script':
+        Slurm_vars = vh.get_slurm_variables()
+        exec_command = []
+
+        if mode == 'Batch': # Handle Batch scripts
+            # Parse parameters from Batchscript
+            file = open(f'{get_clone_path()}/{script}', 'r')
+            batch_parameters = []
+            for line in file.readlines():
+                if line.startswith('#SBATCH'):
+                    batch_parameters.append(line.split()[1])
+            file.close()
+            #Define Batchscript run
+            exec_command += ['srun'] + batch_parameters + [f'{get_clone_path()}/{script}']
             print('Warning: The contents of the script section in the CI definition '
                   'will be ignored in the batch mode. If you want to work on the results '
                   'please create additional stages and connect them via artifacts.')
         else:
-            command += ['srun', '--job-name=CI']
+            # Define Slurm parameters
+            exec_command += ['srun', '--job-name=CI']
             for x in Slurm_vars:
-                command += [f'{x[0]}{x[1]}']
+                exec_command += [f'{x[0]}{x[1]}']
 
             # Handle Slurm shell and singularity shell environment 
             if mode == "Slurm":
-                command += [f'{HOME}/Runner/scripts/script{script_hash}', 'step_script']
+                exec_command += [f'{runner_path}/scripts/script{script_hash}', 'step_script']
             elif mode == "Singularity":
                 if os.path.exists(container):
                     container = f'{get_clone_path()}/{script}'
-                    command += [f'{HOME}/Runner/singularityLocalRunstep.sh',
+                    exec_command += [f'{HOME}/Runner/singularityLocalRunstep.sh',
                                 f'{get_clone_path()}/{container}', script_hash]
                 else:
-                    command += [f'{HOME}/Runner/singularityRunstep.sh', container, script_hash]
+                    exec_command += [f'{HOME}/Runner/singularityRunstep.sh', container, script_hash]
 
+        exec_command = ' '.join(exec_command)
+        #print(exec_command)
+        command_wrapper_ds = f"sudo su --shell /bin/bash --login {account} -c ".split()
+
+    else: #run small scripts on local machine
+        command_wrapper_ds = f"sudo su --shell /bin/bash --login {account} -c ".split()
+        exec_command = f'{runner_path}/scripts/script{script_hash} {argv[3]}'
+
+    command_wrapper_ds.append(f"{exec_command}")
+
+    # check for downscoping
+    command = command_wrapper_ds
+    if not down_scoping:
+        command = exec_command.split()
+
+    # Run command
     print(command)
     cmd_ret = subprocess.run(command)
-        os.remove(HOME + '/Runner/scripts/script' + script_hash)
-        if int(cmd_ret.returncode) != 0:
+    return_code = cmd_ret.returncode
+    os.remove(f'{runner_path}/scripts/script{script_hash}')
+
+    if int(return_code) != 0:
         exit(1)
     else:
         exit(0)
 
-    else:
-        os.system('. ' + '$HOME/Runner/scripts/script' + script_hash + ' ' + argv[3])
+
 
 def handle_cleanup():
     os.system("echo cleanup")

variableHandle.py

 import os
+import subprocess
 from posixpath import split
 
 # Gather Slurm job parameters
@@ -52,9 +53,9 @@ def get_CI_mode():
     if mode == None:
         mode = 'Slurm'
     if mode != 'Slurm' and mode != 'Singularity' and mode != 'Batch':
-        print("Error: only modes Slurm and Singularity are supported!")
+        print("Error: only modes Batch, Slurm and Singularity are supported!")
         os.system('echo mode: ' + mode)
-        exit(1)
+        #exit(1)
 
     # get container for singularity
     container = os.getenv('CUSTOM_ENV_CONTAINER')
@@ -86,4 +87,13 @@ def get_num_nodes():
         num_nodes = "1"
     return num_nodes
 
-
+def set_slurm_env():
+    res = ""
+    proc = subprocess.run('env', stdout=subprocess.PIPE)
+    for variable in proc.stdout.decode().splitlines():
+        if variable.startswith("CUSTOM_ENV_SLURM_ENV_"):
+            value = variable.split("=")
+            value[0] = value[0].replace("CUSTOM_ENV_SLURM_ENV_", "")
+            os.putenv(value[0], value[1])
+            #res = f"{res}export {value[0]}={value[1]}; "
+    #return res
\ No newline at end of file