Merge branch 'srun-driver' into 'main'

Use srun instead of salloc+ssh

See merge request ci-playground/customdriver!3

.gitlab-ci.yml

@@ -29,7 +29,7 @@ fail-exit-code-job-Singularity:       # This job runs in the build stage, which
     CI_MODE: "Singularity"
     CONTAINER: "tensorflow"
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - cd rewagha
 
@@ -41,7 +41,7 @@ fail-timeout-job-Singularity:       # This job runs in the build stage, which ru
     CI_MODE: "Singularity"
     CONTAINER: "tensorflow"
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - echo "Compiling the code..."
     - sleep 1200
@@ -51,7 +51,7 @@ fail-exit-code-job:       # This job runs in the build stage, which runs first.
   variables:
     SLURM_CPUS_PER_TASK: "1" 
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - cd rewagha
 
@@ -61,7 +61,7 @@ fail-timeout-job:       # This job runs in the build stage, which runs first.
     SLURM_CPUS_PER_TASK: "1"
     SLURM_TIME: "00:01:00"   
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - echo "Compiling the code..."
     - sleep 1200
@@ -71,7 +71,7 @@ build-job:       # This job runs in the build stage, which runs first.
   variables:
     SLURM_CPUS_PER_TASK: "2"
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - echo "Compiling the code..."
     - echo "Compile complete."
@@ -83,7 +83,7 @@ build-job-Singularity:       # This job runs in the build stage, which runs firs
     CI_MODE: "Singularity"
     CONTAINER: "tensorflow"
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - echo "Compiling the code..."
     - echo "Compile complete."
@@ -94,7 +94,7 @@ batch-job:       # This job runs in the build stage, which runs first.
     CI_MODE: "Batch"
     BATCH_SCRIPT: "batch.sh"
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - echo "I do nothing"
 
@@ -104,7 +104,7 @@ fail-batch-job:       # This job runs in the build stage, which runs first.
     CI_MODE: "Batch"
     BATCH_SCRIPT: "doesntexist.sh"
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - echo "I do nothing"
 
@@ -113,7 +113,7 @@ unit-test-job:   # This job runs in the test stage.
   variables:
     SLURM_CPUS_PER_TASK: "4"
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - echo "Running unit tests... This will take about 60 seconds."
     - sleep 60
@@ -124,7 +124,7 @@ lint-test-job:   # This job also runs in the test stage.
   variables:
     SLURM_CPUS_PER_TASK: "8"
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - echo "Linting code... This will take about 10 seconds."
     - sleep 10
@@ -133,7 +133,7 @@ lint-test-job:   # This job also runs in the test stage.
 deploy-job:      # This job runs in the deploy stage.
   stage: deploy  # It only runs when *both* jobs in the test stage complete successfully.
   tags:
-    - "custom" 
+    - "custom2" 
   script:
     - echo "Deploying application..."
     - echo "Application successfully deployed."

driver.py

@@ -3,6 +3,7 @@ from re import I
 import sys
 import subprocess
 import variableHandle as vh
+import subprocess
 
 import random
 import string
@@ -97,37 +98,36 @@ def handle_run():
 
     if  argv[3] == 'build_script' or argv[3] == 'step_script':
         Slurm_vars = vh.get_slurm_variables()
+        command = []
 
         if mode == 'Batch':
-            full_script_path = get_clone_path() + '/' + script
-            print(full_script_path)
-            os.system('. '+ HOME + '/Runner/batchRunstep.sh ' + full_script_path + ' ' + script_hash)
-            print('Warning: The contents of the script section in the CI definition will be ignored in the batch mode. If you want to work on the results please create additional stages and connect them via artifacts.')
+            command += ['sbatch', f'{get_clone_path()}/{script}']
+            print('Warning: The contents of the script section in the CI definition '
+                  'will be ignored in the batch mode. If you want to work on the results ' 
+                  'please create additional stages and connect them via artifacts.')
         else:
-            command = 'salloc --job-name=CI'
+            command += ['srun', '--job-name=CI']
             for x in Slurm_vars:
-                command = command + ' ' + x[0] + x[1]
+                command += [f'{x[0]}{x[1]}']
         
             # Handle Slurm shell and singularity shell environment 
             if mode == "Slurm":
-                command = command + ' ' + HOME + '/Runner/sshRunstep.sh ' + script_hash
+                command += [f'{HOME}/Runner/scripts/script{script_hash}', 'step_script']
             elif mode =="Singularity":
                 if os.path.exists(container):
-                    container = get_clone_path() + '/' + script
-                    command = command + ' ' + HOME + '/Runner/singularityLocalRunstep.sh ' + get_clone_path() + '/' + container + ' ' + script_hash
+                    container = f'{get_clone_path()}/{script}'
+                    command += [f'{HOME}/Runner/singularityLocalRunstep.sh',
+                                f'{get_clone_path()}/{container}', script_hash]
                 else:
-                    command = command + ' ' + HOME + '/Runner/singularityRunstep.sh ' + container + ' ' + script_hash
-            print(command)
-            os.system(command)
+                    command += [f'{HOME}/Runner/singularityRunstep.sh', container, script_hash]
 
-        file1 = open(HOME + '/Runner/errorCodes/' + script_hash + '.txt', 'r')
-        Lines = file1.readlines()
-        error_code = Lines[0].rstrip('\n') # $HOME/Runner/errorCodes/$1.txt first line
-        print(error_code)
-        file1.close()
+        print(command)
+        cmd_ret = subprocess.run(command)
         os.remove(HOME + '/Runner/scripts/script' + script_hash)
-        os.remove(HOME + '/Runner/errorCodes/' + script_hash + '.txt')
-        exit(int(error_code))
+        if int(cmd_ret.returncode) != 0:
+            exit(1)
+        else:
+            exit(0)
 
     else:
         os.system('. ' + '$HOME/Runner/scripts/script' + script_hash + ' ' + argv[3])

main.sh

@@ -2,4 +2,4 @@
 
 module load python
 
-python $HOME/Runner/driver.py $@
+python3 $HOME/Runner/driver.py $@
\ No newline at end of file

singularityLocalRunstep.sh

@@ -5,23 +5,10 @@ then
     exit 1
 fi
 
-NODES=($SLURM_JOB_NODELIST)
-
-# starting node
-RUNNODE=(${NODES[@]:0:1})
-
-#echo 'target node: '
-
-#echo $RUNNODE
-
 #echo "singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 "
 
 module load CONTAINERS
 
 dos2unix $HOME/Runner/scripts/script$2
 
-touch $HOME/Runner/errorCodes/$2.txt
-
-ssh -T $RUNNODE singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 step_script
-
-echo $? > $HOME/Runner/errorCodes/$2.txt
+singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 step_script
\ No newline at end of file

singularityRunstep.sh

@@ -5,15 +5,6 @@ then
     exit 1
 fi
 
-NODES=($SLURM_JOB_NODELIST)
-
-# starting node
-RUNNODE=(${NODES[@]:0:1})
-
-#echo 'target node: '
-
-#echo $RUNNODE
-
 #echo "singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 "
 
 module load CONTAINERS
@@ -22,8 +13,4 @@ module load $1
 
 dos2unix $HOME/Runner/scripts/script$2
 
-touch $HOME/Runner/errorCodes/$2.txt
-
-ssh -T $RUNNODE singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 step_script
-
-echo $? > $HOME/Runner/errorCodes/$2.txt
+singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 step_script
\ No newline at end of file