diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fbf3660a9c31249577a2aa57c3573ff287235c6e..c9fa4e96993adee7ff993156a61848a6ea5a5c96 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -29,7 +29,7 @@ fail-exit-code-job-Singularity: # This job runs in the build stage, which
CI_MODE: "Singularity"
CONTAINER: "tensorflow"
tags:
- - "custom"
+ - "custom2"
script:
- cd rewagha
@@ -41,7 +41,7 @@ fail-timeout-job-Singularity: # This job runs in the build stage, which ru
CI_MODE: "Singularity"
CONTAINER: "tensorflow"
tags:
- - "custom"
+ - "custom2"
script:
- echo "Compiling the code..."
- sleep 1200
@@ -51,7 +51,7 @@ fail-exit-code-job: # This job runs in the build stage, which runs first.
variables:
SLURM_CPUS_PER_TASK: "1"
tags:
- - "custom"
+ - "custom2"
script:
- cd rewagha
@@ -61,7 +61,7 @@ fail-timeout-job: # This job runs in the build stage, which runs first.
SLURM_CPUS_PER_TASK: "1"
SLURM_TIME: "00:01:00"
tags:
- - "custom"
+ - "custom2"
script:
- echo "Compiling the code..."
- sleep 1200
@@ -71,7 +71,7 @@ build-job: # This job runs in the build stage, which runs first.
variables:
SLURM_CPUS_PER_TASK: "2"
tags:
- - "custom"
+ - "custom2"
script:
- echo "Compiling the code..."
- echo "Compile complete."
@@ -83,7 +83,7 @@ build-job-Singularity: # This job runs in the build stage, which runs firs
CI_MODE: "Singularity"
CONTAINER: "tensorflow"
tags:
- - "custom"
+ - "custom2"
script:
- echo "Compiling the code..."
- echo "Compile complete."
@@ -94,7 +94,7 @@ batch-job: # This job runs in the build stage, which runs first.
CI_MODE: "Batch"
BATCH_SCRIPT: "batch.sh"
tags:
- - "custom"
+ - "custom2"
script:
- echo "I do nothing"
@@ -104,7 +104,7 @@ fail-batch-job: # This job runs in the build stage, which runs first.
CI_MODE: "Batch"
BATCH_SCRIPT: "doesntexist.sh"
tags:
- - "custom"
+ - "custom2"
script:
- echo "I do nothing"
@@ -113,7 +113,7 @@ unit-test-job: # This job runs in the test stage.
variables:
SLURM_CPUS_PER_TASK: "4"
tags:
- - "custom"
+ - "custom2"
script:
- echo "Running unit tests... This will take about 60 seconds."
- sleep 60
@@ -124,7 +124,7 @@ lint-test-job: # This job also runs in the test stage.
variables:
SLURM_CPUS_PER_TASK: "8"
tags:
- - "custom"
+ - "custom2"
script:
- echo "Linting code... This will take about 10 seconds."
- sleep 10
@@ -133,7 +133,7 @@ lint-test-job: # This job also runs in the test stage.
deploy-job: # This job runs in the deploy stage.
stage: deploy # It only runs when *both* jobs in the test stage complete successfully.
tags:
- - "custom"
+ - "custom2"
script:
- echo "Deploying application..."
- echo "Application successfully deployed."
diff --git a/batch.sh b/batch.sh
old mode 100644
new mode 100755
diff --git a/batchRunstep.sh b/batchRunstep.sh
old mode 100644
new mode 100755
diff --git a/driver.py b/driver.py
index 96c37535cf69737c7dc8a5408dda1dd288bb832c..486869a45a781a0674993e3b1e71adfcdfb28af9 100644
--- a/driver.py
+++ b/driver.py
@@ -3,6 +3,7 @@ from re import I
import sys
import subprocess
import variableHandle as vh
+import subprocess
import random
import string
@@ -97,37 +98,36 @@ def handle_run():
if argv[3] == 'build_script' or argv[3] == 'step_script':
Slurm_vars = vh.get_slurm_variables()
+ command = []
if mode == 'Batch':
- full_script_path = get_clone_path() + '/' + script
- print(full_script_path)
- os.system('. '+ HOME + '/Runner/batchRunstep.sh ' + full_script_path + ' ' + script_hash)
- print('Warning: The contents of the script section in the CI definition will be ignored in the batch mode. If you want to work on the results please create additional stages and connect them via artifacts.')
+ command += ['sbatch', f'{get_clone_path()}/{script}']
+ print('Warning: The contents of the script section in the CI definition '
+ 'will be ignored in the batch mode. If you want to work on the results '
+ 'please create additional stages and connect them via artifacts.')
else:
- command = 'salloc --job-name=CI'
+ command += ['srun', '--job-name=CI']
for x in Slurm_vars:
- command = command + ' ' + x[0] + x[1]
+ command += [f'{x[0]}{x[1]}']
# Handle Slurm shell and singularity shell environment
if mode == "Slurm":
- command = command + ' ' + HOME + '/Runner/sshRunstep.sh ' + script_hash
+ command += [f'{HOME}/Runner/scripts/script{script_hash}', 'step_script']
elif mode =="Singularity":
if os.path.exists(container):
- container = get_clone_path() + '/' + script
- command = command + ' ' + HOME + '/Runner/singularityLocalRunstep.sh ' + get_clone_path() + '/' + container + ' ' + script_hash
+ container = f'{get_clone_path()}/{script}'
+ command += [f'{HOME}/Runner/singularityLocalRunstep.sh',
+ f'{get_clone_path()}/{container}', script_hash]
else:
- command = command + ' ' + HOME + '/Runner/singularityRunstep.sh ' + container + ' ' + script_hash
- print(command)
- os.system(command)
-
- file1 = open(HOME + '/Runner/errorCodes/' + script_hash + '.txt', 'r')
- Lines = file1.readlines()
- error_code = Lines[0].rstrip('\n') # $HOME/Runner/errorCodes/$1.txt first line
- print(error_code)
- file1.close()
+ command += [f'{HOME}/Runner/singularityRunstep.sh', container, script_hash]
+
+ print(command)
+ cmd_ret = subprocess.run(command)
os.remove(HOME + '/Runner/scripts/script' + script_hash)
- os.remove(HOME + '/Runner/errorCodes/' + script_hash + '.txt')
- exit(int(error_code))
+ if int(cmd_ret.returncode) != 0:
+ exit(1)
+ else:
+ exit(0)
else:
os.system('. ' + '$HOME/Runner/scripts/script' + script_hash + ' ' + argv[3])
diff --git a/main.sh b/main.sh
old mode 100644
new mode 100755
index f54ad61dc1466aec601c15ef7c5dcfb5f448b5da..57f64f1248a1ca1aaaf0dd5b1e35008acf614d51
--- a/main.sh
+++ b/main.sh
@@ -2,4 +2,4 @@
module load python
-python $HOME/Runner/driver.py $@
\ No newline at end of file
+python3 $HOME/Runner/driver.py $@
diff --git a/singularityLocalRunstep.sh b/singularityLocalRunstep.sh
old mode 100644
new mode 100755
index 42ba8cfbaad294d248621bd88f92a006757d192f..7813217c90812db1969af8992c1bf5b56440d31e
--- a/singularityLocalRunstep.sh
+++ b/singularityLocalRunstep.sh
@@ -5,23 +5,10 @@ then
exit 1
fi
-NODES=($SLURM_JOB_NODELIST)
-
-# starting node
-RUNNODE=(${NODES[@]:0:1})
-
-#echo 'target node: '
-
-#echo $RUNNODE
-
#echo "singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 "
module load CONTAINERS
dos2unix $HOME/Runner/scripts/script$2
-touch $HOME/Runner/errorCodes/$2.txt
-
-ssh -T $RUNNODE singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 step_script
-
-echo $? > $HOME/Runner/errorCodes/$2.txt
\ No newline at end of file
+singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 step_script
diff --git a/singularityRunstep.sh b/singularityRunstep.sh
old mode 100644
new mode 100755
index e53d6d21f13998ea29f0186ab3d6f976afedd478..a82d87ed64b5495c3e2952530046af53ffead26a
--- a/singularityRunstep.sh
+++ b/singularityRunstep.sh
@@ -5,15 +5,6 @@ then
exit 1
fi
-NODES=($SLURM_JOB_NODELIST)
-
-# starting node
-RUNNODE=(${NODES[@]:0:1})
-
-#echo 'target node: '
-
-#echo $RUNNODE
-
#echo "singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 "
module load CONTAINERS
@@ -22,8 +13,4 @@ module load $1
dos2unix $HOME/Runner/scripts/script$2
-touch $HOME/Runner/errorCodes/$2.txt
-
-ssh -T $RUNNODE singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 step_script
-
-echo $? > $HOME/Runner/errorCodes/$2.txt
\ No newline at end of file
+singularity exec --nv $R_CONTAINER $HOME/Runner/scripts/script$2 step_script
diff --git a/sshRunstep.sh b/sshRunstep.sh
old mode 100644
new mode 100755