diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3af6cf4ecd130dd55ae9e822d75c113103894319..2df517d8ab63d9b3cc943cd706b00fd99a1b52c5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,11 +22,11 @@ variables:
.downscope-template:
variables:
- RUNNER_TAG: "downscope2"
+ RUNNER_TAG: "testing"
.local-template:
variables:
- RUNNER_TAG: "custom2"
+ RUNNER_TAG: "ja664344-dev"
.run-template:
stage: run
diff --git a/core/modes/slurm/batch.py b/core/modes/slurm/batch.py
index cd1ea3f65768dd2cefa447a803ae3b6904215a56..df3d64a98bcdf1f2ac7ae129e490ed9e1bc555a9 100644
--- a/core/modes/slurm/batch.py
+++ b/core/modes/slurm/batch.py
@@ -41,7 +41,7 @@ class Sbatch(Slurm, ABC):
self.executor.management_handler(helper_script=f"{self.job.scripts_path}/xPipeHelper.sh",
wrapper_add=f"/usr/bin/cp /dev/stdin "
f"{self.job.clone_path}/chmodPath{self.job.jobid}.sh",
- script=f"{self.job.runner_path}/core/scripts/chmodPath.sh")
+ script=f"{self.job.scripts_path}/chmodPath.sh")
self.executor.management_handler(helper_script=f"{self.job.scripts_path}/execHelper.sh",
wrapper_add=f"{self.job.shell_path} "
f"{self.job.clone_path}/chmodPath{self.job.jobid}.sh",
diff --git a/core/modes/ssh.py b/core/modes/ssh.py
index 9fea7bd834ec119c0a962723cfa934fcc2e4cfa9..b1e544e6f40851312b6c3e72384694165085dae8 100644
--- a/core/modes/ssh.py
+++ b/core/modes/ssh.py
@@ -1,5 +1,6 @@
from core.modes.common import *
from core.modes.base import ModeBase
+from core.utility.executor import SshExecutor
class SSH(ModeBase):
@@ -8,6 +9,7 @@ class SSH(ModeBase):
self.dest_node = get_cenv('CI_SSH_HOST')
if not self.dest_node:
ModeBase.abort(self, "Using ssh mode but no node specified. Specify: CI_SSH_HOST")
+ self.executor = SshExecutor(job, job.down_scoping)
def get_env_setup(self):
return f' {self.job.driver_path}/core/scripts/ssh.env '
@@ -24,8 +26,15 @@ class SSH(ModeBase):
self._combiner_script = f"{self.job.driver_path}/core/scripts/xPipeHelper.sh"
return self._combiner_script
- def get_simple_script_exec(self):
- return f"ssh -T {self.dest_node}"
+ def run_simple_script(self):
+ out = self.executor.management_handler(helper_script=f"{self.job.scripts_path}/xPipeHelper.sh",
+ params=f'-T {self.dest_node}',
+ script=self.get_simple_run_script())
+
+ def run_main_script(self):
+ out = self.executor.run_direct(params=f'-T {self.dest_node}', script=self.get_run_script())
+ print(out)
+
def cleanup(self):
ModeBase.cleanup(self)
diff --git a/core/utility/executor.py b/core/utility/executor.py
index fa352a8db77e147b73c2bd03d18a030611d8e3d8..f00ea573e651fa1b12bb67ae02ef32bd402fa511 100644
--- a/core/utility/executor.py
+++ b/core/utility/executor.py
@@ -10,52 +10,33 @@ def async_process(file):
class Executor(ABC):
downscope_add = ""
- simple_job_id = ""
-
def __init__(self, job, downscope=False):
self.job = job
if downscope:
self.downscope_add = f"sudo -u {self.job.account}"
- def set_simple_job_id(self, job_id):
- self.simple_job_id = job_id
-
- # Allocates a batch job with optional user specifications and returns the string
- @abstractmethod
- def allocator(self, params=""):
- pass
-
# Executes internal management functions, e.g., setup scripts etc.
@abstractmethod
def management_handler(self, params="", wrapper_add="", script=""):
pass
- # Cancels a batch job based on its id
- @abstractmethod
- def cancel(self, jobid):
- pass
-
# runs a script in the batch system with direct output
@abstractmethod
def run_direct(self, params="", wrapper_add="", script=""):
pass
- # runs a script without direct output, e.g., a batch script or multinode jobs
- @abstractmethod
- def run_batched(self, params="", wrapper_add="", script=""):
- pass
-
- def execute(self, helper_script='', allocator='', params='', wrapper_add='',
+ def execute(self, helper_script='', allocator='', params='', wrapper_add='', pre_exec_scripts=[],
target_script='', skip_env=False, run_async=False, main_script=False, install_env=False, **kwargs):
if main_script:
self.job.mode.custom_run_setup(main_script=main_script, run_async=run_async, **kwargs)
logging.info(f'Executing with env: {str(self.job.custom_env)}')
else:
- params += (f' --jobid={self.job.mode.slurm_simple_job_id} ' if self.job.mode.slurm_simple_job_id else ' ')
if install_env:
params += f' --export=CUSTOM_SHELL_CONFIG={self.job.shell_config}'
- command = [helper_script, f'{self.downscope_add} {allocator} {params} {wrapper_add}',
- f'{target_script}']
+ command = [helper_script]
+ command.extend([f'{self.downscope_add} {allocator} {params} {wrapper_add}'])
+ command.extend(pre_exec_scripts)
+ command.append(target_script)
logging.info(f'Executing command: {str(command)}')
os.chdir('/tmp')
main_proc = subprocess.Popen(command,
@@ -91,6 +72,7 @@ class Slurm_Executor(Executor, ABC):
sbatch_path = "sbatch" # "/usr/local_host/bin/sbatch"
salloc_path = "/opt/slurm/current/bin/salloc"
scancel_path = "scancel"
+ simple_job_id = ""
def _get_slurm_cmd(self, base):
add_args = ''
@@ -195,3 +177,30 @@ class Slurm_Executor(Executor, ABC):
main_script=True, run_async=True,
text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout
logging.debug(f'sbatch output: {sbatch_out}')
+
+
+class SshExecutor(Executor, ABC):
+ def __init__(self, job, downscope=False):
+ Executor.__init__(self, job, downscope=downscope)
+
+ def run_direct(self, params="", wrapper_add="", script=""):
+ self.execute(helper_script=f"{self.job.scripts_path}/xPipeHelper.sh",
+ allocator='ssh',
+ params=params,
+ target_script=script,
+ wrapper_add=wrapper_add, main_script=True)
+ return ''
+
+ def management_handler(self, helper_script="", params="", wrapper_add="", script="", install_env=False):
+ if helper_script == '':
+ helper_script = f"{self.job.scripts_path}/runHelper.sh"
+
+ management_out = self.execute(helper_script=helper_script,
+ allocator='ssh',
+ params=params,
+ wrapper_add=wrapper_add,
+ pre_exec_scripts=[f'{self.job.scripts_path}/ssh.env'],
+ target_script=script, install_env=install_env,
+ text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout
+ logging.debug(management_out)
+ return management_out
diff --git a/utility/.gitlab/.localTemplate.yml b/utility/.gitlab/.localTemplate.yml
index b2533bfa3f4bf1368fca0c7a0911fae4fe332f88..bec27cf88910dd4737898c8f39a1a9aeeeaa8f97 100644
--- a/utility/.gitlab/.localTemplate.yml
+++ b/utility/.gitlab/.localTemplate.yml
@@ -2,7 +2,7 @@
variables:
CI_MODE: "SSH"
-.ssh-build-job:
+ssh-build-job:
extends: .ssh-job
stage: build
script:
@@ -16,5 +16,5 @@
- path.env
parallel:
matrix:
- - CI_SSH_HOST: ['login18-1', 'login18-beta']
+ - CI_SSH_HOST: ['login18-1']
diff --git a/utility/.gitlab/.template.yml b/utility/.gitlab/.template.yml
index 687db5a8ffd956a395777fc6beacee2afd8e3dba..f7a50cece5ab1d8779ce53af6b9b848f4af9e39c 100644
--- a/utility/.gitlab/.template.yml
+++ b/utility/.gitlab/.template.yml
@@ -197,6 +197,7 @@ build-job: # This job runs in the build stage, which runs first.
variables:
SLURM_PARAM_CPUS: "-c 2"
script:
+ - module list
- echo "Compiling the code..."
- echo "Compile complete."
diff --git a/utility/.gitlab/batch.sh b/utility/.gitlab/batch.sh
index 7d0b762e41dfa7d54eaae45d6987bd1c3414c0a2..6ec20a87cb17068f7edc2277cb3f72d0a2ef7c19 100755
--- a/utility/.gitlab/batch.sh
+++ b/utility/.gitlab/batch.sh
@@ -16,5 +16,7 @@
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=2
+module list
+module load Python
echo 'Hello World'
diff --git a/utility/.gitlab/sbatch.sh b/utility/.gitlab/sbatch.sh
index b5ad31fddd583a9f37ed2db0fdac3d688f3c327e..8bd094cf9a896927e0a38870f457944fabe251a8 100755
--- a/utility/.gitlab/sbatch.sh
+++ b/utility/.gitlab/sbatch.sh
@@ -9,6 +9,8 @@
#SBATCH --nodes=1
#SBATCH --ntasks=1
+module list
+module load Python
for i in $(seq 20);
do