Skip to content
Snippets Groups Projects
Commit a5097aae authored by Felix Tomski's avatar Felix Tomski
Browse files

Use xPipeHelper instead of pipeHelper

Fixes issues with PATH in ssh mode.
parent 7eaa1e14
No related branches found
No related tags found
No related merge requests found
...@@ -261,10 +261,12 @@ class Job: ...@@ -261,10 +261,12 @@ class Job:
run_properties = f'sudo -u {self.account} {wrapper} {run_properties}' run_properties = f'sudo -u {self.account} {wrapper} {run_properties}'
os.chdir("/work") os.chdir("/work")
command = [helper_script] command = [helper_script]
if run_properties == '': if run_properties:
command.extend([run_script]) command.extend([run_properties])
else: setup_env_scripts = self._mode.get_env_setup()
command.extend([run_properties, run_script]) if setup_env_scripts:
command.extend(setup_env_scripts.split())
command.extend(run_script.split(' '))
if script_execution: if script_execution:
self._mode.custom_run_setup(install_env=install_env, skip_env=skip_env, script_execution=script_execution, self._mode.custom_run_setup(install_env=install_env, skip_env=skip_env, script_execution=script_execution,
do_inbetween_processing=do_inbetween_processing, srun_wrap=srun_wrap, **kwargs) do_inbetween_processing=do_inbetween_processing, srun_wrap=srun_wrap, **kwargs)
...@@ -300,7 +302,7 @@ class Job: ...@@ -300,7 +302,7 @@ class Job:
utility.update_json_kv(self.error_code_file, self.jobid, -1) utility.update_json_kv(self.error_code_file, self.jobid, -1)
self.exec_script = self.args[2] self.exec_script = self.args[2]
command = f"{self.driver_path}/core/scripts/pipeHelper.sh" command = f"{self.driver_path}/core/scripts/xPipeHelper.sh"
run_properties = '' run_properties = ''
do_inbetween_processing = False do_inbetween_processing = False
if self.args[3] not in ['build_script', 'step_script']: if self.args[3] not in ['build_script', 'step_script']:
...@@ -315,7 +317,7 @@ class Job: ...@@ -315,7 +317,7 @@ class Job:
do_inbetween_processing=do_inbetween_processing) do_inbetween_processing=do_inbetween_processing)
if self.args[3] in ['build_script', 'step_script'] and self._mode.has_post_run_script(): if self.args[3] in ['build_script', 'step_script'] and self._mode.has_post_run_script():
command = f"{self.driver_path}/core/scripts/pipeHelper.sh" command = f"{self.driver_path}/core/scripts/xPipeHelper.sh"
run_properties = self._mode.get_post_run_properties() run_properties = self._mode.get_post_run_properties()
run_script = self._mode.get_post_run_script() run_script = self._mode.get_post_run_script()
self.execute(run_properties, run_script, command, script_execution=True) self.execute(run_properties, run_script, command, script_execution=True)
......
...@@ -206,13 +206,13 @@ class Slurm(ModeBase): ...@@ -206,13 +206,13 @@ class Slurm(ModeBase):
self.job.execute(f'/usr/bin/cp /dev/stdin ' self.job.execute(f'/usr/bin/cp /dev/stdin '
f'{self.job.user_path}/wrapper{self.job.jobid}.sh', f'{self.job.user_path}/wrapper{self.job.jobid}.sh',
f"{self.job.driver_path}/core/scripts/zshWrapper.sh", f"{self.job.driver_path}/core/scripts/zshWrapper.sh",
f"{self.job.driver_path}/core/scripts/pipeHelper.sh", f"{self.job.driver_path}/core/scripts/xPipeHelper.sh",
skip_env=True, srun_wrap=True) skip_env=True, srun_wrap=True)
git_runner_command = [f'{self.job.shell_path} -l', git_runner_command = [f'{self.job.shell_path} -l',
f"{self.job.user_path}/wrapper{self.job.jobid}.sh"] f"{self.job.user_path}/wrapper{self.job.jobid}.sh"]
self.job.execute(' '.join(git_runner_command), self.job.execute(' '.join(git_runner_command),
f'{self.job.driver_path}/core/scripts/runnerInstallHelper.sh', f'{self.job.driver_path}/core/scripts/runnerInstallHelper.sh',
f"{self.job.driver_path}/core/scripts/pipeHelper.sh", srun_wrap=True, install_env=True) f"{self.job.driver_path}/core/scripts/xPipeHelper.sh", srun_wrap=True, install_env=True)
def get_simple_script_exec(self): def get_simple_script_exec(self):
return f"{srun_path} --jobid={self.slurm_simple_job_id} {self.job.shell_path} -l " \ return f"{srun_path} --jobid={self.slurm_simple_job_id} {self.job.shell_path} -l " \
...@@ -228,7 +228,7 @@ class Slurm(ModeBase): ...@@ -228,7 +228,7 @@ class Slurm(ModeBase):
return export_env[:-1] return export_env[:-1]
def get_combiner_script(self): def get_combiner_script(self):
self._combiner_script = f"{self.job.driver_path}/core/scripts/pipeHelper.sh" self._combiner_script = f"{self.job.driver_path}/core/scripts/xPipeHelper.sh"
return self._combiner_script return self._combiner_script
def set_slurm_env(self): def set_slurm_env(self):
...@@ -284,7 +284,7 @@ class Sbatch(Slurm): ...@@ -284,7 +284,7 @@ class Sbatch(Slurm):
return cmd_out return cmd_out
def get_combiner_script(self): def get_combiner_script(self):
self._combiner_script = f"{self.job.driver_path}/core/scripts/pipeHelper.sh" self._combiner_script = f"{self.job.driver_path}/core/scripts/xPipeHelper.sh"
return self._combiner_script return self._combiner_script
...@@ -310,7 +310,7 @@ class Sbatch(Slurm): ...@@ -310,7 +310,7 @@ class Sbatch(Slurm):
self.job.execute(f'/usr/bin/cp /dev/stdin ' self.job.execute(f'/usr/bin/cp /dev/stdin '
f'{self.job.build_path}/chmodPath{self.job.jobid}.sh', f'{self.job.build_path}/chmodPath{self.job.jobid}.sh',
f"{self.job.driver_path}/core/scripts/chmodPath.sh", f"{self.job.driver_path}/core/scripts/chmodPath.sh",
f"{self.job.driver_path}/core/scripts/pipeHelper.sh", f"{self.job.driver_path}/core/scripts/xPipeHelper.sh",
skip_env=True, srun_wrap=True) skip_env=True, srun_wrap=True)
self.job.execute(f'/usr/local_rwth/bin/zsh {self.job.build_path}/chmodPath{self.job.jobid}.sh', self.job.execute(f'/usr/local_rwth/bin/zsh {self.job.build_path}/chmodPath{self.job.jobid}.sh',
f'{self.slurm_output_dir}', f'{self.slurm_output_dir}',
...@@ -344,7 +344,7 @@ class Sbatch(Slurm): ...@@ -344,7 +344,7 @@ class Sbatch(Slurm):
class Default(Sbatch): class Default(Sbatch):
def get_combiner_script(self): def get_combiner_script(self):
self._combiner_script = f"{self.job.driver_path}/core/scripts/pipeHelper.sh" self._combiner_script = f"{self.job.driver_path}/core/scripts/xPipeHelper.sh"
return self._combiner_script return self._combiner_script
def get_run_properties(self): def get_run_properties(self):
...@@ -356,7 +356,7 @@ class Default(Sbatch): ...@@ -356,7 +356,7 @@ class Default(Sbatch):
self.job.execute(f'/usr/bin/cp /dev/stdin ' self.job.execute(f'/usr/bin/cp /dev/stdin '
f'{self.job.clone_path}/script.', f'{self.job.clone_path}/script.',
f"{Slurm.get_run_script(self)}", f"{Slurm.get_run_script(self)}",
f"{self.job.driver_path}/core/scripts/pipeHelper.sh", f"{self.job.driver_path}/core/scripts/xPipeHelper.sh",
skip_env=True, srun_wrap=True) skip_env=True, srun_wrap=True)
tmp = os.getenv("TMP") tmp = os.getenv("TMP")
with open(f'{self.job.driver_path}/core/scripts/batchWrapper.sh', 'r') as file: with open(f'{self.job.driver_path}/core/scripts/batchWrapper.sh', 'r') as file:
...@@ -442,17 +442,17 @@ class Singularity_Batch(Default): ...@@ -442,17 +442,17 @@ class Singularity_Batch(Default):
self.job.execute(f'/usr/bin/cp /dev/stdin ' self.job.execute(f'/usr/bin/cp /dev/stdin '
f'{self.slurm_output_dir}/batchEnv{self.job.jobid}', f'{self.slurm_output_dir}/batchEnv{self.job.jobid}',
f"{self.job.tmp_dir}/env{self.job.jobid}", f"{self.job.tmp_dir}/env{self.job.jobid}",
f"{self.job.driver_path}/core/scripts/pipeHelper.sh", f"{self.job.driver_path}/core/scripts/xPipeHelper.sh",
skip_env=True, srun_wrap=True) skip_env=True, srun_wrap=True)
def get_combiner_script(self): def get_combiner_script(self):
self._combiner_script = f"{self.job.driver_path}/core/scripts/pipeHelper.sh" self._combiner_script = f"{self.job.driver_path}/core/scripts/xPipeHelper.sh"
return self._combiner_script return self._combiner_script
def get_run_script(self): def get_run_script(self):
self.job.execute(f'/usr/bin/cp /dev/stdin ' self.job.execute(f'/usr/bin/cp /dev/stdin '
f'{self.job.clone_path}/script{self.job.jobid}.sh', f'{self.job.clone_path}/script{self.job.jobid}.sh',
f'{Slurm.get_run_script(self)}', f"{self.job.driver_path}/core/scripts/pipeHelper.sh", f'{Slurm.get_run_script(self)}', f"{self.job.driver_path}/core/scripts/xPipeHelper.sh",
skip_env=True, srun_wrap=True) skip_env=True, srun_wrap=True)
# move wrapper to user # move wrapper to user
tmp = os.getenv("TMP") tmp = os.getenv("TMP")
...@@ -474,7 +474,7 @@ class Singularity_Batch(Default): ...@@ -474,7 +474,7 @@ class Singularity_Batch(Default):
# Generation of the singularity script within user space # Generation of the singularity script within user space
self.job.execute(f'/usr/bin/cp /dev/stdin ' self.job.execute(f'/usr/bin/cp /dev/stdin '
f'{self.job.clone_path}/singularity{self.job.jobid}.sh', f'{self.job.clone_path}/singularity{self.job.jobid}.sh',
self.get_run_properties_postfix(), f"{self.job.driver_path}/core/scripts/pipeHelper.sh", self.get_run_properties_postfix(), f"{self.job.driver_path}/core/scripts/xPipeHelper.sh",
skip_env=True, srun_wrap=True) skip_env=True, srun_wrap=True)
# add singularity specific parameter to the properties # add singularity specific parameter to the properties
property_split = self._run_properties.split(" ") property_split = self._run_properties.split(" ")
...@@ -503,7 +503,7 @@ class Singularity(Slurm): ...@@ -503,7 +503,7 @@ class Singularity(Slurm):
container = '' container = ''
def get_combiner_script(self): def get_combiner_script(self):
self._combiner_script = f"{self.job.driver_path}/core/scripts/pipeHelper.sh" self._combiner_script = f"{self.job.driver_path}/core/scripts/xPipeHelper.sh"
return self._combiner_script return self._combiner_script
def __init__(self, job): def __init__(self, job):
...@@ -524,7 +524,7 @@ class Singularity(Slurm): ...@@ -524,7 +524,7 @@ class Singularity(Slurm):
# Generation of the singularity script within user space # Generation of the singularity script within user space
self.job.execute(f'/usr/bin/cp /dev/stdin ' self.job.execute(f'/usr/bin/cp /dev/stdin '
f'{self.job.user_path}/script{self.job.jobid}.sh', f'{self.job.user_path}/script{self.job.jobid}.sh',
self.get_run_properties_postfix(), f"{self.job.driver_path}/core/scripts/pipeHelper.sh", self.get_run_properties_postfix(), f"{self.job.driver_path}/core/scripts/xPipeHelper.sh",
skip_env=True, srun_wrap=True) skip_env=True, srun_wrap=True)
# add singularity specific parameter to the properties # add singularity specific parameter to the properties
property_split = self._run_properties.split(" ") property_split = self._run_properties.split(" ")
...@@ -558,7 +558,7 @@ class Singularity(Slurm): ...@@ -558,7 +558,7 @@ class Singularity(Slurm):
class SingleSlurmJobAcrossStages(Slurm): class SingleSlurmJobAcrossStages(Slurm):
def get_combiner_script(self): def get_combiner_script(self):
self._combiner_script = f"{self.job.driver_path}/core/scripts/pipeHelper.sh" self._combiner_script = f"{self.job.driver_path}/core/scripts/xPipeHelper.sh"
return self._combiner_script return self._combiner_script
def get_run_properties_postfix(self): def get_run_properties_postfix(self):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment