Skip to content
Snippets Groups Projects
Commit 4f48e7eb authored by Adrian Schmitz's avatar Adrian Schmitz
Browse files

merge fix

parent 6928c50f
No related branches found
No related tags found
No related merge requests found
......@@ -146,7 +146,8 @@ class Job:
encoding='utf-8',
level=defines.log_level)
logging.getLogger('filelock').setLevel(logging.INFO)
if not get_cenv('CI_LOG_STDOUT', 'False') in ['0', 'NULL', 'False', 'false', 'FALSE'] and defines.stdout_logging and self.args[1] != 'config':
if not get_cenv('CI_LOG_STDOUT', 'False') in ['0', 'NULL', 'False', 'false',
'FALSE'] and defines.stdout_logging and self.args[1] != 'config':
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
def __init__(self, args, driver_path):
......@@ -213,21 +214,16 @@ class Job:
logging.info(f'ci_mode: {get_cenv("CI_MODE")}')
logging.info(f'ci_job_id: {get_cenv("CI_JOB_ID")}')
def generate_sudo_env(self, skip_env=False):
if not skip_env and self.custom_env:
logging.debug(f'custom_env size = {self.custom_env.__sizeof__()}, len = {len(self.custom_env)}')
env_string = ' '
logging.debug(self.custom_env)
for x in self.custom_env:
env_string += x + "=" + self.custom_env[x] + ' '
return env_string + ' '
return ''
def get_parameters(self):
parameters = {k: v for k, v in os.environ.items() if k.startswith("CUSTOM_ENV_SLURM_PARAM")}
parameter_string = ''
for _, v in parameters.items():
parameter_string += f'{v} '
return parameter_string
def popen_wrapper(self, run_properties, run_script, helper_script, skip_env=False, **kwargs):
if self.down_scoping:
logging.info(f'custom_env is "{self.generate_sudo_env(skip_env=skip_env)}"')
run_properties = f'sudo {self.generate_sudo_env(skip_env=skip_env)}-u {self.account} ' + run_properties
run_properties = f'sudo -u {self.account} ' + run_properties
os.chdir("/work")
command = []
if helper_script:
......@@ -249,8 +245,8 @@ class Job:
else os.environ),
**kwargs)
def execute(self, run_properties, run_script, helper_script, install_env=False ,skip_env=False, script_execution=False, do_inbetween_processing=False, srun_wrap=False, **kwargs):
def execute(self, run_properties, run_script, helper_script, install_env=False, skip_env=False,
script_execution=False, do_inbetween_processing=False, srun_wrap=False, **kwargs):
if self.down_scoping:
wrapper = ''
if srun_wrap:
......@@ -258,14 +254,15 @@ class Job:
wrapper += (f' --jobid={self._mode.slurm_simple_job_id} ' if self._mode.slurm_simple_job_id else ' ')
if install_env:
wrapper += f' --export=CUSTOM_SHELL_CONFIG={self.shell_config}'
run_properties = f'sudo {self.generate_sudo_env(skip_env=skip_env)} -u {self.account} {wrapper} {run_properties}'
run_properties = f'sudo -u {self.account} {wrapper} {run_properties}'
os.chdir("/work")
command = [helper_script]
if run_properties == '':
command.extend([run_script])
else:
command.extend([run_properties, run_script])
self._mode.custom_run_setup(install_env=install_env ,skip_env=skip_env, script_execution=script_execution, do_inbetween_processing=do_inbetween_processing, srun_wrap=srun_wrap,**kwargs)
self._mode.custom_run_setup(install_env=install_env, skip_env=skip_env, script_execution=script_execution,
do_inbetween_processing=do_inbetween_processing, srun_wrap=srun_wrap, **kwargs)
logging.info(f'Executing with env: {str(self.custom_env)}')
logging.info(f'Executing command: {str(command)}')
os.chdir('/tmp')
......@@ -293,7 +290,6 @@ class Job:
# FIXME: do not rely on internal implementation of subprocess.run
return subprocess.CompletedProcess(main_proc.args, main_proc.returncode, stdout, stderr)
def run(self):
if not self.allow_failure:
self.error_code_file = f'{self.stage_tmp_dir}/{self.args[3]}.json'
......@@ -312,7 +308,8 @@ class Job:
run_properties += self._mode.get_run_properties()
run_script = self._mode.get_run_script()
do_inbetween_processing = True
self.execute(run_properties, run_script, command, script_execution=True, do_inbetween_processing=do_inbetween_processing)
self.execute(run_properties, run_script, command, script_execution=True,
do_inbetween_processing=do_inbetween_processing)
if self.args[3] in ['build_script', 'step_script'] and self._mode.has_post_run_script():
command = f"{self.driver_path}/core/scripts/pipeHelper.sh"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment