From ab374a080dbe4bc23a4db1d14c72c676b78e0e8d Mon Sep 17 00:00:00 2001 From: Adrian Schmitz <a.schmitz@itc.rwth-aachen.de> Date: Mon, 9 Jan 2023 10:31:34 +0100 Subject: [PATCH] merge fix Signed-off-by: Adrian Schmitz <a.schmitz@itc.rwth-aachen.de> --- core/modes/__init__.py | 41 ++++------------------------------------- 1 file changed, 4 insertions(+), 37 deletions(-) diff --git a/core/modes/__init__.py b/core/modes/__init__.py index 948c947..76ed01a 100644 --- a/core/modes/__init__.py +++ b/core/modes/__init__.py @@ -207,33 +207,7 @@ class Slurm(ModeBase): return self._combiner_script def set_slurm_env(self): - for variable in os.environ: - if variable.startswith("CUSTOM_ENV_SLURM_ENV_"): - temp = variable.replace("CUSTOM_ENV_SLURM_ENV_", "") - if any(temp.startswith(s) for s in self.considered_env_prefixes): - self._custom_env[temp] = os.environ[variable] - else: - ModeBase.abort(self, f"Error: Only Slurm environment variables allowed: {variable}") - elif (any(variable.startswith(s) for s in [f'CUSTOM_ENV_{s}_' - for s in self.considered_env_prefixes])): - self._custom_env[variable.replace('CUSTOM_ENV_', '')] = os.environ[variable] - - - def get_slurm_variables(self): - self.slurm_vars.append(f"--time={get_cenv('SLURM_TIME', '00:10:00')}") - cpus_per_task = get_cenv('SRUN_CPUS_PER_TASK') - if not cpus_per_task: - cpus_per_task = get_cenv('SLURM_ENV_SRUN_CPUS_PER_TASK', '1') - self.slurm_vars.append(f"--cpus-per-task={cpus_per_task}") - self.slurm_vars.append(f"--nodes={get_cenv('SLURM_NUM_NODES', '1')}") - - num_gpus = get_cenv('SLURM_NUM_GPUS') - if num_gpus is not None: - self.slurm_vars.append(f"--gres=gpu:{num_gpus}") - - partition = get_cenv('SLURM_PARTITION') - if partition is not None: - self.slurm_vars.append(f"--partition={partition}") + self._run_properties = self.job.get_parameters() def set_srun_cmd(self): prop_list = [f'{srun_path}', f'--job-name=CI_{self.job.jobid}'] @@ -282,7 +256,7 @@ class Sbatch(Slurm): def get_run_properties(self): - self._run_properties = f'{sbatch_path} --wait --output={self.slurm_output_file}' + self._run_properties = f'{sbatch_path} --wait --output={self.slurm_output_file} {self.job.get_parameters()}' return self._run_properties def custom_run_setup(self, **kwargs): @@ -338,15 +312,8 @@ class Default(Sbatch): self._combiner_script = f"{self.job.driver_path}/core/scripts/pipeHelper.sh" return self._combiner_script - def get_parameters(self): - parameters = {k: v for k, v in os.environ.items() if k.startswith("CUSTOM_ENV_SLURM_PARAM")} - parameter_string = '' - for _, v in parameters.items(): - parameter_string += f'{v} ' - return parameter_string - def get_run_properties(self): - parameter_string = self.get_parameters() + parameter_string = self.job.get_parameters() self._run_properties = f'{sbatch_path} --wait --output={self.slurm_output_file} {parameter_string}' return self._run_properties @@ -430,7 +397,7 @@ class Singularity_Batch(Default): logging.debug('Creating param file') # write env_file with open(f'{self.slurm_output_dir}/env{self.job.jobid}', 'w') as file: - file.write(f"CONTAINER={self.container}\0EXEC_WRAPPER={srun_path}\0PARAMS={Default.get_parameters(self)}\0OUTPUT_FILE=--output={self.slurm_output_file}") + file.write(f"CONTAINER={self.container}\0EXEC_WRAPPER={srun_path}\0PARAMS={self.job.get_parameters()}\0OUTPUT_FILE=--output={self.slurm_output_file}") def get_combiner_script(self): self._combiner_script = f"{self.job.driver_path}/core/scripts/pipeHelper.sh" -- GitLab