Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
R
Runner Mirror
Manage
Activity
Members
Plan
Wiki
Code
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Deploy
Package registry
Model registry
Operate
Terraform modules
Analyze
Contributor analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Adrian Schmitz
Runner Mirror
Commits
a5097aae
Commit
a5097aae
authored
2 years ago
by
Felix Tomski
Browse files
Options
Downloads
Patches
Plain Diff
Use xPipeHelper instead of pipeHelper
Fixes issues with PATH in ssh mode.
parent
7eaa1e14
No related branches found
No related tags found
No related merge requests found
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
core/job.py
+8
-6
8 additions, 6 deletions
core/job.py
core/modes/__init__.py
+14
-14
14 additions, 14 deletions
core/modes/__init__.py
with
22 additions
and
20 deletions
core/job.py
+
8
−
6
View file @
a5097aae
...
@@ -261,10 +261,12 @@ class Job:
...
@@ -261,10 +261,12 @@ class Job:
run_properties
=
f
'
sudo -u
{
self
.
account
}
{
wrapper
}
{
run_properties
}
'
run_properties
=
f
'
sudo -u
{
self
.
account
}
{
wrapper
}
{
run_properties
}
'
os
.
chdir
(
"
/work
"
)
os
.
chdir
(
"
/work
"
)
command
=
[
helper_script
]
command
=
[
helper_script
]
if
run_properties
==
''
:
if
run_properties
:
command
.
extend
([
run_script
])
command
.
extend
([
run_properties
])
else
:
setup_env_scripts
=
self
.
_mode
.
get_env_setup
()
command
.
extend
([
run_properties
,
run_script
])
if
setup_env_scripts
:
command
.
extend
(
setup_env_scripts
.
split
())
command
.
extend
(
run_script
.
split
(
'
'
))
if
script_execution
:
if
script_execution
:
self
.
_mode
.
custom_run_setup
(
install_env
=
install_env
,
skip_env
=
skip_env
,
script_execution
=
script_execution
,
self
.
_mode
.
custom_run_setup
(
install_env
=
install_env
,
skip_env
=
skip_env
,
script_execution
=
script_execution
,
do_inbetween_processing
=
do_inbetween_processing
,
srun_wrap
=
srun_wrap
,
**
kwargs
)
do_inbetween_processing
=
do_inbetween_processing
,
srun_wrap
=
srun_wrap
,
**
kwargs
)
...
@@ -300,7 +302,7 @@ class Job:
...
@@ -300,7 +302,7 @@ class Job:
utility
.
update_json_kv
(
self
.
error_code_file
,
self
.
jobid
,
-
1
)
utility
.
update_json_kv
(
self
.
error_code_file
,
self
.
jobid
,
-
1
)
self
.
exec_script
=
self
.
args
[
2
]
self
.
exec_script
=
self
.
args
[
2
]
command
=
f
"
{
self
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
command
=
f
"
{
self
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
run_properties
=
''
run_properties
=
''
do_inbetween_processing
=
False
do_inbetween_processing
=
False
if
self
.
args
[
3
]
not
in
[
'
build_script
'
,
'
step_script
'
]:
if
self
.
args
[
3
]
not
in
[
'
build_script
'
,
'
step_script
'
]:
...
@@ -315,7 +317,7 @@ class Job:
...
@@ -315,7 +317,7 @@ class Job:
do_inbetween_processing
=
do_inbetween_processing
)
do_inbetween_processing
=
do_inbetween_processing
)
if
self
.
args
[
3
]
in
[
'
build_script
'
,
'
step_script
'
]
and
self
.
_mode
.
has_post_run_script
():
if
self
.
args
[
3
]
in
[
'
build_script
'
,
'
step_script
'
]
and
self
.
_mode
.
has_post_run_script
():
command
=
f
"
{
self
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
command
=
f
"
{
self
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
run_properties
=
self
.
_mode
.
get_post_run_properties
()
run_properties
=
self
.
_mode
.
get_post_run_properties
()
run_script
=
self
.
_mode
.
get_post_run_script
()
run_script
=
self
.
_mode
.
get_post_run_script
()
self
.
execute
(
run_properties
,
run_script
,
command
,
script_execution
=
True
)
self
.
execute
(
run_properties
,
run_script
,
command
,
script_execution
=
True
)
...
...
This diff is collapsed.
Click to expand it.
core/modes/__init__.py
+
14
−
14
View file @
a5097aae
...
@@ -206,13 +206,13 @@ class Slurm(ModeBase):
...
@@ -206,13 +206,13 @@ class Slurm(ModeBase):
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
f
'
{
self
.
job
.
user_path
}
/wrapper
{
self
.
job
.
jobid
}
.sh
'
,
f
'
{
self
.
job
.
user_path
}
/wrapper
{
self
.
job
.
jobid
}
.sh
'
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/zshWrapper.sh
"
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/zshWrapper.sh
"
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
,
skip_env
=
True
,
srun_wrap
=
True
)
skip_env
=
True
,
srun_wrap
=
True
)
git_runner_command
=
[
f
'
{
self
.
job
.
shell_path
}
-l
'
,
git_runner_command
=
[
f
'
{
self
.
job
.
shell_path
}
-l
'
,
f
"
{
self
.
job
.
user_path
}
/wrapper
{
self
.
job
.
jobid
}
.sh
"
]
f
"
{
self
.
job
.
user_path
}
/wrapper
{
self
.
job
.
jobid
}
.sh
"
]
self
.
job
.
execute
(
'
'
.
join
(
git_runner_command
),
self
.
job
.
execute
(
'
'
.
join
(
git_runner_command
),
f
'
{
self
.
job
.
driver_path
}
/core/scripts/runnerInstallHelper.sh
'
,
f
'
{
self
.
job
.
driver_path
}
/core/scripts/runnerInstallHelper.sh
'
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
,
srun_wrap
=
True
,
install_env
=
True
)
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
,
srun_wrap
=
True
,
install_env
=
True
)
def
get_simple_script_exec
(
self
):
def
get_simple_script_exec
(
self
):
return
f
"
{
srun_path
}
--jobid=
{
self
.
slurm_simple_job_id
}
{
self
.
job
.
shell_path
}
-l
"
\
return
f
"
{
srun_path
}
--jobid=
{
self
.
slurm_simple_job_id
}
{
self
.
job
.
shell_path
}
-l
"
\
...
@@ -228,7 +228,7 @@ class Slurm(ModeBase):
...
@@ -228,7 +228,7 @@ class Slurm(ModeBase):
return
export_env
[:
-
1
]
return
export_env
[:
-
1
]
def
get_combiner_script
(
self
):
def
get_combiner_script
(
self
):
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
return
self
.
_combiner_script
return
self
.
_combiner_script
def
set_slurm_env
(
self
):
def
set_slurm_env
(
self
):
...
@@ -284,7 +284,7 @@ class Sbatch(Slurm):
...
@@ -284,7 +284,7 @@ class Sbatch(Slurm):
return
cmd_out
return
cmd_out
def
get_combiner_script
(
self
):
def
get_combiner_script
(
self
):
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
return
self
.
_combiner_script
return
self
.
_combiner_script
...
@@ -310,7 +310,7 @@ class Sbatch(Slurm):
...
@@ -310,7 +310,7 @@ class Sbatch(Slurm):
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
f
'
{
self
.
job
.
build_path
}
/chmodPath
{
self
.
job
.
jobid
}
.sh
'
,
f
'
{
self
.
job
.
build_path
}
/chmodPath
{
self
.
job
.
jobid
}
.sh
'
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/chmodPath.sh
"
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/chmodPath.sh
"
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
,
skip_env
=
True
,
srun_wrap
=
True
)
skip_env
=
True
,
srun_wrap
=
True
)
self
.
job
.
execute
(
f
'
/usr/local_rwth/bin/zsh
{
self
.
job
.
build_path
}
/chmodPath
{
self
.
job
.
jobid
}
.sh
'
,
self
.
job
.
execute
(
f
'
/usr/local_rwth/bin/zsh
{
self
.
job
.
build_path
}
/chmodPath
{
self
.
job
.
jobid
}
.sh
'
,
f
'
{
self
.
slurm_output_dir
}
'
,
f
'
{
self
.
slurm_output_dir
}
'
,
...
@@ -344,7 +344,7 @@ class Sbatch(Slurm):
...
@@ -344,7 +344,7 @@ class Sbatch(Slurm):
class
Default
(
Sbatch
):
class
Default
(
Sbatch
):
def
get_combiner_script
(
self
):
def
get_combiner_script
(
self
):
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
return
self
.
_combiner_script
return
self
.
_combiner_script
def
get_run_properties
(
self
):
def
get_run_properties
(
self
):
...
@@ -356,7 +356,7 @@ class Default(Sbatch):
...
@@ -356,7 +356,7 @@ class Default(Sbatch):
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
f
'
{
self
.
job
.
clone_path
}
/script.
'
,
f
'
{
self
.
job
.
clone_path
}
/script.
'
,
f
"
{
Slurm
.
get_run_script
(
self
)
}
"
,
f
"
{
Slurm
.
get_run_script
(
self
)
}
"
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
,
skip_env
=
True
,
srun_wrap
=
True
)
skip_env
=
True
,
srun_wrap
=
True
)
tmp
=
os
.
getenv
(
"
TMP
"
)
tmp
=
os
.
getenv
(
"
TMP
"
)
with
open
(
f
'
{
self
.
job
.
driver_path
}
/core/scripts/batchWrapper.sh
'
,
'
r
'
)
as
file
:
with
open
(
f
'
{
self
.
job
.
driver_path
}
/core/scripts/batchWrapper.sh
'
,
'
r
'
)
as
file
:
...
@@ -442,17 +442,17 @@ class Singularity_Batch(Default):
...
@@ -442,17 +442,17 @@ class Singularity_Batch(Default):
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
f
'
{
self
.
slurm_output_dir
}
/batchEnv
{
self
.
job
.
jobid
}
'
,
f
'
{
self
.
slurm_output_dir
}
/batchEnv
{
self
.
job
.
jobid
}
'
,
f
"
{
self
.
job
.
tmp_dir
}
/env
{
self
.
job
.
jobid
}
"
,
f
"
{
self
.
job
.
tmp_dir
}
/env
{
self
.
job
.
jobid
}
"
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
,
skip_env
=
True
,
srun_wrap
=
True
)
skip_env
=
True
,
srun_wrap
=
True
)
def
get_combiner_script
(
self
):
def
get_combiner_script
(
self
):
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
return
self
.
_combiner_script
return
self
.
_combiner_script
def
get_run_script
(
self
):
def
get_run_script
(
self
):
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
f
'
{
self
.
job
.
clone_path
}
/script
{
self
.
job
.
jobid
}
.sh
'
,
f
'
{
self
.
job
.
clone_path
}
/script
{
self
.
job
.
jobid
}
.sh
'
,
f
'
{
Slurm
.
get_run_script
(
self
)
}
'
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
,
f
'
{
Slurm
.
get_run_script
(
self
)
}
'
,
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
,
skip_env
=
True
,
srun_wrap
=
True
)
skip_env
=
True
,
srun_wrap
=
True
)
# move wrapper to user
# move wrapper to user
tmp
=
os
.
getenv
(
"
TMP
"
)
tmp
=
os
.
getenv
(
"
TMP
"
)
...
@@ -474,7 +474,7 @@ class Singularity_Batch(Default):
...
@@ -474,7 +474,7 @@ class Singularity_Batch(Default):
# Generation of the singularity script within user space
# Generation of the singularity script within user space
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
f
'
{
self
.
job
.
clone_path
}
/singularity
{
self
.
job
.
jobid
}
.sh
'
,
f
'
{
self
.
job
.
clone_path
}
/singularity
{
self
.
job
.
jobid
}
.sh
'
,
self
.
get_run_properties_postfix
(),
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
,
self
.
get_run_properties_postfix
(),
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
,
skip_env
=
True
,
srun_wrap
=
True
)
skip_env
=
True
,
srun_wrap
=
True
)
# add singularity specific parameter to the properties
# add singularity specific parameter to the properties
property_split
=
self
.
_run_properties
.
split
(
"
"
)
property_split
=
self
.
_run_properties
.
split
(
"
"
)
...
@@ -503,7 +503,7 @@ class Singularity(Slurm):
...
@@ -503,7 +503,7 @@ class Singularity(Slurm):
container
=
''
container
=
''
def
get_combiner_script
(
self
):
def
get_combiner_script
(
self
):
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
return
self
.
_combiner_script
return
self
.
_combiner_script
def
__init__
(
self
,
job
):
def
__init__
(
self
,
job
):
...
@@ -524,7 +524,7 @@ class Singularity(Slurm):
...
@@ -524,7 +524,7 @@ class Singularity(Slurm):
# Generation of the singularity script within user space
# Generation of the singularity script within user space
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
self
.
job
.
execute
(
f
'
/usr/bin/cp /dev/stdin
'
f
'
{
self
.
job
.
user_path
}
/script
{
self
.
job
.
jobid
}
.sh
'
,
f
'
{
self
.
job
.
user_path
}
/script
{
self
.
job
.
jobid
}
.sh
'
,
self
.
get_run_properties_postfix
(),
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
,
self
.
get_run_properties_postfix
(),
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
,
skip_env
=
True
,
srun_wrap
=
True
)
skip_env
=
True
,
srun_wrap
=
True
)
# add singularity specific parameter to the properties
# add singularity specific parameter to the properties
property_split
=
self
.
_run_properties
.
split
(
"
"
)
property_split
=
self
.
_run_properties
.
split
(
"
"
)
...
@@ -558,7 +558,7 @@ class Singularity(Slurm):
...
@@ -558,7 +558,7 @@ class Singularity(Slurm):
class
SingleSlurmJobAcrossStages
(
Slurm
):
class
SingleSlurmJobAcrossStages
(
Slurm
):
def
get_combiner_script
(
self
):
def
get_combiner_script
(
self
):
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
p
ipeHelper.sh
"
self
.
_combiner_script
=
f
"
{
self
.
job
.
driver_path
}
/core/scripts/
xP
ipeHelper.sh
"
return
self
.
_combiner_script
return
self
.
_combiner_script
def
get_run_properties_postfix
(
self
):
def
get_run_properties_postfix
(
self
):
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment