diff --git a/prule/daemon/__main__.py b/prule/daemon/__main__.py index aa7881b261ee9830cf97676b4badef7083ee61b7..e6ba646dfd004141e3c6a0e2bcac82febab36e09 100644 --- a/prule/daemon/__main__.py +++ b/prule/daemon/__main__.py @@ -631,7 +631,7 @@ class PruleThread(threading.Thread): headers["Access-Control-Request-Headers"] = "x-auth-token" headers["X-Auth-Token"] = config.config["CC_TOKEN"] req = urllib.request.Request(url, headers=headers, method="GET") - tdir = tempfile.TemporaryDirectory(prefix="prule_jobarchive_{}_".format(id)) + tdir = tempfile.TemporaryDirectory(prefix="prule_jobarchive_{}_".format(id), delete="NO_TMPDIR_CLEAN" not in self.config.config) try: with urllib.request.urlopen(req, timeout=10) as response: if response.status == 200: @@ -825,12 +825,18 @@ class PruleThread(threading.Thread): print("Job path:",job_path) + # check if input files actually exist + if os.path.exists(os.path.join(job_path, "meta.json")) == False or (os.path.exists(os.path.join(job_path, "data.json")) == False and os.path.exists(os.path.join(job_path, "data.json.gz")) == False): + if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config: + job_tempdir.cleanup() + print("Process: job {} Missing files in {}".format(job.ccjobid, job_path)) + return "failure-requeue" result_json, process_time = self.prule_job({"job-dir":job_path}) if result_json == None or result_json == False: - if job_tempdir != None: + if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config: job_tempdir.cleanup() if result_json == None: return "failure-shutdown" @@ -840,7 +846,7 @@ class PruleThread(threading.Thread): print("Process: job {} jobId {} time {:.6f}".format(job.ccjobid, job_slurmid, process_time)) if self.processTerminated == True: print("Job {} process was terminated.".format(job.ccjobid)) - if job_tempdir != None: + if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config: job_tempdir.cleanup() return "failure-requeue" @@ -875,7 +881,7 @@ class PruleThread(threading.Thread): traceback.print_exc() print(e) print("Failed to write result to {}".format(job_result_path)) - if job_tempdir != None: + if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config: job_tempdir.cleanup() return "failure-requeue" @@ -928,7 +934,7 @@ class PruleThread(threading.Thread): with prule.debug.Timing("prulethread.request_job_meta", "PRINT_TIMING" in config.config): job_res = self.request_job_meta(job.ccjobid) if type(job_res) != dict: - if job_tempdir != None: + if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config: job_tempdir.cleanup() if job_res == None: return "failure-shutdown" @@ -950,7 +956,7 @@ class PruleThread(threading.Thread): process_result = "failure-shutdown" # cleanup temp directory - if job_tempdir != None: + if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config: job_tempdir.cleanup() return process_result @@ -1115,7 +1121,8 @@ if __name__ == "__main__": parser.add_argument('--cache-db', action='store_true', help='Store results in sqlite3 database. Overwrites the CACHE_DB value from the config file.') parser.add_argument('--job-process-quiet', action='store_true', help='Pass --quiet to prule and do not print full result json') parser.add_argument('--job-process-no-quiet', dest='job_process_quiet', action='store_false', help='Print full result json') - parser.add_argument('--print-timing', action='store_false', help='Print debug timings') + parser.add_argument('--no-tmpdir-clean', action='store_true', help='Keep temporary directories') + parser.add_argument('--print-timing', action='store_true', help='Print debug timings') args = parser.parse_args() args = vars(args) @@ -1147,6 +1154,9 @@ if __name__ == "__main__": if 'print_timing' in args: config.config['PRINT_TIMING'] = True + if 'no_tmpdir_clean' in args: + config.config['NO_TMPDIR_CLEAN'] = True + # load rules rules = [] with open(config.config['PRULE_RULES_FILE_PATH'], 'r') as jsonf: