Prule.daemon: Add --no-tmpdir-clean argument and fix --print-timing argparse

prule/daemon/__main__.py

@@ -631,7 +631,7 @@ class PruleThread(threading.Thread):
         headers["Access-Control-Request-Headers"] = "x-auth-token"
         headers["X-Auth-Token"] = config.config["CC_TOKEN"]
         req = urllib.request.Request(url, headers=headers, method="GET")
-        tdir = tempfile.TemporaryDirectory(prefix="prule_jobarchive_{}_".format(id))
+        tdir = tempfile.TemporaryDirectory(prefix="prule_jobarchive_{}_".format(id), delete="NO_TMPDIR_CLEAN" not in self.config.config)
         try:
             with urllib.request.urlopen(req, timeout=10) as response:
                 if response.status == 200:
@@ -825,12 +825,18 @@ class PruleThread(threading.Thread):
 
         print("Job path:",job_path)
 
+        # check if input files actually exist
+        if os.path.exists(os.path.join(job_path, "meta.json")) == False or (os.path.exists(os.path.join(job_path, "data.json")) == False and os.path.exists(os.path.join(job_path, "data.json.gz")) == False):
+            if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config:
+                job_tempdir.cleanup()
+            print("Process: job {} Missing files in {}".format(job.ccjobid, job_path))
+            return "failure-requeue"
 
 
         result_json, process_time = self.prule_job({"job-dir":job_path})
 
         if result_json == None or result_json == False:
-            if job_tempdir != None:
+            if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config:
                 job_tempdir.cleanup()
             if result_json == None:
                 return "failure-shutdown"
@@ -840,7 +846,7 @@ class PruleThread(threading.Thread):
         print("Process: job {} jobId {} time {:.6f}".format(job.ccjobid, job_slurmid, process_time))
         if self.processTerminated == True:
             print("Job {} process was terminated.".format(job.ccjobid))
-            if job_tempdir != None:
+            if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config:
                 job_tempdir.cleanup()
             return "failure-requeue"
 
@@ -875,7 +881,7 @@ class PruleThread(threading.Thread):
                 traceback.print_exc()
                 print(e)
                 print("Failed to write result to {}".format(job_result_path))
-                if job_tempdir != None:
+                if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config:
                     job_tempdir.cleanup()
                 return "failure-requeue"
 
@@ -928,7 +934,7 @@ class PruleThread(threading.Thread):
                 with prule.debug.Timing("prulethread.request_job_meta", "PRINT_TIMING" in config.config):
                     job_res = self.request_job_meta(job.ccjobid)
                 if type(job_res) != dict:
-                    if job_tempdir != None:
+                    if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config:
                         job_tempdir.cleanup()
                     if job_res == None:
                         return "failure-shutdown"
@@ -950,7 +956,7 @@ class PruleThread(threading.Thread):
                 process_result = "failure-shutdown"
 
         # cleanup temp directory
-        if job_tempdir != None:
+        if job_tempdir != None and "NO_TMPDIR_CLEAN" not in config.config:
             job_tempdir.cleanup()
 
         return process_result
@@ -1115,7 +1121,8 @@ if __name__ == "__main__":
     parser.add_argument('--cache-db', action='store_true', help='Store results in sqlite3 database. Overwrites the CACHE_DB value from the config file.')
     parser.add_argument('--job-process-quiet', action='store_true', help='Pass --quiet to prule and do not print full result json')
     parser.add_argument('--job-process-no-quiet', dest='job_process_quiet', action='store_false', help='Print full result json')
-    parser.add_argument('--print-timing', action='store_false', help='Print debug timings')
+    parser.add_argument('--no-tmpdir-clean', action='store_true', help='Keep temporary directories')
+    parser.add_argument('--print-timing', action='store_true', help='Print debug timings')
 
     args = parser.parse_args()
     args = vars(args)
@@ -1147,6 +1154,9 @@ if __name__ == "__main__":
     if 'print_timing' in args:
         config.config['PRINT_TIMING'] = True
 
+    if 'no_tmpdir_clean' in args:
+        config.config['NO_TMPDIR_CLEAN'] = True
+
     # load rules
     rules = []
     with open(config.config['PRULE_RULES_FILE_PATH'], 'r') as jsonf: