diff --git a/scripts/tools/must.py b/scripts/tools/must.py
index 9d93e3b41c7091427e79ab2022adb75941be5cdb..5858db43b3986114899a5dcd5f628a48bbe006da 100644
--- a/scripts/tools/must.py
+++ b/scripts/tools/must.py
@@ -5,6 +5,7 @@ import shutil
import json
from MBIutils import *
+
def must_filter(line, process):
if re.search("ERROR: MUST detected a deadlock", line):
pid = process.pid
@@ -16,6 +17,7 @@ def must_filter(line, process):
except ProcessLookupError:
pass # Ok, it's gone now
+
class V18(AbstractTool):
def identify(self):
return "MUST v1.9.2 wrapper"
@@ -25,25 +27,32 @@ class V18(AbstractTool):
def build(self, rootdir, cached=True):
if cached and os.path.exists("/MBI-builds/MUST192/bin/mustrun"):
- os.environ['PATH'] = os.environ['PATH'] + ":/MBI-builds/MUST192/bin/"
+ os.environ['PATH'] = os.environ['PATH'] + \
+ ":/MBI-builds/MUST192/bin/"
os.environ['OMPI_CC'] = "clang-14"
return
- subprocess.run(f"rm -rf /MBI-builds/MUST192", shell=True, check=True) # MUST sometimes fails when reinstalling over the same dir
+ # MUST sometimes fails when reinstalling over the same dir
+ subprocess.run(f"rm -rf /MBI-builds/MUST192", shell=True, check=True)
# Build it
- here = os.getcwd() # Save where we were
- subprocess.run(f"rm -rf /tmp/build-must ; mkdir /tmp/build-must", shell=True, check=True)
+ here = os.getcwd() # Save where we were
+ subprocess.run(
+ f"rm -rf /tmp/build-must ; mkdir /tmp/build-must", shell=True, check=True)
os.chdir("/tmp/build-must")
- subprocess.run(f"wget https://hpc.rwth-aachen.de/must/files/MUST-v1.9.2.tar.gz", shell=True, check=True)
+ subprocess.run(
+ f"wget https://hpc.rwth-aachen.de/must/files/MUST-v1.9.2.tar.gz", shell=True, check=True)
subprocess.run(f"tar xfz MUST-*.tar.gz", shell=True, check=True)
- subprocess.run(f"mkdir -p /tmp/build-must/build", shell=True, check=True)
+ subprocess.run(f"mkdir -p /tmp/build-must/build",
+ shell=True, check=True)
os.chdir("/tmp/build-must/build")
subprocess.run(f"CC=$(which gcc) CXX=$(which gcc++) cmake ../MUST-v1.9.2 -DCMAKE_INSTALL_PREFIX=/MBI-builds/MUST192 -DCMAKE_BUILD_TYPE=Release -DENABLE_FORTRAN=OFF -DENABLE_TYPEART=On", shell=True, check=True)
- #subprocess.run(f"CC=$(which clang) CXX=$(which clang++) cmake ../MUST-v1.9.2 -DCMAKE_INSTALL_PREFIX=/MBI-builds/MUST192 -DCMAKE_BUILD_TYPE=Release -DENABLE_FORTRAN=OFF -DENABLE_TYPEART=On", shell=True, check=True)
- subprocess.run(f"make -j$(nproc) install VERBOSE=1", shell=True, check=True)
- subprocess.run(f"make -j$(nproc) install-prebuilds VERBOSE=1", shell=True, check=True)
+ # subprocess.run(f"CC=$(which clang) CXX=$(which clang++) cmake ../MUST-v1.9.2 -DCMAKE_INSTALL_PREFIX=/MBI-builds/MUST192 -DCMAKE_BUILD_TYPE=Release -DENABLE_FORTRAN=OFF -DENABLE_TYPEART=On", shell=True, check=True)
+ subprocess.run(f"make -j$(nproc) install VERBOSE=1",
+ shell=True, check=True)
+ subprocess.run(
+ f"make -j$(nproc) install-prebuilds VERBOSE=1", shell=True, check=True)
subprocess.run(f"rm -rf /tmp/build-must", shell=True, check=True)
# Back to our previous directory
@@ -57,27 +66,29 @@ class V18(AbstractTool):
def run(self, execcmd, filename, binary, id, number, timeout, batchinfo, loglevel=logging.INFO):
cachefile = f'{binary}_{id}'
- execcmd = re.sub("mpirun", "mustrun --must:distributed --must:typeart --must:output json", execcmd)
+ execcmd = re.sub(
+ "mpirun", "mustrun --must:distributed --must:typeart --must:output json", execcmd)
execcmd = re.sub('\${EXE}', f'./{binary}', execcmd)
execcmd = re.sub('\$zero_buffer', "", execcmd)
execcmd = re.sub('\$infty_buffer', "", execcmd)
with tempfile.TemporaryDirectory() as tmpdirname:
ran = self.run_cmd(
- buildcmd=f"typeart-mpicc -g -fPIC {filename} -o {tmpdirname}/{binary} -L/MBI-builds/MUST192/lib -lpnmpi",
- execcmd=execcmd,
- cachefile=cachefile,
- filename=filename,
- number=number,
- binary=binary,
- timeout=timeout,
- batchinfo=batchinfo,
- loglevel=loglevel,
- cwd=tmpdirname,
- read_line_lambda=must_filter)
+ buildcmd=f"typeart-mpicc -g -fPIC {filename} -o {tmpdirname}/{binary} -L/MBI-builds/MUST192/lib -lpnmpi",
+ execcmd=execcmd,
+ cachefile=cachefile,
+ filename=filename,
+ number=number,
+ binary=binary,
+ timeout=timeout,
+ batchinfo=batchinfo,
+ loglevel=loglevel,
+ cwd=tmpdirname,
+ read_line_lambda=must_filter)
if os.path.isfile(f"{tmpdirname}/MUST_Output.html"):
- shutil.copyfile(f"{tmpdirname}/MUST_Output.html", f"{cachefile}.html")
+ shutil.copyfile(
+ f"{tmpdirname}/MUST_Output.html", f"{cachefile}.html")
if os.path.isfile(f"{tmpdirname}/MUST_Output.json"):
# pretty print JSON output
@@ -96,11 +107,128 @@ class V18(AbstractTool):
print(open(f"{tmpdirname}/MUST_Output.json").read())
def teardown(self):
- subprocess.run("find -type f -a -executable | xargs rm -f", shell=True, check=True) # Remove generated (binary files)
+ subprocess.run("find -type f -a -executable | xargs rm -f",
+ # Remove generated (binary files)
+ shell=True, check=True)
subprocess.run("rm -rf must_temp core", shell=True, check=True)
def get_mbb_error_label(self):
- mbb_error_dict = { # TODO
+ return {
+ "MUST_ERROR_INTEGER_NEGATIVE": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_ZERO": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_NEGATIVE_ARRAY": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_ENTRY_GREATER_OR_EQUAL": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_NEGATIVE_NOT_PROC_NULL_ANY_SOURCE": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_NEGATIVE_NOT_PROC_NULL": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_NEGATIVE_NOT_PROC_NULL_ARRAY": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_NEGATIVE_PROC_NULL_ANY_SOURCE": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_NEGATIVE_UNDEFINED": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_NOT_WITHIN_ZERO_TAG_UB": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_NOT_WITHIN_ZERO_TAG_UB_ANY_TAG": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_GREATER_COMM_SIZE": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_GREATER_EQUAL_COMM_SIZE": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_PRODUCT_GREATER_COMM_SIZE": ["InvalidParam"],
+ "MUST_ERROR_GROUP_RANGE_RANK": ["InvalidParam"],
+ "MUST_ERROR_GROUP_RANGE_STRIDE": ["InvalidParam"],
+ "MUST_ERROR_REQUEST_ACTIVE": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_ACTIVE_ARRAY": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_PARTITION_ACTIVE": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_NOT_PARTITIONED_SEND": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_NOT_PARTITIONED_RECV": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_NOT_KNOWN": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_NOT_KNOWN_ARRAY": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_NULL": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_NULL_ARRAY": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_PERSISTENT_BUT_INACTIVE": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_COMM_UNKNWOWN": ["InvalidParam"],
+ "MUST_ERROR_COMM_NULL": ["InvalidParam"],
+ "MUST_ERROR_NOT_CART_COMM": ["InvalidParam"],
+ "MUST_ERROR_NOT_GRAPH_COMM": ["InvalidParam"],
+ "MUST_ERROR_INTER_COMM": ["InvalidParam"],
+ "MUST_ERROR_INTER_COMM_MPI1": ["InvalidParam"],
+ "MUST_ERROR_ROOT_NOT_IN_COMM": ["InvalidParam"],
+ "MUST_ERROR_PREDEFINED_COMM": ["InvalidParam"],
+ "MUST_ERROR_NOT_INTER_COMM": ["InvalidParam"],
+ "MUST_ERROR_POINTER_NULL": ["InvalidParam"],
+ "MUST_ERROR_LEAK_COMM": ["CallOrdering"],
+ "MUST_ERROR_LEAK_DATATYPE": ["CallOrdering"],
+ "MUST_ERROR_LEAK_REQUEST": ["RequestLifeCycle"],
+ "MUST_ERROR_LEAK_GROUP": ["CallOrdering"],
+ "MUST_ERROR_LEAK_ERR": ["InvalidParam"],
+ "MUST_ERROR_LEAK_KEYVAL": ["InvalidParam"],
+ "MUST_ERROR_LEAK_OP": ["InvalidParam"],
+ "MUST_ERROR_DIRECTION_GREATER_NDIMS": ["TODO"],
+ "MUST_ERROR_DATATYPE_NULL": ["InvalidParam"],
+ "MUST_ERROR_DATATYPE_UNKNOWN": ["InvalidParam"],
+ "MUST_ERROR_DATATYPE_NOT_COMMITED": ["InvalidParam"],
+ "MUST_ERROR_GROUP_NULL": ["InvalidParam"],
+ "MUST_ERROR_GROUP_UNKNOWN": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_GREATER_GROUP_SIZE": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_DUPLICATION_ARRAY": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_GREATER_GROUP_SIZE_ARRAY": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_DUPLICATION_ARRAY_TRIPLET": ["InvalidParam"],
+ "MUST_ERROR_RANK_FROM_RANGES_NOT_IN_GROUP": ["InvalidParam"],
+ "MUST_ERROR_OPERATION_PREDEFINED": ["InvalidParam"],
+ "MUST_ERROR_OPERATION_UNKNOWN": ["InvalidParam"],
+ "MUST_ERROR_OPERATION_NULL": ["InvalidParam"],
+ "MUST_ERROR_POINTER_NULL_NOT_BOTTOM": ["InvalidParam"],
+ "MUST_ERROR_POINTER_NULL_COMM_SIZE": ["InvalidParam"],
+ "MUST_ERROR_POINTER_NULL_COMM_SIZE_ARRAY": ["InvalidParam"],
+ "MUST_ERROR_POINTER_NULL_COMM_SIZE_ARRAY_AT_INDEX": ["InvalidParam"],
+ "MUST_ERROR_MPI_IN_PLACE_USED": ["InvalidParam"],
+ "MUST_ERROR_SELFOVERLAPPED": ["LocalParameterMissmatch"],
+ "MUST_ERROR_OVERLAPPED_SEND": ["LocalParameterMissmatch"],
+ "MUST_ERROR_OVERLAPPED_RECV": ["LocalParameterMissmatch"],
+ "MUST_ERROR_POINTER_NULL_STATUS_IGNORE": ["InvalidParam"],
+ "MUST_ERROR_TYPEMATCH_INTERNAL_NOTYPE": ["InvalidParam", "LocalParameterMissmatch"],
+ "MUST_ERROR_TYPEMATCH_INTERNAL_TYPESIG": ["InvalidParam", "LocalParameterMissmatch"],
+ "MUST_ERROR_TYPEMATCH_MISMATCH": ["GlobalParameterMissmatch"],
+ "MUST_ERROR_TYPEMATCH_MISMATCH_BYTE": ["GlobalParameterMissmatch"],
+ "MUST_ERROR_TYPEMATCH_LENGTH": ["InvalidParam", "LocalParameterMissmatch"],
+ "MUST_ERROR_TYPEMATCH_ALIGNMENT": ["InvalidParam", "LocalParameterMissmatch"],
+ "MUST_ERROR_MESSAGE_LOST": ["DEADLOCK", "CallOrdering"],
+ "MUST_ERROR_COLLECTIVE_CALL_MISMATCH": ["DEADLOCK", "CallOrdering"],
+ "MUST_ERROR_COLLECTIVE_OP_MISMATCH": ["GlobalParameterMissmatch"],
+ "MUST_ERROR_COLLECTIVE_ROOT_MISMATCH": ["GlobalParameterMissmatch"],
+ "MUST_ERROR_COLLECTIVE_BLOCKING_NONBLOCKING_MISMATCH": ["GlobalParameterMissmatch"],
+ "MUST_ERROR_DEADLOCK": ["DEADLOCK"],
+ "MUST_ERROR_BUFFER_REATTACH": ["TODO"],
+ "MUST_ERROR_BUFFER_NOATTACHED": ["TODO"],
+ "MUST_ERROR_COUNTS_ARRAYS_DIFFER": ["TODO"],
+ "MUST_ERROR_MPI_MULTIPLE_THREADS": ["TODO"],
+ "MUST_ERROR_UNSUPPORTED": ["TODO"],
+ "MUST_ERROR_OPENMP": ["TODO"],
+ "MUST_INFO_UNIMPLEMENTED_FEATURE": [],
+ # Ignore warnings so far
+ "MUST_WARNING_INTEGER_ZERO": [],
+ "MUST_WARNING_INTEGER_ZERO_ARRAY": [],
+ "MUST_WARNING_INTEGER_NOT_ONE_OR_ZERO": [],
+ "MUST_WARNING_INTEGER_NOT_ONE_OR_ZERO_ARRAY": [],
+ "MUST_WARNING_INTEGER_HIGH_BUT_LESS_TAG_UB": [],
+ "MUST_WARNING_INTEGER_PRODUCT_LESS_COMM_SIZE": [],
+ "MUST_WARNING_INTER_COMM": [],
+ "MUST_WARNING_REQUEST_ACTIVE_RECV": [],
+ "MUST_WARNING_REQUEST_CANCELED": [],
+ "MUST_WARNING_REQUEST_NULL": [],
+ "MUST_WARNING_REQUEST_NULL_OR_INACTIVE_ARRAY": [],
+ "MUST_WARNING_REQUEST_INACTIVE": [],
+ "MUST_WARNING_NOT_CART_COMM": [],
+ "MUST_WARNING_INTER_COMM_MPI2": [],
+ "MUST_WARNING_COMM_NULL": [],
+ "MUST_WARNING_MAXDIMS_GREATER_NDIMS": [],
+ "MUST_WARNING_MAXNEIGHBORS_TO_SMALL": [],
+ "MUST_WARNING_MAXINDICES_TO_SMALL": [],
+ "MUST_WARNING_MAXEDGES_TO_SMALL": [],
+ "MUST_WARNING_DATATYPE_PREDEFINED": [],
+ "MUST_WARNING_DATATYPE_COMMITED": [],
+ "MUST_WARNING_DATATYPE_BAD_ALIGNMENT": [],
+ "MUST_WARNING_IF_EMPTY": [],
+ "MUST_WARNING_GROUP_NULL": [],
+ "MUST_WARNING_POINTER_NULL": [],
+ "MUST_WARNING_SELFOVERLAPPED": [],
+ "MUST_WARNING_BUFFER_OUTSIZED": [],
+ "MUST_WARNING_THREADLEVEL": [],
+ "MUST_WARNING_DATARACE": []
}
def parse(self, cachefile, logs_dir):
@@ -122,9 +250,13 @@ class V18(AbstractTool):
# if nothing was found, parsing will not continue
if json_output is None or len(output_strings) == 0:
return {"status": "failure"}
-
+
output_string = ''.join(output_strings)
- # TODO: catch segfaults
+
+ if re.search('Compilation of .*? raised an error \(retcode: ', output_string):
+ output = {}
+ output["status"] = "UNIMPLEMENTED"
+ return output_string
# No interesting output found, so return the timeout as is if it exists
if os.path.exists(f'{cachefile}.timeout') or os.path.exists(f'{logs_dir}/must/{cachefile}.timeout'):
@@ -140,7 +272,8 @@ class V18(AbstractTool):
# parse JSON output and convert to MBB format
for message in json_output["messages"]:
parsed_report = {}
- parsed_report["error_class"] = "TODO"
+ parsed_report["error_class"] = self.get_mbb_error_label()[
+ message["error_id"]]
parsed_report["calls"] = []
parsed_report["lines"] = []
parsed_report["ranks"] = []
@@ -156,8 +289,8 @@ class V18(AbstractTool):
# extract line numbers of the test file from reported MPI call
for stack_item in message["from"]["stacktrace"]:
if test_file_name in stack_item["file"]:
- parsed_report["lines"].append(stack_item["line"])
-
+ parsed_report["lines"].append(stack_item["line"])
+
# extract other affected calls and line numbers from references in error message
for reference in message["references"]:
parsed_report["calls"].append(reference["call"])
@@ -172,63 +305,7 @@ class V18(AbstractTool):
return output
- # if re.search('MUST_ERROR_DEADLOCK', output):
- # return 'deadlock'
-
- # if re.search('not freed', output):
- # return 'resleak'
-
- # if re.search('conflicting roots', output):
- # return 'various'
-
- # if re.search('unknown datatype', output) or re.search('has to be a non-negative integer', output) or re.search('must use equal type signatures', output):
- # return 'conflicting roots'
-
- # if re.search('MUST_ERROR_COMM', output):
- # return 'error communicator'
-
- # if re.search('MUST_ERROR_COLLECTIVE_(.*)_MISMATCH', output):
- # # ignore collective mismatch for RMA test cases
- # print(os.getcwd())
- # source = open(f'{logs_dir}/must/{cachefile[:-2]}.c').read()
- # if 'Category: RMA' in source:
- # pass
- # else:
- # return 'call mismatch'
-
- # if re.search('MUST_ERROR_(.*)_NULL', output):
- # return 'null parameter'
-
- # if re.search('MUST_ERROR_BUFFER_(.*)', output):
- # return 'buffer error'
-
- # if re.search('MUST_ERROR_INTEGER_(.*)', output):
- # return 'integer error'
-
- # if re.search('MUST_ERROR_TYPEMATCH_MISMATCH', output):
- # return 'type mismatch'
-
- # if re.search('MUST_ERROR_TYPEMATCH_LENGTH', output):
- # return 'type length'
-
- # # catch any remaining error
- # if re.search('MUST_ERROR_(.*)', output):
- # # ignore collective mismatch for RMA test cases
- # source = open(f'{logs_dir}/must/{cachefile[:-2]}.c').read()
- # if 'Category: RMA' in source and 'MUST_ERROR_COLLECTIVE_CALL_MISMATCH' in output:
- # pass
- # else:
- # return 'error'
-
- # if re.search('MBI_MSG_RACE', output):
- # return 'MBI_MSG_RACE'
-
- # if re.search('MUST detected no MPI usage errors nor any suspicious behavior during this application run', output):
- # return 'OK'
-
- # if re.search('Compilation of .*? raised an error \(retcode: ', output):
- # return 'UNIMPLEMENTED'
-
+ # TODO: Catch segfaults?
# if re.search('YOUR APPLICATION TERMINATED WITH THE EXIT STRING: Segmentation fault', output):
# return 'segfault'
# if re.search('caught signal nr 11', output) or re.search('caught signal nr 6', output):
@@ -236,55 +313,11 @@ class V18(AbstractTool):
# if re.search('internal ABORT - process ', output):
# return 'failure'
-
- # # special handling for RMA test cases
- # source = open(f'{logs_dir}/must/{cachefile[:-2]}.c').read()
- # if 'Category: RMA' in source and 'MUST_ERROR_COLLECTIVE_CALL_MISMATCH' in output:
- # return 'OK'
-
+
# if re.search('caught MPI error', output):
# # if we arrive here we, then MUST just found an MPI error, but did not detect anything, so we return 'OK'
# return 'OK'
-
+
# if re.search('Fatal error in internal_Comm_size', output):
# # if we arrive here, nothing relevant has been detected, MPI just crashed internally
- # return 'OK'
-
- # print (f">>>>[ INCONCLUSIVE ]>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> ({self.identify()}/{cachefile})")
- # print(output)
- # print ("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
- # return 'other'
-
-# No beta release of MUST for now
-#
-#class V19(V18):
-# def identify(self):
-# return "MUST v1.8.0 wrapper"
-#
-# def ensure_image(self):
-# AbstractTool.ensure_image(self, "-x must18")
-#
-# def build(self, rootdir, cached=True):
-# if cached and os.path.exists("/MBI-builds/MUST18/bin/mustrun"):
-# return
-#
-# subprocess.run(f"rm -rf /MBI-builds/MUST18", shell=True, check=True) # MUST sometimes fails when reinstalling over the same dir
-#
-# # Build it
-# here = os.getcwd() # Save where we were
-# if not os.path.exists((f"{rootdir}/tools/MUST-v1.8.0.tar.gz")):
-# subprocess.run(f"cd {rootdir}/tools; wget https://hpc.rwth-aachen.de/must/files/MUST-v1.8.0.tar.gz", shell=True, check=True)
-# subprocess.run(f"rm -rf /tmp/build-must ; mkdir /tmp/build-must", shell=True, check=True)
-# os.chdir("/tmp/build-must")
-# subprocess.run(f"tar xfz {rootdir}/tools/MUST-v1.8.0.tar.gz", shell=True, check=True)
-#
-# subprocess.run(f"CC=$(which clang) CXX=$(which clang++) OMPI_CC=$(which clang) OMPI_CXX=$(which clang++) FC=$(which gfortran) cmake MUST-v1.8.0 -DCMAKE_INSTALL_PREFIX=/MBI-builds/MUST18 -DCMAKE_BUILD_TYPE=Release", shell=True, check=True)
-# subprocess.run(f"make -j$(nproc) install VERBOSE=1", shell=True, check=True)
-# subprocess.run(f"make -j$(nproc) install-prebuilds VERBOSE=1", shell=True, check=True)
-#
-# # Back to our previous directory
-# os.chdir(here)
-#
-# def setup(self):
-# os.environ['PATH'] = os.environ['PATH'] + ":/MBI-builds/MUST18/bin/"
-#
+ # return 'OK'
\ No newline at end of file