diff --git a/scripts/ExampleUsage.py b/scripts/ExampleUsage.py index 02146e85f259b33f51561fec37c8a6c28a5df4b3..12c0122f1a8a396402442f3ed6eec6615738f5e5 100644 --- a/scripts/ExampleUsage.py +++ b/scripts/ExampleUsage.py @@ -18,7 +18,7 @@ class Invalid_negative_rank_error: # include the buffer allocation in the template (all ranks execute it) alloc_block = InstructionBlock("alloc") - alloc_block.register_operation(correct_params.get_buffer_alloc()) + alloc_block.register_instruction(correct_params.get_buffer_alloc()) tm.register_instruction_block(alloc_block) send = MPI_Call_Factory().mpi_send( @@ -33,9 +33,9 @@ class Invalid_negative_rank_error: b = InstructionBlock() # only rank 0 execute the send - b.register_operation(send, 0) + b.register_instruction(send, 0) # only rank 1 execute the recv - b.register_operation(get_matching_recv(send), 1) + b.register_instruction(get_matching_recv(send), 1) tm.register_instruction_block(b) return tm diff --git a/scripts/Infrastructure/AllocCall.py b/scripts/Infrastructure/AllocCall.py index 25dea717c1ddfb4af428ebf44562deedfa9ad7ef..69303b69963d91a26e585f861b893debc75c7796 100644 --- a/scripts/Infrastructure/AllocCall.py +++ b/scripts/Infrastructure/AllocCall.py @@ -1,5 +1,8 @@ #! /usr/bin/python3 +from typing_extensions import override +from scripts.Infrastructure.Instruction import Instruction +from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT alloc_template = """ @{TYPE}@* @{NAME}@ = (@{TYPE}@*) @{FUNCTION}@(@{NUM}@ @{SEP}@ sizeof(@{TYPE}@)); @@ -20,9 +23,9 @@ alloc_template = """ """ -class AllocCall: - - def __init__(self, type, num_elements, name="buf", use_malloc=False): +class AllocCall(Instruction): + @override + def __init__(self, type: str, num_elements: str, name: str = "buf", use_malloc: bool = False): """ Creates a New allocation Call @@ -32,11 +35,13 @@ class AllocCall: name: name of buffer variable use_malloc: True: use Malloc, False: use calloc for allocation """ + super().__init__("") self._use_malloc = use_malloc self._type = type self._num_elements = num_elements self._name = name + @override def __str__(self): if self._use_malloc: delim = '*' @@ -45,20 +50,25 @@ class AllocCall: delim = ',' func = "calloc" - return (alloc_template - .replace("@{NAME}@", self._name) - .replace("@{TYPE}@", self._type) - .replace("@{FUNCTION}@", func) - .replace("@{NUM}@", str(self._num_elements)) - .replace("@{SEP}@", delim)) + s = (alloc_template + .replace("@{NAME}@", self._name) + .replace("@{TYPE}@", self._type) + .replace("@{FUNCTION}@", func) + .replace("@{NUM}@", str(self._num_elements)) + .replace("@{SEP}@", delim)) + + if self._has_error: + s += ERROR_MARKER_COMMENT + + return s - def set_num_elements(self, num_elements): + def set_num_elements(self, num_elements: str): self._num_elements = num_elements - def set_name(self, name): + def set_name(self, name: str): self._name = name - def set_type(self, type): + def set_type(self, type: str): self._type = type def set_use_malloc(self): @@ -67,19 +77,19 @@ class AllocCall: def set_use_calloc(self): self._use_malloc = False - def get_num_elements(self): + def get_num_elements(self) -> str: return self._num_elements - def get_name(self): + def get_name(self) -> str: return self._name - def get_type(self): + def get_type(self) -> str: return self._type - def get_use_malloc(self): + def get_use_malloc(self) -> bool: return self._use_malloc -def get_free(alloc_call): +def get_free(alloc_call: AllocCall) -> Instruction: assert isinstance(alloc_call, AllocCall) - return "free(" + alloc_call.get_name() + ");\n" + return Instruction("free(" + alloc_call.get_name() + ");") diff --git a/scripts/Infrastructure/CorrectParameter.py b/scripts/Infrastructure/CorrectParameter.py index 7eea7f606f794c9d9ade66e0eabe7e68529db470..c202cb8ff726b46461959e5cd1051142d46755de 100644 --- a/scripts/Infrastructure/CorrectParameter.py +++ b/scripts/Infrastructure/CorrectParameter.py @@ -1,4 +1,6 @@ #! /usr/bin/python3 +from scripts.Infrastructure import MPICall +from scripts.Infrastructure.Instruction import Instruction from scripts.Infrastructure.MPICallFactory import MPICallFactory from scripts.Infrastructure.Template import InstructionBlock from scripts.Infrastructure.AllocCall import AllocCall, get_free @@ -6,28 +8,30 @@ from scripts.Infrastructure.AllocCall import AllocCall, get_free class CorrectParameterFactory: # default params - buf_size = 10 + buf_size = "10" dtype = ['int', 'MPI_INT'] + buf_size_bytes = f"{buf_size}*sizeof({dtype[0]})" tag = 0 buf_var_name = "buf" + winbuf_var_name = "winbuf" def __init__(self): pass - def get_buffer_alloc(self): + def get_buffer_alloc(self)-> AllocCall: return AllocCall(self.dtype[0], self.buf_size, self.buf_var_name, use_malloc=False) - def get_buffer_free(self): + def get_buffer_free(self)->Instruction: return get_free(AllocCall(self.dtype[0], self.buf_size, self.buf_var_name, use_malloc=False)) - def get(self, param, func=None): - if param in ["BUFFER", "buf", "buffer", "sendbuf", "recvbuf"]: + def get(self, param:str)->str: + if param in ["BUFFER", "buf", "buffer", "sendbuf", "recvbuf", "origin_addr"]: return self.buf_var_name - if param in ["COUNT", "count", "sendcount", "recvcount"]: + if param in ["COUNT", "count", "sendcount", "recvcount", "origin_count", "target_count", "result_count"]: return str(self.buf_size) - if param in ["DATATYPE", "datatype", "sendtype", "recvtype"]: + if param in ["DATATYPE", "datatype", "sendtype", "recvtype", "origin_datatype", "target_datatype", "result_datatype"]: return self.dtype[1] - if param in ["DEST", "dest"]: + if param in ["DEST", "dest", "target_rank"]: return "0" if param in ["SRC", "source"]: return "1" @@ -71,12 +75,30 @@ class CorrectParameterFactory: return "MPI_COMM_WORLD" if param in ["remote_leader"]: return "0" + if param in ["target_disp"]: + return "0" + if param in ["win"]: + return "win" + if param in ["baseptr"]: + return "&" + self.winbuf_var_name + if param in ["base"]: + return self.winbuf_var_name + if param in ["size"]: + return self.buf_size_bytes + if param in ["disp_unit"]: + return "sizeof(int)" + if param in ["info"]: + return "MPI_INFO_NULL" + if param in ["result_addr"]: + return "resultbuf" + if param in ["compare_addr"]: + return "comparebuf" print("Not Implemented: " + param) assert False, "Param not known" # todo also for send and non default args -def get_matching_recv(call): +def get_matching_recv(call:MPICall)->MPICall: correct_params = CorrectParameterFactory() recv = MPICallFactory().mpi_recv( correct_params.get("BUFFER"), diff --git a/scripts/Infrastructure/GeneratorManager.py b/scripts/Infrastructure/GeneratorManager.py index 4752d1b33617a44ecfdb4e08391a9a9c690543d3..dad021d978f7452b279ae7d9080dcdc6e42a5580 100644 --- a/scripts/Infrastructure/GeneratorManager.py +++ b/scripts/Infrastructure/GeneratorManager.py @@ -1,9 +1,13 @@ #! /usr/bin/python3 +from __future__ import annotations + import inspect import os import importlib import importlib.util import subprocess +import typing +from pathlib import Path # for printing a nice progress bar import tqdm @@ -65,8 +69,8 @@ class GeneratorManager: return case_name + "-" + str(num).zfill(digits_to_use) + suffix - def generate(self, outpath, filterlist_=None, print_progress_bar=True, overwrite=True, generate_full_set=False, - try_compile=False, max_mpi_version=4.0, use_clang_format=True): + def generate(self, outpath: str | Path | os.PathLike[str], filterlist_:typing.Sequence[str]=None, print_progress_bar:bool=True, overwrite:bool=True, generate_full_set:bool=False, + try_compile:bool=False, max_mpi_version:str="4.0", use_clang_format:bool=True): """ Generates test cases based on the specified parameters. Parameters: diff --git a/scripts/Infrastructure/Instruction.py b/scripts/Infrastructure/Instruction.py new file mode 100644 index 0000000000000000000000000000000000000000..475c19823061485cd752b8605af0b2919d07b80f --- /dev/null +++ b/scripts/Infrastructure/Instruction.py @@ -0,0 +1,21 @@ +#! /usr/bin/python3 +from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT + + +class Instruction(object): + """ + Base class to represent an Instruction + """ + + def __init__(self, str_representation): + self._str_representation = str_representation + self._has_error = False + + def set_has_error(self, has_error: bool = True): + self._has_error = has_error + + def __str__(self): + if self._has_error: + return self._str_representation + ERROR_MARKER_COMMENT + else: + return self._str_representation diff --git a/scripts/Infrastructure/InstructionBlock.py b/scripts/Infrastructure/InstructionBlock.py index ec5310b924b9854cac885450e52aec40184d465f..f30545dea3a2db4cb2c300d03b0a4df331772ce8 100644 --- a/scripts/Infrastructure/InstructionBlock.py +++ b/scripts/Infrastructure/InstructionBlock.py @@ -1,3 +1,8 @@ +from __future__ import annotations + +import typing + +from scripts.Infrastructure.Instruction import Instruction from scripts.Infrastructure.MPICall import MPI_Call @@ -16,7 +21,7 @@ class InstructionBlock: - `__str__(self)`: Converts the InstructionBlock instance to a string, replacing placeholders. """ - def __init__(self, name=None): + def __init__(self, name: str = None): """ Initialize an empty InstructionBlock @@ -28,29 +33,41 @@ class InstructionBlock: assert not isinstance(name, int) self.name = name - # TODO test if op also can be a list of operations - def register_operation(self, op, kind='all'): + def register_instruction(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all'): """ Registers an operation based on rank. Parameters: - - op: The operation to register. + - op: The operation (or list of Operations) to register. - kind: Rank to execute the operation ('all', 'not0', or integer). - all: all Ranks execute this operation - not0: all Ranks but the Root (rank 0) execute - Or the integer of the rank that should execute + Note: if a str is passed as the operation, it will create a new Instruction from the given string """ + if isinstance(op, str): + op = Instruction(op) + if kind == 'all': - self.operations['all'].append(op) + if isinstance(op, list): + self.operations['all'].extend(op) + else: + self.operations['all'].append(op) elif kind == 'not0': - self.operations['not0'].append(op) + if isinstance(op, list): + self.operations['not0'].extend(op) + else: + self.operations['not0'].append(op) else: as_int = int(kind) # will Raise ValueError if not integer if as_int not in self.operations: self.operations[as_int] = [] - self.operations[as_int].append(op) + if isinstance(op, list): + self.operations[as_int].extend(op) + else: + self.operations[as_int].append(op) - def get_version(self): + def get_version(self) -> str: """ Retrieves the minimum required MPI version. Returns: @@ -89,63 +106,96 @@ class InstructionBlock: return result_str - def has_operation(self, kind='all', index=0): + def has_instruction(self, kind: int | str = 'all', index: int = 0) -> bool: """ Checks if the Block has an operation with the given index and kind Parameters: - kind ('all','not0' or integer): which ranks should execute the operation - - index (int): the index of the operation within the given kind + - index (int ): the index of the operation within the given kind Returns: boolean """ try: result = self.operations[kind][index] return True - except (KeyError,IndexError) as e: + except (KeyError, IndexError) as e: return False - def get_operation(self, kind='all', index=0): + def get_instruction(self, kind: int | str = 'all', index: str | int = 0) -> Instruction | typing.List[Instruction]: """ Retrieve the operation registered. will Raise IndexError if not present Parameters: - kind ('all','not0' or integer): which ranks should execute the operation - - index (int): the index of the operation within the given kind + - index ('all' or int): the index of the operation within the given kind; 'all' means that the list of all operations for the kind is returned Returns: str: The operation specified by kind and index """ - return self.operations[kind][index] + if index == 'all': + return self.operations[kind] + else: + as_int = int(index) # will Raise ValueError if not integer + return self.operations[kind][as_int] - def replace_operation(self, op, kind='all', index=0): + def replace_instruction(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all', + index: str | int = 0): """ Replace the operation registered. will Raise IndexError if not present Parameters: - - op (str or MPICall) the new operation + - op the new operation or list of operations - kind ('all','not0' or integer): which ranks should execute the operation - - index (int): the index of the operation within the given kind + - index ('all' or int): the index of the operation within the given kind; 'all' means all operations will be replaced with the given list + Notes : if one wants to replace all operations one needs to provide a list + if one only wants to replace one operation: no list of operations is allowed + if a string is passed as the operation, it will create a new Instruction """ - if len(self.operations[kind]) < index: - raise IndexError("Operation Not Found") - self.operations[kind][index] = op + if isinstance(op, str): + op = Instruction(op) + + if index == 'all': + if not isinstance(op, list): + raise ValueError('Provide List for replacement') + self.operations[kind] = op + else: + as_int = int(index) # will Raise ValueError if not integer + print(op) + if not isinstance(op, Instruction): + raise ValueError('Provide Instruction') + if len(self.operations[kind]) < as_int: + raise IndexError("Operation Not Found") + self.operations[kind][as_int] = op - def insert_operation(self, op, kind='all', before_index=0): + def insert_instruction(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all', + before_index: int = 0): """ Inserts an operation before the specified one. will Raise IndexError if not present Parameters: - - op (str or MPICall) the new operation + - op the new operation or list of operations - kind ('all','not0' or integer): which ranks should execute the operation - index (int): the index of the operation within the given kind + note: if str is passed as the operation, it will Create a New Instruction """ + if isinstance(op, str): + op = Instruction(op) + as_int = int(before_index) # will Raise ValueError if not integer if len(self.operations[kind]) < before_index: raise IndexError("Operation Not Found") - self.operations[kind].insert(before_index,op) + if isinstance(op, list): + self.operations[kind] = ( + self.operations[kind][0:before_index - 1] + op + self.operations[kind][before_index:]) + else: + self.operations[kind].insert(before_index, op) - def remove_operation(self, kind='all', index=0): + def remove_instruction(self, kind: str | int = 'all', index: str | int = 0): """ Removes the operation registered. will Raise IndexError if not present Parameters: - kind ('all','not0' or integer): which ranks should execute the operation - - index (int): the index of the operation within the given kind + - index ('all' or int): the index of the operation within the given kind """ - if len(self.operations[kind]) < index: - raise IndexError("Operation Not Found") - del self.operations[kind][index] + if index == 'all': + self.operations[kind] = [] + else: + as_int = int(index) # will Raise ValueError if not integer + if len(self.operations[kind]) < index: + raise IndexError("Operation Not Found") + del self.operations[kind][index] diff --git a/scripts/Infrastructure/MPIAPIInfo/GenerateCallFactory.py b/scripts/Infrastructure/MPIAPIInfo/GenerateCallFactory.py index 3618e8b0c86c98eb5b587bbe05fa7a9f1edd782a..f519427bee13ddf84ff437ae5f7c51cc73fe91f8 100644 --- a/scripts/Infrastructure/MPIAPIInfo/GenerateCallFactory.py +++ b/scripts/Infrastructure/MPIAPIInfo/GenerateCallFactory.py @@ -18,6 +18,10 @@ from scripts.Infrastructure.MPICall import MPI_Call class MPICallFactory: + def get(self, func: str, *args): + f_to_call = getattr(self, func) + return f_to_call(*args) + """ correct_call_factory_header=""" @@ -26,6 +30,10 @@ from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory class CorrectMPICallFactory: + def get(self, func: str, *args): + f_to_call = getattr(self, func) + return f_to_call(*args) + """ template_correct = """ def @{FUNC_KEY}@(self): diff --git a/scripts/Infrastructure/MPICall.py b/scripts/Infrastructure/MPICall.py index 02c1cf359b454f8e2f2c4f5f28e3cae3bdbe5fc0..f86987744a3408256ad3ea2e745746ab8462c289 100644 --- a/scripts/Infrastructure/MPICall.py +++ b/scripts/Infrastructure/MPICall.py @@ -1,16 +1,22 @@ #! /usr/bin/python3 +import typing +from typing_extensions import override +from scripts.Infrastructure.Instruction import Instruction from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT -class MPI_Call: +class MPI_Call(Instruction): - def __init__(self, function, args, version): + @override + def __init__(self, function: str, args: typing.OrderedDict[str, str], version: str): + super().__init__("") self._function = function self._args = args self._version = version self._has_error = False + @override def __str__(self): s = self._function + "(" for k, v in self._args.items(): @@ -23,16 +29,12 @@ class MPI_Call: return s - def set_arg(self, arg, value): + def set_arg(self, arg: str, value: str): assert self.has_arg(arg) self._args[arg] = value - def has_arg(self, arg): + def has_arg(self, arg: str) -> bool: return arg in self._args - def get_version(self): + def get_version(self) -> str: return self._version - - def set_has_error(self, has_error=True): - self._has_error = has_error - diff --git a/scripts/Infrastructure/MPICallFactory.py b/scripts/Infrastructure/MPICallFactory.py index 474919f1f589fafcde1d64d69eb27ff85a076442..442192c9c95f2103e81794b563c0b6eb0505afff 100644 --- a/scripts/Infrastructure/MPICallFactory.py +++ b/scripts/Infrastructure/MPICallFactory.py @@ -6,6 +6,10 @@ from scripts.Infrastructure.MPICall import MPI_Call class MPICallFactory: + def get(self, func: str, *args): + f_to_call = getattr(self, func) + return f_to_call(*args) + def mpi_abort(self, *args): return MPI_Call("MPI_Abort", OrderedDict([("comm", args[0]), ("errorcode", args[1]), ]), "1.0") @@ -2174,6 +2178,9 @@ from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory class CorrectMPICallFactory: + def get(self, func: str, *args): + f_to_call = getattr(self, func) + return f_to_call(*args) def mpi_abort(self): correct_params = CorrectParameterFactory() diff --git a/scripts/Infrastructure/Template.py b/scripts/Infrastructure/Template.py index f547026053cbed390c03209ade643d947a40dc41..330c671d876c1a81bb1d6d057d8dc047d0e2f17a 100644 --- a/scripts/Infrastructure/Template.py +++ b/scripts/Infrastructure/Template.py @@ -71,7 +71,7 @@ class TemplateManager: - `get_short_descr(self)`: Retrieves the short description . """ - def __init__(self, min_ranks=2, thread_level=None, has_finalize=True, has_init=True): + def __init__(self, min_ranks: int = 2, thread_level: str = None, has_finalize: bool = True, has_init: bool = True): """ Initialize the TemplateManager @@ -117,7 +117,7 @@ class TemplateManager: .replace("@{version}@", version) .replace("@{test_code}@", block_string)) - def register_instruction_block(self, block): + def register_instruction_block(self, block: InstructionBlock): """ Registers an instruction block with the template. inserting it at the end, before the mpi finalize Parameters: @@ -125,7 +125,7 @@ class TemplateManager: """ self._blocks.append(block) - def get_version(self): + def get_version(self) -> str: """ Retrieves the minimum required MPI version. Returns: @@ -137,7 +137,7 @@ class TemplateManager: max_v = max(block.get_version(), max_v) return max_v - def set_description(self, descr_short, descr_full): + def set_description(self, descr_short: str, descr_full: str): """ Sets the short and full descriptions for the template. Parameters: @@ -148,7 +148,7 @@ class TemplateManager: self._descr_short = descr_short # TODO one could write a function to check if short desc = filename is conforming with the naming sceme - def get_short_descr(self): + def get_short_descr(self) -> str: """ Retrieves the short description to use as a filename. Returns: @@ -157,7 +157,7 @@ class TemplateManager: assert self._descr_short != "" return self._descr_short - def get_block(self, block_name=None, idx=None): + def get_block(self, block_name: str = None, idx: int = None) -> InstructionBlock: """ Retrieves the given Instruction Block Either by name or by index Raises IndexError if the specified block is not found @@ -187,7 +187,7 @@ class TemplateManager: raise ValueError("Neither Both block name nor index is given") - def insert_block(self, new_block, after_block_name=None, after_idx=None): + def insert_block(self, new_block: InstructionBlock, after_block_name: str = None, after_idx: int = None): """ inserts the given Instruction Block AFTER the one specified Either by name or by index Raises IndexError if the specified block is not found @@ -217,7 +217,7 @@ class TemplateManager: raise ValueError("Neither Both block name nor index is given") - def remove_block(self, block_name=None, idx=None): + def remove_block(self, block_name: str = None, idx: int = None): """ Removes the given Instruction Block Either by name or by index Raises IndexError if the specified block is not found @@ -244,7 +244,7 @@ class TemplateManager: raise ValueError("Neither Both block name nor index is given") - def replace_block(self, new_block, block_name=None, idx=None): + def replace_block(self, new_block: InstructionBlock, block_name: str = None, idx: int = None): """ Removes the given Instruction Block Either by name or by index Raises IndexError if the specified block is not found diff --git a/scripts/Infrastructure/TemplateFactory.py b/scripts/Infrastructure/TemplateFactory.py index 1790c08df22c85b416ee9b6b6a30e95283616ef5..4c595ea17de9e9a1e572e3be126842d5a0354880 100644 --- a/scripts/Infrastructure/TemplateFactory.py +++ b/scripts/Infrastructure/TemplateFactory.py @@ -1,4 +1,8 @@ #! /usr/bin/python3 +from __future__ import annotations + +import typing + from scripts.Infrastructure.AllocCall import AllocCall from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory from scripts.Infrastructure.InstructionBlock import InstructionBlock @@ -18,7 +22,7 @@ def get_default_template(mpi_func): pass -def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"): +def get_send_recv_template(send_func: str = "mpi_isend", recv_func: str | typing.Tuple[str, str] = "mpi_irecv"): """ Contructs a default template for the given mpi send recv function pair Returns: @@ -27,50 +31,75 @@ def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"): """ # currently supported: - assert send_func in ["mpi_send", "mpi_ssend", "mpi_isend", "mpi_issend", "mpi_sendrecv", "mpi_rsend", "mpi_irsend", - "mpi_bsend", "mpi_ibsend", "mpi_sendrecv", "mpi_sendrecv_replace", "mpi_isendrecv", - "mpi_isendrecv_replace", "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", - "mpi_psend_init"] - assert recv_func in ["mpi_recv", "mpi_irecv", "mpi_sendrecv", "mpi_sendrecv_replace", "mpi_isendrecv", - "mpi_isendrecv_replace", "mpi_recv_init", "mpi_precv_init"] + + # mprobe and mrecv combinations allowed + probe_pairs = [["mpi_mprobe", "mpi_mrecv"], ["mpi_mprobe", "mpi_imrecv"], ["mpi_improbe", "mpi_mrecv"], + ["mpi_improbe", "mpi_imrecv"]] sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"] persistent_send_funcs = ["mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init"] persistent_recv_funcs = ["mpi_recv_init", "mpi_precv_init"] + isend_funcs = ["mpi_isend", "mpi_issend", "mpi_irsend", "mpi_ibsend"] + irecv_funcs = ["mpi_irecv"] + + assert (send_func in ["mpi_send", "mpi_ssend", "mpi_rsend", "mpi_bsend"] + + sendrecv_funcs + isend_funcs + persistent_send_funcs) + assert recv_func in ["mpi_recv"] + sendrecv_funcs + irecv_funcs + persistent_recv_funcs + probe_pairs + if send_func in sendrecv_funcs or recv_func == sendrecv_funcs: assert recv_func == send_func # default template generation only supports if both use same mechanism if send_func in ["mpi_rsend", "mpi_irsend", "mpi_rsend_init"]: - assert recv_func in ["mpi_irecv", "mpi_recv_init"] # else: deadlock + assert recv_func in ["mpi_irecv", "mpi_recv_init", "mpi_precv_init"] # else: deadlock tm = TemplateManager() cf = CorrectParameterFactory() alloc_block = InstructionBlock("alloc") - alloc_block.register_operation(cf.get_buffer_alloc()) + alloc_block.register_instruction(cf.get_buffer_alloc()) if send_func in sendrecv_funcs: # spilt send and recv buf alloc = cf.get_buffer_alloc() alloc.set_name("recv_buf") - alloc_block.register_operation(alloc) - tm.register_instruction_block(alloc_block) + alloc_block.register_instruction(alloc) + if recv_func in probe_pairs: + alloc_block.register_instruction("MPI_Message msg;") if send_func in ["mpi_bsend", "mpi_ibsend", "mpi_bsend_init"]: - b = InstructionBlock("buf_attach") buf_size = "sizeof(int)*10 + MPI_BSEND_OVERHEAD" - b.register_operation(AllocCall("char", buf_size, "mpi_buf")) - b.register_operation(MPICallFactory().mpi_buffer_attach("mpi_buf", buf_size)) + alloc_block.register_instruction(AllocCall("char", buf_size, "mpi_buf")) + alloc_block.register_instruction(MPICallFactory().mpi_buffer_attach("mpi_buf", buf_size)) + + if (send_func in isend_funcs + persistent_send_funcs or + recv_func in persistent_recv_funcs + irecv_funcs + probe_pairs): + alloc_block.register_instruction("MPI_Request request;", 'all') + if recv_func in probe_pairs: + alloc_block.register_instruction("int flag=0;") + + tm.register_instruction_block(alloc_block) + # end preperation of all local variables + + # before the send/recv block + if send_func in ["mpi_rsend", "mpi_irsend", "mpi_rsend_init"]: + b = InstructionBlock("SYNC") + # sender needs to wait until recv is started + b.register_instruction(CorrectMPICallFactory().mpi_barrier(), 1) tm.register_instruction_block(b) cmpicf = CorrectMPICallFactory() - send_func_creator_function = getattr(cmpicf, send_func) - s = send_func_creator_function() - recv_func_creator_function = getattr(cmpicf, recv_func) - r = recv_func_creator_function() + # get the send and recv block + + recv_to_use = recv_func + + if recv_func in probe_pairs: + recv_to_use = recv_func[0] + + s = cmpicf.get(send_func) + r = cmpicf.get(recv_to_use) if send_func in sendrecv_funcs: # sending the second msg @@ -81,65 +110,68 @@ def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"): if r.has_arg("recvbuf"): r.set_arg("recvbuf", "recv_buf") - if (send_func.startswith("mpi_i") or recv_func.startswith("mpi_i") - or send_func in persistent_send_funcs or recv_func in persistent_recv_funcs): - b = InstructionBlock("MPI_REQUEST") - b.register_operation("MPI_Request request;", 'all') - tm.register_instruction_block(b) + b = InstructionBlock("MPICALL") + b.register_instruction(s, 1) + b.register_instruction(r, 0) - if send_func in ["mpi_rsend", "mpi_irsend", "mpi_rsend_init"]: - b = InstructionBlock("SYNC") - b.register_operation(CorrectMPICallFactory().mpi_barrier(), 1) - tm.register_instruction_block(b) + if recv_func in probe_pairs: + if recv_func in [["mpi_improbe", "mpi_mrecv"], + ["mpi_improbe", "mpi_imrecv"]]: + b.insert_instruction("while (!flag){", 0, 0) + # insertion before the improbe call + b.register_instruction("}", 0) # end while + b.register_instruction(CorrectMPICallFactory().get(recv_func[1]), 0) - b = InstructionBlock("MPICALL") - b.register_operation(s, 1) - b.register_operation(r, 0) if send_func in persistent_send_funcs: - b.register_operation(cmpicf.mpi_start(), 1) + b.register_instruction(cmpicf.mpi_start(), 1) if send_func == "mpi_psend_init": # the pready takes a Request NOt a request* - b.register_operation(MPICallFactory().mpi_pready("0", cf.get("request")[1:]), 1) + b.register_instruction(MPICallFactory().mpi_pready("0", cf.get("request")[1:]), 1) if recv_func in persistent_recv_funcs: - b.register_operation(cmpicf.mpi_start(), 0) # + b.register_instruction(cmpicf.mpi_start(), 0) # # parrived is not necessary tm.register_instruction_block(b) + # after send/recv + if send_func in ["mpi_rsend", "mpi_irsend", "mpi_rsend_init"]: b = InstructionBlock("SYNC") - b.register_operation(CorrectMPICallFactory().mpi_barrier(), 1) + # barrier indicating recv has started + b.register_instruction(CorrectMPICallFactory().mpi_barrier(), 0) tm.register_instruction_block(b) - if send_func.startswith("mpi_i") or send_func in persistent_send_funcs: + # wait for op to complete + if send_func in isend_funcs + persistent_send_funcs: b = InstructionBlock("WAIT") - b.register_operation(CorrectMPICallFactory().mpi_wait(), 1) + b.register_instruction(CorrectMPICallFactory().mpi_wait(), 1) tm.register_instruction_block(b) - if recv_func.startswith("mpi_i") or recv_func in persistent_recv_funcs: + if recv_func in irecv_funcs + persistent_recv_funcs + [["mpi_mprobe", "mpi_imrecv"], + ["mpi_improbe", "mpi_imrecv"]]: b = InstructionBlock("WAIT") - b.register_operation(CorrectMPICallFactory().mpi_wait(), 0) + b.register_instruction(CorrectMPICallFactory().mpi_wait(), 0) tm.register_instruction_block(b) + # end MPI operation + # cleanup + free_block = InstructionBlock("buf_free") if send_func in ["mpi_bsend", "mpi_ibsend", "mpi_bsend_init"]: - b = InstructionBlock("buf_detach") - b.register_operation("int freed_size;") - b.register_operation(MPICallFactory().mpi_buffer_detach("mpi_buf", "&freed_size")) - b.register_operation("free(mpi_buf);") - tm.register_instruction_block(b) + free_block = InstructionBlock("buf_detach") + free_block.register_instruction("int freed_size;") + free_block.register_instruction(MPICallFactory().mpi_buffer_detach("mpi_buf", "&freed_size")) + free_block.register_instruction("free(mpi_buf);") + + free_block.register_instruction(cf.get_buffer_free()) - free_block = InstructionBlock("buf_free") - free_block.register_operation(cf.get_buffer_free()) if send_func in sendrecv_funcs: # spilt send and recv buf - b.register_operation("free(recv_buf);") - tm.register_instruction_block(free_block) + free_block.register_instruction("free(recv_buf);") if send_func in persistent_send_funcs: - # spilt send and recv buf - b = InstructionBlock("req_free") - b.register_operation(cmpicf.mpi_request_free()) - tm.register_instruction_block(b) + free_block.register_instruction(cmpicf.mpi_request_free()) + + tm.register_instruction_block(free_block) return tm @@ -156,12 +188,12 @@ def get_collective_template(collective_func, seperate=True): cf = CorrectParameterFactory() alloc_block = InstructionBlock("alloc") - alloc_block.register_operation(cf.get_buffer_alloc()) + alloc_block.register_instruction(cf.get_buffer_alloc()) if False: # spilt send and recv buf alloc = cf.get_buffer_alloc() alloc.set_name("recv_buf") - alloc_block.register_operation(alloc) + alloc_block.register_instruction(alloc) tm.register_instruction_block(alloc_block) cmpicf = CorrectMPICallFactory() @@ -170,20 +202,85 @@ def get_collective_template(collective_func, seperate=True): b = InstructionBlock("MPICALL") if seperate: - b.register_operation(c, 1) - b.register_operation(c, 0) + b.register_instruction(c, 1) + b.register_instruction(c, 0) else: - b.register_operation(c, 'all') + b.register_instruction(c, 'all') tm.register_instruction_block(b) free_block = InstructionBlock("buf_free") - free_block.register_operation(cf.get_buffer_free()) + free_block.register_instruction(cf.get_buffer_free()) tm.register_instruction_block(free_block) return tm +def get_allocated_window(win_alloc_func, name, bufname, ctype, num_elements): + """ + Constructs a window allocation using Win_allocate or Win_create. + :param win_alloc_func: The window allocation to use (mpi_win_allocate or mpi_win_create). + :param name: name of the window + """ + + b = InstructionBlock("win_allocate") + + # declare window + b.register_instruction(f"MPI_Win {name};") + + # extract C data type and window buffer name + # dtype = CorrectParameterFactory().dtype[0] + # winbuf_name = CorrectParameterFactory().winbuf_var_name + # winbuf_size = CorrectParameterFactory().buf_size_bytes + win_allocate_call = None + + if win_alloc_func == "mpi_win_allocate": + # MPI allocate, only declaration required + b.register_instruction(f"{ctype}* {bufname};") + win_allocate_call = CorrectMPICallFactory().mpi_win_allocate() + win_allocate_call.set_arg("baseptr", "&" + bufname) + elif win_alloc_func == "mpi_win_create": + # allocate buffer for win_create + b.register_instruction(AllocCall(ctype, num_elements, bufname)) + win_allocate_call = CorrectMPICallFactory().mpi_win_create() + win_allocate_call.set_arg("base", bufname) + else: + assert False + + # set common parameters for both calls + win_allocate_call.set_arg("win", "&" + name) + + buf_size_bytes = num_elements + "*sizeof(" + ctype + ")" + win_allocate_call.set_arg("size", buf_size_bytes) + + win_allocate_call.set_arg("disp_unit", f"sizeof({ctype})") + b.register_instruction(win_allocate_call) + + return b + + +def get_rma_call(rma_func, rank): + b = InstructionBlock(rma_func.replace('mpi_', '')) + + cf = CorrectParameterFactory() + cfmpi = CorrectMPICallFactory() + + # request-based RMA call, add request + if rma_func.startswith("mpi_r"): + b.register_instruction(f"MPI_Request " + cf.get("request")[1:] + ";", kind=rank) + + # some RMA ops require result_addr + if rma_func in ["mpi_get_accumulate", "mpi_rget_accumulate", "mpi_fetch_and_op", "mpi_compare_and_swap"]: + b.register_instruction(AllocCall(cf.dtype[0], cf.buf_size, cf.get("result_addr")), kind=rank) + + # some RMA ops require compare_addr + if rma_func in ["mpi_fetch_and_op", "mpi_compare_and_swap"]: + b.register_instruction(AllocCall(cf.dtype[0], cf.buf_size, cf.get("compare_addr")), kind=rank) + + b.register_instruction(getattr(cfmpi, rma_func)(), kind=rank) + return b + + def get_communicator(comm_create_func, name): """ :param comm_create_func: teh function used to create the new communicator @@ -194,12 +291,12 @@ def get_communicator(comm_create_func, name): "mpi_comm_idup_with_info", "mpi_comm_create", "mpi_comm_create_group", "mpi_comm_split", "mpi_comm_split_type", "mpi_comm_create_from_group"] b = InstructionBlock("comm_create") - b.register_operation("MPI_Comm " + name + ";") + b.register_instruction("MPI_Comm " + name + ";") if comm_create_func.startswith("mpi_comm_i"): - b.register_operation("MPI_Request comm_create_req;") + b.register_instruction("MPI_Request comm_create_req;") if comm_create_func in ["mpi_comm_create", "mpi_comm_create_group"]: - b.register_operation("MPI_Group group;") - b.register_operation(CorrectMPICallFactory().mpi_comm_group()) + b.register_instruction("MPI_Group group;") + b.register_instruction(CorrectMPICallFactory().mpi_comm_group()) cmpicf = CorrectMPICallFactory() call_creator_function = getattr(cmpicf, comm_create_func) call = call_creator_function() @@ -208,11 +305,11 @@ def get_communicator(comm_create_func, name): call.set_arg("request", "&comm_create_req") if comm_create_func in ["mpi_comm_create", "mpi_comm_create_group"]: call.set_arg("group", "group") # not &group - b.register_operation(call) + b.register_instruction(call) if comm_create_func.startswith("mpi_comm_i"): - b.register_operation(MPICallFactory().mpi_wait("&comm_create_req", "MPI_STATUS_IGNORE")) + b.register_instruction(MPICallFactory().mpi_wait("&comm_create_req", "MPI_STATUS_IGNORE")) if comm_create_func in ["mpi_comm_create", "mpi_comm_create_group"]: - b.register_operation(cmpicf.mpi_group_free()) + b.register_instruction(cmpicf.mpi_group_free()) return b @@ -229,31 +326,31 @@ def get_intercomm(comm_create_func, name): assert name != "intercomm_base_comm" if comm_create_func == "mpi_intercomm_create": b = InstructionBlock("comm_create") - b.register_operation("MPI_Comm intercomm_base_comm;") - b.register_operation( + b.register_instruction("MPI_Comm intercomm_base_comm;") + b.register_instruction( MPICallFactory().mpi_comm_split("MPI_COMM_WORLD", "rank % 2", "rank", "&intercomm_base_comm")) - b.register_operation("MPI_Comm " + name + ";") - b.register_operation( + b.register_instruction("MPI_Comm " + name + ";") + b.register_instruction( MPICallFactory().mpi_intercomm_create("intercomm_base_comm", "0", "MPI_COMM_WORLD", "!(rank %2)", CorrectParameterFactory().get("tag"), "&" + name)) - b.register_operation(MPICallFactory().mpi_comm_free("&intercomm_base_comm")) + b.register_instruction(MPICallFactory().mpi_comm_free("&intercomm_base_comm")) return b if comm_create_func == "mpi_intercomm_create_from_groups": b = InstructionBlock("comm_create") - b.register_operation("MPI_Group world_group,even_group,odd_group;") - b.register_operation(MPICallFactory().mpi_comm_group("MPI_COMM_WORLD", "&world_group")) - b.register_operation( + b.register_instruction("MPI_Group world_group,even_group,odd_group;") + b.register_instruction(MPICallFactory().mpi_comm_group("MPI_COMM_WORLD", "&world_group")) + b.register_instruction( MPICallFactory().mpi_comm_group("intercomm_base_comm", "&intercomm_base_comm_group")) - b.register_operation("int[3] triplet;") - b.register_operation("triplet[0] =0;") - b.register_operation("triplet[1] =size;") - b.register_operation("triplet[2] =2;") - b.register_operation(MPICallFactory().mpi_group_incl("world_group", "1","&triplet", "even_group")) - b.register_operation("triplet[0] =1;") - b.register_operation(MPICallFactory().mpi_group_incl("world_group", "1","&triplet", "odd_group")) - b.register_operation("MPI_Comm " + name + ";") - b.register_operation( + b.register_instruction("int[3] triplet;") + b.register_instruction("triplet[0] =0;") + b.register_instruction("triplet[1] =size;") + b.register_instruction("triplet[2] =2;") + b.register_instruction(MPICallFactory().mpi_group_incl("world_group", "1", "&triplet", "even_group")) + b.register_instruction("triplet[0] =1;") + b.register_instruction(MPICallFactory().mpi_group_incl("world_group", "1", "&triplet", "odd_group")) + b.register_instruction("MPI_Comm " + name + ";") + b.register_instruction( MPICallFactory().mpi_intercomm_create_from_groups("(rank % 2 ? even_group:odd_group)", "0", "(!(rank % 2) ? even_group:odd_group)", "0", CorrectParameterFactory().get("stringtag"), @@ -263,18 +360,18 @@ def get_intercomm(comm_create_func, name): return b if comm_create_func == "mpi_intercomm_merge": b = InstructionBlock("comm_create") - b.register_operation("MPI_Comm intercomm_base_comm;") - b.register_operation("MPI_Comm to_merge_intercomm_comm;") - b.register_operation( + b.register_instruction("MPI_Comm intercomm_base_comm;") + b.register_instruction("MPI_Comm to_merge_intercomm_comm;") + b.register_instruction( MPICallFactory().mpi_comm_split("MPI_COMM_WORLD", "rank % 2", "rank", "&intercomm_base_comm")) - b.register_operation("MPI_Comm " + name + ";") - b.register_operation( + b.register_instruction("MPI_Comm " + name + ";") + b.register_instruction( MPICallFactory().mpi_intercomm_create("intercomm_base_comm", "0", "MPI_COMM_WORLD", "!(rank %2)", CorrectParameterFactory().get("tag"), "&to_merge_intercomm_comm")) - b.register_operation(MPICallFactory().mpi_intercomm_merge("to_merge_intercomm_comm", "rank %2", "&" + name)) + b.register_instruction(MPICallFactory().mpi_intercomm_merge("to_merge_intercomm_comm", "rank %2", "&" + name)) - b.register_operation(MPICallFactory().mpi_comm_free("&to_merge_intercomm_comm")) - b.register_operation(MPICallFactory().mpi_comm_free("&intercomm_base_comm")) + b.register_instruction(MPICallFactory().mpi_comm_free("&to_merge_intercomm_comm")) + b.register_instruction(MPICallFactory().mpi_comm_free("&intercomm_base_comm")) return b return None diff --git a/scripts/errors/coll/InvalidRank.py b/scripts/errors/coll/InvalidRank.py deleted file mode 100644 index 3033e1bfa36c141ee0b94a553f6efda22b55cb4f..0000000000000000000000000000000000000000 --- a/scripts/errors/coll/InvalidRank.py +++ /dev/null @@ -1,35 +0,0 @@ -#! /usr/bin/python3 - -from scripts.Infrastructure.ErrorGenerator import ErrorGenerator -from scripts.Infrastructure.InstructionBlock import InstructionBlock -from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory -from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv -from scripts.Infrastructure.Template import TemplateManager -from scripts.Infrastructure.TemplateFactory import get_send_recv_template, get_collective_template - -class InvalidRankErrorColl(ErrorGenerator): - invalid_ranks = ["-1", "nprocs", "MPI_PROC_NULL"] - functions_to_use = ["mpi_reduce", "mpi_bcast"] - - def __init__(self): - pass - - - def get_feature(self): - return ["COLL"] - - def generate(self, generate_full_set): - - for func_to_use in self.functions_to_use: - for rank_to_use in self.invalid_ranks: - tm = get_collective_template(func_to_use, seperate=False) - arg_to_replace = "root" - - tm.set_description("InvalidParam-Rank-"+func_to_use, "Invalid Rank: %s" % rank_to_use) - - tm.get_block("MPICALL").get_operation(kind='all', index=0).set_arg(arg_to_replace, rank_to_use) - tm.get_block("MPICALL").get_operation(kind='all', index=0).set_has_error() - - yield tm - if not generate_full_set: - return diff --git a/scripts/errors/coll/__init__.py b/scripts/errors/coll/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/scripts/errors/pt2pt/InvalidBuf.py b/scripts/errors/pt2pt/InvalidBuf.py deleted file mode 100644 index f4ba6037879bf5e2e17ef5b057d9b59276f64df1..0000000000000000000000000000000000000000 --- a/scripts/errors/pt2pt/InvalidBuf.py +++ /dev/null @@ -1,96 +0,0 @@ -#! /usr/bin/python3 - -from scripts.Infrastructure.ErrorGenerator import ErrorGenerator -from scripts.Infrastructure.InstructionBlock import InstructionBlock -from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory -from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv -from scripts.Infrastructure.Template import TemplateManager -from scripts.Infrastructure.TemplateFactory import get_send_recv_template - -from itertools import chain - - -class InvalidBufErrorP2P(ErrorGenerator): - invalid_bufs = [CorrectParameterFactory().buf_var_name, "NULL"] - send_funcs = ["mpi_send", - "mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend", - "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init", - ] - - recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"] - - sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"] - - def __init__(self): - pass - - def get_feature(self): - return ["P2P"] - - def generate_impl(self, send_func, recv_func, check_receive): - - for buf_to_use in self.invalid_bufs: - tm = get_send_recv_template(send_func, recv_func) - - if buf_to_use == CorrectParameterFactory().buf_var_name: - if tm.get_block("alloc").has_operation(kind='all', index=1) and check_receive: - name = tm.get_block("alloc").get_operation(kind='all', index=1).get_name() - type = tm.get_block("alloc").get_operation(kind='all', index=1).get_type() - tm.get_block("alloc").replace_operation( - CorrectParameterFactory.dtype[0] + "* " + name + ";", kind='all', index=1) - else: - tm.get_block("alloc").replace_operation( - CorrectParameterFactory.dtype[0] + "* " + CorrectParameterFactory.buf_var_name + ";") - # without allocation: usage is undefined behaviour - - if check_receive: - tm.set_description("InvalidParam-Buffer-" + recv_func, "Invalid Buffer: %s" % buf_to_use) - else: - tm.set_description("InvalidParam-Buffer-" + send_func, "Invalid Buffer: %s" % buf_to_use) - - if check_receive: - if tm.get_block("MPICALL").get_operation(kind=0, index=0).has_arg("buf"): - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("buf", buf_to_use) - else: - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("recvbuf", buf_to_use) - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error() - else: - if tm.get_block("MPICALL").get_operation(kind=1, index=0).has_arg("buf"): - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("buf", buf_to_use) - else: - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("recvbuf", buf_to_use) - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error() - - yield tm - - def generate(self, generate_full_set): - for func in self.send_funcs: - yield from self.generate_impl(func, 'mpi_irecv', False) - if not generate_full_set: - return - for func in self.recv_funcs: - yield from self.generate_impl("mpi_send", func, True) - - yield from self.generate_impl("mpi_sendrecv", "mpi_sendrecv", True) - yield from self.generate_impl("mpi_sendrecv", "mpi_sendrecv", False) - yield from self.generate_impl("mpi_sendrecv_replace", "mpi_sendrecv_replace", True) - - -class MessageRaceErrorSendRecv(ErrorGenerator): - - def __init__(self): - pass - - def get_feature(self): - return ["P2P"] - - def generate(self, generate_full_set): - - for buf_to_use in ["buf", "MPI_IN_PLACE"]: - tm = get_send_recv_template("mpi_sendrecv", "mpi_sendrecv") - tm.set_description("InvalidParam-Buffer-mpi_sendrecv", "send and recv buffer must be disjoint in sendrecv") - - for k in [0, 1]: - tm.get_block("MPICALL").get_operation(kind=k, index=0).set_arg("recvbuf", buf_to_use) - tm.get_block("MPICALL").get_operation(kind=k, index=0).set_has_error() - yield tm diff --git a/scripts/errors/pt2pt/InvalidComm.py b/scripts/errors/pt2pt/InvalidComm.py deleted file mode 100644 index 24a41046aef1d25a0839e81b7480cbec838df08b..0000000000000000000000000000000000000000 --- a/scripts/errors/pt2pt/InvalidComm.py +++ /dev/null @@ -1,107 +0,0 @@ -#! /usr/bin/python3 - -from scripts.Infrastructure.ErrorGenerator import ErrorGenerator -from scripts.Infrastructure.InstructionBlock import InstructionBlock -from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory -from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv -from scripts.Infrastructure.Template import TemplateManager -from scripts.Infrastructure.TemplateFactory import get_send_recv_template, get_communicator, get_intercomm - -from itertools import chain - -sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"] - - -class InvalidCommErrorP2P(ErrorGenerator): - invalid_comm = ["MPI_COMM_NULL", "NULL"] - - missmatching_comms = ["MPI_COMM_SELF", "mpi_comm_dup", "mpi_comm_dup_with_info", "mpi_comm_idup", - "mpi_comm_idup_with_info", "mpi_comm_create", "mpi_comm_create_group", "mpi_comm_split", - "mpi_comm_split_type", "mpi_comm_create_from_group" - ] - intercomms = ["mpi_intercomm_create", "mpi_intercomm_merge", "mpi_intercomm_create_from_groups"] - - # as extended testcases - - comms_to_check = invalid_comm + missmatching_comms + intercomms - - functions_to_check = ["mpi_send", - "mpi_recv", "mpi_irecv", - "mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend", - "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init", - "mpi_precv_init", "mpi_recv_init" - ] + sendrecv_funcs - - recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"] + sendrecv_funcs - - def __init__(self): - pass - - def get_feature(self): - return ["P2P"] - - def generate(self, generate_full_set): - - for send_func in self.functions_to_check: - for comm_to_use in self.comms_to_check: - - check_receive = False - recv_func = "mpi_irecv" - if send_func in self.recv_funcs: - check_receive = True - recv_func = send_func - send_func = "mpi_send" - if recv_func in sendrecv_funcs: - send_func = recv_func - - tm = get_send_recv_template(send_func, recv_func) - - if comm_to_use in self.missmatching_comms and comm_to_use != "MPI_COMM_SELF": - b = get_communicator(comm_to_use, comm_to_use) - tm.insert_block(b, after_block_name="alloc") - if comm_to_use in self.intercomms: - b = get_intercomm(comm_to_use, comm_to_use) - tm.insert_block(b, after_block_name="alloc") - - error_string = "ParamMatching" - if comm_to_use in self.invalid_comm: - error_string = "InvalidParam" - - if check_receive: - if comm_to_use in self.missmatching_comms and recv_func == "mpi_irecv": - # combination repeated - continue - tm.set_description(error_string + "-Comm-" + recv_func, "Invalid Rank: %s" % comm_to_use) - else: - tm.set_description(error_string + "-Comm-" + send_func, "Invalid Rank: %s" % comm_to_use) - - if check_receive: - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("comm", comm_to_use) - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error() - if comm_to_use in self.missmatching_comms: - # missmatch is between both - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error() - else: - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("comm", comm_to_use) - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error() - if comm_to_use in self.missmatching_comms: - # missmatch is between both - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error() - - # an intercomm has only one rank (the other group) - if comm_to_use in self.intercomms and not comm_to_use == "mpi_intercomm_merge": - # intercomm merge results in an equivalent comm again - if tm.get_block("MPICALL").get_operation(kind=0, index=0).has_arg("source"): - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("source", "0") - if tm.get_block("MPICALL").get_operation(kind=1, index=0).has_arg("source"): - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("source", "0") - - if comm_to_use in self.missmatching_comms + self.intercomms and comm_to_use != "MPI_COMM_SELF": - b = InstructionBlock("comm_free") - b.register_operation(MPICallFactory().mpi_comm_free("&" + comm_to_use)) - tm.register_instruction_block(b) - - yield tm - # end for comm to check - if not generate_full_set: - return diff --git a/scripts/errors/pt2pt/InvalidRank.py b/scripts/errors/pt2pt/InvalidRank.py deleted file mode 100644 index c049e79dc8b241a25198e379f45cc7644d171051..0000000000000000000000000000000000000000 --- a/scripts/errors/pt2pt/InvalidRank.py +++ /dev/null @@ -1,59 +0,0 @@ -#! /usr/bin/python3 - -from scripts.Infrastructure.ErrorGenerator import ErrorGenerator -from scripts.Infrastructure.InstructionBlock import InstructionBlock -from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory -from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv -from scripts.Infrastructure.Template import TemplateManager -from scripts.Infrastructure.TemplateFactory import get_send_recv_template - -from itertools import chain - - -class InvalidRankErrorP2P(ErrorGenerator): - invalid_ranks = ["-1", "nprocs", "MPI_PROC_NULL"] - send_funcs = ["mpi_send", - "mpi_irecv", - "mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend", - "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init" - ] - recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"] - - sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"] - - def __init__(self): - pass - - def get_feature(self): - return ["P2P"] - - def generate_impl(self, send_func, recv_func, check_receive): - for rank_to_use in self.invalid_ranks: - tm = get_send_recv_template(send_func, recv_func) - - if check_receive: - tm.set_description("InvalidParam-Rank-" + recv_func, "Invalid Rank: %s" % rank_to_use) - else: - tm.set_description("InvalidParam-Rank-" + send_func, "Invalid Rank: %s" % rank_to_use) - - if check_receive: - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("source", rank_to_use) - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error() - else: - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("dest", rank_to_use) - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error() - - yield tm - - def generate(self, generate_full_set): - - for send_func in self.send_funcs: - yield from self.generate_impl(send_func, "mpi_irecv", False) - if not generate_full_set: - return - for func in self.recv_funcs: - yield from self.generate_impl("mpi_send", func, True) - - for func in self.sendrecv_funcs: - yield from self.generate_impl(func, func, True) - yield from self.generate_impl(func, func, False) diff --git a/scripts/errors/pt2pt/InvalidRequest.py b/scripts/errors/pt2pt/InvalidRequest.py deleted file mode 100644 index d18888778e2eb9581f0f2a88cc73cb1479c42f84..0000000000000000000000000000000000000000 --- a/scripts/errors/pt2pt/InvalidRequest.py +++ /dev/null @@ -1,67 +0,0 @@ -#! /usr/bin/python3 - -from scripts.Infrastructure.ErrorGenerator import ErrorGenerator -from scripts.Infrastructure.InstructionBlock import InstructionBlock -from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory -from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv -from scripts.Infrastructure.Template import TemplateManager -from scripts.Infrastructure.TemplateFactory import get_send_recv_template - -from itertools import chain - - -class InvalidRequestErrorP2P(ErrorGenerator): - invalid_requests = ["MPI_REQUEST_NULL", # probably triggers compiler warning - "NULL"] - functions_to_check = ["mpi_irecv", "mpi_isend", - "mpi_issend", "mpi_irsend", "mpi_ibsend", - "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init", - "mpi_precv_init", "mpi_recv_init" - ] - - recv_funcs = ["mpi_irecv", "mpi_recv_init", "mpi_precv_init"] - - def __init__(self): - pass - - def get_num_errors(self): - # send + receive = only check the first two functions - return len(self.invalid_requests) * 2 - - # the number of errors to produce in the extended mode (all possible combinations) - def get_num_errors_extended(self): - return len(self.invalid_requests) * len(self.functions_to_check) - - def get_feature(self): - return ["P2P"] - - def generate(self, generate_full_set): - - for send_func in self.functions_to_check: - for req_to_use in self.invalid_requests: - - check_receive = False - recv_func = "mpi_irecv" - if send_func in self.recv_funcs: - check_receive = True - recv_func = send_func - send_func = "mpi_isend" - - tm = get_send_recv_template(send_func, recv_func) - - if check_receive: - tm.set_description("InvalidParam-Request-" + recv_func, "Invalid Request: %s" % req_to_use) - else: - tm.set_description("InvalidParam-Request-" + send_func, "Invalid Request: %s" % req_to_use) - - if check_receive: - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("request", req_to_use) - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error() - else: - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("request", req_to_use) - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error() - - yield tm - - if not generate_full_set: - return diff --git a/scripts/errors/pt2pt/InvalidTag.py b/scripts/errors/pt2pt/InvalidTag.py deleted file mode 100644 index e5c360806065d708f3046ea8319572a7faa1015e..0000000000000000000000000000000000000000 --- a/scripts/errors/pt2pt/InvalidTag.py +++ /dev/null @@ -1,80 +0,0 @@ -#! /usr/bin/python3 - -from scripts.Infrastructure.ErrorGenerator import ErrorGenerator -from scripts.Infrastructure.InstructionBlock import InstructionBlock -from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory -from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv -from scripts.Infrastructure.Template import TemplateManager -from scripts.Infrastructure.TemplateFactory import get_send_recv_template - -from itertools import chain - - -class InvalidTagErrorP2P(ErrorGenerator): - invalid_tags = ["-1", "MPI_TAG_UB+1", CorrectParameterFactory.tag * 2, "MPI_ANY_TAG"] - send_funcs = ["mpi_send", - "mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend", - "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init", - ] - - recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"] - sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"] - - def __init__(self): - pass - - def get_feature(self): - return ["P2P"] - - def generate_impl(self, send_func, recv_func, check_receive): - for tag_to_use in self.invalid_tags: - tm = get_send_recv_template(send_func, recv_func) - - error_string = "InvalidParam" - if tag_to_use == CorrectParameterFactory.tag * 2: - error_string = "ParamMatching" - - if check_receive: - if tag_to_use == "MPI_ANY_TAG": - # correct case - continue - if tag_to_use == CorrectParameterFactory().tag * 2 and recv_func == "mpi_irecv": - # combination repeated - continue - tm.set_description(error_string + "-Tag-" + recv_func, "Invalid Rank: %s" % tag_to_use) - else: - tm.set_description(error_string + "-Tag-" + send_func, "Invalid Rank: %s" % tag_to_use) - - if check_receive: - if tm.get_block("MPICALL").get_operation(kind=0, index=0).has_arg("tag"): - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("tag", tag_to_use) - else: - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("recvtag", tag_to_use) - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error() - if tag_to_use == CorrectParameterFactory.tag * 2: - # missmatch is between both - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error() - else: - if tm.get_block("MPICALL").get_operation(kind=1, index=0).has_arg("tag"): - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("tag", tag_to_use) - else: - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("sendtag", tag_to_use) - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error() - if tag_to_use == CorrectParameterFactory.tag * 2: - # missmatch is between both - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error() - - yield tm - - def generate(self, generate_full_set): - - for send_func in self.send_funcs: - yield from self.generate_impl(self, send_func, "mpi_irecv", False) - if not generate_full_set: - return - for recv_func in self.recv_funcs: - yield from self.generate_impl(self, "mpi_send", recv_func, True) - - for func in self.sendrecv_funcs: - yield from self.generate_impl(func, func, True) - yield from self.generate_impl(func, func, False) diff --git a/scripts/errors/pt2pt/LocalConcurrency.py b/scripts/errors/pt2pt/LocalConcurrency.py deleted file mode 100644 index a12ca70e5f04652a7060b6e7780999a8e001b951..0000000000000000000000000000000000000000 --- a/scripts/errors/pt2pt/LocalConcurrency.py +++ /dev/null @@ -1,59 +0,0 @@ -#! /usr/bin/python3 - -from scripts.Infrastructure.ErrorGenerator import ErrorGenerator -from scripts.Infrastructure.InstructionBlock import InstructionBlock -from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory -from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv -from scripts.Infrastructure.Template import TemplateManager -from scripts.Infrastructure.TemplateFactory import get_send_recv_template - -from itertools import chain - -from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT - -sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"] - - -class LocalConcurrencyErrorP2P(ErrorGenerator): - functions_to_check = ["mpi_irecv", - "mpi_isend", "mpi_issend", "mpi_irsend", "mpi_ibsend", - "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init", - "mpi_precv_init", "mpi_recv_init" - ] - recv_funcs = ["mpi_irecv", "mpi_recv_init", "mpi_precv_init"] - - def __init__(self): - pass - - def get_feature(self): - return ["P2P"] - - def generate(self, generate_full_set): - for send_func in self.functions_to_check: - - check_receive = False - recv_func = "mpi_irecv" - if send_func in self.recv_funcs: - check_receive = True - recv_func = send_func - send_func = "mpi_isend" - - tm = get_send_recv_template(send_func, recv_func) - - if check_receive: - tm.set_description("LocalConcurrency-receive-" + recv_func, - "usage of receive buffer before operation is completed") - else: - tm.set_description("LocalConcurrency-send-" + send_func, - "usage of send buffer before operation is completed") - - if check_receive: - tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error() - tm.get_block("MPICALL").register_operation("buf[2]=1;" + ERROR_MARKER_COMMENT, kind=1) - else: - tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error() - tm.get_block("MPICALL").register_operation("buf[2]=1;" + ERROR_MARKER_COMMENT, kind=1) - - yield tm - if not generate_full_set: - return diff --git a/scripts/errors/pt2pt/MessageRace.py b/scripts/errors/pt2pt/MessageRace.py deleted file mode 100644 index e102fe09cddc736f061b34a7a5d0b32a94d689b6..0000000000000000000000000000000000000000 --- a/scripts/errors/pt2pt/MessageRace.py +++ /dev/null @@ -1,100 +0,0 @@ -#! /usr/bin/python3 - -from scripts.Infrastructure.ErrorGenerator import ErrorGenerator -from scripts.Infrastructure.InstructionBlock import InstructionBlock -from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory -from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv -from scripts.Infrastructure.Template import TemplateManager -from scripts.Infrastructure.TemplateFactory import get_send_recv_template - -from itertools import chain - - -class MessageRaceErrorAnyTag(ErrorGenerator): - # TODO do we need to generate it for all combinations of send and recv? - - def __init__(self): - pass - - def get_feature(self): - return ["P2P"] - - def generate(self, generate_full_set): - tm = TemplateManager() - tm.set_description("MsgRace-ANY_TAG", "order of messages is indeterministic, may lead to a deadlock") - - b = InstructionBlock("alloc") - b.register_operation(CorrectParameterFactory().get_buffer_alloc()) - tm.register_instruction_block(b) - - b = InstructionBlock("MPICALL") - # send part - b.register_operation("for(int i =0; i < 10; ++i) {", kind=1) - b.register_operation("buf[0]=i;", kind=1) - send_call = CorrectMPICallFactory().mpi_send() - send_call.set_arg("tag", "i") - b.register_operation(send_call, kind=1) - b.register_operation("}", kind=1) - # recv part - b.register_operation("for(int i =0; i < 10; ++i) {", kind=0) - recv_call = CorrectMPICallFactory().mpi_recv() - recv_call.set_arg("tag", "MPI_ANY_TAG") - b.register_operation(recv_call, kind=0) - b.register_operation("if(buf[0]!=i){", kind=0) - additional_recv = CorrectMPICallFactory().mpi_recv() - additional_recv.set_has_error() # additional recv leads to deadlock - b.register_operation(additional_recv, kind=0) - b.register_operation(" }", kind=0) # end if - b.register_operation("}", kind=0) # end for - - tm.register_instruction_block(b) - - b = InstructionBlock("free") - b.register_operation(CorrectParameterFactory().get_buffer_free()) - tm.register_instruction_block(b) - - yield tm - - -class MessageRaceErrorAnysource(ErrorGenerator): - # TODO do we need to generate it for all combinations of send and recv? - - def __init__(self): - pass - - def get_feature(self): - return ["P2P"] - - def generate(self, generate_full_set): - tm = TemplateManager(min_ranks=3) - tm.set_description("MsgRace-ANY_SOURCE", "order of messages is indeterministic, may lead to a deadlock") - - b = InstructionBlock("alloc") - b.register_operation(CorrectParameterFactory().get_buffer_alloc()) - tm.register_instruction_block(b) - - b = InstructionBlock("MPICALL") - # send part - b.register_operation("buf[0]=rank;", kind='not0') - send_call = CorrectMPICallFactory().mpi_send() - b.register_operation(send_call, kind='not0') - - # recv part - b.register_operation("for(int i =1; i < nprocs; ++i) {", kind=0) - recv_call = CorrectMPICallFactory().mpi_recv() - recv_call.set_arg("source", "MPI_ANY_SOURCE") - b.register_operation(recv_call, kind=0) - b.register_operation("if(buf[0]!=i){", kind=0) - additional_recv = CorrectMPICallFactory().mpi_recv() - additional_recv.set_has_error() # additional recv leads to deadlock - b.register_operation(additional_recv, kind=0) - b.register_operation(" }", kind=0) # end if - b.register_operation("}", kind=0) # end for - - tm.register_instruction_block(b) - - b = InstructionBlock("free") - b.register_operation(CorrectParameterFactory().get_buffer_free()) - tm.register_instruction_block(b) - - yield tm diff --git a/scripts/errors/pt2pt/__init__.py b/scripts/errors/pt2pt/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000