From 1747635ae3b07a8d2e0508d3c2e55c5936d64d65 Mon Sep 17 00:00:00 2001
From: Simon Schwitanski <schwitanski@itc.rwth-aachen.de>
Date: Sun, 4 Feb 2024 22:03:35 +0100
Subject: [PATCH 01/18] InstructionBlock: Add get_operations and
register_operations for lists of operations
---
scripts/Infrastructure/InstructionBlock.py | 24 ++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/scripts/Infrastructure/InstructionBlock.py b/scripts/Infrastructure/InstructionBlock.py
index ec5310b92..dfb40d40a 100644
--- a/scripts/Infrastructure/InstructionBlock.py
+++ b/scripts/Infrastructure/InstructionBlock.py
@@ -50,6 +50,20 @@ class InstructionBlock:
self.operations[as_int] = []
self.operations[as_int].append(op)
+ def register_operations(self, ops, kind='all'):
+ """
+ Registers a list of operations based on rank.
+
+ Parameters:
+ - ops: The operations to register.
+ - kind: Rank to execute the operation ('all', 'not0', or integer).
+ - all: all Ranks execute this operation
+ - not0: all Ranks but the Root (rank 0) execute
+ - Or the integer of the rank that should execute
+ """
+ for op in ops:
+ self.register_operation(op, kind)
+
def get_version(self):
"""
Retrieves the minimum required MPI version.
@@ -114,6 +128,16 @@ class InstructionBlock:
str: The operation specified by kind and index
"""
return self.operations[kind][index]
+
+ def get_operations(self, kind='all'):
+ """
+ Retrieve all operations registered for the given kind.
+ Parameters:
+ - kind ('all','not0' or integer): which ranks should execute the operation
+ Returns:
+ str: List of all operations with given kind
+ """
+ return self.operations[kind]
def replace_operation(self, op, kind='all', index=0):
"""
--
GitLab
From f994cc14378b7acaabd074b89b707750e5a87e21 Mon Sep 17 00:00:00 2001
From: Simon Schwitanski <schwitanski@itc.rwth-aachen.de>
Date: Sun, 4 Feb 2024 22:05:41 +0100
Subject: [PATCH 02/18] Add default parameters for RMA calls
---
scripts/Infrastructure/CorrectParameter.py | 28 ++++++++++++++++++----
1 file changed, 24 insertions(+), 4 deletions(-)
diff --git a/scripts/Infrastructure/CorrectParameter.py b/scripts/Infrastructure/CorrectParameter.py
index 7eea7f606..d4aac6405 100644
--- a/scripts/Infrastructure/CorrectParameter.py
+++ b/scripts/Infrastructure/CorrectParameter.py
@@ -8,8 +8,10 @@ class CorrectParameterFactory:
# default params
buf_size = 10
dtype = ['int', 'MPI_INT']
+ buf_size_bytes = f"{buf_size}*sizeof({dtype[0]})"
tag = 0
buf_var_name = "buf"
+ winbuf_var_name = "winbuf"
def __init__(self):
pass
@@ -21,13 +23,13 @@ class CorrectParameterFactory:
return get_free(AllocCall(self.dtype[0], self.buf_size, self.buf_var_name, use_malloc=False))
def get(self, param, func=None):
- if param in ["BUFFER", "buf", "buffer", "sendbuf", "recvbuf"]:
+ if param in ["BUFFER", "buf", "buffer", "sendbuf", "recvbuf", "origin_addr"]:
return self.buf_var_name
- if param in ["COUNT", "count", "sendcount", "recvcount"]:
+ if param in ["COUNT", "count", "sendcount", "recvcount", "origin_count", "target_count", "result_count"]:
return str(self.buf_size)
- if param in ["DATATYPE", "datatype", "sendtype", "recvtype"]:
+ if param in ["DATATYPE", "datatype", "sendtype", "recvtype", "origin_datatype", "target_datatype", "result_datatype"]:
return self.dtype[1]
- if param in ["DEST", "dest"]:
+ if param in ["DEST", "dest", "target_rank"]:
return "0"
if param in ["SRC", "source"]:
return "1"
@@ -71,6 +73,24 @@ class CorrectParameterFactory:
return "MPI_COMM_WORLD"
if param in ["remote_leader"]:
return "0"
+ if param in ["target_disp"]:
+ return "0"
+ if param in ["win"]:
+ return "win"
+ if param in ["baseptr"]:
+ return "&" + self.winbuf_var_name
+ if param in ["base"]:
+ return self.winbuf_var_name
+ if param in ["size"]:
+ return self.buf_size_bytes
+ if param in ["disp_unit"]:
+ return "sizeof(int)"
+ if param in ["info"]:
+ return "MPI_INFO_NULL"
+ if param in ["result_addr"]:
+ return "resultbuf"
+ if param in ["compare_addr"]:
+ return "comparebuf"
print("Not Implemented: " + param)
assert False, "Param not known"
--
GitLab
From 17730c7d8519ee95c6a6517124f907d7a7357c71 Mon Sep 17 00:00:00 2001
From: Simon Schwitanski <schwitanski@itc.rwth-aachen.de>
Date: Sun, 4 Feb 2024 22:06:15 +0100
Subject: [PATCH 03/18] Add templates for window allocations and RMA calls
---
scripts/Infrastructure/TemplateFactory.py | 68 +++++++++++++++++++++++
1 file changed, 68 insertions(+)
diff --git a/scripts/Infrastructure/TemplateFactory.py b/scripts/Infrastructure/TemplateFactory.py
index 1790c08df..47abb7688 100644
--- a/scripts/Infrastructure/TemplateFactory.py
+++ b/scripts/Infrastructure/TemplateFactory.py
@@ -184,6 +184,74 @@ def get_collective_template(collective_func, seperate=True):
return tm
+def get_allocated_window(win_alloc_func, name, bufname, ctype, num_elements):
+ """
+ Constructs a window allocation using Win_allocate or Win_create.
+ :param win_alloc_func: The window allocation to use (mpi_win_allocate or mpi_win_create).
+ :param name: name of the window
+ """
+
+ b = InstructionBlock("win_allocate")
+
+ # declare window
+ b.register_operation(f"MPI_Win {name};")
+
+ # extract C data type and window buffer name
+ # dtype = CorrectParameterFactory().dtype[0]
+ # winbuf_name = CorrectParameterFactory().winbuf_var_name
+ # winbuf_size = CorrectParameterFactory().buf_size_bytes
+ win_allocate_call = None
+
+ if win_alloc_func == "mpi_win_allocate":
+ # MPI allocate, only declaration required
+ b.register_operation(f"{ctype}* {bufname};")
+ win_allocate_call = CorrectMPICallFactory().mpi_win_allocate()
+ win_allocate_call.set_arg("baseptr", "&" + bufname)
+ elif win_alloc_func == "mpi_win_create":
+ # allocate buffer for win_create
+ b.register_operation(AllocCall(ctype, num_elements, bufname))
+ win_allocate_call = CorrectMPICallFactory().mpi_win_create()
+ win_allocate_call.set_arg("base", bufname)
+ else:
+ assert False
+
+ # set common parameters for both calls
+ win_allocate_call.set_arg("win", "&" + name)
+
+ buf_size_bytes = num_elements + "*sizeof(" + ctype + ")"
+ win_allocate_call.set_arg("size", buf_size_bytes)
+
+ win_allocate_call.set_arg("disp_unit", f"sizeof({ctype})")
+ b.register_operation(win_allocate_call)
+
+ return b
+
+
+
+def get_rma_call(rma_func, rank):
+
+ b = InstructionBlock(rma_func.replace('mpi_',''))
+
+ cf = CorrectParameterFactory()
+ cfmpi = CorrectMPICallFactory()
+
+ # request-based RMA call, add request
+ if rma_func.startswith("mpi_r"):
+ b.register_operation(f"MPI_Request " + cf.get("request")[1:] + ";", kind=rank)
+
+ # some RMA ops require result_addr
+ if rma_func in ["mpi_get_accumulate", "mpi_rget_accumulate", "mpi_fetch_and_op", "mpi_compare_and_swap"]:
+ b.register_operation(AllocCall(cf.dtype[0], cf.buf_size, cf.get("result_addr")), kind=rank)
+
+ # some RMA ops require compare_addr
+ if rma_func in ["mpi_fetch_and_op", "mpi_compare_and_swap"]:
+ b.register_operation(AllocCall(cf.dtype[0], cf.buf_size, cf.get("compare_addr")), kind=rank)
+
+ b.register_operation(getattr(cfmpi, rma_func)(), kind=rank)
+ return b
+
+
+
def get_communicator(comm_create_func, name):
"""
:param comm_create_func: teh function used to create the new communicator
--
GitLab
From 683de0c2a613e8b6d736a0c70b3046e208fc67cc Mon Sep 17 00:00:00 2001
From: Simon Schwitanski <schwitanski@itc.rwth-aachen.de>
Date: Sun, 4 Feb 2024 22:07:09 +0100
Subject: [PATCH 04/18] Add initial version of LocalConcurrency RMA generator
---
scripts/errors/rma/LocalConcurrency.py | 146 +++++++++++++++++++++++++
1 file changed, 146 insertions(+)
create mode 100644 scripts/errors/rma/LocalConcurrency.py
diff --git a/scripts/errors/rma/LocalConcurrency.py b/scripts/errors/rma/LocalConcurrency.py
new file mode 100644
index 000000000..152203dc6
--- /dev/null
+++ b/scripts/errors/rma/LocalConcurrency.py
@@ -0,0 +1,146 @@
+#! /usr/bin/python3
+
+from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
+from scripts.Infrastructure.InstructionBlock import InstructionBlock
+from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
+from scripts.Infrastructure.CorrectParameter import (
+ CorrectParameterFactory,
+ get_matching_recv,
+)
+from scripts.Infrastructure.Template import TemplateManager
+from scripts.Infrastructure.TemplateFactory import get_allocated_window, get_rma_call
+from scripts.Infrastructure.AllocCall import AllocCall
+from scripts.Infrastructure.MPICall import MPI_Call
+
+import itertools
+
+from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT
+
+from typing import Tuple, List
+
+class LocalConcurrencyErrorRMA(ErrorGenerator):
+ local_origin_addr_write = ["mpi_get", "mpi_rget"]
+ local_origin_addr_read = [
+ "mpi_put",
+ "mpi_rput",
+ "mpi_accumulate",
+ "mpi_raccumulate",
+ "mpi_get_accumulate",
+ "mpi_rget_accumulate",
+ "mpi_fetch_and_op",
+ "mpi_compare_and_swap",
+ ]
+ functions_to_check = ["mpi_put", "mpi_get", "mpi_rput", "mpi_rget"]
+
+ # recv_funcs = ["mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
+
+ def __init__(self):
+ pass
+
+ def get_feature(self):
+ return ["RMA"]
+
+ def generate(self, generate_full_set):
+
+ cf = CorrectParameterFactory()
+ cfmpi = CorrectMPICallFactory()
+
+ mpi_buf_read = [
+ get_rma_call("mpi_put", 0),
+ get_rma_call("mpi_rput", 0),
+ get_rma_call("mpi_accumulate", 0),
+ get_rma_call("mpi_raccumulate", 0),
+ get_rma_call("mpi_get_accumulate", 0),
+ get_rma_call("mpi_rget_accumulate", 0),
+ get_rma_call("mpi_fetch_and_op", 0),
+ get_rma_call("mpi_compare_and_swap", 0),
+ ]
+ mpi_buf_write = [get_rma_call("mpi_get", 0), get_rma_call("mpi_rget", 0)]
+
+ bufread = InstructionBlock("bufread")
+ bufread.register_operation(f'printf("buf is %d\\n", {cf.buf_var_name}[1]);', 0)
+ bufwrite = InstructionBlock("write")
+ bufwrite.register_operation(f'{cf.buf_var_name}[1] = 42;', 0)
+
+ # 7 possible combinations of local buffer accesses (hasconflict = True | False)
+ local_access_combinations: List[Tuple[List[InstructionBlock], List[InstructionBlock], bool]] = [
+ (mpi_buf_read, [bufread], False),
+ (mpi_buf_read, [bufwrite], True),
+ (mpi_buf_write, [bufread], True),
+ (mpi_buf_write, [bufwrite], True),
+ (mpi_buf_read, mpi_buf_read, False),
+ (mpi_buf_read, mpi_buf_write, True),
+ (mpi_buf_write, mpi_buf_write, True),
+ ]
+
+ for ops1, ops2, hasconflict in local_access_combinations:
+ for (op1, op2) in itertools.product(ops1, ops2):
+ tm = TemplateManager()
+ # window allocation boilerplate
+ b = get_allocated_window("mpi_win_create", "win", "winbuf", "int", "2")
+ tm.register_instruction_block(b)
+
+ # local buffer allocation
+ alloc = InstructionBlock("alloc")
+ alloc.register_operation(
+ AllocCall(cf.dtype[0], cf.buf_size, cf.buf_var_name)
+ )
+ tm.register_instruction_block(alloc)
+
+ if hasconflict:
+ if isinstance(op1.get_operation(kind=0, index=-1), MPI_Call):
+ op1.get_operation(kind=0, index=-1).set_has_error()
+ else:
+ op1.replace_operation(op1.get_operation(kind=0, index=-1) + ERROR_MARKER_COMMENT, 0, 0)
+
+ if isinstance(op2.get_operation(kind=0, index=-1), MPI_Call):
+ op2.get_operation(kind=0, index=-1).set_has_error()
+ else:
+ op2.replace_operation(op2.get_operation(kind=0, index=-1) + ERROR_MARKER_COMMENT, 0, 0)
+
+ # fuse instructions blocks
+ # combined_ops = InstructionBlock("COMBINED")
+ # combined_ops.register_operations(op1.get_operations(kind=0), kind=0)
+ # combined_ops.register_operations(op2.get_operations(kind=0), kind=0)
+ tm.register_instruction_block(op1)
+ tm.register_instruction_block(op2)
+
+ tm.set_description(
+ ("LocalConcurrency" if hasconflict else "Correct") +
+ "-"
+ + op1.name
+ + "_"
+ + op2.name,
+ "full description",
+ )
+ yield tm
+
+ # get RMA call
+ # rmaop = get_rma_call(function_to_check, 0)
+
+ # tm.register_instruction_block(rmaop)
+
+ # bufstring = ""
+ # if bufop == "read": # local buffer access is read
+ # bufstring = f'printf("buf is %d\\n", {cf.buf_var_name}[1]);'
+ # # if RMA call performs local buffer write, this is a race, otherwise no race
+ # if function_to_check in local_origin_addr_write:
+ # bufstring += ERROR_MARKER_COMMENT
+ # # mark RMA call as erroneous
+ # tm.get_block("RMACALL").get_operation(
+ # kind=0, index=-1
+ # ).set_has_error()
+
+ # if bufop == "write":
+ # # a buffer write is always a race
+ # bufstring = f"{cf.buf_var_name}[1] = 42;" + ERROR_MARKER_COMMENT
+ # # mark RMA call as erroneous
+ # tm.get_block("RMACALL").get_operation(
+ # kind=0, index=-1
+ # ).set_has_error()
+
+ # # finally register buffer access
+ # tm.get_block("RMACALL").register_operation(bufstring, 0)
+
+ # if not generate_full_set:
+ # return
--
GitLab
From 5e272eefd3b7b8ed898baab4bd8ae798150f5074 Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 12:46:14 +0100
Subject: [PATCH 05/18] fix wrong generators
---
scripts/errors/pt2pt/InvalidRank.py | 1 -
scripts/errors/pt2pt/InvalidTag.py | 4 ++--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/scripts/errors/pt2pt/InvalidRank.py b/scripts/errors/pt2pt/InvalidRank.py
index c049e79dc..95f6db52d 100644
--- a/scripts/errors/pt2pt/InvalidRank.py
+++ b/scripts/errors/pt2pt/InvalidRank.py
@@ -13,7 +13,6 @@ from itertools import chain
class InvalidRankErrorP2P(ErrorGenerator):
invalid_ranks = ["-1", "nprocs", "MPI_PROC_NULL"]
send_funcs = ["mpi_send",
- "mpi_irecv",
"mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend",
"mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init"
]
diff --git a/scripts/errors/pt2pt/InvalidTag.py b/scripts/errors/pt2pt/InvalidTag.py
index e5c360806..74041139b 100644
--- a/scripts/errors/pt2pt/InvalidTag.py
+++ b/scripts/errors/pt2pt/InvalidTag.py
@@ -69,11 +69,11 @@ class InvalidTagErrorP2P(ErrorGenerator):
def generate(self, generate_full_set):
for send_func in self.send_funcs:
- yield from self.generate_impl(self, send_func, "mpi_irecv", False)
+ yield from self.generate_impl(send_func, "mpi_irecv", False)
if not generate_full_set:
return
for recv_func in self.recv_funcs:
- yield from self.generate_impl(self, "mpi_send", recv_func, True)
+ yield from self.generate_impl("mpi_send", recv_func, True)
for func in self.sendrecv_funcs:
yield from self.generate_impl(func, func, True)
--
GitLab
From 92f7fa9c788d925518ebcb7ab65fa45816700bbc Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 13:00:22 +0100
Subject: [PATCH 06/18] added Instruction class (#6)
---
scripts/Infrastructure/AllocCall.py | 7 +++++--
scripts/Infrastructure/Instruction.py | 13 +++++++++++++
scripts/Infrastructure/MPICall.py | 6 +++++-
3 files changed, 23 insertions(+), 3 deletions(-)
create mode 100644 scripts/Infrastructure/Instruction.py
diff --git a/scripts/Infrastructure/AllocCall.py b/scripts/Infrastructure/AllocCall.py
index 25dea717c..0e11f2462 100644
--- a/scripts/Infrastructure/AllocCall.py
+++ b/scripts/Infrastructure/AllocCall.py
@@ -1,5 +1,7 @@
#! /usr/bin/python3
+from typing_extensions import override
+from scripts.Infrastructure.Instruction import Instruction
alloc_template = """
@{TYPE}@* @{NAME}@ = (@{TYPE}@*) @{FUNCTION}@(@{NUM}@ @{SEP}@ sizeof(@{TYPE}@));
@@ -20,8 +22,8 @@ alloc_template = """
"""
-class AllocCall:
-
+class AllocCall (Instruction):
+ @override
def __init__(self, type, num_elements, name="buf", use_malloc=False):
"""
Creates a New allocation Call
@@ -37,6 +39,7 @@ class AllocCall:
self._num_elements = num_elements
self._name = name
+ @override
def __str__(self):
if self._use_malloc:
delim = '*'
diff --git a/scripts/Infrastructure/Instruction.py b/scripts/Infrastructure/Instruction.py
new file mode 100644
index 000000000..16165d5b0
--- /dev/null
+++ b/scripts/Infrastructure/Instruction.py
@@ -0,0 +1,13 @@
+#! /usr/bin/python3
+
+
+class Instruction(object):
+ """
+ Base class to represent an Instruction
+ """
+
+ def __init__(self, str_representation):
+ self._str_representation = str_representation
+
+ def __str__(self):
+ return self._str_representation
diff --git a/scripts/Infrastructure/MPICall.py b/scripts/Infrastructure/MPICall.py
index 02c1cf359..05f525928 100644
--- a/scripts/Infrastructure/MPICall.py
+++ b/scripts/Infrastructure/MPICall.py
@@ -1,16 +1,20 @@
#! /usr/bin/python3
+from typing_extensions import override
+from scripts.Infrastructure.Instruction import Instruction
from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT
-class MPI_Call:
+class MPI_Call (Instruction):
+ @override
def __init__(self, function, args, version):
self._function = function
self._args = args
self._version = version
self._has_error = False
+ @override
def __str__(self):
s = self._function + "("
for k, v in self._args.items():
--
GitLab
From 6bf6c24572352416918186f9af51742b91e178a2 Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 13:09:50 +0100
Subject: [PATCH 07/18] added Type hints (#7) for the Instruction Classes
---
scripts/Infrastructure/AllocCall.py | 22 +++++++++++-----------
scripts/Infrastructure/Instruction.py | 2 +-
scripts/Infrastructure/MPICall.py | 14 +++++++-------
3 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/scripts/Infrastructure/AllocCall.py b/scripts/Infrastructure/AllocCall.py
index 0e11f2462..26e8a8c3c 100644
--- a/scripts/Infrastructure/AllocCall.py
+++ b/scripts/Infrastructure/AllocCall.py
@@ -22,9 +22,9 @@ alloc_template = """
"""
-class AllocCall (Instruction):
+class AllocCall(Instruction):
@override
- def __init__(self, type, num_elements, name="buf", use_malloc=False):
+ def __init__(self, type: str, num_elements: int, name: str = "buf", use_malloc: bool = False):
"""
Creates a New allocation Call
@@ -55,13 +55,13 @@ class AllocCall (Instruction):
.replace("@{NUM}@", str(self._num_elements))
.replace("@{SEP}@", delim))
- def set_num_elements(self, num_elements):
+ def set_num_elements(self, num_elements: int):
self._num_elements = num_elements
- def set_name(self, name):
+ def set_name(self, name: str):
self._name = name
- def set_type(self, type):
+ def set_type(self, type: str):
self._type = type
def set_use_malloc(self):
@@ -70,19 +70,19 @@ class AllocCall (Instruction):
def set_use_calloc(self):
self._use_malloc = False
- def get_num_elements(self):
+ def get_num_elements(self) -> int:
return self._num_elements
- def get_name(self):
+ def get_name(self) -> str:
return self._name
- def get_type(self):
+ def get_type(self) -> str:
return self._type
- def get_use_malloc(self):
+ def get_use_malloc(self) -> bool:
return self._use_malloc
-def get_free(alloc_call):
+def get_free(alloc_call: AllocCall) -> Instruction:
assert isinstance(alloc_call, AllocCall)
- return "free(" + alloc_call.get_name() + ");\n"
+ return Instruction("free(" + alloc_call.get_name() + ");")
diff --git a/scripts/Infrastructure/Instruction.py b/scripts/Infrastructure/Instruction.py
index 16165d5b0..17ee42ccf 100644
--- a/scripts/Infrastructure/Instruction.py
+++ b/scripts/Infrastructure/Instruction.py
@@ -10,4 +10,4 @@ class Instruction(object):
self._str_representation = str_representation
def __str__(self):
- return self._str_representation
+ return self._str_representation + "\n"
diff --git a/scripts/Infrastructure/MPICall.py b/scripts/Infrastructure/MPICall.py
index 05f525928..0d5386803 100644
--- a/scripts/Infrastructure/MPICall.py
+++ b/scripts/Infrastructure/MPICall.py
@@ -1,14 +1,15 @@
#! /usr/bin/python3
+import typing
from typing_extensions import override
from scripts.Infrastructure.Instruction import Instruction
from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT
-class MPI_Call (Instruction):
+class MPI_Call(Instruction):
@override
- def __init__(self, function, args, version):
+ def __init__(self, function: str, args: typing.OrderedDict[str, str], version: str):
self._function = function
self._args = args
self._version = version
@@ -27,16 +28,15 @@ class MPI_Call (Instruction):
return s
- def set_arg(self, arg, value):
+ def set_arg(self, arg: str, value: str):
assert self.has_arg(arg)
self._args[arg] = value
- def has_arg(self, arg):
+ def has_arg(self, arg: str) -> bool:
return arg in self._args
- def get_version(self):
+ def get_version(self) -> str:
return self._version
- def set_has_error(self, has_error=True):
+ def set_has_error(self, has_error: bool = True):
self._has_error = has_error
-
--
GitLab
From 54801c07c4f77ffd7004f8d9e4b765176b7979fe Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 13:20:56 +0100
Subject: [PATCH 08/18] added Type hints (#7) for the Template Manager and
InstructionBlock classes
---
scripts/Infrastructure/InstructionBlock.py | 56 +++++++++-------------
scripts/Infrastructure/Template.py | 18 +++----
2 files changed, 32 insertions(+), 42 deletions(-)
diff --git a/scripts/Infrastructure/InstructionBlock.py b/scripts/Infrastructure/InstructionBlock.py
index dfb40d40a..a0119fa48 100644
--- a/scripts/Infrastructure/InstructionBlock.py
+++ b/scripts/Infrastructure/InstructionBlock.py
@@ -1,3 +1,8 @@
+from __future__ import annotations
+
+import typing
+
+from scripts.Infrastructure.Instruction import Instruction
from scripts.Infrastructure.MPICall import MPI_Call
@@ -16,7 +21,7 @@ class InstructionBlock:
- `__str__(self)`: Converts the InstructionBlock instance to a string, replacing placeholders.
"""
- def __init__(self, name=None):
+ def __init__(self, name: str = None):
"""
Initialize an empty InstructionBlock
@@ -29,7 +34,7 @@ class InstructionBlock:
self.name = name
# TODO test if op also can be a list of operations
- def register_operation(self, op, kind='all'):
+ def register_operation(self, op: Instruction | typing.List[Instruction], kind: str | int = 'all'):
"""
Registers an operation based on rank.
@@ -50,21 +55,9 @@ class InstructionBlock:
self.operations[as_int] = []
self.operations[as_int].append(op)
- def register_operations(self, ops, kind='all'):
- """
- Registers a list of operations based on rank.
-
- Parameters:
- - ops: The operations to register.
- - kind: Rank to execute the operation ('all', 'not0', or integer).
- - all: all Ranks execute this operation
- - not0: all Ranks but the Root (rank 0) execute
- - Or the integer of the rank that should execute
- """
- for op in ops:
- self.register_operation(op, kind)
+ # TODO implement list part
- def get_version(self):
+ def get_version(self) -> str:
"""
Retrieves the minimum required MPI version.
Returns:
@@ -103,7 +96,7 @@ class InstructionBlock:
return result_str
- def has_operation(self, kind='all', index=0):
+ def has_operation(self, kind: int | str = 'all', index: int = 0) -> bool:
"""
Checks if the Block has an operation with the given index and kind
Parameters:
@@ -115,10 +108,10 @@ class InstructionBlock:
try:
result = self.operations[kind][index]
return True
- except (KeyError,IndexError) as e:
+ except (KeyError, IndexError) as e:
return False
- def get_operation(self, kind='all', index=0):
+ def get_operation(self, kind: int | str = 'all', index: int = 0) -> Instruction | typing.List[Instruction]:
"""
Retrieve the operation registered. will Raise IndexError if not present
Parameters:
@@ -128,18 +121,10 @@ class InstructionBlock:
str: The operation specified by kind and index
"""
return self.operations[kind][index]
-
- def get_operations(self, kind='all'):
- """
- Retrieve all operations registered for the given kind.
- Parameters:
- - kind ('all','not0' or integer): which ranks should execute the operation
- Returns:
- str: List of all operations with given kind
- """
- return self.operations[kind]
- def replace_operation(self, op, kind='all', index=0):
+ # todo implement list case
+
+ def replace_operation(self, op: Instruction | typing.List[Instruction], kind: str | int = 'all', index: int = 0):
"""
Replace the operation registered. will Raise IndexError if not present
Parameters:
@@ -151,7 +136,10 @@ class InstructionBlock:
raise IndexError("Operation Not Found")
self.operations[kind][index] = op
- def insert_operation(self, op, kind='all', before_index=0):
+ # todo implement list caee
+
+ def insert_operation(self, op: Instruction | typing.List[Instruction], kind: str | int = 'all',
+ before_index: int = 0):
"""
Inserts an operation before the specified one. will Raise IndexError if not present
Parameters:
@@ -161,9 +149,11 @@ class InstructionBlock:
"""
if len(self.operations[kind]) < before_index:
raise IndexError("Operation Not Found")
- self.operations[kind].insert(before_index,op)
+ self.operations[kind].insert(before_index, op)
+
+ # todo implement list caee
- def remove_operation(self, kind='all', index=0):
+ def remove_operation(self, kind: str | int = 'all', index: int = 0):
"""
Removes the operation registered. will Raise IndexError if not present
Parameters:
diff --git a/scripts/Infrastructure/Template.py b/scripts/Infrastructure/Template.py
index f54702605..330c671d8 100644
--- a/scripts/Infrastructure/Template.py
+++ b/scripts/Infrastructure/Template.py
@@ -71,7 +71,7 @@ class TemplateManager:
- `get_short_descr(self)`: Retrieves the short description .
"""
- def __init__(self, min_ranks=2, thread_level=None, has_finalize=True, has_init=True):
+ def __init__(self, min_ranks: int = 2, thread_level: str = None, has_finalize: bool = True, has_init: bool = True):
"""
Initialize the TemplateManager
@@ -117,7 +117,7 @@ class TemplateManager:
.replace("@{version}@", version)
.replace("@{test_code}@", block_string))
- def register_instruction_block(self, block):
+ def register_instruction_block(self, block: InstructionBlock):
"""
Registers an instruction block with the template. inserting it at the end, before the mpi finalize
Parameters:
@@ -125,7 +125,7 @@ class TemplateManager:
"""
self._blocks.append(block)
- def get_version(self):
+ def get_version(self) -> str:
"""
Retrieves the minimum required MPI version.
Returns:
@@ -137,7 +137,7 @@ class TemplateManager:
max_v = max(block.get_version(), max_v)
return max_v
- def set_description(self, descr_short, descr_full):
+ def set_description(self, descr_short: str, descr_full: str):
"""
Sets the short and full descriptions for the template.
Parameters:
@@ -148,7 +148,7 @@ class TemplateManager:
self._descr_short = descr_short
# TODO one could write a function to check if short desc = filename is conforming with the naming sceme
- def get_short_descr(self):
+ def get_short_descr(self) -> str:
"""
Retrieves the short description to use as a filename.
Returns:
@@ -157,7 +157,7 @@ class TemplateManager:
assert self._descr_short != ""
return self._descr_short
- def get_block(self, block_name=None, idx=None):
+ def get_block(self, block_name: str = None, idx: int = None) -> InstructionBlock:
"""
Retrieves the given Instruction Block Either by name or by index
Raises IndexError if the specified block is not found
@@ -187,7 +187,7 @@ class TemplateManager:
raise ValueError("Neither Both block name nor index is given")
- def insert_block(self, new_block, after_block_name=None, after_idx=None):
+ def insert_block(self, new_block: InstructionBlock, after_block_name: str = None, after_idx: int = None):
"""
inserts the given Instruction Block AFTER the one specified Either by name or by index
Raises IndexError if the specified block is not found
@@ -217,7 +217,7 @@ class TemplateManager:
raise ValueError("Neither Both block name nor index is given")
- def remove_block(self, block_name=None, idx=None):
+ def remove_block(self, block_name: str = None, idx: int = None):
"""
Removes the given Instruction Block Either by name or by index
Raises IndexError if the specified block is not found
@@ -244,7 +244,7 @@ class TemplateManager:
raise ValueError("Neither Both block name nor index is given")
- def replace_block(self, new_block, block_name=None, idx=None):
+ def replace_block(self, new_block: InstructionBlock, block_name: str = None, idx: int = None):
"""
Removes the given Instruction Block Either by name or by index
Raises IndexError if the specified block is not found
--
GitLab
From b3054538d81d217413dd4a14bbd5f743c1eca85c Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 13:40:01 +0100
Subject: [PATCH 09/18] implemented List cases for managing list of
Instructions (see !7)
---
scripts/Infrastructure/InstructionBlock.py | 86 ++++++++++++++--------
1 file changed, 55 insertions(+), 31 deletions(-)
diff --git a/scripts/Infrastructure/InstructionBlock.py b/scripts/Infrastructure/InstructionBlock.py
index a0119fa48..05b8a16b3 100644
--- a/scripts/Infrastructure/InstructionBlock.py
+++ b/scripts/Infrastructure/InstructionBlock.py
@@ -33,29 +33,35 @@ class InstructionBlock:
assert not isinstance(name, int)
self.name = name
- # TODO test if op also can be a list of operations
def register_operation(self, op: Instruction | typing.List[Instruction], kind: str | int = 'all'):
"""
Registers an operation based on rank.
Parameters:
- - op: The operation to register.
+ - op: The operation (or list of Operations) to register.
- kind: Rank to execute the operation ('all', 'not0', or integer).
- all: all Ranks execute this operation
- not0: all Ranks but the Root (rank 0) execute
- Or the integer of the rank that should execute
"""
if kind == 'all':
- self.operations['all'].append(op)
+ if isinstance(op, list):
+ self.operations['all'].extend(op)
+ else:
+ self.operations['all'].append(op)
elif kind == 'not0':
- self.operations['not0'].append(op)
+ if isinstance(op, list):
+ self.operations['not0'].extend(op)
+ else:
+ self.operations['not0'].append(op)
else:
as_int = int(kind) # will Raise ValueError if not integer
if as_int not in self.operations:
self.operations[as_int] = []
- self.operations[as_int].append(op)
-
- # TODO implement list part
+ if isinstance(op, list):
+ self.operations[as_int].extend(op)
+ else:
+ self.operations[as_int].append(op)
def get_version(self) -> str:
"""
@@ -101,7 +107,7 @@ class InstructionBlock:
Checks if the Block has an operation with the given index and kind
Parameters:
- kind ('all','not0' or integer): which ranks should execute the operation
- - index (int): the index of the operation within the given kind
+ - index (int ): the index of the operation within the given kind
Returns:
boolean
"""
@@ -111,55 +117,73 @@ class InstructionBlock:
except (KeyError, IndexError) as e:
return False
- def get_operation(self, kind: int | str = 'all', index: int = 0) -> Instruction | typing.List[Instruction]:
+ def get_operation(self, kind: int | str = 'all', index: str | int = 0) -> Instruction | typing.List[Instruction]:
"""
Retrieve the operation registered. will Raise IndexError if not present
Parameters:
- kind ('all','not0' or integer): which ranks should execute the operation
- - index (int): the index of the operation within the given kind
+ - index ('all' or int): the index of the operation within the given kind; 'all' means that the list of all operations for the kind is returned
Returns:
str: The operation specified by kind and index
"""
- return self.operations[kind][index]
-
- # todo implement list case
+ if index == 'all':
+ return self.operations[kind]
+ else:
+ as_int = int(index) # will Raise ValueError if not integer
+ return self.operations[kind][as_int]
- def replace_operation(self, op: Instruction | typing.List[Instruction], kind: str | int = 'all', index: int = 0):
+ def replace_operation(self, op: Instruction | typing.List[Instruction], kind: str | int = 'all',
+ index: str | int = 0):
"""
Replace the operation registered. will Raise IndexError if not present
Parameters:
- - op (str or MPICall) the new operation
+ - op the new operation or list of operations
- kind ('all','not0' or integer): which ranks should execute the operation
- - index (int): the index of the operation within the given kind
- """
- if len(self.operations[kind]) < index:
- raise IndexError("Operation Not Found")
- self.operations[kind][index] = op
-
- # todo implement list caee
+ - index ('all' or int): the index of the operation within the given kind; 'all' means all operations will be replaced with the given list
+ Notes : if one wants to replace all operations one needs to provide a list
+ if one only wants to replace one operation: no list of operations is allowed
+ """
+ if index == 'all':
+ if not isinstance(op, list):
+ raise ValueError('Provide List for replacement')
+ self.operations[kind] = op
+ else:
+ as_int = int(index) # will Raise ValueError if not integer
+ if not isinstance(op, Instruction):
+ raise ValueError('Provide Instruction')
+ if len(self.operations[kind]) < as_int:
+ raise IndexError("Operation Not Found")
+ self.operations[kind][as_int] = op
def insert_operation(self, op: Instruction | typing.List[Instruction], kind: str | int = 'all',
before_index: int = 0):
"""
Inserts an operation before the specified one. will Raise IndexError if not present
Parameters:
- - op (str or MPICall) the new operation
+ - op the new operation or list of operations
- kind ('all','not0' or integer): which ranks should execute the operation
- index (int): the index of the operation within the given kind
"""
+ as_int = int(before_index) # will Raise ValueError if not integer
if len(self.operations[kind]) < before_index:
raise IndexError("Operation Not Found")
- self.operations[kind].insert(before_index, op)
-
- # todo implement list caee
+ if isinstance(op, list):
+ self.operations[kind] = (
+ self.operations[kind][0:before_index - 1] + op + self.operations[kind][before_index:])
+ else:
+ self.operations[kind].insert(before_index, op)
- def remove_operation(self, kind: str | int = 'all', index: int = 0):
+ def remove_operation(self, kind: str | int = 'all', index: str | int = 0):
"""
Removes the operation registered. will Raise IndexError if not present
Parameters:
- kind ('all','not0' or integer): which ranks should execute the operation
- - index (int): the index of the operation within the given kind
+ - index ('all' or int): the index of the operation within the given kind
"""
- if len(self.operations[kind]) < index:
- raise IndexError("Operation Not Found")
- del self.operations[kind][index]
+ if index == 'all':
+ self.operations[kind] = []
+ else:
+ as_int = int(index) # will Raise ValueError if not integer
+ if len(self.operations[kind]) < index:
+ raise IndexError("Operation Not Found")
+ del self.operations[kind][index]
--
GitLab
From 8fb974512cc238c8eef98a3ad896be8353f91603 Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 13:42:46 +0100
Subject: [PATCH 10/18] added more type hints
---
scripts/Infrastructure/CorrectParameter.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/scripts/Infrastructure/CorrectParameter.py b/scripts/Infrastructure/CorrectParameter.py
index d4aac6405..7304b0484 100644
--- a/scripts/Infrastructure/CorrectParameter.py
+++ b/scripts/Infrastructure/CorrectParameter.py
@@ -1,4 +1,6 @@
#! /usr/bin/python3
+from scripts.Infrastructure import MPICall
+from scripts.Infrastructure.Instruction import Instruction
from scripts.Infrastructure.MPICallFactory import MPICallFactory
from scripts.Infrastructure.Template import InstructionBlock
from scripts.Infrastructure.AllocCall import AllocCall, get_free
@@ -16,13 +18,13 @@ class CorrectParameterFactory:
def __init__(self):
pass
- def get_buffer_alloc(self):
+ def get_buffer_alloc(self)-> AllocCall:
return AllocCall(self.dtype[0], self.buf_size, self.buf_var_name, use_malloc=False)
- def get_buffer_free(self):
+ def get_buffer_free(self)->Instruction:
return get_free(AllocCall(self.dtype[0], self.buf_size, self.buf_var_name, use_malloc=False))
- def get(self, param, func=None):
+ def get(self, param:str)->str:
if param in ["BUFFER", "buf", "buffer", "sendbuf", "recvbuf", "origin_addr"]:
return self.buf_var_name
if param in ["COUNT", "count", "sendcount", "recvcount", "origin_count", "target_count", "result_count"]:
@@ -96,7 +98,7 @@ class CorrectParameterFactory:
# todo also for send and non default args
-def get_matching_recv(call):
+def get_matching_recv(call:MPICall)->MPICall:
correct_params = CorrectParameterFactory()
recv = MPICallFactory().mpi_recv(
correct_params.get("BUFFER"),
--
GitLab
From f8da17157ed80b5f2c18d661bb3f418488b99744 Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 13:53:50 +0100
Subject: [PATCH 11/18] Refactoring: set_has_error into base class
---
scripts/Infrastructure/AllocCall.py | 19 +++++++++++++------
scripts/Infrastructure/Instruction.py | 10 +++++++++-
scripts/Infrastructure/MPICall.py | 4 +---
3 files changed, 23 insertions(+), 10 deletions(-)
diff --git a/scripts/Infrastructure/AllocCall.py b/scripts/Infrastructure/AllocCall.py
index 26e8a8c3c..983993fba 100644
--- a/scripts/Infrastructure/AllocCall.py
+++ b/scripts/Infrastructure/AllocCall.py
@@ -2,6 +2,7 @@
from typing_extensions import override
from scripts.Infrastructure.Instruction import Instruction
+from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT
alloc_template = """
@{TYPE}@* @{NAME}@ = (@{TYPE}@*) @{FUNCTION}@(@{NUM}@ @{SEP}@ sizeof(@{TYPE}@));
@@ -34,6 +35,7 @@ class AllocCall(Instruction):
name: name of buffer variable
use_malloc: True: use Malloc, False: use calloc for allocation
"""
+ super().__init__("")
self._use_malloc = use_malloc
self._type = type
self._num_elements = num_elements
@@ -48,12 +50,17 @@ class AllocCall(Instruction):
delim = ','
func = "calloc"
- return (alloc_template
- .replace("@{NAME}@", self._name)
- .replace("@{TYPE}@", self._type)
- .replace("@{FUNCTION}@", func)
- .replace("@{NUM}@", str(self._num_elements))
- .replace("@{SEP}@", delim))
+ s = (alloc_template
+ .replace("@{NAME}@", self._name)
+ .replace("@{TYPE}@", self._type)
+ .replace("@{FUNCTION}@", func)
+ .replace("@{NUM}@", str(self._num_elements))
+ .replace("@{SEP}@", delim))
+
+ if self._has_error:
+ s += ERROR_MARKER_COMMENT
+
+ return s
def set_num_elements(self, num_elements: int):
self._num_elements = num_elements
diff --git a/scripts/Infrastructure/Instruction.py b/scripts/Infrastructure/Instruction.py
index 17ee42ccf..475c19823 100644
--- a/scripts/Infrastructure/Instruction.py
+++ b/scripts/Infrastructure/Instruction.py
@@ -1,4 +1,5 @@
#! /usr/bin/python3
+from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT
class Instruction(object):
@@ -8,6 +9,13 @@ class Instruction(object):
def __init__(self, str_representation):
self._str_representation = str_representation
+ self._has_error = False
+
+ def set_has_error(self, has_error: bool = True):
+ self._has_error = has_error
def __str__(self):
- return self._str_representation + "\n"
+ if self._has_error:
+ return self._str_representation + ERROR_MARKER_COMMENT
+ else:
+ return self._str_representation
diff --git a/scripts/Infrastructure/MPICall.py b/scripts/Infrastructure/MPICall.py
index 0d5386803..f86987744 100644
--- a/scripts/Infrastructure/MPICall.py
+++ b/scripts/Infrastructure/MPICall.py
@@ -10,6 +10,7 @@ class MPI_Call(Instruction):
@override
def __init__(self, function: str, args: typing.OrderedDict[str, str], version: str):
+ super().__init__("")
self._function = function
self._args = args
self._version = version
@@ -37,6 +38,3 @@ class MPI_Call(Instruction):
def get_version(self) -> str:
return self._version
-
- def set_has_error(self, has_error: bool = True):
- self._has_error = has_error
--
GitLab
From 15c33d89c17a3f6eb9d3d65e3a118ab424516494 Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 14:02:45 +0100
Subject: [PATCH 12/18] Allowed the Instruction block to create instructions in
place from srings for convenience
---
scripts/Infrastructure/InstructionBlock.py | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/scripts/Infrastructure/InstructionBlock.py b/scripts/Infrastructure/InstructionBlock.py
index 05b8a16b3..ebc3d1ba3 100644
--- a/scripts/Infrastructure/InstructionBlock.py
+++ b/scripts/Infrastructure/InstructionBlock.py
@@ -33,7 +33,7 @@ class InstructionBlock:
assert not isinstance(name, int)
self.name = name
- def register_operation(self, op: Instruction | typing.List[Instruction], kind: str | int = 'all'):
+ def register_operation(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all'):
"""
Registers an operation based on rank.
@@ -43,7 +43,11 @@ class InstructionBlock:
- all: all Ranks execute this operation
- not0: all Ranks but the Root (rank 0) execute
- Or the integer of the rank that should execute
+ Note: if a str is passed as the operation, it will create a new Instruction from the given string
"""
+ if isinstance(op, str):
+ op = Instruction(op)
+
if kind == 'all':
if isinstance(op, list):
self.operations['all'].extend(op)
@@ -132,7 +136,7 @@ class InstructionBlock:
as_int = int(index) # will Raise ValueError if not integer
return self.operations[kind][as_int]
- def replace_operation(self, op: Instruction | typing.List[Instruction], kind: str | int = 'all',
+ def replace_operation(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all',
index: str | int = 0):
"""
Replace the operation registered. will Raise IndexError if not present
@@ -142,20 +146,25 @@ class InstructionBlock:
- index ('all' or int): the index of the operation within the given kind; 'all' means all operations will be replaced with the given list
Notes : if one wants to replace all operations one needs to provide a list
if one only wants to replace one operation: no list of operations is allowed
+ if a string is passed as the operation, it will create a new Instruction
"""
+ if isinstance(op, str):
+ op = Instruction(op)
+
if index == 'all':
if not isinstance(op, list):
raise ValueError('Provide List for replacement')
self.operations[kind] = op
else:
as_int = int(index) # will Raise ValueError if not integer
+ print(op)
if not isinstance(op, Instruction):
raise ValueError('Provide Instruction')
if len(self.operations[kind]) < as_int:
raise IndexError("Operation Not Found")
self.operations[kind][as_int] = op
- def insert_operation(self, op: Instruction | typing.List[Instruction], kind: str | int = 'all',
+ def insert_operation(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all',
before_index: int = 0):
"""
Inserts an operation before the specified one. will Raise IndexError if not present
@@ -163,7 +172,10 @@ class InstructionBlock:
- op the new operation or list of operations
- kind ('all','not0' or integer): which ranks should execute the operation
- index (int): the index of the operation within the given kind
+ note: if str is passed as the operation, it will Create a New Instruction
"""
+ if isinstance(op, str):
+ op = Instruction(op)
as_int = int(before_index) # will Raise ValueError if not integer
if len(self.operations[kind]) < before_index:
raise IndexError("Operation Not Found")
--
GitLab
From e19da0fce8858221cd0f5950db7b0d9a042d4b5b Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 14:03:47 +0100
Subject: [PATCH 13/18] Simplified Local Concurrency RMA testcase
---
scripts/errors/rma/LocalConcurrency.py | 16 +++++-----------
1 file changed, 5 insertions(+), 11 deletions(-)
diff --git a/scripts/errors/rma/LocalConcurrency.py b/scripts/errors/rma/LocalConcurrency.py
index 152203dc6..a047c1cb8 100644
--- a/scripts/errors/rma/LocalConcurrency.py
+++ b/scripts/errors/rma/LocalConcurrency.py
@@ -18,6 +18,7 @@ from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT
from typing import Tuple, List
+
class LocalConcurrencyErrorRMA(ErrorGenerator):
local_origin_addr_write = ["mpi_get", "mpi_rget"]
local_origin_addr_read = [
@@ -88,15 +89,8 @@ class LocalConcurrencyErrorRMA(ErrorGenerator):
tm.register_instruction_block(alloc)
if hasconflict:
- if isinstance(op1.get_operation(kind=0, index=-1), MPI_Call):
- op1.get_operation(kind=0, index=-1).set_has_error()
- else:
- op1.replace_operation(op1.get_operation(kind=0, index=-1) + ERROR_MARKER_COMMENT, 0, 0)
-
- if isinstance(op2.get_operation(kind=0, index=-1), MPI_Call):
- op2.get_operation(kind=0, index=-1).set_has_error()
- else:
- op2.replace_operation(op2.get_operation(kind=0, index=-1) + ERROR_MARKER_COMMENT, 0, 0)
+ op1.get_operation(kind=0, index=-1).set_has_error()
+ op2.get_operation(kind=0, index=-1).set_has_error()
# fuse instructions blocks
# combined_ops = InstructionBlock("COMBINED")
@@ -106,7 +100,7 @@ class LocalConcurrencyErrorRMA(ErrorGenerator):
tm.register_instruction_block(op2)
tm.set_description(
- ("LocalConcurrency" if hasconflict else "Correct") +
+ ("LocalConcurrency" if hasconflict else "Correct") +
"-"
+ op1.name
+ "_"
@@ -114,7 +108,7 @@ class LocalConcurrencyErrorRMA(ErrorGenerator):
"full description",
)
yield tm
-
+
# get RMA call
# rmaop = get_rma_call(function_to_check, 0)
--
GitLab
From 2e9823d8af7a4433c1558d7a78f3e193af5d90d4 Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 14:05:42 +0100
Subject: [PATCH 14/18] fix wrong type hint
---
scripts/Infrastructure/AllocCall.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/scripts/Infrastructure/AllocCall.py b/scripts/Infrastructure/AllocCall.py
index 983993fba..69303b699 100644
--- a/scripts/Infrastructure/AllocCall.py
+++ b/scripts/Infrastructure/AllocCall.py
@@ -25,7 +25,7 @@ alloc_template = """
class AllocCall(Instruction):
@override
- def __init__(self, type: str, num_elements: int, name: str = "buf", use_malloc: bool = False):
+ def __init__(self, type: str, num_elements: str, name: str = "buf", use_malloc: bool = False):
"""
Creates a New allocation Call
@@ -62,7 +62,7 @@ class AllocCall(Instruction):
return s
- def set_num_elements(self, num_elements: int):
+ def set_num_elements(self, num_elements: str):
self._num_elements = num_elements
def set_name(self, name: str):
@@ -77,7 +77,7 @@ class AllocCall(Instruction):
def set_use_calloc(self):
self._use_malloc = False
- def get_num_elements(self) -> int:
+ def get_num_elements(self) -> str:
return self._num_elements
def get_name(self) -> str:
--
GitLab
From 1eb914fbd6bc23f48b25f845881d2c9fc5b93496 Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 14:18:29 +0100
Subject: [PATCH 15/18] fix warnings about unexpected types
---
scripts/Infrastructure/CorrectParameter.py | 2 +-
scripts/Infrastructure/GeneratorManager.py | 8 ++++++--
scripts/Infrastructure/InstructionBlock.py | 2 +-
3 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/scripts/Infrastructure/CorrectParameter.py b/scripts/Infrastructure/CorrectParameter.py
index 7304b0484..c202cb8ff 100644
--- a/scripts/Infrastructure/CorrectParameter.py
+++ b/scripts/Infrastructure/CorrectParameter.py
@@ -8,7 +8,7 @@ from scripts.Infrastructure.AllocCall import AllocCall, get_free
class CorrectParameterFactory:
# default params
- buf_size = 10
+ buf_size = "10"
dtype = ['int', 'MPI_INT']
buf_size_bytes = f"{buf_size}*sizeof({dtype[0]})"
tag = 0
diff --git a/scripts/Infrastructure/GeneratorManager.py b/scripts/Infrastructure/GeneratorManager.py
index 4752d1b33..dad021d97 100644
--- a/scripts/Infrastructure/GeneratorManager.py
+++ b/scripts/Infrastructure/GeneratorManager.py
@@ -1,9 +1,13 @@
#! /usr/bin/python3
+from __future__ import annotations
+
import inspect
import os
import importlib
import importlib.util
import subprocess
+import typing
+from pathlib import Path
# for printing a nice progress bar
import tqdm
@@ -65,8 +69,8 @@ class GeneratorManager:
return case_name + "-" + str(num).zfill(digits_to_use) + suffix
- def generate(self, outpath, filterlist_=None, print_progress_bar=True, overwrite=True, generate_full_set=False,
- try_compile=False, max_mpi_version=4.0, use_clang_format=True):
+ def generate(self, outpath: str | Path | os.PathLike[str], filterlist_:typing.Sequence[str]=None, print_progress_bar:bool=True, overwrite:bool=True, generate_full_set:bool=False,
+ try_compile:bool=False, max_mpi_version:str="4.0", use_clang_format:bool=True):
"""
Generates test cases based on the specified parameters.
Parameters:
diff --git a/scripts/Infrastructure/InstructionBlock.py b/scripts/Infrastructure/InstructionBlock.py
index ebc3d1ba3..e6b7d28a3 100644
--- a/scripts/Infrastructure/InstructionBlock.py
+++ b/scripts/Infrastructure/InstructionBlock.py
@@ -43,7 +43,7 @@ class InstructionBlock:
- all: all Ranks execute this operation
- not0: all Ranks but the Root (rank 0) execute
- Or the integer of the rank that should execute
- Note: if a str is passed as the operation, it will create a new Instruction from the given string
+ Note: if a str is passed as the operation, it will create a new Instruction from the given string
"""
if isinstance(op, str):
op = Instruction(op)
--
GitLab
From 135dbf4afbc370afb4eb13162673b5bb123894c9 Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 14:27:04 +0100
Subject: [PATCH 16/18] removed the testcases to develop them in extra branches
---
scripts/errors/coll/InvalidRank.py | 35 ------
scripts/errors/coll/__init__.py | 0
scripts/errors/pt2pt/InvalidBuf.py | 96 ----------------
scripts/errors/pt2pt/InvalidComm.py | 107 -----------------
scripts/errors/pt2pt/InvalidRank.py | 58 ----------
scripts/errors/pt2pt/InvalidRequest.py | 67 -----------
scripts/errors/pt2pt/InvalidTag.py | 80 -------------
scripts/errors/pt2pt/LocalConcurrency.py | 59 ----------
scripts/errors/pt2pt/MessageRace.py | 100 ----------------
scripts/errors/pt2pt/__init__.py | 0
scripts/errors/rma/LocalConcurrency.py | 140 -----------------------
11 files changed, 742 deletions(-)
delete mode 100644 scripts/errors/coll/InvalidRank.py
delete mode 100644 scripts/errors/coll/__init__.py
delete mode 100644 scripts/errors/pt2pt/InvalidBuf.py
delete mode 100644 scripts/errors/pt2pt/InvalidComm.py
delete mode 100644 scripts/errors/pt2pt/InvalidRank.py
delete mode 100644 scripts/errors/pt2pt/InvalidRequest.py
delete mode 100644 scripts/errors/pt2pt/InvalidTag.py
delete mode 100644 scripts/errors/pt2pt/LocalConcurrency.py
delete mode 100644 scripts/errors/pt2pt/MessageRace.py
delete mode 100644 scripts/errors/pt2pt/__init__.py
delete mode 100644 scripts/errors/rma/LocalConcurrency.py
diff --git a/scripts/errors/coll/InvalidRank.py b/scripts/errors/coll/InvalidRank.py
deleted file mode 100644
index 3033e1bfa..000000000
--- a/scripts/errors/coll/InvalidRank.py
+++ /dev/null
@@ -1,35 +0,0 @@
-#! /usr/bin/python3
-
-from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
-from scripts.Infrastructure.InstructionBlock import InstructionBlock
-from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
-from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
-from scripts.Infrastructure.Template import TemplateManager
-from scripts.Infrastructure.TemplateFactory import get_send_recv_template, get_collective_template
-
-class InvalidRankErrorColl(ErrorGenerator):
- invalid_ranks = ["-1", "nprocs", "MPI_PROC_NULL"]
- functions_to_use = ["mpi_reduce", "mpi_bcast"]
-
- def __init__(self):
- pass
-
-
- def get_feature(self):
- return ["COLL"]
-
- def generate(self, generate_full_set):
-
- for func_to_use in self.functions_to_use:
- for rank_to_use in self.invalid_ranks:
- tm = get_collective_template(func_to_use, seperate=False)
- arg_to_replace = "root"
-
- tm.set_description("InvalidParam-Rank-"+func_to_use, "Invalid Rank: %s" % rank_to_use)
-
- tm.get_block("MPICALL").get_operation(kind='all', index=0).set_arg(arg_to_replace, rank_to_use)
- tm.get_block("MPICALL").get_operation(kind='all', index=0).set_has_error()
-
- yield tm
- if not generate_full_set:
- return
diff --git a/scripts/errors/coll/__init__.py b/scripts/errors/coll/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scripts/errors/pt2pt/InvalidBuf.py b/scripts/errors/pt2pt/InvalidBuf.py
deleted file mode 100644
index f4ba60378..000000000
--- a/scripts/errors/pt2pt/InvalidBuf.py
+++ /dev/null
@@ -1,96 +0,0 @@
-#! /usr/bin/python3
-
-from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
-from scripts.Infrastructure.InstructionBlock import InstructionBlock
-from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
-from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
-from scripts.Infrastructure.Template import TemplateManager
-from scripts.Infrastructure.TemplateFactory import get_send_recv_template
-
-from itertools import chain
-
-
-class InvalidBufErrorP2P(ErrorGenerator):
- invalid_bufs = [CorrectParameterFactory().buf_var_name, "NULL"]
- send_funcs = ["mpi_send",
- "mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend",
- "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init",
- ]
-
- recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
-
- sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
-
- def __init__(self):
- pass
-
- def get_feature(self):
- return ["P2P"]
-
- def generate_impl(self, send_func, recv_func, check_receive):
-
- for buf_to_use in self.invalid_bufs:
- tm = get_send_recv_template(send_func, recv_func)
-
- if buf_to_use == CorrectParameterFactory().buf_var_name:
- if tm.get_block("alloc").has_operation(kind='all', index=1) and check_receive:
- name = tm.get_block("alloc").get_operation(kind='all', index=1).get_name()
- type = tm.get_block("alloc").get_operation(kind='all', index=1).get_type()
- tm.get_block("alloc").replace_operation(
- CorrectParameterFactory.dtype[0] + "* " + name + ";", kind='all', index=1)
- else:
- tm.get_block("alloc").replace_operation(
- CorrectParameterFactory.dtype[0] + "* " + CorrectParameterFactory.buf_var_name + ";")
- # without allocation: usage is undefined behaviour
-
- if check_receive:
- tm.set_description("InvalidParam-Buffer-" + recv_func, "Invalid Buffer: %s" % buf_to_use)
- else:
- tm.set_description("InvalidParam-Buffer-" + send_func, "Invalid Buffer: %s" % buf_to_use)
-
- if check_receive:
- if tm.get_block("MPICALL").get_operation(kind=0, index=0).has_arg("buf"):
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("buf", buf_to_use)
- else:
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("recvbuf", buf_to_use)
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
- else:
- if tm.get_block("MPICALL").get_operation(kind=1, index=0).has_arg("buf"):
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("buf", buf_to_use)
- else:
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("recvbuf", buf_to_use)
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
-
- yield tm
-
- def generate(self, generate_full_set):
- for func in self.send_funcs:
- yield from self.generate_impl(func, 'mpi_irecv', False)
- if not generate_full_set:
- return
- for func in self.recv_funcs:
- yield from self.generate_impl("mpi_send", func, True)
-
- yield from self.generate_impl("mpi_sendrecv", "mpi_sendrecv", True)
- yield from self.generate_impl("mpi_sendrecv", "mpi_sendrecv", False)
- yield from self.generate_impl("mpi_sendrecv_replace", "mpi_sendrecv_replace", True)
-
-
-class MessageRaceErrorSendRecv(ErrorGenerator):
-
- def __init__(self):
- pass
-
- def get_feature(self):
- return ["P2P"]
-
- def generate(self, generate_full_set):
-
- for buf_to_use in ["buf", "MPI_IN_PLACE"]:
- tm = get_send_recv_template("mpi_sendrecv", "mpi_sendrecv")
- tm.set_description("InvalidParam-Buffer-mpi_sendrecv", "send and recv buffer must be disjoint in sendrecv")
-
- for k in [0, 1]:
- tm.get_block("MPICALL").get_operation(kind=k, index=0).set_arg("recvbuf", buf_to_use)
- tm.get_block("MPICALL").get_operation(kind=k, index=0).set_has_error()
- yield tm
diff --git a/scripts/errors/pt2pt/InvalidComm.py b/scripts/errors/pt2pt/InvalidComm.py
deleted file mode 100644
index 24a41046a..000000000
--- a/scripts/errors/pt2pt/InvalidComm.py
+++ /dev/null
@@ -1,107 +0,0 @@
-#! /usr/bin/python3
-
-from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
-from scripts.Infrastructure.InstructionBlock import InstructionBlock
-from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
-from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
-from scripts.Infrastructure.Template import TemplateManager
-from scripts.Infrastructure.TemplateFactory import get_send_recv_template, get_communicator, get_intercomm
-
-from itertools import chain
-
-sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
-
-
-class InvalidCommErrorP2P(ErrorGenerator):
- invalid_comm = ["MPI_COMM_NULL", "NULL"]
-
- missmatching_comms = ["MPI_COMM_SELF", "mpi_comm_dup", "mpi_comm_dup_with_info", "mpi_comm_idup",
- "mpi_comm_idup_with_info", "mpi_comm_create", "mpi_comm_create_group", "mpi_comm_split",
- "mpi_comm_split_type", "mpi_comm_create_from_group"
- ]
- intercomms = ["mpi_intercomm_create", "mpi_intercomm_merge", "mpi_intercomm_create_from_groups"]
-
- # as extended testcases
-
- comms_to_check = invalid_comm + missmatching_comms + intercomms
-
- functions_to_check = ["mpi_send",
- "mpi_recv", "mpi_irecv",
- "mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend",
- "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init",
- "mpi_precv_init", "mpi_recv_init"
- ] + sendrecv_funcs
-
- recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"] + sendrecv_funcs
-
- def __init__(self):
- pass
-
- def get_feature(self):
- return ["P2P"]
-
- def generate(self, generate_full_set):
-
- for send_func in self.functions_to_check:
- for comm_to_use in self.comms_to_check:
-
- check_receive = False
- recv_func = "mpi_irecv"
- if send_func in self.recv_funcs:
- check_receive = True
- recv_func = send_func
- send_func = "mpi_send"
- if recv_func in sendrecv_funcs:
- send_func = recv_func
-
- tm = get_send_recv_template(send_func, recv_func)
-
- if comm_to_use in self.missmatching_comms and comm_to_use != "MPI_COMM_SELF":
- b = get_communicator(comm_to_use, comm_to_use)
- tm.insert_block(b, after_block_name="alloc")
- if comm_to_use in self.intercomms:
- b = get_intercomm(comm_to_use, comm_to_use)
- tm.insert_block(b, after_block_name="alloc")
-
- error_string = "ParamMatching"
- if comm_to_use in self.invalid_comm:
- error_string = "InvalidParam"
-
- if check_receive:
- if comm_to_use in self.missmatching_comms and recv_func == "mpi_irecv":
- # combination repeated
- continue
- tm.set_description(error_string + "-Comm-" + recv_func, "Invalid Rank: %s" % comm_to_use)
- else:
- tm.set_description(error_string + "-Comm-" + send_func, "Invalid Rank: %s" % comm_to_use)
-
- if check_receive:
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("comm", comm_to_use)
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
- if comm_to_use in self.missmatching_comms:
- # missmatch is between both
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
- else:
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("comm", comm_to_use)
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
- if comm_to_use in self.missmatching_comms:
- # missmatch is between both
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
-
- # an intercomm has only one rank (the other group)
- if comm_to_use in self.intercomms and not comm_to_use == "mpi_intercomm_merge":
- # intercomm merge results in an equivalent comm again
- if tm.get_block("MPICALL").get_operation(kind=0, index=0).has_arg("source"):
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("source", "0")
- if tm.get_block("MPICALL").get_operation(kind=1, index=0).has_arg("source"):
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("source", "0")
-
- if comm_to_use in self.missmatching_comms + self.intercomms and comm_to_use != "MPI_COMM_SELF":
- b = InstructionBlock("comm_free")
- b.register_operation(MPICallFactory().mpi_comm_free("&" + comm_to_use))
- tm.register_instruction_block(b)
-
- yield tm
- # end for comm to check
- if not generate_full_set:
- return
diff --git a/scripts/errors/pt2pt/InvalidRank.py b/scripts/errors/pt2pt/InvalidRank.py
deleted file mode 100644
index 95f6db52d..000000000
--- a/scripts/errors/pt2pt/InvalidRank.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#! /usr/bin/python3
-
-from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
-from scripts.Infrastructure.InstructionBlock import InstructionBlock
-from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
-from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
-from scripts.Infrastructure.Template import TemplateManager
-from scripts.Infrastructure.TemplateFactory import get_send_recv_template
-
-from itertools import chain
-
-
-class InvalidRankErrorP2P(ErrorGenerator):
- invalid_ranks = ["-1", "nprocs", "MPI_PROC_NULL"]
- send_funcs = ["mpi_send",
- "mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend",
- "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init"
- ]
- recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
-
- sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
-
- def __init__(self):
- pass
-
- def get_feature(self):
- return ["P2P"]
-
- def generate_impl(self, send_func, recv_func, check_receive):
- for rank_to_use in self.invalid_ranks:
- tm = get_send_recv_template(send_func, recv_func)
-
- if check_receive:
- tm.set_description("InvalidParam-Rank-" + recv_func, "Invalid Rank: %s" % rank_to_use)
- else:
- tm.set_description("InvalidParam-Rank-" + send_func, "Invalid Rank: %s" % rank_to_use)
-
- if check_receive:
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("source", rank_to_use)
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
- else:
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("dest", rank_to_use)
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
-
- yield tm
-
- def generate(self, generate_full_set):
-
- for send_func in self.send_funcs:
- yield from self.generate_impl(send_func, "mpi_irecv", False)
- if not generate_full_set:
- return
- for func in self.recv_funcs:
- yield from self.generate_impl("mpi_send", func, True)
-
- for func in self.sendrecv_funcs:
- yield from self.generate_impl(func, func, True)
- yield from self.generate_impl(func, func, False)
diff --git a/scripts/errors/pt2pt/InvalidRequest.py b/scripts/errors/pt2pt/InvalidRequest.py
deleted file mode 100644
index d18888778..000000000
--- a/scripts/errors/pt2pt/InvalidRequest.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#! /usr/bin/python3
-
-from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
-from scripts.Infrastructure.InstructionBlock import InstructionBlock
-from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
-from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
-from scripts.Infrastructure.Template import TemplateManager
-from scripts.Infrastructure.TemplateFactory import get_send_recv_template
-
-from itertools import chain
-
-
-class InvalidRequestErrorP2P(ErrorGenerator):
- invalid_requests = ["MPI_REQUEST_NULL", # probably triggers compiler warning
- "NULL"]
- functions_to_check = ["mpi_irecv", "mpi_isend",
- "mpi_issend", "mpi_irsend", "mpi_ibsend",
- "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init",
- "mpi_precv_init", "mpi_recv_init"
- ]
-
- recv_funcs = ["mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
-
- def __init__(self):
- pass
-
- def get_num_errors(self):
- # send + receive = only check the first two functions
- return len(self.invalid_requests) * 2
-
- # the number of errors to produce in the extended mode (all possible combinations)
- def get_num_errors_extended(self):
- return len(self.invalid_requests) * len(self.functions_to_check)
-
- def get_feature(self):
- return ["P2P"]
-
- def generate(self, generate_full_set):
-
- for send_func in self.functions_to_check:
- for req_to_use in self.invalid_requests:
-
- check_receive = False
- recv_func = "mpi_irecv"
- if send_func in self.recv_funcs:
- check_receive = True
- recv_func = send_func
- send_func = "mpi_isend"
-
- tm = get_send_recv_template(send_func, recv_func)
-
- if check_receive:
- tm.set_description("InvalidParam-Request-" + recv_func, "Invalid Request: %s" % req_to_use)
- else:
- tm.set_description("InvalidParam-Request-" + send_func, "Invalid Request: %s" % req_to_use)
-
- if check_receive:
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("request", req_to_use)
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
- else:
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("request", req_to_use)
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
-
- yield tm
-
- if not generate_full_set:
- return
diff --git a/scripts/errors/pt2pt/InvalidTag.py b/scripts/errors/pt2pt/InvalidTag.py
deleted file mode 100644
index 74041139b..000000000
--- a/scripts/errors/pt2pt/InvalidTag.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#! /usr/bin/python3
-
-from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
-from scripts.Infrastructure.InstructionBlock import InstructionBlock
-from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
-from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
-from scripts.Infrastructure.Template import TemplateManager
-from scripts.Infrastructure.TemplateFactory import get_send_recv_template
-
-from itertools import chain
-
-
-class InvalidTagErrorP2P(ErrorGenerator):
- invalid_tags = ["-1", "MPI_TAG_UB+1", CorrectParameterFactory.tag * 2, "MPI_ANY_TAG"]
- send_funcs = ["mpi_send",
- "mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend",
- "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init",
- ]
-
- recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
- sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
-
- def __init__(self):
- pass
-
- def get_feature(self):
- return ["P2P"]
-
- def generate_impl(self, send_func, recv_func, check_receive):
- for tag_to_use in self.invalid_tags:
- tm = get_send_recv_template(send_func, recv_func)
-
- error_string = "InvalidParam"
- if tag_to_use == CorrectParameterFactory.tag * 2:
- error_string = "ParamMatching"
-
- if check_receive:
- if tag_to_use == "MPI_ANY_TAG":
- # correct case
- continue
- if tag_to_use == CorrectParameterFactory().tag * 2 and recv_func == "mpi_irecv":
- # combination repeated
- continue
- tm.set_description(error_string + "-Tag-" + recv_func, "Invalid Rank: %s" % tag_to_use)
- else:
- tm.set_description(error_string + "-Tag-" + send_func, "Invalid Rank: %s" % tag_to_use)
-
- if check_receive:
- if tm.get_block("MPICALL").get_operation(kind=0, index=0).has_arg("tag"):
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("tag", tag_to_use)
- else:
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("recvtag", tag_to_use)
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
- if tag_to_use == CorrectParameterFactory.tag * 2:
- # missmatch is between both
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
- else:
- if tm.get_block("MPICALL").get_operation(kind=1, index=0).has_arg("tag"):
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("tag", tag_to_use)
- else:
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("sendtag", tag_to_use)
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
- if tag_to_use == CorrectParameterFactory.tag * 2:
- # missmatch is between both
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
-
- yield tm
-
- def generate(self, generate_full_set):
-
- for send_func in self.send_funcs:
- yield from self.generate_impl(send_func, "mpi_irecv", False)
- if not generate_full_set:
- return
- for recv_func in self.recv_funcs:
- yield from self.generate_impl("mpi_send", recv_func, True)
-
- for func in self.sendrecv_funcs:
- yield from self.generate_impl(func, func, True)
- yield from self.generate_impl(func, func, False)
diff --git a/scripts/errors/pt2pt/LocalConcurrency.py b/scripts/errors/pt2pt/LocalConcurrency.py
deleted file mode 100644
index a12ca70e5..000000000
--- a/scripts/errors/pt2pt/LocalConcurrency.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#! /usr/bin/python3
-
-from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
-from scripts.Infrastructure.InstructionBlock import InstructionBlock
-from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
-from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
-from scripts.Infrastructure.Template import TemplateManager
-from scripts.Infrastructure.TemplateFactory import get_send_recv_template
-
-from itertools import chain
-
-from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT
-
-sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
-
-
-class LocalConcurrencyErrorP2P(ErrorGenerator):
- functions_to_check = ["mpi_irecv",
- "mpi_isend", "mpi_issend", "mpi_irsend", "mpi_ibsend",
- "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init",
- "mpi_precv_init", "mpi_recv_init"
- ]
- recv_funcs = ["mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
-
- def __init__(self):
- pass
-
- def get_feature(self):
- return ["P2P"]
-
- def generate(self, generate_full_set):
- for send_func in self.functions_to_check:
-
- check_receive = False
- recv_func = "mpi_irecv"
- if send_func in self.recv_funcs:
- check_receive = True
- recv_func = send_func
- send_func = "mpi_isend"
-
- tm = get_send_recv_template(send_func, recv_func)
-
- if check_receive:
- tm.set_description("LocalConcurrency-receive-" + recv_func,
- "usage of receive buffer before operation is completed")
- else:
- tm.set_description("LocalConcurrency-send-" + send_func,
- "usage of send buffer before operation is completed")
-
- if check_receive:
- tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
- tm.get_block("MPICALL").register_operation("buf[2]=1;" + ERROR_MARKER_COMMENT, kind=1)
- else:
- tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
- tm.get_block("MPICALL").register_operation("buf[2]=1;" + ERROR_MARKER_COMMENT, kind=1)
-
- yield tm
- if not generate_full_set:
- return
diff --git a/scripts/errors/pt2pt/MessageRace.py b/scripts/errors/pt2pt/MessageRace.py
deleted file mode 100644
index e102fe09c..000000000
--- a/scripts/errors/pt2pt/MessageRace.py
+++ /dev/null
@@ -1,100 +0,0 @@
-#! /usr/bin/python3
-
-from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
-from scripts.Infrastructure.InstructionBlock import InstructionBlock
-from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
-from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
-from scripts.Infrastructure.Template import TemplateManager
-from scripts.Infrastructure.TemplateFactory import get_send_recv_template
-
-from itertools import chain
-
-
-class MessageRaceErrorAnyTag(ErrorGenerator):
- # TODO do we need to generate it for all combinations of send and recv?
-
- def __init__(self):
- pass
-
- def get_feature(self):
- return ["P2P"]
-
- def generate(self, generate_full_set):
- tm = TemplateManager()
- tm.set_description("MsgRace-ANY_TAG", "order of messages is indeterministic, may lead to a deadlock")
-
- b = InstructionBlock("alloc")
- b.register_operation(CorrectParameterFactory().get_buffer_alloc())
- tm.register_instruction_block(b)
-
- b = InstructionBlock("MPICALL")
- # send part
- b.register_operation("for(int i =0; i < 10; ++i) {", kind=1)
- b.register_operation("buf[0]=i;", kind=1)
- send_call = CorrectMPICallFactory().mpi_send()
- send_call.set_arg("tag", "i")
- b.register_operation(send_call, kind=1)
- b.register_operation("}", kind=1)
- # recv part
- b.register_operation("for(int i =0; i < 10; ++i) {", kind=0)
- recv_call = CorrectMPICallFactory().mpi_recv()
- recv_call.set_arg("tag", "MPI_ANY_TAG")
- b.register_operation(recv_call, kind=0)
- b.register_operation("if(buf[0]!=i){", kind=0)
- additional_recv = CorrectMPICallFactory().mpi_recv()
- additional_recv.set_has_error() # additional recv leads to deadlock
- b.register_operation(additional_recv, kind=0)
- b.register_operation(" }", kind=0) # end if
- b.register_operation("}", kind=0) # end for
-
- tm.register_instruction_block(b)
-
- b = InstructionBlock("free")
- b.register_operation(CorrectParameterFactory().get_buffer_free())
- tm.register_instruction_block(b)
-
- yield tm
-
-
-class MessageRaceErrorAnysource(ErrorGenerator):
- # TODO do we need to generate it for all combinations of send and recv?
-
- def __init__(self):
- pass
-
- def get_feature(self):
- return ["P2P"]
-
- def generate(self, generate_full_set):
- tm = TemplateManager(min_ranks=3)
- tm.set_description("MsgRace-ANY_SOURCE", "order of messages is indeterministic, may lead to a deadlock")
-
- b = InstructionBlock("alloc")
- b.register_operation(CorrectParameterFactory().get_buffer_alloc())
- tm.register_instruction_block(b)
-
- b = InstructionBlock("MPICALL")
- # send part
- b.register_operation("buf[0]=rank;", kind='not0')
- send_call = CorrectMPICallFactory().mpi_send()
- b.register_operation(send_call, kind='not0')
-
- # recv part
- b.register_operation("for(int i =1; i < nprocs; ++i) {", kind=0)
- recv_call = CorrectMPICallFactory().mpi_recv()
- recv_call.set_arg("source", "MPI_ANY_SOURCE")
- b.register_operation(recv_call, kind=0)
- b.register_operation("if(buf[0]!=i){", kind=0)
- additional_recv = CorrectMPICallFactory().mpi_recv()
- additional_recv.set_has_error() # additional recv leads to deadlock
- b.register_operation(additional_recv, kind=0)
- b.register_operation(" }", kind=0) # end if
- b.register_operation("}", kind=0) # end for
-
- tm.register_instruction_block(b)
-
- b = InstructionBlock("free")
- b.register_operation(CorrectParameterFactory().get_buffer_free())
- tm.register_instruction_block(b)
-
- yield tm
diff --git a/scripts/errors/pt2pt/__init__.py b/scripts/errors/pt2pt/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scripts/errors/rma/LocalConcurrency.py b/scripts/errors/rma/LocalConcurrency.py
deleted file mode 100644
index a047c1cb8..000000000
--- a/scripts/errors/rma/LocalConcurrency.py
+++ /dev/null
@@ -1,140 +0,0 @@
-#! /usr/bin/python3
-
-from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
-from scripts.Infrastructure.InstructionBlock import InstructionBlock
-from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
-from scripts.Infrastructure.CorrectParameter import (
- CorrectParameterFactory,
- get_matching_recv,
-)
-from scripts.Infrastructure.Template import TemplateManager
-from scripts.Infrastructure.TemplateFactory import get_allocated_window, get_rma_call
-from scripts.Infrastructure.AllocCall import AllocCall
-from scripts.Infrastructure.MPICall import MPI_Call
-
-import itertools
-
-from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT
-
-from typing import Tuple, List
-
-
-class LocalConcurrencyErrorRMA(ErrorGenerator):
- local_origin_addr_write = ["mpi_get", "mpi_rget"]
- local_origin_addr_read = [
- "mpi_put",
- "mpi_rput",
- "mpi_accumulate",
- "mpi_raccumulate",
- "mpi_get_accumulate",
- "mpi_rget_accumulate",
- "mpi_fetch_and_op",
- "mpi_compare_and_swap",
- ]
- functions_to_check = ["mpi_put", "mpi_get", "mpi_rput", "mpi_rget"]
-
- # recv_funcs = ["mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
-
- def __init__(self):
- pass
-
- def get_feature(self):
- return ["RMA"]
-
- def generate(self, generate_full_set):
-
- cf = CorrectParameterFactory()
- cfmpi = CorrectMPICallFactory()
-
- mpi_buf_read = [
- get_rma_call("mpi_put", 0),
- get_rma_call("mpi_rput", 0),
- get_rma_call("mpi_accumulate", 0),
- get_rma_call("mpi_raccumulate", 0),
- get_rma_call("mpi_get_accumulate", 0),
- get_rma_call("mpi_rget_accumulate", 0),
- get_rma_call("mpi_fetch_and_op", 0),
- get_rma_call("mpi_compare_and_swap", 0),
- ]
- mpi_buf_write = [get_rma_call("mpi_get", 0), get_rma_call("mpi_rget", 0)]
-
- bufread = InstructionBlock("bufread")
- bufread.register_operation(f'printf("buf is %d\\n", {cf.buf_var_name}[1]);', 0)
- bufwrite = InstructionBlock("write")
- bufwrite.register_operation(f'{cf.buf_var_name}[1] = 42;', 0)
-
- # 7 possible combinations of local buffer accesses (hasconflict = True | False)
- local_access_combinations: List[Tuple[List[InstructionBlock], List[InstructionBlock], bool]] = [
- (mpi_buf_read, [bufread], False),
- (mpi_buf_read, [bufwrite], True),
- (mpi_buf_write, [bufread], True),
- (mpi_buf_write, [bufwrite], True),
- (mpi_buf_read, mpi_buf_read, False),
- (mpi_buf_read, mpi_buf_write, True),
- (mpi_buf_write, mpi_buf_write, True),
- ]
-
- for ops1, ops2, hasconflict in local_access_combinations:
- for (op1, op2) in itertools.product(ops1, ops2):
- tm = TemplateManager()
- # window allocation boilerplate
- b = get_allocated_window("mpi_win_create", "win", "winbuf", "int", "2")
- tm.register_instruction_block(b)
-
- # local buffer allocation
- alloc = InstructionBlock("alloc")
- alloc.register_operation(
- AllocCall(cf.dtype[0], cf.buf_size, cf.buf_var_name)
- )
- tm.register_instruction_block(alloc)
-
- if hasconflict:
- op1.get_operation(kind=0, index=-1).set_has_error()
- op2.get_operation(kind=0, index=-1).set_has_error()
-
- # fuse instructions blocks
- # combined_ops = InstructionBlock("COMBINED")
- # combined_ops.register_operations(op1.get_operations(kind=0), kind=0)
- # combined_ops.register_operations(op2.get_operations(kind=0), kind=0)
- tm.register_instruction_block(op1)
- tm.register_instruction_block(op2)
-
- tm.set_description(
- ("LocalConcurrency" if hasconflict else "Correct") +
- "-"
- + op1.name
- + "_"
- + op2.name,
- "full description",
- )
- yield tm
-
- # get RMA call
- # rmaop = get_rma_call(function_to_check, 0)
-
- # tm.register_instruction_block(rmaop)
-
- # bufstring = ""
- # if bufop == "read": # local buffer access is read
- # bufstring = f'printf("buf is %d\\n", {cf.buf_var_name}[1]);'
- # # if RMA call performs local buffer write, this is a race, otherwise no race
- # if function_to_check in local_origin_addr_write:
- # bufstring += ERROR_MARKER_COMMENT
- # # mark RMA call as erroneous
- # tm.get_block("RMACALL").get_operation(
- # kind=0, index=-1
- # ).set_has_error()
-
- # if bufop == "write":
- # # a buffer write is always a race
- # bufstring = f"{cf.buf_var_name}[1] = 42;" + ERROR_MARKER_COMMENT
- # # mark RMA call as erroneous
- # tm.get_block("RMACALL").get_operation(
- # kind=0, index=-1
- # ).set_has_error()
-
- # # finally register buffer access
- # tm.get_block("RMACALL").register_operation(bufstring, 0)
-
- # if not generate_full_set:
- # return
--
GitLab
From 636ff42fad4c6efd6b06f714a5fc9c23bd9e5f0f Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 16:00:56 +0100
Subject: [PATCH 17/18] added get method to MPI call factory, so that one can
get an MPI call from a string (basically a wrapper for getattr)
---
scripts/Infrastructure/MPIAPIInfo/GenerateCallFactory.py | 8 ++++++++
scripts/Infrastructure/MPICallFactory.py | 7 +++++++
2 files changed, 15 insertions(+)
diff --git a/scripts/Infrastructure/MPIAPIInfo/GenerateCallFactory.py b/scripts/Infrastructure/MPIAPIInfo/GenerateCallFactory.py
index 3618e8b0c..f519427be 100644
--- a/scripts/Infrastructure/MPIAPIInfo/GenerateCallFactory.py
+++ b/scripts/Infrastructure/MPIAPIInfo/GenerateCallFactory.py
@@ -18,6 +18,10 @@ from scripts.Infrastructure.MPICall import MPI_Call
class MPICallFactory:
+ def get(self, func: str, *args):
+ f_to_call = getattr(self, func)
+ return f_to_call(*args)
+
"""
correct_call_factory_header="""
@@ -26,6 +30,10 @@ from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory
class CorrectMPICallFactory:
+ def get(self, func: str, *args):
+ f_to_call = getattr(self, func)
+ return f_to_call(*args)
+
"""
template_correct = """
def @{FUNC_KEY}@(self):
diff --git a/scripts/Infrastructure/MPICallFactory.py b/scripts/Infrastructure/MPICallFactory.py
index 474919f1f..442192c9c 100644
--- a/scripts/Infrastructure/MPICallFactory.py
+++ b/scripts/Infrastructure/MPICallFactory.py
@@ -6,6 +6,10 @@ from scripts.Infrastructure.MPICall import MPI_Call
class MPICallFactory:
+ def get(self, func: str, *args):
+ f_to_call = getattr(self, func)
+ return f_to_call(*args)
+
def mpi_abort(self, *args):
return MPI_Call("MPI_Abort", OrderedDict([("comm", args[0]), ("errorcode", args[1]), ]), "1.0")
@@ -2174,6 +2178,9 @@ from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory
class CorrectMPICallFactory:
+ def get(self, func: str, *args):
+ f_to_call = getattr(self, func)
+ return f_to_call(*args)
def mpi_abort(self):
correct_params = CorrectParameterFactory()
--
GitLab
From eb6b0e3c2123cd550280ebb8d4997d2a2a3b87c0 Mon Sep 17 00:00:00 2001
From: Tim Jammer <tim.jammer@tu-darmstadt.de>
Date: Mon, 5 Feb 2024 16:39:44 +0100
Subject: [PATCH 18/18] Renamed operation with instruction (#6)
---
scripts/ExampleUsage.py | 6 +-
scripts/Infrastructure/InstructionBlock.py | 16 +-
scripts/Infrastructure/TemplateFactory.py | 235 ++++++++++++---------
3 files changed, 143 insertions(+), 114 deletions(-)
diff --git a/scripts/ExampleUsage.py b/scripts/ExampleUsage.py
index 02146e85f..12c0122f1 100644
--- a/scripts/ExampleUsage.py
+++ b/scripts/ExampleUsage.py
@@ -18,7 +18,7 @@ class Invalid_negative_rank_error:
# include the buffer allocation in the template (all ranks execute it)
alloc_block = InstructionBlock("alloc")
- alloc_block.register_operation(correct_params.get_buffer_alloc())
+ alloc_block.register_instruction(correct_params.get_buffer_alloc())
tm.register_instruction_block(alloc_block)
send = MPI_Call_Factory().mpi_send(
@@ -33,9 +33,9 @@ class Invalid_negative_rank_error:
b = InstructionBlock()
# only rank 0 execute the send
- b.register_operation(send, 0)
+ b.register_instruction(send, 0)
# only rank 1 execute the recv
- b.register_operation(get_matching_recv(send), 1)
+ b.register_instruction(get_matching_recv(send), 1)
tm.register_instruction_block(b)
return tm
diff --git a/scripts/Infrastructure/InstructionBlock.py b/scripts/Infrastructure/InstructionBlock.py
index e6b7d28a3..f30545dea 100644
--- a/scripts/Infrastructure/InstructionBlock.py
+++ b/scripts/Infrastructure/InstructionBlock.py
@@ -33,7 +33,7 @@ class InstructionBlock:
assert not isinstance(name, int)
self.name = name
- def register_operation(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all'):
+ def register_instruction(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all'):
"""
Registers an operation based on rank.
@@ -106,7 +106,7 @@ class InstructionBlock:
return result_str
- def has_operation(self, kind: int | str = 'all', index: int = 0) -> bool:
+ def has_instruction(self, kind: int | str = 'all', index: int = 0) -> bool:
"""
Checks if the Block has an operation with the given index and kind
Parameters:
@@ -121,7 +121,7 @@ class InstructionBlock:
except (KeyError, IndexError) as e:
return False
- def get_operation(self, kind: int | str = 'all', index: str | int = 0) -> Instruction | typing.List[Instruction]:
+ def get_instruction(self, kind: int | str = 'all', index: str | int = 0) -> Instruction | typing.List[Instruction]:
"""
Retrieve the operation registered. will Raise IndexError if not present
Parameters:
@@ -136,8 +136,8 @@ class InstructionBlock:
as_int = int(index) # will Raise ValueError if not integer
return self.operations[kind][as_int]
- def replace_operation(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all',
- index: str | int = 0):
+ def replace_instruction(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all',
+ index: str | int = 0):
"""
Replace the operation registered. will Raise IndexError if not present
Parameters:
@@ -164,8 +164,8 @@ class InstructionBlock:
raise IndexError("Operation Not Found")
self.operations[kind][as_int] = op
- def insert_operation(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all',
- before_index: int = 0):
+ def insert_instruction(self, op: str | Instruction | typing.List[Instruction], kind: str | int = 'all',
+ before_index: int = 0):
"""
Inserts an operation before the specified one. will Raise IndexError if not present
Parameters:
@@ -185,7 +185,7 @@ class InstructionBlock:
else:
self.operations[kind].insert(before_index, op)
- def remove_operation(self, kind: str | int = 'all', index: str | int = 0):
+ def remove_instruction(self, kind: str | int = 'all', index: str | int = 0):
"""
Removes the operation registered. will Raise IndexError if not present
Parameters:
diff --git a/scripts/Infrastructure/TemplateFactory.py b/scripts/Infrastructure/TemplateFactory.py
index 47abb7688..4c595ea17 100644
--- a/scripts/Infrastructure/TemplateFactory.py
+++ b/scripts/Infrastructure/TemplateFactory.py
@@ -1,4 +1,8 @@
#! /usr/bin/python3
+from __future__ import annotations
+
+import typing
+
from scripts.Infrastructure.AllocCall import AllocCall
from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory
from scripts.Infrastructure.InstructionBlock import InstructionBlock
@@ -18,7 +22,7 @@ def get_default_template(mpi_func):
pass
-def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"):
+def get_send_recv_template(send_func: str = "mpi_isend", recv_func: str | typing.Tuple[str, str] = "mpi_irecv"):
"""
Contructs a default template for the given mpi send recv function pair
Returns:
@@ -27,50 +31,75 @@ def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"):
"""
# currently supported:
- assert send_func in ["mpi_send", "mpi_ssend", "mpi_isend", "mpi_issend", "mpi_sendrecv", "mpi_rsend", "mpi_irsend",
- "mpi_bsend", "mpi_ibsend", "mpi_sendrecv", "mpi_sendrecv_replace", "mpi_isendrecv",
- "mpi_isendrecv_replace", "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init",
- "mpi_psend_init"]
- assert recv_func in ["mpi_recv", "mpi_irecv", "mpi_sendrecv", "mpi_sendrecv_replace", "mpi_isendrecv",
- "mpi_isendrecv_replace", "mpi_recv_init", "mpi_precv_init"]
+
+ # mprobe and mrecv combinations allowed
+ probe_pairs = [["mpi_mprobe", "mpi_mrecv"], ["mpi_mprobe", "mpi_imrecv"], ["mpi_improbe", "mpi_mrecv"],
+ ["mpi_improbe", "mpi_imrecv"]]
sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
persistent_send_funcs = ["mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init"]
persistent_recv_funcs = ["mpi_recv_init", "mpi_precv_init"]
+ isend_funcs = ["mpi_isend", "mpi_issend", "mpi_irsend", "mpi_ibsend"]
+ irecv_funcs = ["mpi_irecv"]
+
+ assert (send_func in ["mpi_send", "mpi_ssend", "mpi_rsend", "mpi_bsend"]
+ + sendrecv_funcs + isend_funcs + persistent_send_funcs)
+ assert recv_func in ["mpi_recv"] + sendrecv_funcs + irecv_funcs + persistent_recv_funcs + probe_pairs
+
if send_func in sendrecv_funcs or recv_func == sendrecv_funcs:
assert recv_func == send_func
# default template generation only supports if both use same mechanism
if send_func in ["mpi_rsend", "mpi_irsend", "mpi_rsend_init"]:
- assert recv_func in ["mpi_irecv", "mpi_recv_init"] # else: deadlock
+ assert recv_func in ["mpi_irecv", "mpi_recv_init", "mpi_precv_init"] # else: deadlock
tm = TemplateManager()
cf = CorrectParameterFactory()
alloc_block = InstructionBlock("alloc")
- alloc_block.register_operation(cf.get_buffer_alloc())
+ alloc_block.register_instruction(cf.get_buffer_alloc())
if send_func in sendrecv_funcs:
# spilt send and recv buf
alloc = cf.get_buffer_alloc()
alloc.set_name("recv_buf")
- alloc_block.register_operation(alloc)
- tm.register_instruction_block(alloc_block)
+ alloc_block.register_instruction(alloc)
+ if recv_func in probe_pairs:
+ alloc_block.register_instruction("MPI_Message msg;")
if send_func in ["mpi_bsend", "mpi_ibsend", "mpi_bsend_init"]:
- b = InstructionBlock("buf_attach")
buf_size = "sizeof(int)*10 + MPI_BSEND_OVERHEAD"
- b.register_operation(AllocCall("char", buf_size, "mpi_buf"))
- b.register_operation(MPICallFactory().mpi_buffer_attach("mpi_buf", buf_size))
+ alloc_block.register_instruction(AllocCall("char", buf_size, "mpi_buf"))
+ alloc_block.register_instruction(MPICallFactory().mpi_buffer_attach("mpi_buf", buf_size))
+
+ if (send_func in isend_funcs + persistent_send_funcs or
+ recv_func in persistent_recv_funcs + irecv_funcs + probe_pairs):
+ alloc_block.register_instruction("MPI_Request request;", 'all')
+ if recv_func in probe_pairs:
+ alloc_block.register_instruction("int flag=0;")
+
+ tm.register_instruction_block(alloc_block)
+ # end preperation of all local variables
+
+ # before the send/recv block
+ if send_func in ["mpi_rsend", "mpi_irsend", "mpi_rsend_init"]:
+ b = InstructionBlock("SYNC")
+ # sender needs to wait until recv is started
+ b.register_instruction(CorrectMPICallFactory().mpi_barrier(), 1)
tm.register_instruction_block(b)
cmpicf = CorrectMPICallFactory()
- send_func_creator_function = getattr(cmpicf, send_func)
- s = send_func_creator_function()
- recv_func_creator_function = getattr(cmpicf, recv_func)
- r = recv_func_creator_function()
+ # get the send and recv block
+
+ recv_to_use = recv_func
+
+ if recv_func in probe_pairs:
+ recv_to_use = recv_func[0]
+
+ s = cmpicf.get(send_func)
+ r = cmpicf.get(recv_to_use)
if send_func in sendrecv_funcs:
# sending the second msg
@@ -81,65 +110,68 @@ def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"):
if r.has_arg("recvbuf"):
r.set_arg("recvbuf", "recv_buf")
- if (send_func.startswith("mpi_i") or recv_func.startswith("mpi_i")
- or send_func in persistent_send_funcs or recv_func in persistent_recv_funcs):
- b = InstructionBlock("MPI_REQUEST")
- b.register_operation("MPI_Request request;", 'all')
- tm.register_instruction_block(b)
+ b = InstructionBlock("MPICALL")
+ b.register_instruction(s, 1)
+ b.register_instruction(r, 0)
- if send_func in ["mpi_rsend", "mpi_irsend", "mpi_rsend_init"]:
- b = InstructionBlock("SYNC")
- b.register_operation(CorrectMPICallFactory().mpi_barrier(), 1)
- tm.register_instruction_block(b)
+ if recv_func in probe_pairs:
+ if recv_func in [["mpi_improbe", "mpi_mrecv"],
+ ["mpi_improbe", "mpi_imrecv"]]:
+ b.insert_instruction("while (!flag){", 0, 0)
+ # insertion before the improbe call
+ b.register_instruction("}", 0) # end while
+ b.register_instruction(CorrectMPICallFactory().get(recv_func[1]), 0)
- b = InstructionBlock("MPICALL")
- b.register_operation(s, 1)
- b.register_operation(r, 0)
if send_func in persistent_send_funcs:
- b.register_operation(cmpicf.mpi_start(), 1)
+ b.register_instruction(cmpicf.mpi_start(), 1)
if send_func == "mpi_psend_init":
# the pready takes a Request NOt a request*
- b.register_operation(MPICallFactory().mpi_pready("0", cf.get("request")[1:]), 1)
+ b.register_instruction(MPICallFactory().mpi_pready("0", cf.get("request")[1:]), 1)
if recv_func in persistent_recv_funcs:
- b.register_operation(cmpicf.mpi_start(), 0) #
+ b.register_instruction(cmpicf.mpi_start(), 0) #
# parrived is not necessary
tm.register_instruction_block(b)
+ # after send/recv
+
if send_func in ["mpi_rsend", "mpi_irsend", "mpi_rsend_init"]:
b = InstructionBlock("SYNC")
- b.register_operation(CorrectMPICallFactory().mpi_barrier(), 1)
+ # barrier indicating recv has started
+ b.register_instruction(CorrectMPICallFactory().mpi_barrier(), 0)
tm.register_instruction_block(b)
- if send_func.startswith("mpi_i") or send_func in persistent_send_funcs:
+ # wait for op to complete
+ if send_func in isend_funcs + persistent_send_funcs:
b = InstructionBlock("WAIT")
- b.register_operation(CorrectMPICallFactory().mpi_wait(), 1)
+ b.register_instruction(CorrectMPICallFactory().mpi_wait(), 1)
tm.register_instruction_block(b)
- if recv_func.startswith("mpi_i") or recv_func in persistent_recv_funcs:
+ if recv_func in irecv_funcs + persistent_recv_funcs + [["mpi_mprobe", "mpi_imrecv"],
+ ["mpi_improbe", "mpi_imrecv"]]:
b = InstructionBlock("WAIT")
- b.register_operation(CorrectMPICallFactory().mpi_wait(), 0)
+ b.register_instruction(CorrectMPICallFactory().mpi_wait(), 0)
tm.register_instruction_block(b)
+ # end MPI operation
+ # cleanup
+ free_block = InstructionBlock("buf_free")
if send_func in ["mpi_bsend", "mpi_ibsend", "mpi_bsend_init"]:
- b = InstructionBlock("buf_detach")
- b.register_operation("int freed_size;")
- b.register_operation(MPICallFactory().mpi_buffer_detach("mpi_buf", "&freed_size"))
- b.register_operation("free(mpi_buf);")
- tm.register_instruction_block(b)
+ free_block = InstructionBlock("buf_detach")
+ free_block.register_instruction("int freed_size;")
+ free_block.register_instruction(MPICallFactory().mpi_buffer_detach("mpi_buf", "&freed_size"))
+ free_block.register_instruction("free(mpi_buf);")
+
+ free_block.register_instruction(cf.get_buffer_free())
- free_block = InstructionBlock("buf_free")
- free_block.register_operation(cf.get_buffer_free())
if send_func in sendrecv_funcs:
# spilt send and recv buf
- b.register_operation("free(recv_buf);")
- tm.register_instruction_block(free_block)
+ free_block.register_instruction("free(recv_buf);")
if send_func in persistent_send_funcs:
- # spilt send and recv buf
- b = InstructionBlock("req_free")
- b.register_operation(cmpicf.mpi_request_free())
- tm.register_instruction_block(b)
+ free_block.register_instruction(cmpicf.mpi_request_free())
+
+ tm.register_instruction_block(free_block)
return tm
@@ -156,12 +188,12 @@ def get_collective_template(collective_func, seperate=True):
cf = CorrectParameterFactory()
alloc_block = InstructionBlock("alloc")
- alloc_block.register_operation(cf.get_buffer_alloc())
+ alloc_block.register_instruction(cf.get_buffer_alloc())
if False:
# spilt send and recv buf
alloc = cf.get_buffer_alloc()
alloc.set_name("recv_buf")
- alloc_block.register_operation(alloc)
+ alloc_block.register_instruction(alloc)
tm.register_instruction_block(alloc_block)
cmpicf = CorrectMPICallFactory()
@@ -170,15 +202,15 @@ def get_collective_template(collective_func, seperate=True):
b = InstructionBlock("MPICALL")
if seperate:
- b.register_operation(c, 1)
- b.register_operation(c, 0)
+ b.register_instruction(c, 1)
+ b.register_instruction(c, 0)
else:
- b.register_operation(c, 'all')
+ b.register_instruction(c, 'all')
tm.register_instruction_block(b)
free_block = InstructionBlock("buf_free")
- free_block.register_operation(cf.get_buffer_free())
+ free_block.register_instruction(cf.get_buffer_free())
tm.register_instruction_block(free_block)
return tm
@@ -190,11 +222,11 @@ def get_allocated_window(win_alloc_func, name, bufname, ctype, num_elements):
:param win_alloc_func: The window allocation to use (mpi_win_allocate or mpi_win_create).
:param name: name of the window
"""
-
+
b = InstructionBlock("win_allocate")
# declare window
- b.register_operation(f"MPI_Win {name};")
+ b.register_instruction(f"MPI_Win {name};")
# extract C data type and window buffer name
# dtype = CorrectParameterFactory().dtype[0]
@@ -204,17 +236,17 @@ def get_allocated_window(win_alloc_func, name, bufname, ctype, num_elements):
if win_alloc_func == "mpi_win_allocate":
# MPI allocate, only declaration required
- b.register_operation(f"{ctype}* {bufname};")
+ b.register_instruction(f"{ctype}* {bufname};")
win_allocate_call = CorrectMPICallFactory().mpi_win_allocate()
win_allocate_call.set_arg("baseptr", "&" + bufname)
elif win_alloc_func == "mpi_win_create":
# allocate buffer for win_create
- b.register_operation(AllocCall(ctype, num_elements, bufname))
+ b.register_instruction(AllocCall(ctype, num_elements, bufname))
win_allocate_call = CorrectMPICallFactory().mpi_win_create()
win_allocate_call.set_arg("base", bufname)
else:
assert False
-
+
# set common parameters for both calls
win_allocate_call.set_arg("win", "&" + name)
@@ -222,36 +254,33 @@ def get_allocated_window(win_alloc_func, name, bufname, ctype, num_elements):
win_allocate_call.set_arg("size", buf_size_bytes)
win_allocate_call.set_arg("disp_unit", f"sizeof({ctype})")
- b.register_operation(win_allocate_call)
+ b.register_instruction(win_allocate_call)
return b
-
def get_rma_call(rma_func, rank):
-
- b = InstructionBlock(rma_func.replace('mpi_',''))
+ b = InstructionBlock(rma_func.replace('mpi_', ''))
cf = CorrectParameterFactory()
cfmpi = CorrectMPICallFactory()
# request-based RMA call, add request
if rma_func.startswith("mpi_r"):
- b.register_operation(f"MPI_Request " + cf.get("request")[1:] + ";", kind=rank)
+ b.register_instruction(f"MPI_Request " + cf.get("request")[1:] + ";", kind=rank)
# some RMA ops require result_addr
if rma_func in ["mpi_get_accumulate", "mpi_rget_accumulate", "mpi_fetch_and_op", "mpi_compare_and_swap"]:
- b.register_operation(AllocCall(cf.dtype[0], cf.buf_size, cf.get("result_addr")), kind=rank)
+ b.register_instruction(AllocCall(cf.dtype[0], cf.buf_size, cf.get("result_addr")), kind=rank)
# some RMA ops require compare_addr
if rma_func in ["mpi_fetch_and_op", "mpi_compare_and_swap"]:
- b.register_operation(AllocCall(cf.dtype[0], cf.buf_size, cf.get("compare_addr")), kind=rank)
+ b.register_instruction(AllocCall(cf.dtype[0], cf.buf_size, cf.get("compare_addr")), kind=rank)
- b.register_operation(getattr(cfmpi, rma_func)(), kind=rank)
+ b.register_instruction(getattr(cfmpi, rma_func)(), kind=rank)
return b
-
def get_communicator(comm_create_func, name):
"""
:param comm_create_func: teh function used to create the new communicator
@@ -262,12 +291,12 @@ def get_communicator(comm_create_func, name):
"mpi_comm_idup_with_info", "mpi_comm_create", "mpi_comm_create_group", "mpi_comm_split",
"mpi_comm_split_type", "mpi_comm_create_from_group"]
b = InstructionBlock("comm_create")
- b.register_operation("MPI_Comm " + name + ";")
+ b.register_instruction("MPI_Comm " + name + ";")
if comm_create_func.startswith("mpi_comm_i"):
- b.register_operation("MPI_Request comm_create_req;")
+ b.register_instruction("MPI_Request comm_create_req;")
if comm_create_func in ["mpi_comm_create", "mpi_comm_create_group"]:
- b.register_operation("MPI_Group group;")
- b.register_operation(CorrectMPICallFactory().mpi_comm_group())
+ b.register_instruction("MPI_Group group;")
+ b.register_instruction(CorrectMPICallFactory().mpi_comm_group())
cmpicf = CorrectMPICallFactory()
call_creator_function = getattr(cmpicf, comm_create_func)
call = call_creator_function()
@@ -276,11 +305,11 @@ def get_communicator(comm_create_func, name):
call.set_arg("request", "&comm_create_req")
if comm_create_func in ["mpi_comm_create", "mpi_comm_create_group"]:
call.set_arg("group", "group") # not &group
- b.register_operation(call)
+ b.register_instruction(call)
if comm_create_func.startswith("mpi_comm_i"):
- b.register_operation(MPICallFactory().mpi_wait("&comm_create_req", "MPI_STATUS_IGNORE"))
+ b.register_instruction(MPICallFactory().mpi_wait("&comm_create_req", "MPI_STATUS_IGNORE"))
if comm_create_func in ["mpi_comm_create", "mpi_comm_create_group"]:
- b.register_operation(cmpicf.mpi_group_free())
+ b.register_instruction(cmpicf.mpi_group_free())
return b
@@ -297,31 +326,31 @@ def get_intercomm(comm_create_func, name):
assert name != "intercomm_base_comm"
if comm_create_func == "mpi_intercomm_create":
b = InstructionBlock("comm_create")
- b.register_operation("MPI_Comm intercomm_base_comm;")
- b.register_operation(
+ b.register_instruction("MPI_Comm intercomm_base_comm;")
+ b.register_instruction(
MPICallFactory().mpi_comm_split("MPI_COMM_WORLD", "rank % 2", "rank", "&intercomm_base_comm"))
- b.register_operation("MPI_Comm " + name + ";")
- b.register_operation(
+ b.register_instruction("MPI_Comm " + name + ";")
+ b.register_instruction(
MPICallFactory().mpi_intercomm_create("intercomm_base_comm", "0", "MPI_COMM_WORLD", "!(rank %2)",
CorrectParameterFactory().get("tag"), "&" + name))
- b.register_operation(MPICallFactory().mpi_comm_free("&intercomm_base_comm"))
+ b.register_instruction(MPICallFactory().mpi_comm_free("&intercomm_base_comm"))
return b
if comm_create_func == "mpi_intercomm_create_from_groups":
b = InstructionBlock("comm_create")
- b.register_operation("MPI_Group world_group,even_group,odd_group;")
- b.register_operation(MPICallFactory().mpi_comm_group("MPI_COMM_WORLD", "&world_group"))
- b.register_operation(
+ b.register_instruction("MPI_Group world_group,even_group,odd_group;")
+ b.register_instruction(MPICallFactory().mpi_comm_group("MPI_COMM_WORLD", "&world_group"))
+ b.register_instruction(
MPICallFactory().mpi_comm_group("intercomm_base_comm", "&intercomm_base_comm_group"))
- b.register_operation("int[3] triplet;")
- b.register_operation("triplet[0] =0;")
- b.register_operation("triplet[1] =size;")
- b.register_operation("triplet[2] =2;")
- b.register_operation(MPICallFactory().mpi_group_incl("world_group", "1","&triplet", "even_group"))
- b.register_operation("triplet[0] =1;")
- b.register_operation(MPICallFactory().mpi_group_incl("world_group", "1","&triplet", "odd_group"))
- b.register_operation("MPI_Comm " + name + ";")
- b.register_operation(
+ b.register_instruction("int[3] triplet;")
+ b.register_instruction("triplet[0] =0;")
+ b.register_instruction("triplet[1] =size;")
+ b.register_instruction("triplet[2] =2;")
+ b.register_instruction(MPICallFactory().mpi_group_incl("world_group", "1", "&triplet", "even_group"))
+ b.register_instruction("triplet[0] =1;")
+ b.register_instruction(MPICallFactory().mpi_group_incl("world_group", "1", "&triplet", "odd_group"))
+ b.register_instruction("MPI_Comm " + name + ";")
+ b.register_instruction(
MPICallFactory().mpi_intercomm_create_from_groups("(rank % 2 ? even_group:odd_group)", "0",
"(!(rank % 2) ? even_group:odd_group)", "0",
CorrectParameterFactory().get("stringtag"),
@@ -331,18 +360,18 @@ def get_intercomm(comm_create_func, name):
return b
if comm_create_func == "mpi_intercomm_merge":
b = InstructionBlock("comm_create")
- b.register_operation("MPI_Comm intercomm_base_comm;")
- b.register_operation("MPI_Comm to_merge_intercomm_comm;")
- b.register_operation(
+ b.register_instruction("MPI_Comm intercomm_base_comm;")
+ b.register_instruction("MPI_Comm to_merge_intercomm_comm;")
+ b.register_instruction(
MPICallFactory().mpi_comm_split("MPI_COMM_WORLD", "rank % 2", "rank", "&intercomm_base_comm"))
- b.register_operation("MPI_Comm " + name + ";")
- b.register_operation(
+ b.register_instruction("MPI_Comm " + name + ";")
+ b.register_instruction(
MPICallFactory().mpi_intercomm_create("intercomm_base_comm", "0", "MPI_COMM_WORLD", "!(rank %2)",
CorrectParameterFactory().get("tag"), "&to_merge_intercomm_comm"))
- b.register_operation(MPICallFactory().mpi_intercomm_merge("to_merge_intercomm_comm", "rank %2", "&" + name))
+ b.register_instruction(MPICallFactory().mpi_intercomm_merge("to_merge_intercomm_comm", "rank %2", "&" + name))
- b.register_operation(MPICallFactory().mpi_comm_free("&to_merge_intercomm_comm"))
- b.register_operation(MPICallFactory().mpi_comm_free("&intercomm_base_comm"))
+ b.register_instruction(MPICallFactory().mpi_comm_free("&to_merge_intercomm_comm"))
+ b.register_instruction(MPICallFactory().mpi_comm_free("&intercomm_base_comm"))
return b
return None
--
GitLab