Skip to content
Snippets Groups Projects
Commit b3a7aed6 authored by Jammer, Tim's avatar Jammer, Tim
Browse files

Merge branch 'devel-TJ' into 'main'

More Work on infrastructure IV

See merge request !6
parents e22aad47 96408975
No related branches found
No related tags found
1 merge request!6More Work on infrastructure IV
Showing
with 624 additions and 414 deletions
......@@ -35,9 +35,11 @@ class CorrectParameterFactory:
return "0"
if param in ["TAG", "tag", "sendtag", "recvtag"]:
return str(self.tag)
if param in "stringtag":
return "\"" + str(self.tag) + "\""
if param in ["COMM", "comm"]:
return "MPI_COMM_WORLD"
if param in ["newcomm"]:
if param in ["newcomm", "newintercomm"]:
return "newcomm"
if param in ["STATUS", "status"]:
return "MPI_STATUS_IGNORE"
......@@ -51,7 +53,24 @@ class CorrectParameterFactory:
return "0"
if param in ["REQUEST", "request"]:
return "&request"
if param in ["GROUP", "group"]:
return "&group"
if param in ["color"]:
return "1"
if param in ["split_type"]:
return "MPI_COMM_TYPE_SHARED"
if param in ["key"]:
return "rank"
if param in ["errhandler"]:
return "MPI_ERRORS_ARE_FATAL" #
if param in ["local_comm"]:
return "MPI_COMM_SELF"
if param in ["local_leader"]:
return "0"
if param in ["peer_comm"]:
return "MPI_COMM_WORLD"
if param in ["remote_leader"]:
return "0"
print("Not Implemented: " + param)
assert False, "Param not known"
......
......@@ -32,43 +32,17 @@ class ErrorGenerator(ABC):
"""
pass
# the number of errors to produce in the normal mode
@abstractmethod
def get_num_errors(self):
def generate(self, generate_full_set):
"""
Abstract method to get the number of errors to produce in the normal mode.
Returns:
int: Number of errors in normal mode.
"""
pass
@abstractmethod
def get_num_errors_extended(self):
"""
Abstract method to get the number of errors to produce in the extended mode (all possible combinations).
Returns:
int: Number of errors in extended mode.
"""
pass
@abstractmethod
def generate(self, i):
"""
Abstract method to generate error number i.
may raise CorrectTestcase if the specified case is actually correct
Abstract method to generate error cases. Implemented as a python generator.
Meaning that the cases should be yielded instead of returned
Parameters:
- i: Error number.
- generate_full_set: if the generator should generate the extended full set of cases.
Returns:
yield:
TemplateManager: An instantiated and set up TemplateManager with the error case.
"""
pass
class CorrectTestcase (BaseException):
# exception class to be raised by generators if they generate a correct case
"""
pass
......@@ -8,7 +8,8 @@ import subprocess
# for printing a nice progress bar
import tqdm
from scripts.Infrastructure.ErrorGenerator import ErrorGenerator, CorrectTestcase
from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
from scripts.Infrastructure.Template import TemplateManager
from scripts.Infrastructure.Variables import featurelist
# number of digits to use numbering filenames
......@@ -99,36 +100,21 @@ class GeneratorManager:
mpicc = os.environ.get('MPICC')
assert mpicc and "Environment var MPICC not set"
print("Generate Testcases")
# use generator if at least one feature of the generator matches the filterlist
generators_to_use = [g for g in self.generators if any(elem in filterlist for elem in g.get_feature())]
# collect total number of cases
total_cases = 0
for generator in generators_to_use:
num_errors_in_generator = 0
if generate_full_set:
num_errors_in_generator = generator.get_num_errors_extended()
else:
num_errors_in_generator = generator.get_num_errors()
total_cases = total_cases + num_errors_in_generator
print("Generate Testcases using %d generators" % len(generators_to_use))
# prints a nice progress bar
if print_progress_bar:
progress_bar = tqdm.tqdm(total=total_cases)
progress_bar = tqdm.tqdm(total=len(generators_to_use))
cases_generated = 0
for generator in generators_to_use:
num_errors_in_generator = 0
if generate_full_set:
num_errors_in_generator = generator.get_num_errors_extended()
else:
num_errors_in_generator = generator.get_num_errors()
for i in range(num_errors_in_generator):
try:
result_error = generator.generate(i)
for result_error in generator.generate(generate_full_set):
assert isinstance(result_error, TemplateManager)
if not float(result_error.get_version()) > float(max_mpi_version):
case_name = result_error.get_short_descr()
fname = self.get_filename(case_name)
......@@ -149,9 +135,6 @@ class GeneratorManager:
if try_compile:
subprocess.check_call([mpicc, full_name])
# raises CalledProcessError if code does not compile
except CorrectTestcase:
# ignore a correct case
pass
if print_progress_bar:
progress_bar.update(1)
......
......@@ -127,6 +127,18 @@ class InstructionBlock:
raise IndexError("Operation Not Found")
self.operations[kind][index] = op
def insert_operation(self, op, kind='all', before_index=0):
"""
Inserts an operation before the specified one. will Raise IndexError if not present
Parameters:
- op (str or MPICall) the new operation
- kind ('all','not0' or integer): which ranks should execute the operation
- index (int): the index of the operation within the given kind
"""
if len(self.operations[kind]) < before_index:
raise IndexError("Operation Not Found")
self.operations[kind].insert(before_index,op)
def remove_operation(self, kind='all', index=0):
"""
Removes the operation registered. will Raise IndexError if not present
......
......@@ -40,7 +40,7 @@ int main(int argc, char **argv) {
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (nprocs < @{min_num_ranks}@)
printf("MBB ERROR: This test needs at least 2 processes to produce a bug!\\n");
printf("MBB ERROR: This test needs at least @{min_num_ranks}@ processes to produce a bug!\\n");
@{test_code}@
......@@ -187,7 +187,7 @@ class TemplateManager:
raise ValueError("Neither Both block name nor index is given")
def insert_block(self, new_block, block_name=None, idx=None):
def insert_block(self, new_block, after_block_name=None, after_idx=None):
"""
inserts the given Instruction Block AFTER the one specified Either by name or by index
Raises IndexError if the specified block is not found
......@@ -195,13 +195,13 @@ class TemplateManager:
Raises ValueError if Both a block name and index are given (or none is given)
Args:
new_block (InstructionBlock): the block to insert
block_name (str): The name of the InstructionBlock to receive
idx (int): index of the InstructionBlock to retrieve
after_block_name (str): The name of the InstructionBlock to receive
after_idx (int): index of the InstructionBlock to retrieve
"""
if block_name is not None:
if idx is not None:
if after_block_name is not None:
if after_idx is not None:
raise ValueError("Both block name and index are given")
to_return = [b for b in self._blocks if b.name == block_name]
to_return = [b for b in self._blocks if b.name == after_block_name]
if len(to_return) == 0:
raise IndexError("Block Not Found")
if len(to_return) > 1:
......@@ -209,10 +209,10 @@ class TemplateManager:
self._blocks.insert(self._blocks.index(to_return[0]) + 1, new_block)
return
if idx is not None:
if block_name is not None:
if after_idx is not None:
if after_block_name is not None:
raise ValueError("Both block name and index are given")
self._blocks.insert(idx + 1, new_block)
self._blocks.insert(after_idx + 1, new_block)
return
raise ValueError("Neither Both block name nor index is given")
......
......@@ -29,7 +29,8 @@ def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"):
# currently supported:
assert send_func in ["mpi_send", "mpi_ssend", "mpi_isend", "mpi_issend", "mpi_sendrecv", "mpi_rsend", "mpi_irsend",
"mpi_bsend", "mpi_ibsend", "mpi_sendrecv", "mpi_sendrecv_replace", "mpi_isendrecv",
"mpi_isendrecv_replace","mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init","mpi_psend_init"]
"mpi_isendrecv_replace", "mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init",
"mpi_psend_init"]
assert recv_func in ["mpi_recv", "mpi_irecv", "mpi_sendrecv", "mpi_sendrecv_replace", "mpi_isendrecv",
"mpi_isendrecv_replace", "mpi_recv_init", "mpi_precv_init"]
......@@ -57,7 +58,6 @@ def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"):
alloc_block.register_operation(alloc)
tm.register_instruction_block(alloc_block)
if send_func in ["mpi_bsend", "mpi_ibsend", "mpi_bsend_init"]:
b = InstructionBlock("buf_attach")
buf_size = "sizeof(int)*10 + MPI_BSEND_OVERHEAD"
......@@ -101,13 +101,11 @@ def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"):
# the pready takes a Request NOt a request*
b.register_operation(MPICallFactory().mpi_pready("0", cf.get("request")[1:]), 1)
if recv_func in persistent_recv_funcs:
b.register_operation(cmpicf.mpi_start(),1)#
b.register_operation(cmpicf.mpi_start(), 0) #
# parrived is not necessary
tm.register_instruction_block(b)
if send_func in ["mpi_rsend", "mpi_irsend", "mpi_rsend_init"]:
b = InstructionBlock("SYNC")
b.register_operation(CorrectMPICallFactory().mpi_barrier(), 1)
......@@ -123,8 +121,6 @@ def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"):
b.register_operation(CorrectMPICallFactory().mpi_wait(), 0)
tm.register_instruction_block(b)
if send_func in ["mpi_bsend", "mpi_ibsend", "mpi_bsend_init"]:
b = InstructionBlock("buf_detach")
b.register_operation("int freed_size;")
......@@ -145,7 +141,6 @@ def get_send_recv_template(send_func="mpi_isend", recv_func="mpi_irecv"):
b.register_operation(cmpicf.mpi_request_free())
tm.register_instruction_block(b)
return tm
......@@ -187,3 +182,99 @@ def get_collective_template(collective_func, seperate=True):
tm.register_instruction_block(free_block)
return tm
def get_communicator(comm_create_func, name):
"""
:param comm_create_func: teh function used to create the new communicator
:param name: name of the communicator variable
:return: instruction block with name "comm_create" that will initialize the communicator with the given initialization function, does include the allocation of a stack variable with the procided name for the communicator
"""
assert comm_create_func in ["mpi_comm_dup", "mpi_comm_dup_with_info", "mpi_comm_idup",
"mpi_comm_idup_with_info", "mpi_comm_create", "mpi_comm_create_group", "mpi_comm_split",
"mpi_comm_split_type", "mpi_comm_create_from_group"]
b = InstructionBlock("comm_create")
b.register_operation("MPI_Comm " + name + ";")
if comm_create_func.startswith("mpi_comm_i"):
b.register_operation("MPI_Request comm_create_req;")
if comm_create_func in ["mpi_comm_create", "mpi_comm_create_group"]:
b.register_operation("MPI_Group group;")
b.register_operation(CorrectMPICallFactory().mpi_comm_group())
cmpicf = CorrectMPICallFactory()
call_creator_function = getattr(cmpicf, comm_create_func)
call = call_creator_function()
call.set_arg("newcomm", "&" + name)
if comm_create_func.startswith("mpi_comm_i"):
call.set_arg("request", "&comm_create_req")
if comm_create_func in ["mpi_comm_create", "mpi_comm_create_group"]:
call.set_arg("group", "group") # not &group
b.register_operation(call)
if comm_create_func.startswith("mpi_comm_i"):
b.register_operation(MPICallFactory().mpi_wait("&comm_create_req", "MPI_STATUS_IGNORE"))
if comm_create_func in ["mpi_comm_create", "mpi_comm_create_group"]:
b.register_operation(cmpicf.mpi_group_free())
return b
def get_intercomm(comm_create_func, name):
"""
:param comm_create_func: the function used to create the new communicator
:param name: name of the communicator variable
:return Tuple InstructionBlock, InstructionBlock: instruction block with name "comm_create" that will initialize the communicator with the given initialization function,
does include the allocation of a stack variable with the provided name for the communicator
may also contain other stack variables as needed
and the block containing all the necessary frees
"""
assert comm_create_func in ["mpi_intercomm_create", "mpi_intercomm_create_from_groups", "mpi_intercomm_merge"]
assert name != "intercomm_base_comm"
if comm_create_func == "mpi_intercomm_create":
b = InstructionBlock("comm_create")
b.register_operation("MPI_Comm intercomm_base_comm;")
b.register_operation(
MPICallFactory().mpi_comm_split("MPI_COMM_WORLD", "rank % 2", "rank", "&intercomm_base_comm"))
b.register_operation("MPI_Comm " + name + ";")
b.register_operation(
MPICallFactory().mpi_intercomm_create("intercomm_base_comm", "0", "MPI_COMM_WORLD", "!(rank %2)",
CorrectParameterFactory().get("tag"), "&" + name))
b.register_operation(MPICallFactory().mpi_comm_free("&intercomm_base_comm"))
return b
if comm_create_func == "mpi_intercomm_create_from_groups":
b = InstructionBlock("comm_create")
b.register_operation("MPI_Group world_group,even_group,odd_group;")
b.register_operation(MPICallFactory().mpi_comm_group("MPI_COMM_WORLD", "&world_group"))
b.register_operation(
MPICallFactory().mpi_comm_group("intercomm_base_comm", "&intercomm_base_comm_group"))
b.register_operation("int[3] triplet;")
b.register_operation("triplet[0] =0;")
b.register_operation("triplet[1] =size;")
b.register_operation("triplet[2] =2;")
b.register_operation(MPICallFactory().mpi_group_incl("world_group", "1","&triplet", "even_group"))
b.register_operation("triplet[0] =1;")
b.register_operation(MPICallFactory().mpi_group_incl("world_group", "1","&triplet", "odd_group"))
b.register_operation("MPI_Comm " + name + ";")
b.register_operation(
MPICallFactory().mpi_intercomm_create_from_groups("(rank % 2 ? even_group:odd_group)", "0",
"(!(rank % 2) ? even_group:odd_group)", "0",
CorrectParameterFactory().get("stringtag"),
CorrectParameterFactory().get("INFO"),
CorrectParameterFactory().get("errhandler"),
"&" + name), )
return b
if comm_create_func == "mpi_intercomm_merge":
b = InstructionBlock("comm_create")
b.register_operation("MPI_Comm intercomm_base_comm;")
b.register_operation("MPI_Comm to_merge_intercomm_comm;")
b.register_operation(
MPICallFactory().mpi_comm_split("MPI_COMM_WORLD", "rank % 2", "rank", "&intercomm_base_comm"))
b.register_operation("MPI_Comm " + name + ";")
b.register_operation(
MPICallFactory().mpi_intercomm_create("intercomm_base_comm", "0", "MPI_COMM_WORLD", "!(rank %2)",
CorrectParameterFactory().get("tag"), "&to_merge_intercomm_comm"))
b.register_operation(MPICallFactory().mpi_intercomm_merge("to_merge_intercomm_comm", "rank %2", "&" + name))
b.register_operation(MPICallFactory().mpi_comm_free("&to_merge_intercomm_comm"))
b.register_operation(MPICallFactory().mpi_comm_free("&intercomm_base_comm"))
return b
return None
......@@ -14,19 +14,14 @@ class InvalidRankErrorColl(ErrorGenerator):
def __init__(self):
pass
def get_num_errors(self):
return len(self.invalid_ranks) * len(self.functions_to_use)
# the number of errors to produce in the extended mode (all possible combinations)
def get_num_errors_extended(self):
return len(self.invalid_ranks) * len(self.functions_to_use)
def get_feature(self):
return ["COLL"]
def generate(self, i):
rank_to_use = self.invalid_ranks[i // len(self.functions_to_use)]
func_to_use = self.functions_to_use[i % len(self.functions_to_use)]
def generate(self, generate_full_set):
for func_to_use in self.functions_to_use:
for rank_to_use in self.invalid_ranks:
tm = get_collective_template(func_to_use, seperate=False)
arg_to_replace = "root"
......@@ -35,4 +30,6 @@ class InvalidRankErrorColl(ErrorGenerator):
tm.get_block("MPICALL").get_operation(kind='all', index=0).set_arg(arg_to_replace, rank_to_use)
tm.get_block("MPICALL").get_operation(kind='all', index=0).set_has_error()
return tm
yield tm
if not generate_full_set:
return
......@@ -9,50 +9,27 @@ from scripts.Infrastructure.TemplateFactory import get_send_recv_template
from itertools import chain
sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
class InvalidBufErrorP2P(ErrorGenerator):
invalid_bufs = [CorrectParameterFactory().buf_var_name, "NULL"]
functions_to_check = ["mpi_send",
"mpi_recv", "mpi_irecv",
send_funcs = ["mpi_send",
"mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend",
"mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init",
"mpi_precv_init", "mpi_recv_init"
] + sendrecv_funcs + sendrecv_funcs
# chekc sendrecv funcs two times: the send and recv part
recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"] + sendrecv_funcs
]
def __init__(self):
pass
recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
def get_num_errors(self):
# send + receive = only check the first two functions
return len(self.invalid_bufs) * 2
sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
# the number of errors to produce in the extended mode (all possible combinations)
def get_num_errors_extended(self):
return len(self.invalid_bufs) * len(self.functions_to_check)
def __init__(self):
pass
def get_feature(self):
return ["P2P"]
def generate(self, i):
buf_to_use = self.invalid_bufs[i % len(self.invalid_bufs)]
send_func = self.functions_to_check[i // len(self.invalid_bufs)]
check_receive = False
recv_func = "mpi_irecv"
if send_func in self.recv_funcs:
check_receive = True
recv_func = send_func
send_func = "mpi_send"
if recv_func in sendrecv_funcs:
send_func = recv_func
if i % len(self.functions_to_check) >= len(self.functions_to_check) - len(sendrecv_funcs):
# check the send part of sendrecv
check_receive = False
def generate_impl(self, send_func, recv_func, check_receive):
for buf_to_use in self.invalid_bufs:
tm = get_send_recv_template(send_func, recv_func)
if buf_to_use == CorrectParameterFactory().buf_var_name:
......@@ -61,16 +38,15 @@ class InvalidBufErrorP2P(ErrorGenerator):
type = tm.get_block("alloc").get_operation(kind='all', index=1).get_type()
tm.get_block("alloc").replace_operation(
CorrectParameterFactory.dtype[0] + "* " + name + ";", kind='all', index=1)
else:
tm.get_block("alloc").replace_operation(
CorrectParameterFactory.dtype[0] + "* " + CorrectParameterFactory.buf_var_name + ";")
# without allocation: usage is undefined behaviour
if check_receive:
tm.set_description("InvalidParam-Buffer-" + recv_func, "Invalid Rank: %s" % buf_to_use)
tm.set_description("InvalidParam-Buffer-" + recv_func, "Invalid Buffer: %s" % buf_to_use)
else:
tm.set_description("InvalidParam-Buffer-" + send_func, "Invalid Rank: %s" % buf_to_use)
tm.set_description("InvalidParam-Buffer-" + send_func, "Invalid Buffer: %s" % buf_to_use)
if check_receive:
if tm.get_block("MPICALL").get_operation(kind=0, index=0).has_arg("buf"):
......@@ -85,4 +61,36 @@ class InvalidBufErrorP2P(ErrorGenerator):
tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("recvbuf", buf_to_use)
tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
return tm
yield tm
def generate(self, generate_full_set):
for func in self.send_funcs:
yield from self.generate_impl(func, 'mpi_irecv', False)
if not generate_full_set:
return
for func in self.recv_funcs:
yield from self.generate_impl("mpi_send", func, True)
yield from self.generate_impl("mpi_sendrecv", "mpi_sendrecv", True)
yield from self.generate_impl("mpi_sendrecv", "mpi_sendrecv", False)
yield from self.generate_impl("mpi_sendrecv_replace", "mpi_sendrecv_replace", True)
class MessageRaceErrorSendRecv(ErrorGenerator):
def __init__(self):
pass
def get_feature(self):
return ["P2P"]
def generate(self, generate_full_set):
for buf_to_use in ["buf", "MPI_IN_PLACE"]:
tm = get_send_recv_template("mpi_sendrecv", "mpi_sendrecv")
tm.set_description("InvalidParam-Buffer-mpi_sendrecv", "send and recv buffer must be disjoint in sendrecv")
for k in [0, 1]:
tm.get_block("MPICALL").get_operation(kind=k, index=0).set_arg("recvbuf", buf_to_use)
tm.get_block("MPICALL").get_operation(kind=k, index=0).set_has_error()
yield tm
#! /usr/bin/python3
from scripts.Infrastructure.ErrorGenerator import ErrorGenerator, CorrectTestcase
from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
from scripts.Infrastructure.InstructionBlock import InstructionBlock
from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
from scripts.Infrastructure.Template import TemplateManager
from scripts.Infrastructure.TemplateFactory import get_send_recv_template
from scripts.Infrastructure.TemplateFactory import get_send_recv_template, get_communicator, get_intercomm
from itertools import chain
......@@ -15,25 +15,15 @@ sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
class InvalidCommErrorP2P(ErrorGenerator):
invalid_comm = ["MPI_COMM_NULL", "NULL"]
missmatching_comms = ["MPI_COMM_SELF","mpi_comm_dup"]
#TODO test with:
#'MPI_Comm_dup_with_info',
#'MPI_Comm_idup',
#'MPI_Comm_idup_with_info',
#'MPI_Comm_create',
#'MPI_Comm_create_group',
#'MPI_Comm_split',
#'MPI_Comm_split_type',
#'MPI_Comm_create_from_group',
#'MPI_Intercomm_create',
#'MPI_Intercomm_create_from_groups',
#'MPI_Intercomm_merge'
missmatching_comms = ["MPI_COMM_SELF", "mpi_comm_dup", "mpi_comm_dup_with_info", "mpi_comm_idup",
"mpi_comm_idup_with_info", "mpi_comm_create", "mpi_comm_create_group", "mpi_comm_split",
"mpi_comm_split_type", "mpi_comm_create_from_group"
]
intercomms = ["mpi_intercomm_create", "mpi_intercomm_merge", "mpi_intercomm_create_from_groups"]
# as extended testcases
comms_to_check = invalid_comm + missmatching_comms
comms_to_check = invalid_comm + missmatching_comms + intercomms
functions_to_check = ["mpi_send",
"mpi_recv", "mpi_irecv",
......@@ -47,20 +37,13 @@ class InvalidCommErrorP2P(ErrorGenerator):
def __init__(self):
pass
def get_num_errors(self):
# send + receive = only check the first two functions
return len(self.comms_to_check) * 2
# the number of errors to produce in the extended mode (all possible combinations)
def get_num_errors_extended(self):
return len(self.comms_to_check) * len(self.functions_to_check)
def get_feature(self):
return ["P2P"]
def generate(self, i):
comm_to_use = self.comms_to_check[i % len(self.comms_to_check)]
send_func = self.functions_to_check[i // len(self.comms_to_check)]
def generate(self, generate_full_set):
for send_func in self.functions_to_check:
for comm_to_use in self.comms_to_check:
check_receive = False
recv_func = "mpi_irecv"
......@@ -70,23 +53,15 @@ class InvalidCommErrorP2P(ErrorGenerator):
send_func = "mpi_send"
if recv_func in sendrecv_funcs:
send_func = recv_func
if i % len(self.functions_to_check) >= len(self.functions_to_check) - len(sendrecv_funcs):
# check the send part of sendrecv
check_receive = False
tm = get_send_recv_template(send_func, recv_func)
if comm_to_use in self.missmatching_comms and comm_to_use != "MPI_COMM_SELF":
b = InstructionBlock("comm_create")
b.register_operation("MPI_Comm "+comm_to_use + ";")
cmpicf = CorrectMPICallFactory()
cmpicf = CorrectMPICallFactory()
call_creator_function = getattr(cmpicf, comm_to_use)
call = call_creator_function()
call.set_arg("newcomm","&"+comm_to_use)
b.register_operation(call)
tm.insert_block(b,block_name="alloc")
b = get_communicator(comm_to_use, comm_to_use)
tm.insert_block(b, after_block_name="alloc")
if comm_to_use in self.intercomms:
b = get_intercomm(comm_to_use, comm_to_use)
tm.insert_block(b, after_block_name="alloc")
error_string = "ParamMatching"
if comm_to_use in self.invalid_comm:
......@@ -95,12 +70,11 @@ class InvalidCommErrorP2P(ErrorGenerator):
if check_receive:
if comm_to_use in self.missmatching_comms and recv_func == "mpi_irecv":
# combination repeated
raise CorrectTestcase
continue
tm.set_description(error_string + "-Comm-" + recv_func, "Invalid Rank: %s" % comm_to_use)
else:
tm.set_description(error_string + "-Comm-" + send_func, "Invalid Rank: %s" % comm_to_use)
if check_receive:
tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("comm", comm_to_use)
tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
......@@ -114,9 +88,20 @@ class InvalidCommErrorP2P(ErrorGenerator):
# missmatch is between both
tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
if comm_to_use in self.missmatching_comms and comm_to_use != "MPI_COMM_SELF":
# an intercomm has only one rank (the other group)
if comm_to_use in self.intercomms and not comm_to_use == "mpi_intercomm_merge":
# intercomm merge results in an equivalent comm again
if tm.get_block("MPICALL").get_operation(kind=0, index=0).has_arg("source"):
tm.get_block("MPICALL").get_operation(kind=0, index=0).set_arg("source", "0")
if tm.get_block("MPICALL").get_operation(kind=1, index=0).has_arg("source"):
tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("source", "0")
if comm_to_use in self.missmatching_comms + self.intercomms and comm_to_use != "MPI_COMM_SELF":
b = InstructionBlock("comm_free")
b.register_operation(MPICallFactory().mpi_comm_free("&" + comm_to_use))
tm.register_instruction_block(b)
return tm
yield tm
# end for comm to check
if not generate_full_set:
return
......@@ -9,49 +9,26 @@ from scripts.Infrastructure.TemplateFactory import get_send_recv_template
from itertools import chain
sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
class InvalidRankErrorP2P(ErrorGenerator):
invalid_ranks = ["-1", "nprocs", "MPI_PROC_NULL"]
functions_to_check = ["mpi_send",
"mpi_recv", "mpi_irecv",
send_funcs = ["mpi_send",
"mpi_irecv",
"mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend",
"mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init","mpi_psend_init","mpi_precv_init","mpi_recv_init"
] + sendrecv_funcs + sendrecv_funcs
# chekc sendrecv funcs two times: the send and recv part
recv_funcs = ["mpi_recv", "mpi_irecv","mpi_recv_init","mpi_precv_init"] + sendrecv_funcs
"mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init"
]
recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
def __init__(self):
pass
def get_num_errors(self):
# send + receive = only check the first two functions
return len(self.invalid_ranks) * 2
# the number of errors to produce in the extended mode (all possible combinations)
def get_num_errors_extended(self):
return len(self.invalid_ranks) * len(self.functions_to_check)
def get_feature(self):
return ["P2P"]
def generate(self, i):
rank_to_use = self.invalid_ranks[i % len(self.invalid_ranks)]
send_func = self.functions_to_check[i // len(self.invalid_ranks)]
check_receive = False
recv_func = "mpi_irecv"
if send_func in self.recv_funcs:
check_receive = True
recv_func = send_func
send_func = "mpi_send"
if recv_func in sendrecv_funcs:
send_func = recv_func
if i % len(self.functions_to_check) >= len(self.functions_to_check) - len(sendrecv_funcs):
# check the send part of sendrecv
check_receive = False
def generate_impl(self, send_func, recv_func, check_receive):
for rank_to_use in self.invalid_ranks:
tm = get_send_recv_template(send_func, recv_func)
if check_receive:
......@@ -66,4 +43,17 @@ class InvalidRankErrorP2P(ErrorGenerator):
tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("dest", rank_to_use)
tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
return tm
yield tm
def generate(self, generate_full_set):
for send_func in self.send_funcs:
yield from self.generate_impl(send_func, "mpi_irecv", False)
if not generate_full_set:
return
for func in self.recv_funcs:
yield from self.generate_impl("mpi_send", func, True)
for func in self.sendrecv_funcs:
yield from self.generate_impl(func, func, True)
yield from self.generate_impl(func, func, False)
......@@ -10,7 +10,6 @@ from scripts.Infrastructure.TemplateFactory import get_send_recv_template
from itertools import chain
class InvalidRequestErrorP2P(ErrorGenerator):
invalid_requests = ["MPI_REQUEST_NULL", # probably triggers compiler warning
"NULL"]
......@@ -36,9 +35,10 @@ class InvalidRequestErrorP2P(ErrorGenerator):
def get_feature(self):
return ["P2P"]
def generate(self, i):
req_to_use = self.invalid_requests[i % len(self.invalid_requests)]
send_func = self.functions_to_check[i // len(self.invalid_requests)]
def generate(self, generate_full_set):
for send_func in self.functions_to_check:
for req_to_use in self.invalid_requests:
check_receive = False
recv_func = "mpi_irecv"
......@@ -61,4 +61,7 @@ class InvalidRequestErrorP2P(ErrorGenerator):
tm.get_block("MPICALL").get_operation(kind=1, index=0).set_arg("request", req_to_use)
tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
return tm
yield tm
if not generate_full_set:
return
#! /usr/bin/python3
from scripts.Infrastructure.ErrorGenerator import ErrorGenerator, CorrectTestcase
from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
from scripts.Infrastructure.InstructionBlock import InstructionBlock
from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
......@@ -9,50 +9,25 @@ from scripts.Infrastructure.TemplateFactory import get_send_recv_template
from itertools import chain
sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
class InvalidTagErrorP2P(ErrorGenerator):
invalid_tags = ["-1", "MPI_TAG_UB+1", CorrectParameterFactory.tag * 2, "MPI_ANY_TAG"]
functions_to_check = ["mpi_send",
"mpi_recv", "mpi_irecv",
send_funcs = ["mpi_send",
"mpi_isend", "mpi_ssend", "mpi_issend", "mpi_rsend", "mpi_irsend", "mpi_bsend", "mpi_ibsend",
"mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init",
"mpi_precv_init", "mpi_recv_init"
] + sendrecv_funcs + sendrecv_funcs
# chekc sendrecv funcs two times: the send and recv part
recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"] + sendrecv_funcs
]
recv_funcs = ["mpi_recv", "mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
def __init__(self):
pass
def get_num_errors(self):
# send + receive = only check the first two functions
return len(self.invalid_tags) * 2
# the number of errors to produce in the extended mode (all possible combinations)
def get_num_errors_extended(self):
return len(self.invalid_tags) * len(self.functions_to_check)
def get_feature(self):
return ["P2P"]
def generate(self, i):
tag_to_use = self.invalid_tags[i % len(self.invalid_tags)]
send_func = self.functions_to_check[i // len(self.invalid_tags)]
check_receive = False
recv_func = "mpi_irecv"
if send_func in self.recv_funcs:
check_receive = True
recv_func = send_func
send_func = "mpi_send"
if recv_func in sendrecv_funcs:
send_func = recv_func
if i % len(self.functions_to_check) >= len(self.functions_to_check) - len(sendrecv_funcs):
# check the send part of sendrecv
check_receive = False
def generate_impl(self, send_func, recv_func, check_receive):
for tag_to_use in self.invalid_tags:
tm = get_send_recv_template(send_func, recv_func)
error_string = "InvalidParam"
......@@ -62,10 +37,10 @@ class InvalidTagErrorP2P(ErrorGenerator):
if check_receive:
if tag_to_use == "MPI_ANY_TAG":
# correct case
raise CorrectTestcase
continue
if tag_to_use == CorrectParameterFactory().tag * 2 and recv_func == "mpi_irecv":
# combination repeated
raise CorrectTestcase
continue
tm.set_description(error_string + "-Tag-" + recv_func, "Invalid Rank: %s" % tag_to_use)
else:
tm.set_description(error_string + "-Tag-" + send_func, "Invalid Rank: %s" % tag_to_use)
......@@ -89,4 +64,17 @@ class InvalidTagErrorP2P(ErrorGenerator):
# missmatch is between both
tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
return tm
yield tm
def generate(self, generate_full_set):
for send_func in self.send_funcs:
yield from self.generate_impl(self, send_func, "mpi_irecv", False)
if not generate_full_set:
return
for recv_func in self.recv_funcs:
yield from self.generate_impl(self, "mpi_send", recv_func, True)
for func in self.sendrecv_funcs:
yield from self.generate_impl(func, func, True)
yield from self.generate_impl(func, func, False)
#! /usr/bin/python3
from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
from scripts.Infrastructure.InstructionBlock import InstructionBlock
from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
from scripts.Infrastructure.Template import TemplateManager
from scripts.Infrastructure.TemplateFactory import get_send_recv_template
from itertools import chain
from scripts.Infrastructure.Variables import ERROR_MARKER_COMMENT
sendrecv_funcs = ["mpi_sendrecv", "mpi_sendrecv_replace"]
class LocalConcurrencyErrorP2P(ErrorGenerator):
functions_to_check = ["mpi_irecv",
"mpi_isend", "mpi_issend", "mpi_irsend", "mpi_ibsend",
"mpi_send_init", "mpi_ssend_init", "mpi_bsend_init", "mpi_rsend_init", "mpi_psend_init",
"mpi_precv_init", "mpi_recv_init"
]
recv_funcs = ["mpi_irecv", "mpi_recv_init", "mpi_precv_init"]
def __init__(self):
pass
def get_feature(self):
return ["P2P"]
def generate(self, generate_full_set):
for send_func in self.functions_to_check:
check_receive = False
recv_func = "mpi_irecv"
if send_func in self.recv_funcs:
check_receive = True
recv_func = send_func
send_func = "mpi_isend"
tm = get_send_recv_template(send_func, recv_func)
if check_receive:
tm.set_description("LocalConcurrency-receive-" + recv_func,
"usage of receive buffer before operation is completed")
else:
tm.set_description("LocalConcurrency-send-" + send_func,
"usage of send buffer before operation is completed")
if check_receive:
tm.get_block("MPICALL").get_operation(kind=0, index=0).set_has_error()
tm.get_block("MPICALL").register_operation("buf[2]=1;" + ERROR_MARKER_COMMENT, kind=1)
else:
tm.get_block("MPICALL").get_operation(kind=1, index=0).set_has_error()
tm.get_block("MPICALL").register_operation("buf[2]=1;" + ERROR_MARKER_COMMENT, kind=1)
yield tm
if not generate_full_set:
return
#! /usr/bin/python3
from scripts.Infrastructure.ErrorGenerator import ErrorGenerator
from scripts.Infrastructure.InstructionBlock import InstructionBlock
from scripts.Infrastructure.MPICallFactory import MPICallFactory, CorrectMPICallFactory
from scripts.Infrastructure.CorrectParameter import CorrectParameterFactory, get_matching_recv
from scripts.Infrastructure.Template import TemplateManager
from scripts.Infrastructure.TemplateFactory import get_send_recv_template
from itertools import chain
class MessageRaceErrorAnyTag(ErrorGenerator):
# TODO do we need to generate it for all combinations of send and recv?
def __init__(self):
pass
def get_feature(self):
return ["P2P"]
def generate(self, generate_full_set):
tm = TemplateManager()
tm.set_description("MsgRace-ANY_TAG", "order of messages is indeterministic, may lead to a deadlock")
b = InstructionBlock("alloc")
b.register_operation(CorrectParameterFactory().get_buffer_alloc())
tm.register_instruction_block(b)
b = InstructionBlock("MPICALL")
# send part
b.register_operation("for(int i =0; i < 10; ++i) {", kind=1)
b.register_operation("buf[0]=i;", kind=1)
send_call = CorrectMPICallFactory().mpi_send()
send_call.set_arg("tag", "i")
b.register_operation(send_call, kind=1)
b.register_operation("}", kind=1)
# recv part
b.register_operation("for(int i =0; i < 10; ++i) {", kind=0)
recv_call = CorrectMPICallFactory().mpi_recv()
recv_call.set_arg("tag", "MPI_ANY_TAG")
b.register_operation(recv_call, kind=0)
b.register_operation("if(buf[0]!=i){", kind=0)
additional_recv = CorrectMPICallFactory().mpi_recv()
additional_recv.set_has_error() # additional recv leads to deadlock
b.register_operation(additional_recv, kind=0)
b.register_operation(" }", kind=0) # end if
b.register_operation("}", kind=0) # end for
tm.register_instruction_block(b)
b = InstructionBlock("free")
b.register_operation(CorrectParameterFactory().get_buffer_free())
tm.register_instruction_block(b)
yield tm
class MessageRaceErrorAnysource(ErrorGenerator):
# TODO do we need to generate it for all combinations of send and recv?
def __init__(self):
pass
def get_feature(self):
return ["P2P"]
def generate(self, generate_full_set):
tm = TemplateManager(min_ranks=3)
tm.set_description("MsgRace-ANY_SOURCE", "order of messages is indeterministic, may lead to a deadlock")
b = InstructionBlock("alloc")
b.register_operation(CorrectParameterFactory().get_buffer_alloc())
tm.register_instruction_block(b)
b = InstructionBlock("MPICALL")
# send part
b.register_operation("buf[0]=rank;", kind='not0')
send_call = CorrectMPICallFactory().mpi_send()
b.register_operation(send_call, kind='not0')
# recv part
b.register_operation("for(int i =1; i < nprocs; ++i) {", kind=0)
recv_call = CorrectMPICallFactory().mpi_recv()
recv_call.set_arg("source", "MPI_ANY_SOURCE")
b.register_operation(recv_call, kind=0)
b.register_operation("if(buf[0]!=i){", kind=0)
additional_recv = CorrectMPICallFactory().mpi_recv()
additional_recv.set_has_error() # additional recv leads to deadlock
b.register_operation(additional_recv, kind=0)
b.register_operation(" }", kind=0) # end if
b.register_operation("}", kind=0) # end for
tm.register_instruction_block(b)
b = InstructionBlock("free")
b.register_operation(CorrectParameterFactory().get_buffer_free())
tm.register_instruction_block(b)
yield tm
......@@ -2,8 +2,9 @@
from scripts.Infrastructure.GeneratorManager import GeneratorManager
if __name__ == "__main__":
gm = GeneratorManager("./errors")
gm.generate("../gencodes", try_compile=True, generate_full_set=False) # default
# gm.generate("../gencodes", try_compile=True, generate_full_set=True, max_mpi_version="3.1") #all cases that can compile for my local mpi installation
#gm = GeneratorManager("./errors")
gm = GeneratorManager("./errors/devel")
#gm.generate("../gencodes", try_compile=True, generate_full_set=False) # default
gm.generate("../gencodes", try_compile=True, generate_full_set=True, max_mpi_version="3.1") #all cases that can compile for my local mpi installation
pass
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment