diff --git a/.deactivated.gitlab-ci.yml b/.deactivated.gitlab-ci.yml
index c967c5c54a7c0a137663d05f44b91ab98a714e78..026c6c3fcbf88b431099a1a4d8923d62fa3dcbfe 100755
--- a/.deactivated.gitlab-ci.yml
+++ b/.deactivated.gitlab-ci.yml
@@ -19,7 +19,7 @@ build-must:
stage: build
needs: []
script:
- - scripts/ensure_python3 ./MBI.py -x must -c build
+ - scripts/ensure_python3 ./MBB.py -x must -c build
artifacts:
untracked: false
paths:
@@ -33,8 +33,8 @@ test-must-1:
before_script:
- apt-get install -y psmisc
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must -c run -b 1/10
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must -c run -b 1/10
artifacts:
untracked: false
when: always
@@ -49,8 +49,8 @@ test-must-2:
before_script:
- apt-get install -y psmisc
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must -c run -b 2/10
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must -c run -b 2/10
artifacts:
untracked: false
when: always
@@ -65,8 +65,8 @@ test-must-3:
before_script:
- apt-get install -y psmisc
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must -c run -b 3/10
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must -c run -b 3/10
artifacts:
untracked: false
when: always
@@ -81,8 +81,8 @@ test-must-4:
before_script:
- apt-get install -y psmisc
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must -c run -b 4/10
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must -c run -b 4/10
artifacts:
untracked: false
when: always
@@ -97,8 +97,8 @@ test-must-5:
before_script:
- apt-get install -y psmisc
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must -c run -b 5/10
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must -c run -b 5/10
artifacts:
untracked: false
when: always
@@ -113,8 +113,8 @@ test-must-6:
before_script:
- apt-get install -y psmisc
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must -c run -b 6/10
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must -c run -b 6/10
artifacts:
untracked: false
when: always
@@ -129,8 +129,8 @@ test-must-7:
before_script:
- apt-get install -y psmisc
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must -c run -b 7/10
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must -c run -b 7/10
artifacts:
untracked: false
when: always
@@ -145,8 +145,8 @@ test-must-8:
before_script:
- apt-get install -y psmisc
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must -c run -b 8/10
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must -c run -b 8/10
artifacts:
untracked: false
when: always
@@ -161,8 +161,8 @@ test-must-9:
before_script:
- apt-get install -y psmisc
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must -c run -b 9/10
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must -c run -b 9/10
artifacts:
untracked: false
when: always
@@ -177,8 +177,8 @@ test-must-10:
before_script:
- apt-get install -y psmisc
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must -c run -b 10/10
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must -c run -b 10/10
artifacts:
untracked: false
when: always
@@ -211,8 +211,8 @@ test-must-all:
- job: test-must-10
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x must
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x must
artifacts:
untracked: false
when: always
@@ -224,7 +224,7 @@ build-aislinn:
needs: []
image: ubuntu:18.04
script:
- - scripts/ensure_python3 ./MBI.py -x aislinn -c build
+ - scripts/ensure_python3 ./MBB.py -x aislinn -c build
artifacts:
untracked: false
when: always
@@ -238,8 +238,8 @@ test-aislinn:
- job: build-aislinn
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x aislinn
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x aislinn
artifacts:
untracked: false
when: always
@@ -255,8 +255,8 @@ test-aislinn:
# - baremetal
# image: registry.hub.docker.com/mquinson/mbi
# script:
-# - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
-# - scripts/ensure_python3 ./MBI.py -x mpisv
+# - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+# - scripts/ensure_python3 ./MBB.py -x mpisv
# artifacts:
# untracked: false
# when: always
@@ -267,8 +267,8 @@ test-civl-1:
stage: test
needs: []
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x civl -c run -b 1/4
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x civl -c run -b 1/4
artifacts:
untracked: false
when: always
@@ -278,8 +278,8 @@ test-civl-2:
stage: test
needs: []
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x civl -c run -b 2/4
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x civl -c run -b 2/4
artifacts:
untracked: false
when: always
@@ -289,8 +289,8 @@ test-civl-3:
stage: test
needs: []
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x civl -c run -b 3/4
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x civl -c run -b 3/4
artifacts:
untracked: false
when: always
@@ -300,8 +300,8 @@ test-civl-4:
stage: test
needs: []
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x civl -c run -b 4/4
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x civl -c run -b 4/4
artifacts:
untracked: false
when: always
@@ -320,8 +320,8 @@ test-civl-all:
- job: test-civl-4
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x civl
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x civl
artifacts:
untracked: false
when: always
@@ -332,7 +332,7 @@ build-parcoach:
stage: build
needs: []
script:
- - scripts/ensure_python3 ./MBI.py -x parcoach -c build
+ - scripts/ensure_python3 ./MBB.py -x parcoach -c build
artifacts:
untracked: false
when: always
@@ -345,8 +345,8 @@ test-parcoach:
- job: build-parcoach
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x parcoach
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x parcoach
artifacts:
untracked: false
when: always
@@ -357,7 +357,7 @@ build-simgrid:
stage: build
needs: []
script:
- - scripts/ensure_python3 ./MBI.py -x simgrid -c build
+ - scripts/ensure_python3 ./MBB.py -x simgrid -c build
cache:
paths:
- SimGrid/*
@@ -373,8 +373,8 @@ test-simgrid:
- job: build-simgrid
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x simgrid
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x simgrid
artifacts:
untracked: false
when: always
@@ -386,8 +386,8 @@ test-smpi:
- job: build-simgrid
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x smpi
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x smpi
artifacts:
untracked: false
when: always
@@ -399,8 +399,8 @@ test-smpivg-1:
- job: build-simgrid
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x smpivg -b 1/3
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x smpivg -b 1/3
artifacts:
untracked: false
when: always
@@ -412,8 +412,8 @@ test-smpivg-2:
- job: build-simgrid
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x smpivg -b 2/3
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x smpivg -b 2/3
artifacts:
untracked: false
when: always
@@ -425,8 +425,8 @@ test-smpivg-3:
- job: build-simgrid
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x smpivg -b 3/3
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x smpivg -b 3/3
artifacts:
untracked: false
when: always
@@ -442,8 +442,8 @@ test-smpivg-all:
- job: test-smpivg-3
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x smpivg
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x smpivg
artifacts:
untracked: false
when: always
@@ -457,8 +457,8 @@ test-smpivg-all:
# - baremetal
# image: registry.hub.docker.com/mquinson/mbi
# script:
-# - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
-# - scripts/ensure_python3 ./MBI.py -x itac -c run
+# - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+# - scripts/ensure_python3 ./MBB.py -x itac -c run
# artifacts:
# untracked: false
# when: always
@@ -469,7 +469,7 @@ build-isp:
stage: build
needs: []
script:
- - scripts/ensure_python3 ./MBI.py -x isp -c build
+ - scripts/ensure_python3 ./MBB.py -x isp -c build
artifacts:
untracked: false
when: always
@@ -482,8 +482,8 @@ test-isp-1:
- job: build-isp
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x isp -c run -b 1/2
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x isp -c run -b 1/2
artifacts:
untracked: false
when: always
@@ -496,8 +496,8 @@ test-isp-2:
- job: build-isp
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x isp -c run -b 2/2
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x isp -c run -b 2/2
artifacts:
untracked: false
when: always
@@ -512,8 +512,8 @@ test-isp-all:
- job: test-isp-2
artifacts: true
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -x isp
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -x isp
artifacts:
untracked: false
when: always
@@ -526,8 +526,8 @@ latex:
when: always
script:
- apt update ; apt -y install texlive-latex-base texlive-latex-extra texlive-pictures
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -c latex
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -c latex
- cd script ; ./tools/gen_latex.py ; cd ..
- cd latex ; pdflatex -batch dashboard.tex ; cd ..
artifacts:
@@ -540,8 +540,8 @@ pages:
stage: deploy
when: always
script:
- - rm -rf /MBI/*; scripts/ensure_python3 ./MBI.py -c generate
- - scripts/ensure_python3 ./MBI.py -c html
+ - rm -rf /MBI/*; scripts/ensure_python3 ./MBB.py -c generate
+ - scripts/ensure_python3 ./MBB.py -c html
- pwd ; ls
- mkdir public
- cp -r *.html gencodes img logs public/
diff --git a/MBI.py b/MBB.py
similarity index 93%
rename from MBI.py
rename to MBB.py
index df907a7ec50426e9b03726279019f728980833bd..f67624ae2ee9f8a4a1f8f07f836c4c9da9ac5f3b 100755
--- a/MBI.py
+++ b/MBB.py
@@ -1,6 +1,6 @@
#! /usr/bin/python3
-# autopep8 -i --max-line-length 130 MBI.py
+# autopep8 -i --max-line-length 130 MBB.py
import shutil
import os
@@ -23,6 +23,8 @@ import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
+from scripts.MBIutils import categorize
+
mpl.rcParams['hatch.linewidth'] = 4.5 # hatch linewidth
# Add our lib directory to the PYTHONPATH, and load our utilitary libraries
@@ -595,40 +597,15 @@ def aggregate_metrics_per_category(df_in):
return df[["CE", "RE", "TP", "TN", "FP", "FN", "coverage", "conclusiveness", "specificity", "recall", "precision", "f1", "overallaccuracy"]]
+def read_tool_reports(rootdir, toolname):
+ if not toolname in tools:
+ raise Exception(f"Tool {toolname} does not seem to be a valid name.")
-def cmd_csv(rootdir, toolnames):
- here = os.getcwd()
- os.chdir(rootdir)
- results = {}
- total_elapsed = {}
- used_toolnames = []
- outpath = f'{rootdir}/csv/'
-
- # Create directory for output if not present
- pathlib.Path(outpath).mkdir(parents=True, exist_ok=True)
-
- # select the tools for which we have some results
- print("Produce the stats for:", end='')
- for toolname in toolnames:
- if not toolname in tools:
- raise Exception(f"Tool {toolname} does not seem to be a valid name.")
-
- if os.path.exists(f'{args.logs_dir}/{toolname}'):
- used_toolnames.append(toolname)
- print(f' {toolname}', end="")
-
- # To compute timing statistics
- total_elapsed[toolname] = 0
- print(".")
+ if not os.path.exists(f'{args.logs_dir}/{toolname}'):
+ raise Exception(f"Not found Logs for {toolname}.")
- test_categories = ['COLL', 'P2P', 'RMA', 'other']
+ results=[]
- # Initialize the data structure to gather all results
- results = {}
- for toolname in used_toolnames:
- results[toolname] = []
-
- # Get all data from the caches
for test in todo:
binary = re.sub(r'\.c', '', os.path.basename(test['filename']))
ID = test['id']
@@ -636,29 +613,45 @@ def cmd_csv(rootdir, toolnames):
test_id = f"{binary}_{ID}"
expected = test['expect']
- for toolname in used_toolnames:
- resulting_categorization = categorize(tool=tools[toolname], toolname=toolname,
+ resulting_categorization = categorize(tool=tools[toolname], toolname=toolname,
test=test, test_id=test_id, logs_dir=args.logs_dir,
)
- resulting_categorization["test_id"] = test_id
- resulting_categorization["category"] = test["category"]
- results[toolname].append(resulting_categorization)
-
- pd.set_option('display.max_columns',14)
- for toolname in ["itac", "must", "parcoach"]:
- df = pd.DataFrame(results[toolname])
-
- df["TP_base"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & (df["CE"] == False) & (df["RE"] == False)
- df["TP_class"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & df["correct_class_reported"] & (df["CE"] == False) & (df["RE"] == False)
- df["TP_line"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & df["correct_line_reported"] & (df["CE"] == False) & (df["RE"] == False)
- df["TP_class_line"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & df["correct_class_reported"] & df[
- "correct_line_reported"] & (df["CE"] == False) & (df["RE"] == False)
- df["TP_class_line_no_class_noise"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & df[
- "correct_class_reported"] & df["correct_line_reported"] & (~df["contains_noise_class"]) & (df["CE"] == False) & (df["RE"] == False)
- df["TP_class_line_no_line_noise"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & df[
- "correct_class_reported"] & df["correct_line_reported"] & (~df["contains_noise_line"]) & (df["CE"] == False) & (df["RE"] == False)
+ resulting_categorization["test_id"] = test_id
+ resulting_categorization["category"] = test["category"]
+ results.append(resulting_categorization)
+
+ df = pd.DataFrame(results)
+
+ df["TP_base"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & (df["CE"] == False) & (df["RE"] == False)
+ df["TP_class"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & df["correct_class_reported"] & (df["CE"] == False) & (df["RE"] == False)
+ df["TP_line"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & df["correct_line_reported"] & (df["CE"] == False) & (df["RE"] == False)
+ df["TP_class_line"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & df["correct_class_reported"] & df[
+ "correct_line_reported"] & (df["CE"] == False) & (df["RE"] == False)
+ df["TP_class_line_no_class_noise"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & df[
+ "correct_class_reported"] & df["correct_line_reported"] & (~df["contains_noise_class"]) & (df["CE"] == False) & (df["RE"] == False)
+ df["TP_class_line_no_line_noise"] = df["ERROR_EXPECTED"] & df["any_error_reported"] & df[
+ "correct_class_reported"] & df["correct_line_reported"] & (~df["contains_noise_line"]) & (df["CE"] == False) & (df["RE"] == False)
+
+ return df
+
+def cmd_csv(rootdir, toolnames,print_to_console=False):
+ here = os.getcwd()
+ os.chdir(rootdir)
+ outpath = f'{rootdir}/csv/'
+
+ # Create directory for output if not present
+ pathlib.Path(outpath).mkdir(parents=True, exist_ok=True)
+
+ df_noise_ratio = pd.DataFrame(columns=toolnames)
+ df_overall_noise_ratio = pd.DataFrame(columns=toolnames)
+
+ pd.set_option('display.max_columns', 14)
+
+ for toolname in toolnames:
+ df = read_tool_reports(rootdir, toolname)
df.to_csv(f'{outpath}/{toolname}_raw.csv', index=False)
- print(f"=== {toolname} ===")
+ if print_to_console:
+ print(f"=== {toolname} ===")
# Output for each type of TP
for (colname) in ["base", "class", "line", "class_line", "class_line_no_line_noise", "class_line_no_line_noise", "class_line_no_class_noise"]:
@@ -667,72 +660,80 @@ def cmd_csv(rootdir, toolnames):
df_classified.to_csv(f'{outpath}/{toolname}_{colname}_full.csv', index=False)
df_result = aggregate_metrics_per_category(df_classified)
df_result.to_csv(f'{outpath}/{toolname}_{colname}.csv', index=True)
- if colname == "base":
+ if print_to_console:
print(f"\n{colname}:")
print(df_result[["CE", "RE", "TP", "TN", "FP", "FN", "coverage", "conclusiveness", "specificity", "recall", "precision", "f1", "overallaccuracy"]])
df_result[["CE", "RE", "TP", "TN", "FP", "FN", "coverage", "conclusiveness", "specificity", "recall", "precision", "f1", "overallaccuracy"]].style.format(precision=2).to_latex(f'{outpath}/{toolname}_{colname}.tex')
-
- # todo move this into plots cmd?
- df_plot = df.groupby("category").sum()
- df_plot.loc["ALL"] = df_plot.sum(axis=0)
- df_plot.drop("other", axis=0, inplace=True)
-
- df_plot["TP_class"] = df_plot["TP_class"] - df_plot["TP_class_line"]
- df_plot["TP_line"] = df_plot["TP_line"] - df_plot["TP_class_line"]
- df_plot["TP_base"] = df_plot["TP_base"] - df_plot["TP_class_line"] - df_plot["TP_class"] - df_plot["TP_line"]
-
- colors = ['#66CCEE', '#66CCEE', '#228833', '#EE6677']
- # colors = ['#66CCEE', 'yellow', '#228833', '#EE6677']
-
- fig, ax = plt.subplots()
-
- df_plot = df_plot[["TP_class", "TP_class_line", "TP_line", "TP_base"]]
-
- df_plot[["TP_class", "TP_class_line", "TP_line", "TP_base"]].div(df_plot.sum(axis=1), axis=0).plot.barh(
- stacked=True, color=colors, ax=ax, legend=False)
-
- # Customize bars
- bars = ax.patches
-
- for i in [4, 5, 6, 7]:
- bars[i].set_hatch("//")
- bars[i].set_edgecolor(colors[2])
-
- # Create custom legend handles
- handles = [
- mpatches.Patch(color=colors[0], label="correct error class"),
- mpatches.Patch(facecolor=colors[1], edgecolor=colors[2], hatch='//', label='correct class and line'),
- mpatches.Patch(color=colors[2], label='correct source line'),
- mpatches.Patch(color=colors[3], label='not helpful report')
- ]
-
- # ax.legend(handles=handles, ncol=2, loc='center left', bbox_to_anchor=(0.1, -0.25))
- # ax.set_title(f"Helpfulness of {toolname} Error Reports")
- # ax.set_xlabel("percentage of error reports")
- # ax.set_ylabel("MPI feature")
- # plt.tight_layout()
- # plt.savefig(f'{outpath}/{toolname}_plot.pdf')
-
- # print()
- # print(toolname)
- # print()
-
- # df_plot = df.groupby("category").sum()
- # df_plot.loc["ALL"] = df_plot.sum(axis=0)
- # df_plot.drop("other", axis=0, inplace=True)
- # df_plot["noise_ratio"] = df_plot["num_noise_line"] / df_plot["num_error_reports"]
- # print("overall_noise")
- # print(df_plot["noise_ratio"])
-
- # df_copy = df.copy()
- # df_copy.loc[df_copy['ERROR_EXPECTED'] == False, ['num_noise_class_line', 'num_error_reports']] = 0
- # df_plot = df_copy.groupby("category").sum()
- # df_plot.loc["ALL"] = df_plot.sum(axis=0)
- # df_plot.drop("other", axis=0, inplace=True)
- # df_plot["noise_ratio"] = df_plot["num_noise_line"] / df_plot["num_error_reports"]
- # print("noise_in_cases_where_errors_are_present")
- # print(df_plot[["noise_ratio", "num_noise_class_line", "num_error_reports"]])
+ plot_helpfulness(df, outpath, toolname)
+
+ df_noise_per_tool = df.groupby("category").sum()
+ df_noise_per_tool.loc["ALL"] = df_noise_per_tool.sum(axis=0)
+ df_noise_per_tool.drop("other", axis=0, inplace=True)
+ df_noise_per_tool["noise_ratio"] = df_noise_per_tool["num_noise_line"] / df_noise_per_tool["num_error_reports"]
+ if print_to_console:
+ print("overall_noise")
+ print(df_noise_per_tool["noise_ratio"])
+ df_overall_noise_ratio[toolname] =df_noise_per_tool["noise_ratio"]
+
+ df_copy = df.copy()
+ df_copy.loc[df_copy['ERROR_EXPECTED'] == False, ['num_noise_class_line', 'num_error_reports']] = 0
+ df_noise_per_tool = df_copy.groupby("category").sum()
+ df_noise_per_tool.loc["ALL"] = df_noise_per_tool.sum(axis=0)
+ df_noise_per_tool.drop("other", axis=0, inplace=True)
+ df_noise_per_tool["noise_ratio"] = df_noise_per_tool["num_noise_line"] / df_noise_per_tool["num_error_reports"]
+ if print_to_console:
+ print("noise_in_cases_where_errors_are_present")
+ print(df_noise_per_tool[["noise_ratio", "num_noise_class_line", "num_error_reports"]])
+ df_noise_ratio[toolname] = df_noise_per_tool["noise_ratio"]
+
+ df_noise_ratio.to_csv(f'{outpath}/noise.csv')
+ df_overall_noise_ratio.to_csv(f'{outpath}/overall_noise_including_unexpected.csv')
+
+
+def plot_helpfulness(df, outpath, toolname):
+ SMALL_SIZE = 16
+ MEDIUM_SIZE = 16
+ BIGGER_SIZE = 16
+
+ plt.rc('font', size=SMALL_SIZE) # controls default text sizes
+ plt.rc('axes', titlesize=BIGGER_SIZE) # fontsize of the axes title
+ plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
+ plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels
+ plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels
+ plt.rc('legend', fontsize=SMALL_SIZE) # legend fontsize
+ plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title
+
+ df_plot = df.groupby("category").sum()
+ df_plot.loc["ALL"] = df_plot.sum(axis=0)
+ df_plot.drop("other", axis=0, inplace=True)
+ df_plot["TP_class"] = df_plot["TP_class"] - df_plot["TP_class_line"]
+ df_plot["TP_line"] = df_plot["TP_line"] - df_plot["TP_class_line"]
+ df_plot["TP_base"] = df_plot["TP_base"] - df_plot["TP_class_line"] - df_plot["TP_class"] - df_plot["TP_line"]
+ colors = ['#88CCEE', '#88CCEE', '#44AA99', '#EE6677']
+ # colors = ['#66CCEE', 'yellow', '#228833', '#EE6677']
+ fig, ax = plt.subplots(1, 1, figsize=(9, 6))
+ df_plot = df_plot[["TP_class", "TP_class_line", "TP_line", "TP_base"]]
+ df_plot[["TP_class", "TP_class_line", "TP_line", "TP_base"]].div(df_plot.sum(axis=1), axis=0).plot.barh(
+ stacked=True, color=colors, ax=ax, legend=False)
+ # Customize bars
+ bars = ax.patches
+ for i in [4, 5, 6, 7]:
+ bars[i].set_hatch("//")
+ bars[i].set_edgecolor(colors[2])
+ # Create custom legend handles
+ handles = [
+ mpatches.Patch(color=colors[0], label="correct error class"),
+ mpatches.Patch(facecolor=colors[1], edgecolor=colors[2], hatch='//', label='correct class and line'),
+ mpatches.Patch(color=colors[2], label='correct source line'),
+ mpatches.Patch(color=colors[3], label='not helpful report')
+ ]
+ ax.legend(handles=handles, ncol=2, loc='center left', bbox_to_anchor=(0.05, -0.3))
+ #ax.set_title(f"Helpfulness of {toolname} error reports")
+ ax.set_xlabel("Percentage of error reports")
+ ax.set_ylabel("MPI feature")
+ plt.tight_layout()
+ plt.savefig(f'{outpath}/{toolname}_plot.pdf')
def cmd_latex(rootdir, toolnames):
@@ -1959,15 +1960,18 @@ elif args.c == 'run':
for t in arg_tools:
cmd_run(rootdir=rootdir, toolname=t, batchinfo=args.b)
elif args.c == 'latex':
- extract_all_todo_from_logdir(args.x, args.logs_dir)
+ extract_all_todo_from_logdir(arg_tools[0], args.logs_dir)
# 'smpi','smpivg' are not shown in the paper
# cmd_latex(rootdir, toolnames=['aislinn', 'civl', 'isp','itac', 'simgrid', 'mpisv', 'must', 'hermes', 'parcoach', 'mpi-checker'])
cmd_latex(rootdir, toolnames=['itac', 'must', 'parcoach'])
elif args.c == 'csv':
- extract_all_todo_from_logdir(args.x, args.logs_dir)
- cmd_csv(rootdir, toolnames=['itac', 'must', 'parcoach'])
+ extract_all_todo_from_logdir(arg_tools[0], args.logs_dir)
+ if arg_tools:
+ cmd_csv(rootdir, toolnames=arg_tools)
+ else:
+ cmd_csv(rootdir, toolnames=['itac', 'must', 'parcoach'])
elif args.c == 'html':
- extract_all_todo_from_logdir(args.x, args.logs_dir)
+ extract_all_todo_from_logdir(arg_tools[0], args.logs_dir)
if args.x == 'mpirun':
# toolnames=['itac', 'simgrid','must', 'smpi', 'smpivg', 'aislinn', 'civl', 'isp', 'mpisv', 'parcoach', 'hermes', 'mpi-checker']
toolnames = ['itac', 'must', 'parcoach']
diff --git a/gitlab-simgrid.yml b/gitlab-simgrid.yml
index 6d40ad739946354feaec8a4d497bb360b1fa0804..3b4e500f2af40c02da7423b025e6f16ab3d2072f 100755
--- a/gitlab-simgrid.yml
+++ b/gitlab-simgrid.yml
@@ -15,9 +15,9 @@ pages:
- 7zr x -so logs-*.7z | tar xf -
- mkdir -p logs/simgrid
- touch logs/simgrid/trust_the_installation trust_the_installation # Silence the checks for the right docker image
- - python3 ./MBI.py -c generate
-# - python3 ./MBI.py -x simgrid | grep Test
- - python3 ./MBI.py -c html -x itac,simgrid,must,parcoach,isp,aislinn,mpisv,civl,smpi,smpivg
+ - python3 ./MBB.py -c generate
+# - python3 ./MBB.py -x simgrid | grep Test
+ - python3 ./MBB.py -c html -x itac,simgrid,must,parcoach,isp,aislinn,mpisv,civl,smpi,smpivg
- pwd ; ls ; echo logs:; ls logs
- mkdir public
- cp -r *.html gencodes img logs public/
diff --git a/scripts/result_plot.py b/scripts/result_plot.py
index 22185325ab92d8ae7056035c0e36c6e6d2592540..b62b5ae835dba524d862eecc5ef9e8001285faf3 100644
--- a/scripts/result_plot.py
+++ b/scripts/result_plot.py
@@ -52,10 +52,14 @@ plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title
fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(16, 9)) #
-colors = ['#228833', '#66ccee', '#ee6677', '#aa3377', '#ccbb44', '#bbbbbb']
+#colors = ['#228833', '#66ccee', '#ee6677', '#aa3377', '#ccbb44', '#bbbbbb']
+colors = ['#6699CC', '#EECC66', '#004488', '#997700', '#BBBBBB', '#000000']
+colors_2 = {'TP': '#117733', 'TN': '#88CCEE', 'FP': '#CC6677', 'FN': '#AA4499', 'RE': '#DDCC77', 'CE': '#DDDDDD'}
+
+hatches= ["","",".",".","",""]
((ax1, ax2), (ax3, ax4)) = axs
-df_p2p[["TP", "TN", "FP", "FN", "RE", "CE"]].plot.barh(stacked=True, ax=ax1, legend=False, color=colors)
+df_p2p[["TP", "TN", "FP", "FN", "RE", "CE"]].plot.barh(stacked=True, ax=ax1, legend=False,color=colors)
ax1.set_title('P2P')
handles, labels = ax1.get_legend_handles_labels()
@@ -80,4 +84,5 @@ for ax in [ax1, ax2, ax3, ax4]:
fig.legend(handles, labels, loc='upper center', ncols=6, bbox_to_anchor=(0.5, 1.05), )
plt.tight_layout()
+
plt.savefig(os.path.join(plot_path, "results_per_cat.pdf"), bbox_inches="tight")
diff --git a/scripts/tools/aislinn.py b/scripts/tools/aislinn.py
index 5f45b992c8d2e39b70c8b66e377d2bda582e53d3..ae76700a850156d4cceac743d3c71c1130eddd33 100644
--- a/scripts/tools/aislinn.py
+++ b/scripts/tools/aislinn.py
@@ -16,7 +16,7 @@ class Tool(AbstractTool):
print(f"id: '{id.stdout}'; version: '{ver.stdout}'")
print("Please run this script in a ubuntu:18.04 image. Run these commands:")
print(" docker image pull ubuntu:18.04")
- print(" docker run -it --rm --name MIB --volume $(pwd):/MBI ubuntu:18.04 /MBI/MBI.py -x aislinn")
+ print(" docker run -it --rm --name MIB --volume $(pwd):/MBI ubuntu:18.04 /MBI/MBB.py -x aislinn")
sys.exit(1)
def build(self, rootdir, cached=True):
diff --git a/scripts/tools/mpisv.py b/scripts/tools/mpisv.py
index 5ace8ccf739c922cdd42bddbad3a6f56a6a9cb44..c00103e47e095b7578995c05f34af46e04611c90 100644
--- a/scripts/tools/mpisv.py
+++ b/scripts/tools/mpisv.py
@@ -13,7 +13,7 @@ class Tool(AbstractTool):
else:
print("Please run this script in a mpisv/mpi-sv image. Run these commands:")
print(" docker image pull mpisv/mpi-sv")
- print(" docker run -it --rm --name MIB --shm-size=512m --volume $(pwd):/MBI mpisv/mpi-sv /MBI/scripts/ensure_python3 /MBI/MBI.py -x mpisv")
+ print(" docker run -it --rm --name MIB --shm-size=512m --volume $(pwd):/MBI mpisv/mpi-sv /MBI/scripts/ensure_python3 /MBI/MBB.py -x mpisv")
sys.exit(1)
def run(self, execcmd, filename, binary, id, timeout, batchinfo):
diff --git a/scripts/tools/must.py b/scripts/tools/must.py
index e7a9c50c7672611906f645262de2e16a229c9a57..fe89640841dee2993c94bd33a0a38387410ed21a 100644
--- a/scripts/tools/must.py
+++ b/scripts/tools/must.py
@@ -126,18 +126,18 @@ class V18(AbstractTool):
"MUST_ERROR_INTEGER_NOT_WITHIN_ZERO_TAG_UB": ["InvalidParam"],
"MUST_ERROR_INTEGER_NOT_WITHIN_ZERO_TAG_UB_ANY_TAG": ["InvalidParam"],
"MUST_ERROR_INTEGER_GREATER_COMM_SIZE": ["InvalidParam"],
- "MUST_ERROR_INTEGER_GREATER_EQUAL_COMM_SIZE": ["InvalidParam"],
+ "MUST_ERROR_INTEGER_GREATER_EQUAL_COMM_SIZE": ["InvalidParam", "GlobalParameterMissmatch"],
"MUST_ERROR_INTEGER_PRODUCT_GREATER_COMM_SIZE": ["InvalidParam"],
"MUST_ERROR_GROUP_RANGE_RANK": ["InvalidParam"],
"MUST_ERROR_GROUP_RANGE_STRIDE": ["InvalidParam"],
- "MUST_ERROR_REQUEST_ACTIVE": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_ACTIVE": ["DEADLOCK", "RequestLifeCycle", "InvalidParam"],
"MUST_ERROR_REQUEST_ACTIVE_ARRAY": ["DEADLOCK", "RequestLifeCycle"],
"MUST_ERROR_REQUEST_PARTITION_ACTIVE": ["DEADLOCK", "RequestLifeCycle"],
"MUST_ERROR_REQUEST_NOT_PARTITIONED_SEND": ["DEADLOCK", "RequestLifeCycle"],
"MUST_ERROR_REQUEST_NOT_PARTITIONED_RECV": ["DEADLOCK", "RequestLifeCycle"],
- "MUST_ERROR_REQUEST_NOT_KNOWN": ["DEADLOCK", "RequestLifeCycle"],
- "MUST_ERROR_REQUEST_NOT_KNOWN_ARRAY": ["DEADLOCK", "RequestLifeCycle"],
- "MUST_ERROR_REQUEST_NULL": ["DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_NOT_KNOWN": ["InvalidParam", "DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_NOT_KNOWN_ARRAY": ["InvalidParam", "DEADLOCK", "RequestLifeCycle"],
+ "MUST_ERROR_REQUEST_NULL": ["InvalidParam", "DEADLOCK", "RequestLifeCycle", "CallOrdering"],
"MUST_ERROR_REQUEST_NULL_ARRAY": ["DEADLOCK", "RequestLifeCycle"],
"MUST_ERROR_REQUEST_PERSISTENT_BUT_INACTIVE": ["DEADLOCK", "RequestLifeCycle"],
"MUST_ERROR_COMM_UNKNWOWN": ["InvalidParam"],
@@ -176,24 +176,24 @@ class V18(AbstractTool):
"MUST_ERROR_POINTER_NULL_COMM_SIZE_ARRAY": ["InvalidParam"],
"MUST_ERROR_POINTER_NULL_COMM_SIZE_ARRAY_AT_INDEX": ["InvalidParam"],
"MUST_ERROR_MPI_IN_PLACE_USED": ["InvalidParam"],
- "MUST_ERROR_SELFOVERLAPPED": ["LocalParameterMissmatch"],
- "MUST_ERROR_OVERLAPPED_SEND": ["LocalParameterMissmatch"],
- "MUST_ERROR_OVERLAPPED_RECV": ["LocalParameterMissmatch"],
+ "MUST_ERROR_SELFOVERLAPPED": ["LocalParameterMissmatch", "LocalConcurrency", "GlobalParameterMissmatch"],
+ "MUST_ERROR_OVERLAPPED_SEND": ["LocalParameterMissmatch", "LocalConcurrency"],
+ "MUST_ERROR_OVERLAPPED_RECV": ["LocalParameterMissmatch", "LocalConcurrency"],
"MUST_ERROR_POINTER_NULL_STATUS_IGNORE": ["InvalidParam"],
"MUST_ERROR_TYPEMATCH_INTERNAL_NOTYPE": ["InvalidParam", "LocalParameterMissmatch"],
"MUST_ERROR_TYPEMATCH_INTERNAL_TYPESIG": ["InvalidParam", "LocalParameterMissmatch"],
- "MUST_ERROR_TYPEMATCH_MISMATCH": ["GlobalParameterMissmatch"],
+ "MUST_ERROR_TYPEMATCH_MISMATCH": ["GlobalParameterMissmatch", "LocalParameterMissmatch"],
"MUST_ERROR_TYPEMATCH_MISMATCH_BYTE": ["GlobalParameterMissmatch"],
- "MUST_ERROR_TYPEMATCH_LENGTH": ["InvalidParam", "LocalParameterMissmatch"],
+ "MUST_ERROR_TYPEMATCH_LENGTH": ["InvalidParam", "LocalParameterMissmatch", "GlobalParameterMissmatch"],
"MUST_ERROR_TYPEMATCH_ALIGNMENT": ["InvalidParam", "LocalParameterMissmatch"],
"MUST_ERROR_MESSAGE_LOST": ["DEADLOCK", "CallOrdering"],
"MUST_ERROR_COLLECTIVE_CALL_MISMATCH": ["DEADLOCK", "CallOrdering"],
"MUST_ERROR_COLLECTIVE_OP_MISMATCH": ["GlobalParameterMissmatch"],
"MUST_ERROR_COLLECTIVE_ROOT_MISMATCH": ["GlobalParameterMissmatch"],
- "MUST_ERROR_COLLECTIVE_BLOCKING_NONBLOCKING_MISMATCH": ["GlobalParameterMissmatch"],
- "MUST_ERROR_DEADLOCK": ["DEADLOCK"],
+ "MUST_ERROR_COLLECTIVE_BLOCKING_NONBLOCKING_MISMATCH": ["GlobalParameterMissmatch", "CallOrdering"],
+ "MUST_ERROR_DEADLOCK": ["DEADLOCK", "CallOrdering", "GlobalParameterMissmatch", "MissingCall", "InvalidParam"],
"MUST_ERROR_BUFFER_REATTACH": ["TODO"],
- "MUST_ERROR_BUFFER_NOATTACHED": ["TODO"],
+ "MUST_ERROR_BUFFER_NOATTACHED": ["InvalidParam"],
"MUST_ERROR_COUNTS_ARRAYS_DIFFER": ["TODO"],
"MUST_ERROR_MPI_MULTIPLE_THREADS": ["TODO"],
"MUST_ERROR_UNSUPPORTED": ["TODO"],