Skip to content
Snippets Groups Projects
Verified Commit e56a8ecd authored by Simon Schwitanski's avatar Simon Schwitanski :slight_smile:
Browse files

Fix computation of metrics in LaTeX command

parent 92b39674
Branches
Tags
1 merge request!17Test infrastructure redesign
...@@ -211,17 +211,17 @@ def cmd_run(rootdir, toolname, batchinfo): ...@@ -211,17 +211,17 @@ def cmd_run(rootdir, toolname, batchinfo):
######################## ########################
# cmd_html(): what to do when '-c html' is used (extract the statistics of this tool) # cmd_html(): what to do when '-c html' is used (extract the statistics of this tool)
######################## ########################
def percent(num, den, compl=False, one=False): def percent(num, den, compl=False, one=False, digits=4):
"""Returns the ratio of num/den as a percentage, rounded to 2 digits only. If one=True, then return a ratio of 1 with 4 digits""" """Returns the ratio of num/den as a percentage, rounded to N digits only (default: 4). If one=True, then return a ratio of 1 with 4 digits"""
if den == 0: if den == 0:
return "(error)" return "(error)"
elif compl: # Complementary elif compl: # Complementary
res = round (100 - num/den*100, 2) res = round (100 - num/den*100, digits - 2)
else: else:
res = round (num/den*100, 2) res = round (num/den*100, 2)
if int(res) == 100: if int(res) == 100:
return "1" if one else "100" return "1" if one else "100"
return round(res/100, 4) if one else res return round(res/100, digits) if one else res
def bold_if(val, target): def bold_if(val, target):
"""Returns the value as a bold LaTeX string if it equals the target, or unchanged otherwise.""" """Returns the value as a bold LaTeX string if it equals the target, or unchanged otherwise."""
...@@ -841,24 +841,24 @@ def cmd_latex(rootdir, toolnames): ...@@ -841,24 +841,24 @@ def cmd_latex(rootdir, toolnames):
tout = len(results[test_category][toolname]['timeout']) tout = len(results[test_category][toolname]['timeout'])
total = TP + TN + FP + FN + port + fail + othr + tout total = TP + TN + FP + FN + port + fail + othr + tout
if (TN+FP) != 0 and TP+FN != 0 and TP+FP != 0: if (TN+FP) != 0 and TP+FN != 0 and TP+FP != 0:
coverage = float(percent(port,total,compl=True,one=True)) coverage = float(percent(port,total,compl=True,one=True,digits=2))
if coverage > best['coverage']: if coverage > best['coverage']:
best['coverage'] = coverage best['coverage'] = coverage
completion = float(percent((port+fail+othr+tout),(total),compl=True,one=True)) completion = float(percent((port+fail+othr+tout),(total),compl=True,one=True,digits=2))
if completion > best['completion']: if completion > best['completion']:
best['completion'] = completion best['completion'] = completion
specificity = float(percent(TN,(TN+FP),one=True)) specificity = float(percent(TN,(TN+FP),one=True,digits=2))
if specificity > best['specificity']: if specificity > best['specificity']:
best['specificity'] = specificity best['specificity'] = specificity
recall = float(percent(TP,(TP+FN),one=True)) recall = float(percent(TP,(TP+FN),one=True,digits=2))
if recall > best['recall']: if recall > best['recall']:
best['recall'] = recall best['recall'] = recall
precision = float(percent(TP,(TP+FP),one=True)) precision = float(percent(TP,(TP+FP),one=True,digits=2))
if precision > best['precision']: if precision > best['precision']:
best['precision'] = precision best['precision'] = precision
# Recompute precision & recall without rounding, to match the value computed when displaying the result # Recompute precision & recall without rounding, to match the value computed when displaying the result
precision = TN/(TP+FP) precision = TP/(TP+FP)
recall = TP/(TP+FN) recall = TP/(TP+FN)
F1 = percent(2*precision*recall,(precision+recall),one=True) F1 = percent(2*precision*recall,(precision+recall),one=True)
if F1 > best['F1']: if F1 > best['F1']:
...@@ -894,26 +894,26 @@ def cmd_latex(rootdir, toolnames): ...@@ -894,26 +894,26 @@ def cmd_latex(rootdir, toolnames):
outfile.write(f"&{bold_if(TP,best['TP'])}&{bold_if(TN,best['TN'])}&{bold_if(FP,best['FP'])}&{bold_if(FN,best['FN'])}&") outfile.write(f"&{bold_if(TP,best['TP'])}&{bold_if(TN,best['TN'])}&{bold_if(FP,best['FP'])}&{bold_if(FN,best['FN'])}&")
# Coverage & Completion # Coverage & Completion
coverage = percent(port,total,compl=True,one=True) coverage = percent(port,total,compl=True,one=True,digits=2)
completion = percent((port+fail+othr+tout),(total),compl=True,one=True) completion = percent((port+fail+othr+tout),(total),compl=True,one=True,digits=2)
outfile.write(f"{bold_if(coverage,best['coverage'])} &{bold_if(completion, best['completion'])}&") outfile.write(f"{bold_if(coverage,best['coverage'])} &{bold_if(completion, best['completion'])}&")
# Specificity: recognized {TN} correct codes out of {TN+FP} # Specificity: recognized {TN} correct codes out of {TN+FP}
specificity = percent(TN,(TN+FP),one=True) specificity = percent(TN,(TN+FP),one=True,digits=2)
outfile.write(f'{bold_if(specificity, best["specificity"])}&') outfile.write(f'{bold_if(specificity, best["specificity"])}&')
# Recall: found {TP} errors out of {TP+FN} ;Precision: {TP} diagnostic of error are correct out of {TP+FP}) ; # Recall: found {TP} errors out of {TP+FN} ;Precision: {TP} diagnostic of error are correct out of {TP+FP}) ;
recall = percent(TP,(TP+FN),one=True) recall = percent(TP,(TP+FN),one=True,digits=2)
precision = percent(TP,(TP+FP),one=True) precision = percent(TP,(TP+FP),one=True,digits=2)
outfile.write(f'{bold_if(recall, best["recall"])} & {bold_if(precision, best["precision"])} &') outfile.write(f'{bold_if(recall, best["recall"])} & {bold_if(precision, best["precision"])} &')
# F1 Score # F1 Score
if TP+FP >0 and TP+FN >0: if TP+FP >0 and TP+FN >0:
precision = TN/(TP+FP) precision = TP/(TP+FP)
recall = TP/(TP+FN) recall = TP/(TP+FN)
F1 = percent(2*precision*recall,(precision+recall),one=True) F1 = percent(2*precision*recall,(precision+recall),one=True,digits=2)
outfile.write(f'{bold_if(F1, best["F1"])}&') outfile.write(f'{bold_if(F1, best["F1"])}&')
else: else:
outfile.write('(error)&') outfile.write('(error)&')
# Accuracy: {TP+TN} correct diagnostics in total, out of all tests {TP+TN+FP+FN+port+fail+othr+tout} diagnostics # Accuracy: {TP+TN} correct diagnostics in total, out of all tests {TP+TN+FP+FN+port+fail+othr+tout} diagnostics
accuracy = percent(TP+TN,(TP+TN+FP+FN+port+fail+othr+tout),one=True) accuracy = percent(TP+TN,(TP+TN+FP+FN+port+fail+othr+tout),one=True,digits=2)
outfile.write(f'{bold_if(accuracy, best["accuracy"])}') outfile.write(f'{bold_if(accuracy, best["accuracy"])}')
outfile.write(f'\\\\\\hline\n') outfile.write(f'\\\\\\hline\n')
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment