Skip to content
Snippets Groups Projects
Commit 2e978098 authored by Sebastian Rieger's avatar Sebastian Rieger
Browse files

added global timeout to detect otherwise endless running checks, when nodes...

added global timeout to detect otherwise endless running checks, when nodes crash or resources are depleted
parent b13bf9b3
No related branches found
No related tags found
No related merge requests found
...@@ -27,6 +27,7 @@ PORT = 13080 ...@@ -27,6 +27,7 @@ PORT = 13080
PASSWORD = sys.argv[4] PASSWORD = sys.argv[4]
TIMEOUT = int(sys.argv[5]) TIMEOUT = int(sys.argv[5])
GLOBAL_TIMEOUT = 1000
PROJECT_NAME = sys.argv[1] PROJECT_NAME = sys.argv[1]
# PROJECT_ID = "69e293ac-a339-4d1d-b2e3-047dc2da8566" # PROJECT_ID = "69e293ac-a339-4d1d-b2e3-047dc2da8566"
...@@ -141,6 +142,30 @@ for run in range(1, NUMBER_OF_BENCHMARK_RUNS + 1): ...@@ -141,6 +142,30 @@ for run in range(1, NUMBER_OF_BENCHMARK_RUNS + 1):
# check if nodes are usable # check if nodes are usable
usable_nodes = [] usable_nodes = []
while len(nodes) > 0: while len(nodes) > 0:
# perf output
# print(os.popen("top -b -n 1 | head -20 >> gns3bench-perf.log"))
time_globalTimeout = time.time()
# if usability of nodes is checked longer than global timeout, exit
# this can happen if nodes get stuck, e.g., due to high load, missing resources etc.
if time_globalTimeout - time_active > GLOBAL_TIMEOUT:
date_usable = 0
durationUsable = 0
avgConsoleDelay = 0
durationStart = time_started - time_start
durationActive = time_active - time_start
results.append("%s;%i;%i;%s;%s;%s;%s;%i;%i;%i;%f" % (
(PROJECT_NAME + ": TIMEOUT"), NUMBER_OF_CONCURRENT_BENCHMARK_PROJECTS, nodeCount, date_start,
date_started, date_active, date_usable, durationStart, durationActive, durationUsable, avgConsoleDelay))
results.append(" Occurred while still checking nodes: " + str(nodes))
f = open("gns3bench-error.log", "a")
f.writelines(results)
f.write("\r\n")
f.close()
sys.exit("global timeout")
for entry in nodes: for entry in nodes:
print("Checking node %s in %s..." % (entry["name"], entry["project_id"])) print("Checking node %s in %s..." % (entry["name"], entry["project_id"]))
try: try:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment