new version to support VIRL >1.1 and VIRL clusters using websockets for console access

virl-benchmark/check-reachability

-#!/bin/bash
-
-# Check reachability of all running VIRL VMs by logging in on the telnet console port and expect a prompt that includes the host name
-
-# usage
-if [ ! $# -gt 2 ] ; then
-  echo -e "usage: $0 <username> <timeout for telnet connection to each node> <simulation> [debug level], e.g.,\n"
-  echo "$0 guest VIRLBENCH-223721@smb-WaZLhH 5"
-  exit -1
-fi
-
-# get the IP address of eth0
-IP=$(ip addr show | grep eth0 | grep inet | tr -s " " | cut -d " " -f 3 | cut -d "/" -f 1)
-
-RUNNING=true
-
-USERNAME=$1
-TIMEOUT=$2
-SIMULATION=$3
-if [ $4 ]; then 
-  DEBUG=$4
-else
-  DEBUG=0
-fi
-
-while $RUNNING = true
-do
-# Check if all nodes are ready-to-use using expect
-
-  # get the telnet ports of all nodes in all simulations
-  VM_PORTS=$(./ports-vms.py $USERNAME $SIMULATION)
-  
-  # connect to every telnet port of each node and expect the hostname in the prompt
-  VMS_UNUSABLE=0
-  for VM_PORT in $VM_PORTS
-  do
-    VM_TELNET_PORT=$(echo $VM_PORT | cut -d "=" -f 2)
-    VM_NAME=$(echo $VM_PORT | cut -d "=" -f 1)
-    # connect to every telnet port and check whether it can be used by pressing return
-    # twice and expecting the hostname to appear in the resulting prompt each time
-    if [ $DEBUG -lt 2 ]; then
-    	./test-virl-telnet-connection $IP $VM_TELNET_PORT $VM_NAME $TIMEOUT >/dev/null
-    else
-    	./test-virl-telnet-connection $IP $VM_TELNET_PORT $VM_NAME $TIMEOUT
-    fi
-    EXPECT_EXITCODE=$?
-    if [ $EXPECT_EXITCODE -eq 5 ] ; then
-      VMS_UNUSABLE=$(expr $VMS_UNUSABLE + 1)
-      if [ $DEBUG -gt 0 ]; then echo "$VM_NAME ($VM_TELNET_PORT) still unusable"; fi
-    fi
-  done
-  
-  if [ $VMS_UNUSABLE -gt 0 ]; then
-    if [ $DEBUG -gt 0 ]; then echo "$VMS_UNUSABLE VMs are still unusable"; fi
-  else
-    RUNNING=false
-  fi
-
-  sleep 1
-
-done
-
-DATE=$(date)
-echo "Finished at $DATE"

virl-benchmark/check-reachability-websocket.py

+#!/usr/bin/env python
+
+import sys, time, socket
+from websocket import create_connection, WebSocketTimeoutException
+#websocket._exceptions.WebSocketTimeoutException
+
+def checkws(ws,hostname,timeout,debug):
+        # create websocket
+        ws = create_connection(ws, subprotocols=["binary", "base64"])
+        # set timeout
+        ws.settimeout(timeout);
+        # terminal initialization
+        ws.send("\x0d")
+        ws.send("\xff\xfd\x01")
+        ws.send("\xff\xfb\x03")
+        ws.send("\xff\xfb\x18")
+        ws.send("\xff\xfb\x1f")
+        ws.send("\xff\xfd\x03")
+        ws.send("\xff\xfd\x18")
+        ws.send("\xff\xfd\x1f")
+        ws.send("\xff\xfe\x00")
+        ws.send("\xff\xfc\x00")
+        # initial receive
+        try:
+                result = ""
+                result += ws.recv()
+        except WebSocketTimeoutException:
+                print "timeout"
+        except socket.error:
+                print "socket closed"
+                sys.exit(3)
+        if debug > 1: print result
+        # send three carriage returns
+        ws.send("\r\r\r")
+        start = time.clock()
+        duration = 0
+        # receive result
+        try:
+                result = ""
+                while True:
+                        result += ws.recv()
+                        if duration == 0 and result.count(hostname) >= 3:
+                                duration = time.clock() - start
+                                break
+        except WebSocketTimeoutException:
+                print "timeout"
+        except socket.error:
+                print "socket closed"
+                sys.exit(3)
+        if debug > 1: print result
+        ws.close()
+        # count occurences of hostname
+        if result.count(hostname) >= 3:
+                print "%f *sec elapsed*" % duration
+                return 0
+        else:
+                print "UNUSABLE"
+                return 2
+
+def main():
+        if len(sys.argv) <= 4:
+                sys.stdout.write(str(sys.argv[0]))
+                print ": ws url, hostname, timeout and debug (e.g., \"ws://192.168.76.210:19406/websockify?token=870668d2-8855-4208-9d48-9b1a141271b2\" iosv-1 0.5 0) needed as argument! bailing out"
+                return 1
+        else:
+                ws = str(sys.argv[1]).strip()
+                hostname = str(sys.argv[2]).strip()
+                timeout = float(sys.argv[3])
+                debug = int(sys.argv[4])
+                return checkws(ws,hostname,timeout,debug)
+
+if __name__ == '__main__':
+        sys.exit(main())

virl-benchmark/check-usability-of-sims

+#!/bin/bash
+# check-usability-of-sims
+# HS-Fulda - sebastian.rieger@informatik.hs-fulda.de
+#
+# changelog:
+# V1.0   initial version
+
+# usage
+if [ ! $# -gt 2 ] ; then
+  echo -e "usage: $0 <username> <password> <timeout> [sim-regexp] [debug level], e.g.,\n"
+  echo "$0 guest password 0.5"
+  exit -1
+fi
+
+USERNAME=$1
+PASSWORD=$2
+TIMEOUT=$3
+if [ $4 ]; then
+  SIM_FILTER_REGEXP=$4
+else
+  SIM_FILTER_REGEXP="VIRLBENCH-(.*)@(.*)-[_a-zA-Z0-9]{6}"
+fi
+if [ $5 ]; then
+  DEBUG=$5
+else
+  DEBUG=0
+fi
+
+# get all running benchmark simulations
+SIMS=$(virl_std_client --username $USERNAME --password $PASSWORD --quiet --json simengine-list 2>&1 | egrep -o -e "$SIM_FILTER_REGEXP")
+if [ $DEBUG -gt 1 ]; then
+  echo "Running simulations:"
+  for SIM in $SIMS; do
+    echo $SIM
+  done
+  echo
+fi
+
+# Check if the nodes are ready-for-use using websocket consoles
+TOTAL_USABLE_COUNT=0
+TOTAL_AVG_DURATION=0
+for SIM in $SIMS; do
+  USABLE_COUNT=0
+  AVG_DURATION=0
+  WS_COUNT=$(virl_std_client --username $USERNAME --password $PASSWORD --quiet --json simengine-serial-port --session-id $SIM 2>&1 | grep -c "ws://")
+  WS_NODES=$(virl_std_client --username $USERNAME --password $PASSWORD --quiet --json simengine-serial-port --session-id $SIM 2>&1 | grep "ws://")
+  while [ $USABLE_COUNT -lt $WS_COUNT ]
+  do
+    USABLE_COUNT=0
+    AVG_DURATION=0
+    IFS=$'\n'
+    for WS_NODE in $WS_NODES; do
+      NODE=$(echo $WS_NODE | cut -d "\"" -f 2)
+      WS_URL=$(echo $WS_NODE | cut -d "\"" -f 4)
+      WS_RESULT=$(./check-reachability-websocket.py $WS_URL $NODE $TIMEOUT $DEBUG)
+      if [ $? -eq 0 ]; then
+        if [ $DEBUG -gt 0 ] ; then echo "$WS_RESULT: $NODE in $SIM is usable"; fi
+        DURATION=$(echo $WS_RESULT | egrep -o -e '[0-9]\.[0-9]{6} \*sec elapsed\*' | cut -d ' ' -f 1)
+        AVG_DURATION=$(awk "BEGIN {print $AVG_DURATION + $DURATION}")
+        USABLE_COUNT=$(expr $USABLE_COUNT + 1)
+      else
+        if [ $DEBUG -gt 0 ] ; then echo "$WS_RESULT: $NODE in $SIM is unusable"; fi
+      fi
+    done
+  done
+  # compute average in millisecs
+  TOTAL_AVG_DURATION=$(awk "BEGIN {print $TOTAL_AVG_DURATION + $AVG_DURATION}")
+  TOTAL_USABLE_COUNT=$(expr $TOTAL_USABLE_COUNT + $USABLE_COUNT)
+  AVG_DURATION=$(awk "BEGIN {print $AVG_DURATION / $USABLE_COUNT * 1000}")
+  if [ $DEBUG -gt 0 ] ; then echo "$USABLE_COUNT of $WS_COUNT nodes in $SIM are usable... (avg console delay $AVG_DURATION ms)"; fi
+done
+# compute average in millisecs
+TOTAL_AVG_DURATION=$(awk "BEGIN {print $TOTAL_AVG_DURATION / $TOTAL_USABLE_COUNT * 1000}")
+if [ $DEBUG -gt 0 ] ; then echo "$TOTAL_USABLE_COUNT nodes usable total in all sims... (total avg console delay $TOTAL_AVG_DURATION ms)"; fi
+echo "avgconsoledelay:$TOTAL_AVG_DURATION"

virl-benchmark/ports-vms.py

-#!/usr/bin/env python
-#
-# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
-#
-# Show all ports of all VM instances for
-# all users and topologies.
-# Like the console port and the VNC port
-#
-# rschmied@cisco.com
-#
-# modified by sebastian.rieger@informatik.hs-fulda.de to display telnet console ports of running nodes in a topology
-#
-
-
-import os, libvirt, re, sys
-from keystoneclient import client as keystone
-from novaclient import client as nova
-from xml.dom.minidom import parseString
-
-"""
-
-In [14]: m.group(0)
-Out[14]: '</guest/endpoint>-<Sample_Project@asa-test-topo-yJHnoK>-<iosv-2>-<Multipoint Connection-1>'
-
-In [15]: m.group(1)
-Out[15]: '/guest/endpoint'
-
-In [16]: m.group(2)
-Out[16]: 'Sample_Project'
-
-In [16]: m.group(3)
-Out[16]: 'asa-test-topo'
-
-In [17]: m.group(4)
-Out[17]: 'iosv-2'
-
-In [18]: m.group(5)
-Out[18]: 'Multipoint Connection-1'
-
-will not match jumphost ports!
-not interested in these, anyway
-
-"""
-
-class KeystoneV3NovaAuthPlugin(object):
-    def __init__(self, keystone_client):
-        self.keystone_client = keystone_client
-
-    def authenticate(self, client, fake_auth_url):
-        client.auth_url = fake_auth_url
-        client.service_catalog = self.keystone_client.service_catalog
-        client.auth_token = self.keystone_client.auth_token
-        client.tenant_id = self.keystone_client.tenant_id
-        client.management_url = self.keystone_client.service_catalog.url_for(
-            attr='region',
-            filter_value=client.region_name,
-            endpoint_type=client.endpoint_type,
-            service_type=client.service_type,
-            service_name=client.service_name).rstrip('/')
-
-def getports(user,simulation):
-	# Sample output / field mapping
-	# </guest/endpoint>-<Sample_Topologies@single-server-WO9N_h>-<csr1000v-1>
-	#         USER            PROJECT          TOPOLOGY             NODE
-	
-	# </advcompnet/endpoint>-<advcompnet-lab1-dcn-scenario1-ymaMSJ>-<veos-4>
-	#   USER                  TOPOLOGY                               NODE
-	prog=re.compile(r'</(.*)/endpoint>-<(.*)-[_0-9a-z]{6}>-<(.*)>', re.IGNORECASE)
-
-	# table=list()
-	try:
-		libvirt_uri = os.environ['LIBVIRT_DEFAULT_URI']
-	except:
-		libvirt_uri = "qemu:///system"
-		print "LIBVIRT_DEFAULT_URI env not set!"
-		print "Using default '" + libvirt_uri + "'"
-	conn=libvirt.openReadOnly(libvirt_uri)
-	
-	kc = keystone.Client(auth_url=os.environ['OS_AUTH_URL'],
-                     username=os.environ['OS_USERNAME'], password=os.environ['OS_PASSWORD'],
-                     project_name=os.environ['OS_TENANT_NAME'])
-	kc.session.auth = kc
-	kc.authenticate()
-	nc=nova.Client('2', os.environ['OS_USERNAME'], os.environ['OS_PASSWORD'], 
-		os.environ['OS_TENANT_NAME'], auth_system='keystonev3', auth_plugin=KeystoneV3NovaAuthPlugin(kc), auth_url='http://fake/v2.0')
-
-	for server in nc.servers.list(search_opts={'all_tenants': True}):
-		m=prog.match(server.name)
-		if m:
-			try:
-				domain=conn.lookupByUUIDString(server.id)
-			except:
-				print "Domain not found / not running"
-				return 1
-			else:
-				doc=parseString(domain.XMLDesc(flags=0))
-			# get the VNC port
-			#port=doc.getElementsByTagName('graphics')[0].getAttribute('port')
-			# get the serial console TCP port
-			for i in doc.getElementsByTagName('source'):
-				if i.parentNode.nodeName == u'console':
-					console=i.getAttribute('service')
-			# get the instance name
-			name=doc.getElementsByTagName('name')[0].childNodes[0].nodeValue
-			# print info
-			if simulation == "*":
-			        if m.group(1) == user:
-			                print m.group(3) + "=" + console
-                        else:
-			        if m.group(1) == user and server.name.find(simulation) != -1:
-			                print m.group(3) + "=" + console
-                       
-
-def main():
-        if len(sys.argv) != 3:
-                sys.stdout.write(str(sys.argv[0]))
-                print ": username and simulation (e.g., project name or session-id) needed as argument! bailing out"
-                return 1
-        else:
-                user = str(sys.argv[1]).strip()
-                simulation = str(sys.argv[2]).strip()
-                getports(user,simulation)
-                return 0
-
-if __name__ == '__main__':
-	sys.exit(main())
-

virl-benchmark/start-virl-topology

@@ -4,11 +4,12 @@
 #
 # changelog:
 # V1.0   initial version
+# V2.0   changed benchmark to support clusters and VIRL Version >=1.0.0
 
 # usage
 if [ ! $# -gt 4 ] ; then
   echo -e "usage: $0 <my-topology.virl> <number of concurrent simulations> <username> <password> <timeout> [debug level], e.g.,\n"
-  echo "$0 /home/virl/git-virl-hs-fulda/GIT-VIRL-HS-Fulda/advcompnet-lab2-dcn-fabricpath.virl 6 guest password"
+  echo "$0 /home/virl/git-virl-hs-fulda/GIT-VIRL-HS-Fulda/advcompnet-lab2-dcn-fabricpath.virl 6 guest password 0.5"
   exit -1
 fi
 
@@ -27,8 +28,9 @@ TOPOLOGYFILENAME=$(basename $TOPOLOGY)
 TIMESTAMP=$(date +%H%M%S)
 FILENAME="VIRLBENCH-$TIMESTAMP@$TOPOLOGYFILENAME"
 
-DATE=$(date)
-echo "Script started at $DATE"
+SCRIPTSTART_DATE=$(date)
+SCRIPTSTART_TIMESTAMP=$(date "+%s")
+echo "Script started at $SCRIPTSTART_DATE"
 
 # start the simulations
 RUN=0
@@ -45,8 +47,9 @@ do
   RUN=$(expr $RUN + 1)
 done
 
-DATE=$(date)
-echo "Started at $DATE"
+START_DATE=$(date)
+START_TIMESTAMP=$(date "+%s")
+echo "Started at $START_DATE"
 
 RUNNING=true
 
@@ -91,36 +94,26 @@ do
   sleep 1
 done
 
-DATE=$(date)
-echo "Active at $DATE"
+ACTIVE_DATE=$(date)
+ACTIVE_TIMESTAMP=$(date "+%s")
+echo "Active at $ACTIVE_DATE"
 
-# Check if the nodes are ready-for-use using expect
-./check-reachability $USERNAME $TIMEOUT "VIRLBENCH-$TIMESTAMP" $DEBUG
+# Check if the nodes are ready-for-use using websocket consoles
+./check-usability-of-sims $USERNAME $PASSWORD $TIMEOUT "VIRLBENCH-$TIMESTAMP@(.*)-[_a-zA-Z0-9]{6}" $DEBUG
 
-# Check using load
+USABLE_DATE=$(date)
+USABLE_TIMESTAMP=$(date "+%s")
+echo "Usable at $USABLE_DATE"
 
-# Alternative to check if nodes are ready, use CPU load threshold for VMs (IOSv and IOSvL2 have a high CPU load during initial boot)
-#THRESHOLD="70.0"
+START_TIME=$(( $START_TIMESTAMP - $SCRIPTSTART_TIMESTAMP ))
+ACTIVE_TIME=$(( $ACTIVE_TIMESTAMP - $SCRIPTSTART_TIMESTAMP ))
+USABLE_TIME=$(( $USABLE_TIMESTAMP - $SCRIPTSTART_TIMESTAMP ))
 
-#while $RUNNING = true
-#do
-#  VM_LOADS=$(top -n 1 | grep kvm.real | tr -s " " | cut -d " " -f 9)
-#
-#  VMS_WITH_HIGH_CPU_LOAD=0
-#  for VM_LOAD in $VM_LOADS
-#  do
-#    if [ $(echo "$VM_LOAD > $THRESHOLD" | bc) -eq 1 ] ; then 
-#      VMS_WITH_HIGH_CPU_LOAD=$(expr $VMS_WITH_HIGH_CPU_LOAD + 1)
-#      echo "$VM_LOAD > 6x0.0"
-#    fi
-#  done
-#  
-#  if [ $VMS_WITH_HIGH_CPU_LOAD -gt 0 ]; then
-#    echo "cpu load = high"
-#  else
-#    RUNNING=false
-#  fi
-#done
-
-DATE=$(date)
-echo "Finished at $DATE"
+echo
+echo
+echo "CSV Result:"
+echo "==========="
+echo "Topology;ConcurrentSims;Nodes;Script-Start;Started;Active;Usable;Finished;Start Time (sec);Active Time (sec);Usable Time (sec)"
+echo "$TOPOLOGYFILENAME;$RUNCOUNT;$NODE_COUNT;$SCRIPTSTART_DATE;$STARTDATE;$ACTIVE_DATE;$USABLE_DATE;$START_TIME;$ACTIVE_TIME;$USABLE_TIME"
+echo
+echo

virl-benchmark/test-virl-telnet-connection

-#!/usr/bin/expect
-
-# connect to telnet port, press return and expect argv 2 (hostname) in
-# prompt, press return again and expect to receive argv 2 again 
-
-set timeout [lindex $argv 3]
-spawn telnet [lindex $argv 0] [lindex $argv 1]
-send "\r"
-expect { 
-  [lindex $argv 2]
-}
-send "\r"
-expect { 
-  [lindex $argv 2]
-  { exit 0 }
-}
-exit 5