Fixed per GPU power draw dict being empty

experiment_impact_tracker/compute_tracker.py

@@ -17,7 +17,7 @@ import numpy as np
 import pandas as pd
 import psutil
 import ujson as json
-from pandas.io.json import json_normalize
+from pandas import json_normalize
 
 from experiment_impact_tracker.cpu import rapl
 from experiment_impact_tracker.cpu.common import get_my_cpu_info

experiment_impact_tracker/cpu/intel.py

@@ -550,7 +550,7 @@ def get_rapl_power(pid_list, logger=None, **kwargs):
             ]
         )
 
-    if total_intel_power < total_attributable_power:
+    if total_intel_power*1.05 < total_attributable_power:  # 5% increase to prevent an error where both values are similar
         raise ValueError(
             "For some reason the total intel estimated power is less than the attributable power. This "
             "means there is an error in computing the attribution. Please re-open "

experiment_impact_tracker/data_interface.py

@@ -34,7 +34,7 @@ from experiment_impact_tracker.stats import (get_average_treatment_effect,
                                              run_test)
 from experiment_impact_tracker.utils import gather_additional_info
 
-pd.set_option("display.max_colwidth", -1)
+pd.set_option("display.max_colwidth", None)
 
 
 def _gather_executive_summary(

experiment_impact_tracker/data_utils.py

@@ -5,7 +5,7 @@ import zipfile
 from datetime import datetime
 
 import ujson as json
-from pandas.io.json import json_normalize
+from pandas import json_normalize
 
 BASE_LOG_PATH = "impacttracker/"
 DATAPATH = BASE_LOG_PATH + "data.json"

experiment_impact_tracker/gpu/nvidia.py

@@ -164,6 +164,7 @@ def get_nvidia_gpu_power(pid_list, logger=None, **kwargs):
     per_gpu_relative_percent_usage = {}
     absolute_power = 0
     per_gpu_performance_states = {}
+    per_gpu_power_draw = {}
 
     for gpu_id, gpu in enumerate(xml.findall("gpu")):
         gpu_data = {}
@@ -189,6 +190,11 @@ def get_nvidia_gpu_power(pid_list, logger=None, **kwargs):
         gpu_data["utilization"] = {"gpu_util": gpu_util, "memory_util": memory_util}
 
         # get power
+        try: # newer CUDA versions
+            power_readings = gpu.findall("gpu_power_readings")[0]
+            print("Power readings")
+            print(power_readings)
+        except:
             power_readings = gpu.findall("power_readings")[0]
         power_draw = power_readings.findall("power_draw")[0].text
 
@@ -205,7 +211,7 @@ def get_nvidia_gpu_power(pid_list, logger=None, **kwargs):
         ]
         # what's the total absolute SM for this gpu across all accessible processes
         percentage_of_gpu_used_by_all_processes = float(gpu_based_processes["sm"].sum())
-        per_gpu_power_draw = {}
+        
         for info in processes.findall("process_info"):
             pid = info.findall("pid")[0].text
             process_name = info.findall("process_name")[0].text