diff --git a/experiment_impact_tracker/gpu/nvidia.py b/experiment_impact_tracker/gpu/nvidia.py index aa39d7fc3f6bae1849c74bcb6e33589fd0a8acb2..745385c283e6b634c7094c4500ab085091de57c7 100755 --- a/experiment_impact_tracker/gpu/nvidia.py +++ b/experiment_impact_tracker/gpu/nvidia.py @@ -152,7 +152,10 @@ def get_nvidia_gpu_power(pid_list, logger=None, **kwargs): out_str_final = re.sub("\s+\n", "\n", out_str_final) # else pd will mis-align out_str_final = out_str_final.strip() df = pd.read_csv(StringIO(out_str_final), engine="python", delimiter="\t") - process_percentage_used_gpu = df.groupby(["gpu", "pid"]).mean().reset_index() + try: + process_percentage_used_gpu = df.groupby(["gpu", "pid"])[['sm', 'mem', 'enc', 'dec']].mean().reset_index() + except: # Older pandas versions + process_percentage_used_gpu = df.groupby(["gpu", "pid"]).mean().reset_index() p = Popen(["nvidia-smi", "-q", "-x"], stdout=PIPE) outs, errors = p.communicate()