Add tests for create-compute-appendix

experiment_impact_tracker/compute_tracker.py

@@ -21,18 +21,21 @@ from pandas.io.json import json_normalize
 from experiment_impact_tracker.cpu import rapl
 from experiment_impact_tracker.cpu.common import get_my_cpu_info
 from experiment_impact_tracker.cpu.intel import get_intel_power, get_rapl_power
-from experiment_impact_tracker.data_info_and_router import (DATA_HEADERS,
-                                                            INITIAL_INFO)
+from experiment_impact_tracker.data_info_and_router import DATA_HEADERS, INITIAL_INFO
 from experiment_impact_tracker.data_utils import *
-from experiment_impact_tracker.emissions.common import \
-    is_capable_realtime_carbon_intensity
-from experiment_impact_tracker.emissions.get_region_metrics import \
-    get_current_region_info_cached
-from experiment_impact_tracker.gpu.nvidia import (get_gpu_info,
-                                                  get_nvidia_gpu_power)
-from experiment_impact_tracker.utils import (get_timestamp, processify,
+from experiment_impact_tracker.emissions.common import (
+    is_capable_realtime_carbon_intensity,
+)
+from experiment_impact_tracker.emissions.get_region_metrics import (
+    get_current_region_info_cached,
+)
+from experiment_impact_tracker.gpu.nvidia import get_gpu_info, get_nvidia_gpu_power
+from experiment_impact_tracker.utils import (
+    get_timestamp,
+    processify,
     safe_file_path,
-                                             write_json_data_to_file)
+    write_json_data_to_file,
+)
 
 SLEEP_TIME = 1
 STOP_MESSAGE = "Stop"

experiment_impact_tracker/cpu/powercap.py

@@ -5,19 +5,17 @@ import multiprocessing
 import os
 import platform
 import re
+import shlex
 import shutil
+import subprocess
 import sys
 import tempfile
 
-import pandas
+import pandas as pd
+from pandas import DataFrame
 
 sys.path.append("..")
 
-import shlex
-import subprocess
-
-import pandas as pd
-from pandas import DataFrame
 
 get_long_path = lambda x: x
 try:

experiment_impact_tracker/data_info_and_router.py

@@ -3,20 +3,32 @@ from datetime import datetime
 import experiment_impact_tracker
 from experiment_impact_tracker.cpu import rapl
 from experiment_impact_tracker.cpu.common import (
-    get_cpu_count_adjusted_load_avg, get_cpu_freq, get_my_cpu_info)
+    get_cpu_count_adjusted_load_avg,
+    get_cpu_freq,
+    get_my_cpu_info,
+)
 from experiment_impact_tracker.cpu.intel import (
-    get_intel_power, get_rapl_power, is_intel_compatible)
+    get_intel_power,
+    get_rapl_power,
+    is_intel_compatible,
+)
 from experiment_impact_tracker.disk.common import measure_disk_speed_at_dir
 from experiment_impact_tracker.emissions.common import (
-    get_realtime_carbon, is_capable_realtime_carbon_intensity)
-from experiment_impact_tracker.emissions.get_region_metrics import \
-    get_current_region_info_cached
-from experiment_impact_tracker.gpu.nvidia import (get_gpu_info,
+    get_realtime_carbon,
+    is_capable_realtime_carbon_intensity,
+)
+from experiment_impact_tracker.emissions.get_region_metrics import (
+    get_current_region_info_cached,
+)
+from experiment_impact_tracker.gpu.nvidia import (
+    get_gpu_info,
     get_nvidia_gpu_power,
-                                                  is_nvidia_compatible)
+    is_nvidia_compatible,
+)
 from experiment_impact_tracker.operating_system.common import is_linux
-from experiment_impact_tracker.py_environment.common import \
-    get_python_packages_and_versions
+from experiment_impact_tracker.py_environment.common import (
+    get_python_packages_and_versions,
+)
 from experiment_impact_tracker.utils import *
 
 get_version_number = lambda *args, **kwargs: experiment_impact_tracker.__version__

experiment_impact_tracker/data_interface.py

@@ -21,17 +21,18 @@ from jinja2 import Environment, FileSystemLoader
 
 import experiment_impact_tracker
 from experiment_impact_tracker.create_graph_appendix import (
-    create_graphs, create_scatterplot_from_df)
-from experiment_impact_tracker.data_utils import (load_data_into_frame,
+    create_graphs,
+    create_scatterplot_from_df,
+)
+from experiment_impact_tracker.data_utils import (
+    load_data_into_frame,
     load_initial_info,
-                                                  zip_data_and_info)
-from experiment_impact_tracker.emissions.common import \
-    get_realtime_carbon_source
+    zip_data_and_info,
+)
+from experiment_impact_tracker.emissions.common import get_realtime_carbon_source
 from experiment_impact_tracker.emissions.constants import PUE
-from experiment_impact_tracker.emissions.get_region_metrics import \
-    get_zone_name_by_id
-from experiment_impact_tracker.stats import (get_average_treatment_effect,
-                                             run_test)
+from experiment_impact_tracker.emissions.get_region_metrics import get_zone_name_by_id
+from experiment_impact_tracker.stats import get_average_treatment_effect, run_test
 from experiment_impact_tracker.utils import gather_additional_info
 
 pd.set_option("display.max_colwidth", -1)

experiment_impact_tracker/html_templates/exp_details.html

@@ -45,9 +45,13 @@
                 <a name="cpu_info"></a> 
                 <br> <div class="header"><span>CPU Info</span></div></span>
                 {{ cpu_info.to_html(classes="table table-striped") | safe }}
+
+                {% if gpu_info is defined %}
                 <a name="gpu_info"></a>
                 <div class="header"><span>GPU Info</span></div></span>
                 {{ gpu_info.to_html(classes="table table-striped") | safe }}
+                {% endif %}
+
                 <a name="package_info"></a> 
                 <div class="header"><span>Package Info</span></div></span>
                 {{ package.to_html(classes="table table-striped") | safe }}

experiment_impact_tracker/utils.py

 import atexit
 import csv
+import logging
 import os
 import sys
 import time
@@ -11,6 +12,7 @@ from multiprocessing import Process, Queue
 import numpy as np
 import pandas as pd
 import psutil
+import pytest
 import ujson
 
 from experiment_impact_tracker.data_utils import *
@@ -19,6 +21,8 @@ from experiment_impact_tracker.emissions.constants import PUE
 
 _timer = getattr(time, "monotonic", time.time)
 
+log = logging.getLogger(__name__)
+
 
 def get_timestamp(*args, **kwargs):
     now = datetime.now()
@@ -91,7 +95,7 @@ def processify(func):
 def _get_cpu_hours_from_per_process_data(json_array):
     latest_per_pid = {}
     for datapoint in json_array:
-        cpu_point = datapoint["cpu_time_seconds"]
+        cpu_point = datapoint.get("cpu_time_seconds", {})
         for pid, value in cpu_point.items():
             latest_per_pid[pid] = value["user"] + value["system"]
     return sum(latest_per_pid.values())
@@ -99,6 +103,7 @@ def _get_cpu_hours_from_per_process_data(json_array):
 
 def gather_additional_info(info, logdir):
     df, json_array = load_data_into_frame(logdir)
+    print("GATHER ADDITIONAL INFO")
     cpu_seconds = _get_cpu_hours_from_per_process_data(json_array)
     exp_len = datetime.timestamp(info["experiment_end"]) - datetime.timestamp(
         info["experiment_start"]
@@ -120,7 +125,10 @@ def gather_additional_info(info, logdir):
 
     # elementwise multiplication and sum
     time_differences_in_hours = time_differences / 3600.0
-    power_draw_rapl_kw = df["rapl_estimated_attributable_power_draw"] / 1000.0
+    power_draw_rapl_kw = None
+
+    if "rapl_estimated_attributable_power_draw" in df:
+        power_draw_rapl_kw = df.get["rapl_estimated_attributable_power_draw"] / 1000.0
         power_draw_rapl_kw.loc[len(power_draw_rapl_kw)] = power_draw_rapl_kw.loc[
             len(power_draw_rapl_kw) - 1
         ]
@@ -140,7 +148,11 @@ def gather_additional_info(info, logdir):
         # elementwise multiplication and sum
         kw_hr_nvidia = np.multiply(time_differences_in_hours, nvidia_power_draw_kw)
 
-    kw_hr_rapl = np.multiply(time_differences_in_hours, power_draw_rapl_kw)
+    kw_hr_rapl = (
+        np.multiply(time_differences_in_hours, power_draw_rapl_kw)
+        if power_draw_rapl_kw
+        else np.array([0])
+    )
 
     if has_gpu:
         total_power_per_timestep = PUE * (kw_hr_nvidia + kw_hr_rapl)

scripts/create-compute-appendix

@@ -7,31 +7,24 @@ import json
 import os
 import re
 import sys
-from datetime import datetime
 from importlib import import_module
-from itertools import combinations
-from pprint import pprint
 from shutil import copyfile
 
 import numpy as np
 import pandas as pd
 import scipy
-from deepdiff import DeepDiff  # For Deep Difference of 2 objects
 from jinja2 import Environment, FileSystemLoader
 
 import experiment_impact_tracker
 from experiment_impact_tracker.create_graph_appendix import (
     create_graphs, create_scatterplot_from_df)
-from experiment_impact_tracker.data_utils import (load_data_into_frame,
-                                                  load_initial_info,
+from experiment_impact_tracker.data_utils import (load_initial_info,
                                                   zip_data_and_info)
 from experiment_impact_tracker.emissions.common import \
     get_realtime_carbon_source
 from experiment_impact_tracker.emissions.constants import PUE
 from experiment_impact_tracker.emissions.get_region_metrics import \
     get_zone_name_by_id
-from experiment_impact_tracker.stats import (get_average_treatment_effect,
-                                             run_test)
 from experiment_impact_tracker.utils import gather_additional_info
 
 pd.set_option('display.max_colwidth', -1)
@@ -43,11 +36,14 @@ def _gather_executive_summary(aggregated_info, executive_summary_variables, expe
         if not all_points:
             data = [exp_name]
             for variable in executive_summary_variables:
+                if variable in aggregated_info[exp_name]:
                     values = aggregated_info[exp_name][variable]
                     values_mean = np.mean(values)
                     values_stdder = scipy.stats.sem(values)
                     data.append(
                         "{:.3f} +/- {:.2f}".format(values_mean, values_stdder))
+                else:
+                    data.append("NAN")
             executive_summary.append(data)
         else:
             for j in range(len(aggregated_info[exp_name][executive_summary_variables[0]])):
@@ -214,13 +210,17 @@ def _aggregated_data_for_filterset(output_dir,
                 zip_data_and_info(x, zip_file_name)
 
                 # {k: [v] for k, v in info["gpu_info"].items()})
+                if "gpu_info" in info:
+                    import pdb;pdb.set_trace()
                     gpu_data_frame = pd.DataFrame.from_dict(info["gpu_info"])
                     gpu_infos_all[experiment_set_names[exp_set]].append(
                         gpu_data_frame)
+                if "cpu_info" in info:
                     cpu_data_frame = pd.DataFrame.from_dict(
                         {k: [v] for k, v in info["cpu_info"].items()})
                     cpu_infos_all[experiment_set_names[exp_set]].append(
                         cpu_data_frame)
+
                 carbon_infos_all[experiment_set_names[exp_set]].append(
                     _get_carbon_infos(info, extracted_info))
                 package_infos_all[experiment_set_names[exp_set]].append(
@@ -281,20 +281,26 @@ def _create_leaf_page(output_directory, all_infos, exp_set_name, description, ex
             graph_path, html_output_dir) for graph_path in graph_paths_all[exp_set_name][i]]
         relative_data_zip_paths = os.path.relpath(
             data_zip_paths_all[exp_set_name][i], html_output_dir)
-        output = template.render(
-            exp_set_names_titles=[(_format_setname(experiment_set_names[exp_set]), experiment_set_names[exp_set])
-                                  for exp_set in range(len(experiment_set_filters))],
-            exps=list(
-                range(len(list(aggregated_info[exp_set_name].values())[0]))),
-            cpu_info=cpu_infos_all[exp_set_name][i].T,
-            gpu_info=gpu_infos_all[exp_set_name][i].T,
-            carbon_info=carbon_infos_all[exp_set_name][i].T,
-            package=pd.DataFrame.from_dict(package_infos_all[exp_set_name][i]),
-            stats=pd.DataFrame(summary_info),
-            graph_paths=relative_graph_paths,
-            data_download_path=relative_data_zip_paths,
-            title=exp_set_name,
-            relative_base_dir=os.path.relpath(base_dir, output_directory))
+
+        template_args = {}
+        if gpu_infos_all and gpu_infos_all[exp_set_name] and gpu_infos_all[exp_set_name][i]:
+            template_args["gpu_info"] = gpu_infos_all[exp_set_name][i].T
+
+        template_args["exp_set_names_titles"] = [(_format_setname(experiment_set_names[exp_set]), experiment_set_names[exp_set])
+                                  for exp_set in range(len(experiment_set_filters))]
+        template_args["exps"] = list(
+                range(len(list(aggregated_info[exp_set_name].values())[0])))
+        template_args["cpu_info"] = cpu_infos_all[exp_set_name][i].T
+        template_args["carbon_info"] = carbon_infos_all[exp_set_name][i].T
+        template_args["package"] = pd.DataFrame.from_dict(package_infos_all[exp_set_name][i])
+        template_args["stats"] = pd.DataFrame(summary_info)
+        template_args["graph_paths"] = relative_graph_paths
+        template_args["data_download_path"] = relative_data_zip_paths
+        template_args["title"] = exp_set_name
+        template_args["relative_base_dir"] = os.path.relpath(base_dir, output_directory)
+
+        output = template.render(**template_args)
+
         with open(html_output_path, 'w') as f:
             f.write(output)
 
@@ -390,7 +396,7 @@ def main(arguments):
     parser.add_argument("--description", type=str,
                         default="TODO: description of experimental setups")
     parser.add_argument("--site_spec", type=str, required=True)
-    parser.add_argument('--output_dir')
+    parser.add_argument('--output_dir', type=str, required=True)
     args = parser.parse_args(arguments)
 
     # TODO: add flag for summary stats instead of table for each, this should create a shorter appendix

scripts/generate-carbon-impact-statement

@@ -22,6 +22,7 @@ from jinja2 import Environment, FileSystemLoader
 import experiment_impact_tracker
 from experiment_impact_tracker.create_graph_appendix import (
     create_graphs, create_scatterplot_from_df)
+from experiment_impact_tracker.data_interface import DataInterface
 from experiment_impact_tracker.data_utils import (load_data_into_frame,
                                                   load_initial_info,
                                                   zip_data_and_info)
@@ -36,7 +37,6 @@ from experiment_impact_tracker.utils import gather_additional_info
 
 pd.set_option('display.max_colwidth', -1)
 
-from experiment_impact_tracker.data_interface import DataInterface
 
 
 def main(arguments):

tests/scripts/leaderboard_generation_format.json

+{
+  "Comparing My two test experiments" :
+  {
+    "filter": "",
+    "description" : "A text experiment.",
+    "executive_summary_variables" : ["total_power", "exp_len_hours", "cpu_hours", "gpu_hours", "estimated_carbon_impact_kg"],
+    "child_experiments" :
+        {
+            "Experiment 1" : {
+                                "filter" : "exp1",
+                                "description" : "first run"
+                            },
+            "Experiment 2" : {
+                                "filter" : "exp2",
+                                "description" : "second run"
+                            }
+
+        }
+  }
+}
\ No newline at end of file

tests/test_scripts/myapp.py → tests/scripts/myapp.py

@@ -28,7 +28,7 @@ def train(d: str = "cpu", log_dir: str = tempfile.mkdtemp()):
     w2 = torch.randn(H, D_out, device=device)
 
     learning_rate = 1e-6
-    for t in range(10):
+    for t in range(5):
         # Forward pass: compute predicted y
         h = x.mm(w1)
         h_relu = h.clamp(min=0)

tests/test_context.py

@@ -4,8 +4,9 @@ import tempfile
 import torch
 
 from experiment_impact_tracker.compute_tracker import ImpactTracker
-from experiment_impact_tracker.data_interface import DataInterface
 from experiment_impact_tracker.cpu.intel import is_intel_compatible
+from experiment_impact_tracker.data_interface import DataInterface
+
 
 def _helper_function():
 

tests/test_powercap.py

@@ -3,8 +3,7 @@ import time
 
 import torch
 
-from experiment_impact_tracker.cpu.powercap import (PowerGadget,
-                                                    is_powercap_compatible)
+from experiment_impact_tracker.cpu.powercap import PowerGadget, is_powercap_compatible
 
 
 def _helper_function():

tests/test_runs.py → tests/test_scripts.py

-from pathlib import Path
 import subprocess
 import sys
+from pathlib import Path
 from typing import Any
 
 
-def test_script(tmpdir: Any) -> Any:
-    print(f"INSIDE TEST {tmpdir}")
-    cmd = [
-        sys.executable,
-        "tests/test_scripts/myapp.py",
-        "cpu",
-        tmpdir
-    ]
+def exp(exp_dir: Path) -> None:
+    exp_dir.mkdir(parents=True, exist_ok=True)
+    cmd = [sys.executable, "tests/scripts/myapp.py", "cpu", exp_dir]
     result = subprocess.check_output(cmd)
     assert str(result.decode("utf-8")).strip().split()[-1] == "SUCCESS"
-    assert Path(tmpdir/"impacttracker").exists()
+    assert Path(exp_dir / "impacttracker").exists()
 
 
+def test_script(tmpdir: Any) -> Any:
+    exp1 = Path(tmpdir) / "exp1"
+    exp(exp1)
+
+    exp2 = Path(tmpdir) / "exp2"
+    exp(exp2)
+
 
+    # test create-compute-appendix
+    cmd = [
+        sys.executable,
+        "scripts/create-compute-appendix",
+        str(tmpdir),
+        "--site_spec",
+        "tests/scripts/leaderboard_generation_format.json",
+        "--output_dir",
+        str(tmpdir / "output"),
+    ]
+    # pytest.set_trace()
+    # for now make sure the script runs.
+    subprocess.check_output(cmd)

tests/train_two_layer_net_tensor.py

-# code from pytorch tutorials
-# https://github.com/pytorch/tutorials/blob/master/beginner_source/examples_tensor/two_layer_net_tensor.py
-import os
-import sys
-import tempfile
-
-import torch
-from tqdm import tqdm
-
-from experiment_impact_tracker.compute_tracker import ImpactTracker
-
-
-def train(d: str):
-    temp_dir = tempfile.mkdtemp()
-
-    tracker = ImpactTracker(os.path.join(temp_dir, ""))
-
-    print(f"ImpactTracker log dir: {temp_dir}")
-
-    tracker.launch_impact_monitor()
-    device = torch.device(d)
-
-    # N is batch size; D_in is input dimension;
-    # H is hidden dimension; D_out is output dimension.
-    N, D_in, H, D_out = 1024, 10000, 1000, 100
-
-    # Create random input and output data
-    x = torch.randn(N, D_in, device=device)
-    y = torch.randn(N, D_out, device=device)
-
-    # Randomly initialize weights
-    w1 = torch.randn(D_in, H, device=device)
-    w2 = torch.randn(H, D_out, device=device)
-
-    learning_rate = 1e-6
-    for t in tqdm(range(1000)):
-        # Forward pass: compute predicted y
-        h = x.mm(w1)
-        h_relu = h.clamp(min=0)
-        y_pred = h_relu.mm(w2)
-
-        # Backprop to compute gradients of w1 and w2 with respect to loss
-        grad_y_pred = 2.0 * (y_pred - y)
-        grad_w2 = h_relu.t().mm(grad_y_pred)
-        grad_h_relu = grad_y_pred.mm(w2.t())
-        grad_h = grad_h_relu.clone()
-        grad_h[h < 0] = 0
-        grad_w1 = x.t().mm(grad_h)
-
-        # Update weights using gradient descent
-        w1 -= learning_rate * grad_w1
-        w2 -= learning_rate * grad_w2
-        tracker.get_latest_info_and_check_for_errors()
-
-
-if __name__ == "__main__":
-    train(sys.argv[1])