From 7cb92cb33b163fde2af22a2131d167ee4fd4416e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yu-Jin=20Schr=C3=B6er?= <yujin.schroeer@gmail.com>
Date: Fri, 21 Mar 2025 11:41:14 +0100
Subject: [PATCH] Upload New File
---
.../Optimization_control_rules.ipynb | 817 ++++++++++++++++++
1 file changed, 817 insertions(+)
create mode 100644 Optimization of control rules/Optimization_control_rules.ipynb
diff --git a/Optimization of control rules/Optimization_control_rules.ipynb b/Optimization of control rules/Optimization_control_rules.ipynb
new file mode 100644
index 0000000..4216078
--- /dev/null
+++ b/Optimization of control rules/Optimization_control_rules.ipynb
@@ -0,0 +1,817 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "dc6fe0c6",
+ "metadata": {},
+ "source": [
+ "### Import of libraries"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9f940e4e",
+ "metadata": {},
+ "source": [
+ "General libraries"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ebbcf530",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2024-07-28T14:21:34.156926700Z",
+ "start_time": "2024-07-28T14:21:33.780745600Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import typing\n",
+ "import datetime\n",
+ "import shutil\n",
+ "import subprocess\n",
+ "from multiprocessing.connection import Client\n",
+ "from time import sleep\n",
+ "from warnings import warn\n",
+ "from pathlib import Path\n",
+ "import functools\n",
+ "import threading\n",
+ "from multiprocessing.pool import ThreadPool\n",
+ "import traceback\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import nevergrad as ng\n",
+ "from IPython.utils import io as ipyio\n",
+ "import os\n",
+ "from scipy.ndimage import gaussian_filter1d"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d260d7da",
+ "metadata": {},
+ "source": [
+ "Libraries for the interface with MIKE+"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ebb5c530",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from mikeplus import DataTableAccess\n",
+ "from mikeplus.engines import Engine1D\n",
+ "import matplotlib as mpl\n",
+ "import matplotlib.pyplot as plt\n",
+ "from mikeio1d import Res1D\n",
+ "from mikeio1d.query import QueryDataStructure\n",
+ "from mikeio1d.query import QueryDataNode"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7de86ed5",
+ "metadata": {},
+ "source": [
+ "### Setting variables"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "34c9b436",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# a folder where used files are saved\n",
+ "# note this should be at the same folder depth as the original folder, so that relative paths lead to the same files\n",
+ "model_folder = Path(\"verteilte Steuerung/\")\n",
+ "# path of .mupp file, relative to model_folder\n",
+ "db_path_rel = \"240502_Modell Rodenkirchen.mupp\"\n",
+ "# Number of simulations allowed for optimization\n",
+ "optimization_budget = 60"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "003d8aff",
+ "metadata": {},
+ "source": [
+ "Names of the input Excel files"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "afb44b21",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "parameter_xlsx = \"config_parameter.xlsx\"\n",
+ "event_xlsx = \"config_events.xlsx\"\n",
+ "structure_xlsx = \"structures.xlsx\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "32eb5a4f",
+ "metadata": {},
+ "source": [
+ "Further Variables"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5ffa64b6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# delete result files(as specified by event config) before a sim, assures that no old file is reread in the case that a simulation does not run\n",
+ "DELETE_RESFILE_BEFORE_SIM = True\n",
+ "# output heaps of additional data, may be useful for debugging, Default = False\n",
+ "VERBOSE_OUTPUT = False\n",
+ "# attempt simulation at most so many times, used when a simulation fails\n",
+ "MAX_SIMULATION_TRY = 1\n",
+ "# Consider a simulation as failed and retry if so many values in the result compared to the reference are missing\n",
+ "MAX_MISSING_RATIO = 0.6\n",
+ "# delete the temp folder at end, Default = True\n",
+ "DELETE_TEMP = True\n",
+ "# time string for output folder\n",
+ "time_path_part = datetime.datetime.now().strftime(\"%Y-%m-%dT%H.%M.%S\")\n",
+ "# set path of temp and output folder\n",
+ "temp_folder = Path(\"tmp_\" + time_path_part)\n",
+ "# output folder\n",
+ "out_folder = Path(\"out_\" + time_path_part)\n",
+ "# wheter output from mike simulation get shown on error\n",
+ "show_sim_out_on_error = True\n",
+ "# set the id of the sensors and the timezone\n",
+ "timezone = \"UTC+01:00\"\n",
+ "#Number of seats for the Engine, limis amount of simulation we can run at once\n",
+ "engine_seats = 4\n",
+ "optimizers = [\"BayesOptimBO\"] "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b9dceb58",
+ "metadata": {},
+ "source": [
+ "### Preparation of the database"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b04a0f9f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def file_to_df(path: Path) -> pd.DataFrame:\n",
+ " \"\"\"\n",
+ " Tries to read the file at path as a pandas DataFrame.\n",
+ " Supports Excel, CSV, and .res1d files.\n",
+ " \"\"\"\n",
+ " if path.suffix == \".res1d\":\n",
+ " res1d = Res1D(str(path)) # Create a Res1D instance with the file\n",
+ " df = res1d.read_all() # Read all data from the .res1d file\n",
+ " return df\n",
+ " elif path.suffix == \".xlsx\":\n",
+ " return pd.read_excel(path)\n",
+ " elif path.suffix == \".csv\":\n",
+ " return pd.read_csv(path)\n",
+ " else:\n",
+ " raise NotImplementedError(f\"No method for {path.suffix} implemented\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "14138d91",
+ "metadata": {},
+ "source": [
+ "Creating an output folder and a temporary folder"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fa253ce4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "out_folder.mkdir(exist_ok=True,parents=True)\n",
+ "assert not temp_folder.exists(), \"Make sure not to mess with existing folders\"\n",
+ "shutil.copytree(model_folder,temp_folder)\n",
+ "db_path = temp_folder / db_path_rel\n",
+ "\n",
+ "# print the path of the temp and out folder\n",
+ "temp_folder.absolute()\n",
+ "out_folder.absolute()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8283a2e5",
+ "metadata": {},
+ "source": [
+ "Read the excel config_events"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0efc3c47",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "events = file_to_df(temp_folder / event_xlsx).set_index('Event', drop=False)\n",
+ "# check if start time of events is before the end time.\n",
+ "assert (events[\"Start\"] <= events[\"End\"]).all(\n",
+ " axis=None\n",
+ "), \"Event end needs to be after start.\"\n",
+ "# add timezone. Use same as sensor_date\n",
+ "events[\"Start\"]=events[\"Start\"].dt.tz_localize(timezone)\n",
+ "events[\"End\"]=events[\"End\"].dt.tz_localize(timezone)\n",
+ "# check if there are duplicated events\n",
+ "assert events.index.is_unique, \"Need unique event\"\n",
+ "assert events.drop_duplicates(subset=\"Event\")[\"SimulationName\"].is_unique, \"Need exactly one simulation name per event\"\n",
+ "# Conversion of the SimulationName column to the String data type, if it is not already there\n",
+ "events[\"SimulationName\"] = events[\"SimulationName\"].astype(str)\n",
+ "events"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3db96632",
+ "metadata": {},
+ "source": [
+ "Read the excel config_parameter"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5d1e12ad",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "param_specs = file_to_df(temp_folder / parameter_xlsx)\n",
+ "\n",
+ "# give params_specs a sensible string index, this is later used to pass optimization parameters as a dictionary\n",
+ "param_specs.index = pd.Index(\n",
+ " param_specs[[\"Table\", \"Column\", \"Muid\"]].agg(\";\".join, axis=\"columns\"),\n",
+ " name=\"ParamKey\",\n",
+ ")\n",
+ "assert (\n",
+ " param_specs.index.is_unique\n",
+ "), \"Index needs to be unique. Otherwise indicates dupplicates in the parameters.\"\n",
+ "param_specs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0c455656",
+ "metadata": {},
+ "source": [
+ "Read the excel strucutres"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b60324a6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "structures = file_to_df(temp_folder / structure_xlsx)\n",
+ "structures_list = structures.tolist()\n",
+ "structures_list"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "21eab72c",
+ "metadata": {},
+ "source": [
+ "Open Mike+ database and start engine"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fe607e54",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dta = DataTableAccess(db_path)\n",
+ "# open the database from MIKE+\n",
+ "dta.open_database()\n",
+ "# check if the database is open.\n",
+ "dta.is_database_open()\n",
+ "# Create the Engine1D object that will be used to run MIKE 1D simulations\n",
+ "engine = Engine1D(dta.datatables)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a67a5823",
+ "metadata": {},
+ "source": [
+ "Threading is used to run multiple simulations with differnt engines"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "371fc664",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "engines = {}\n",
+ "\n",
+ "def init_engine():\n",
+ " engines[threading.current_thread()] = Engine1D(dta.datatables)\n",
+ " print(f'initialized engine for {threading.current_thread()}')\n",
+ "\n",
+ "# Use threadpool from the multiprocessing package, this has a rather simple interface\n",
+ "sim_executor = ThreadPool(processes=engine_seats,initializer=init_engine)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "40f2b6da",
+ "metadata": {},
+ "source": [
+ "Functions for the simulation"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a51fa7f3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ " def calculate_discharge_for_structure(res1d, structure):\n",
+ " \"\"\"\n",
+ " Calculates the discharge for a given structure from the loaded Res1D object.\n",
+ " \n",
+ " Parameters\n",
+ " ----------\n",
+ " res1d : Res1D\n",
+ " The loaded Res1D object containing simulation results.\n",
+ " structure : str\n",
+ " The name of the structure for which discharge is to be calculated.\n",
+ " \n",
+ " Returns\n",
+ " -------\n",
+ " float or None\n",
+ " The total discharge in liters, or None if an error occurs.\n",
+ " \"\"\"\n",
+ " try:\n",
+ " values = res1d.structures[structure].Discharge.read()\n",
+ " total_discharge = np.sum(values) * 60 * 1000\n",
+ " return total_discharge\n",
+ " except KeyError:\n",
+ " print(f\"Structure {structure} not found in the Res1D data.\")\n",
+ " return None\n",
+ " except AttributeError:\n",
+ " print(f\"Discharge data not available for structure {structure}.\")\n",
+ " return None\n",
+ " except Exception as e:\n",
+ " print(f\"An error occurred while calculating discharge for {structure}: {e}\")\n",
+ " return None "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f3ab9f06",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def check_flood(res1d) -> bool:\n",
+ " \"\"\"\n",
+ " Checks whether there is flooding in the system by checking all nodes of the .res1d file\n",
+ " are checked for the WaterVolumeAboveGround value.\n",
+ "\n",
+ " Parameters:\n",
+ " -----------\n",
+ " res1d_file_path: str\n",
+ " The file path to the .res1d file containing the simulation results.\n",
+ "\n",
+ " Returns:\n",
+ " --------\n",
+ " bool\n",
+ " Retruns True, if a flooding is detected otherwise False.\n",
+ " \"\"\"\n",
+ " Flood = False\n",
+ " node_names = res1d.nodes\n",
+ " flooded_nodes = []\n",
+ "\n",
+ " # Iterate over all nodes and retrieve the WaterVolumeAboveGround values\n",
+ " for node in node_names:\n",
+ " query_node = QueryDataNode(quantity=\"WaterVolumeAboveGround\", name=node)\n",
+ " \n",
+ " try:\n",
+ " water_level_values = query_node.get_values(res1d)\n",
+ " \n",
+ " if water_level_values is None or len(water_level_values) == 0:\n",
+ " raise ValueError(f\"WaterVolumeAboveGround for Node '{node}' doesn't exist.\")\n",
+ " \n",
+ " if (water_level_values > 0).any(): \n",
+ " Flood = True\n",
+ " flooded_nodes.append(node)\n",
+ " print(f\"Flood detected for Node: {node}\")\n",
+ " \n",
+ " except Exception as e:\n",
+ " print(f\"Fehler beim Abrufen der Werte für Node {node}: {e}\")\n",
+ "\n",
+ " if Flood:\n",
+ " print(\"Flood detected in the system!\")\n",
+ " print(\"Flooded nodes:\", flooded_nodes)\n",
+ " else:\n",
+ " print(\"No flood detected in the system.\")\n",
+ " \n",
+ " return Flood"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "654c4b86",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def single_sim_run(ev):\n",
+ " '''\n",
+ " Runs a single simulation for event ev. Returns a sting message on successfull finish.\n",
+ " Intended for unsage with ThreadPool.map(). Use map to complete all simulations and read results after that\n",
+ " '''\n",
+ " sim_name = events.loc[events[\"Event\"] == ev, \"SimulationName\"].iloc[0]\n",
+ " \n",
+ " print(f\"Running simulation {sim_name} in {threading.current_thread()}.\")\n",
+ " engines[threading.current_thread()].run(sim_name)\n",
+ " return f\"Completed sim for {ev} in {threading.current_thread()}\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b821fda2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def simulation_run(**kwparams):\n",
+ " '''\n",
+ " Takes parameters as arguments, runs simulations as defined by events with them and returns the results as a series with Multiinex [event, time].\n",
+ " Note that kwparams need keys that match the indeyx of the DataFrame param_specs\n",
+ " '''\n",
+ " assert set(kwparams.keys()) == set(\n",
+ " param_specs.index\n",
+ " ), \"parameters need to match param_specs\" \n",
+ " \n",
+ " for k in kwparams.keys():\n",
+ " tab = param_specs.loc[k, \"Table\"]\n",
+ " col = param_specs.loc[k, \"Column\"]\n",
+ " muid = param_specs.loc[k, \"Muid\"]\n",
+ " val = kwparams[k]\n",
+ "\n",
+ " if pd.notna(param_specs.loc[k, \"Expression\"]):\n",
+ " expression = param_specs.loc[k, \"Expression\"].format(ConditionValue=val)\n",
+ " dta.set_value(tab, muid, col, expression)\n",
+ " print(f\"Set value for {tab}.{muid}.{col} to expression: {expression}\")\n",
+ "\n",
+ " else:\n",
+ " dta.set_value(tab, muid, col, val)\n",
+ " print(f\"Set value for {tab}.{muid}.{col} to value: {val}\")\n",
+ "\n",
+ " if DELETE_RESFILE_BEFORE_SIM: \n",
+ " for l in events.index:\n",
+ " delete_result_file(l) \n",
+ " \n",
+ " # Now run each simulation on its own thread\n",
+ " finish_msgs = sim_executor.map(single_sim_run,events[\"Event\"].unique())\n",
+ " print(finish_msgs)\n",
+ " \n",
+ " results_table = pd.DataFrame(columns=['Event', 'Result'])\n",
+ " total_result = 0\n",
+ " \n",
+ " for l in events.index:\n",
+ " result_file_path = temp_folder / events.loc[l, \"ResultFile\"]\n",
+ " result_file_path_str = str(temp_folder / events.loc[l, \"ResultFile\"])\n",
+ " \n",
+ " # Check if the result file exists\n",
+ " if not Path(result_file_path).exists():\n",
+ " print(f\"Result file does not exist: {result_file_path}\")\n",
+ " continue\n",
+ "\n",
+ " try:\n",
+ " res1d = Res1D(str(Path(result_file_path))) # Create Res1D instance for each file\n",
+ " flood = check_flood(res1d)\n",
+ " \n",
+ " if flood == True: # Give a penelty for flooding\n",
+ " results_ev = 10e100\n",
+ " \n",
+ " else:\n",
+ " # Calculate discharge values\n",
+ " discharge_values = [calculate_discharge_for_structure(res1d, structure) for structure in structures_list]\n",
+ " # Entferne None-Werte, falls es Fehler bei der Berechnung gab\n",
+ " discharge_values = [value for value in discharge_values if value is not None]\n",
+ " # Berechne die Gesamtzahl der Discharge-Werte\n",
+ " results_ev = np.sum(discharge_values) if discharge_values else 0\n",
+ " print(f\"Result for result file {result_file_path}: {results_ev}\")\n",
+ " # Update total result\n",
+ " total_result += results_ev\n",
+ " except Exception as e:\n",
+ " print(f\"Error processing result file {result_file_path}: {e}\")\n",
+ " traceback.print_exc()\n",
+ " results_ev = 0\n",
+ " results_table = pd.concat([results_table, pd.DataFrame([{'Event': events.loc[l, \"Event\"], 'Result': results_ev}])], ignore_index=True)\n",
+ " print(f\"Total result: {total_result}\") \n",
+ " print(results_table)\n",
+ " \n",
+ " return total_result"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "012fb1ff",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def delete_result_file(event):\n",
+ " '''\n",
+ " Deletes existing result file before a simulation corresponding to event.\n",
+ " '''\n",
+ " file_path = str(temp_folder / events.loc[event, \"ResultFile\"])\n",
+ " try:\n",
+ " Path(file_path).unlink(missing_ok=True)\n",
+ " except FileNotFoundError:\n",
+ " pass"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "19d16489",
+ "metadata": {},
+ "source": [
+ "### Test simulation run for all events with the default parameters"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dffeeb37",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# test run with default parameters\n",
+ "# the ** in the call unrolls the series into keyword parameters\n",
+ "param_specs.dtypes\n",
+ "_test_res = simulation_run(**param_specs[\"Default\"])\n",
+ "display(\"_test_res:\")\n",
+ "display(_test_res)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "baf6b68a",
+ "metadata": {},
+ "source": [
+ "### Preparations for optimisation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3d54b910",
+ "metadata": {},
+ "source": [
+ "Define optimization parameters as needed by nevergrade"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0dbba878",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ng_params = {}\n",
+ "for i in param_specs.index:\n",
+ " vmin = param_specs.at[i, \"Min\"]\n",
+ " vmax = param_specs.at[i, \"Max\"]\n",
+ " vdef = param_specs.at[i, \"Default\"]\n",
+ " ngp = ng.p.Scalar(init=vdef, lower=vmin, upper=vmax)\n",
+ " ng_params[i] = ngp\n",
+ "# now turn the list into an instrumentation\n",
+ "# the instrumentation is used to tell the optimizer how the paremeters are passed to our function\n",
+ "instrumentation = ng.p.Instrumentation(**ng_params)\n",
+ "instrumentation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6cb4a183",
+ "metadata": {},
+ "source": [
+ "Define constarins for the parameters"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "62e0a803",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def constraint_pid_setpoints(params_val):\n",
+ " \"\"\"Constraint: SetPoints should be less than or equal to the CSO-reduction-301 value\"\"\"\n",
+ " cso_reduction = params_val[1][\"msm_RTCRule;Condition;CSO-reduction-301\"]\n",
+ " setpoint_0301 = params_val[1][\"msm_RTCAction;PIDSetPoint;HUB0301_CSO_red\"]\n",
+ " \n",
+ " # Ensure both setpoints are <= CSO-reduction-301\n",
+ " return cso_reduction - setpoint_0301\n",
+ "\n",
+ "# Register the constraint\n",
+ "instrumentation.register_cheap_constraint(constraint_pid_setpoints)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7d5b751f",
+ "metadata": {},
+ "source": [
+ "Define callback funtions for the optimizers"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f4a870f3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def params_losses_callback():\n",
+ " \"\"\"returns two lists and a callback function.\n",
+ " When the callback function is executed it puts the parameters and loss into the lists.\n",
+ " !!Do not register this function itself as a callback, but the third returned parameter!!\n",
+ " \"\"\"\n",
+ " # create two lists to store the params and the corresponding losses\n",
+ " params = []\n",
+ " losses = []\n",
+ "\n",
+ " def callback(optim, par, loss):\n",
+ " params.append(par)\n",
+ " losses.append(loss)\n",
+ "\n",
+ " # returns two lists and the function\n",
+ " return params, losses, callback"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "aea52f70",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def optim_run(optim_name):\n",
+ " '''\n",
+ " Function to start the optimizer.\n",
+ " input = name of the optimizer\n",
+ " output = dataframe with details from optimization model\n",
+ " '''\n",
+ " # set the optimizer based on the optim_name and instrumentation from params_spec\n",
+ " optimizer = ng.optimizers.registry[optim_name](\n",
+ " parametrization=instrumentation, budget=optimization_budget, num_workers=1\n",
+ " )\n",
+ " # show progressbar. updated when the optimizer gets the result of a step\n",
+ " optimizer.register_callback(\"tell\", ng.callbacks.ProgressBar())\n",
+ " # put params and loss in list\n",
+ " param_list, loss_list, plcb = params_losses_callback()\n",
+ " optimizer.register_callback(\"tell\", plcb)\n",
+ " # get recommendation of parameters after run is finished\n",
+ " reccomendation = optimizer.minimize(simulation_run, verbosity=2)\n",
+ " optim_run_df = pd.concat(\n",
+ " [\n",
+ " pd.DataFrame(data=[p.value[1] for p in param_list]),\n",
+ " pd.DataFrame({\"Optimizer\": optimizer.name, \"Loss\": loss_list}),\n",
+ " ],\n",
+ " axis=\"columns\",\n",
+ " )\n",
+ " # also add a line for the reccomendation\n",
+ " rec_params = reccomendation.value[1]\n",
+ " optim_run_df.loc[\"reccomendation\", rec_params.keys()] = rec_params\n",
+ " optim_run_df.loc[\"reccomendation\", \"Loss\"] = reccomendation.loss\n",
+ " optim_run_df.loc[\"reccomendation\", \"Optimizer\"] = optimizer.name\n",
+ " # save the results of the optimization as a csv and excel file\n",
+ " optim_run_df.to_csv(out_folder / f\"optim_run{optimizer.name}.csv\")\n",
+ " optim_run_df.to_excel(out_folder / f\"optim_run{optimizer.name}.xlsx\")\n",
+ "\n",
+ " return optim_run_df"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "312ebf1f",
+ "metadata": {},
+ "source": [
+ "### Run optimisation"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0517d4fe",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# choose optimizers from nevergrad\n",
+ "# https://en.wikipedia.org/wiki/CMA-ES\n",
+ "# create a list with optimizers\n",
+ "# it is also possible to use multiple optimizers and compare the results\n",
+ "# create dictionary to store the rundata for each optimizer\n",
+ "optim_run_dfs = {}\n",
+ "for o in optimizers:\n",
+ " # start the optimization\n",
+ " optim_run_dfs[o] = optim_run(o)\n",
+ " # display results\n",
+ " display(optim_run_dfs[o])\n",
+ " optim_run_dfs[o].plot(subplots=True, figsize=(5, 2 * len(param_specs)))\n",
+ "\n",
+ "# optim_run_df already has optimizer included, use .values instead of passing the dict, so that the optimizer is not added again.\n",
+ "optim_run_df = pd.concat(optim_run_dfs.values())\n",
+ "optim_run_df.to_csv(out_folder / \"optim_run.csv\")\n",
+ "\n",
+ "# set the index if the optim_run_df\n",
+ "optim_run_df.index.name = \"Step\"\n",
+ "optim_run_df = optim_run_df.reset_index().set_index([\"Optimizer\", \"Step\"])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "64de11d1",
+ "metadata": {},
+ "source": [
+ "Print the results of the Optimization"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7162ffac",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# print the recommendations one last time\n",
+ "for key, df in optim_run_df.groupby(level=\"Optimizer\"):\n",
+ " display(key,df.loc[(key, \"reccomendation\"), param_specs.index])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "06dd3f0e",
+ "metadata": {},
+ "source": [
+ "Close database"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "93579781",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dta.close_database()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python [conda env:anaconda3-entfrachten]",
+ "language": "python",
+ "name": "conda-env-anaconda3-entfrachten-py"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.20"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
--
GitLab