Merge branch 'dev_cvo_input_resampling' into 'main'

Input resampling

See merge request focus/tooling!17

input_profile_processor/input_profile_processor.py

@@ -23,6 +23,7 @@ THE SOFTWARE.
 """
 import pandas as pd
 from datetime import datetime
+from datetime import timedelta
 from Tooling.demand_generator.electrical_demand.ElectricalDemand import ElectricalDemand
 from Tooling.demand_generator.thermal_demand.ThermalDemand import ThermalDemand
 from Tooling.input_profile_processor.calc_irradiance import generate_g_t_series
@@ -30,19 +31,18 @@ from Tooling.modifier import Modifier
 
 
 def process_input_profiles(input_profile_dict, t_start, t_horizon, t_step):
+    # This entire process assumes three things:
+    # 1. t_step devides a day into an integer amount of steps
+    # 2. t_step devides the intervall from YYYY.MM.DD 00:00:00 of the day containing t_start to t_start into an integer amount of steps
+    # 3. The timestamps of the desired timeseries either line up the with timestamps provided by the profiles in the files or the other way around
     input_profiles = {}
     for input_profile_name, input_profile_config in input_profile_dict.items():
         if 'file' in input_profile_config:
-            profile = pd.read_csv(input_profile_config['file'], index_col = 0)
-            try:
-                profile.index = pd.to_datetime(profile.index, format = '%d-%m-%Y %H:%M:%S')
-            except ValueError:
-                profile.index = pd.to_datetime(profile.index, format = '%Y-%m-%d %H:%M:%S')
-            profile = profile.resample(str(t_step) + 'H').mean().interpolate('linear')
+            profile = load_profile(input_profile_name, input_profile_config['file'], t_start, t_horizon, t_step)
         elif 'generate' in input_profile_config:
-            profile = generate_profile(input_profile_config['type'], input_profile_config['generate'], input_profiles, t_start, t_horizon, t_step)
+            profile = generate_profile(input_profile_name, input_profile_config['type'], input_profile_config['generate'], input_profiles, t_start, t_horizon, t_step)
         elif 'modify' in input_profile_config:
-            profile = modify_profile(input_profile_config['type'], input_profile_config['modify'], input_profiles, t_start, t_horizon, t_step)
+            profile = modify_profile(input_profile_name, input_profile_config['type'], input_profile_config['modify'], input_profiles, t_start, t_horizon, t_step)
         input_profiles[input_profile_name] = (input_profile_config['type'], profile)
 
     for input_profile_name, (input_profile_type, input_profile) in input_profiles.items():
@@ -61,19 +61,94 @@ def process_input_profiles(input_profile_dict, t_start, t_horizon, t_step):
         input_profiles[input_profile_name] = input_profile
     return input_profiles
 
+def resample_profile(name, profile, t_start, t_horizon, t_step, t_last):
+    profile = profile.resample(str(t_step) + 'H').mean().interpolate('linear')
+    if t_start < profile.index[0]:
+        missing_indices = pd.date_range(t_start, profile.index[0] - timedelta(hours = t_step), freq = str(t_step) + 'H')
+        print(f'For input profile {name} {len(missing_indices)} {"values are" if len(missing_indices) > 1 else "value is"} extrapolated to the beginning of the profile!')
+        profile = pd.concat([pd.DataFrame([profile.iloc[0]], index = missing_indices), profile])
+    if profile.index[-1] < t_last:
+        missing_indices = pd.date_range(profile.index[-1] + timedelta(hours = t_step), t_last, freq = str(t_step) + 'H')
+        print(f'For input profile {name} {len(missing_indices)} {"values are" if len(missing_indices) > 1 else "value is"} extrapolated to the end of the profile!')
+        profile = pd.concat([profile, pd.DataFrame([profile.iloc[-1]], index = missing_indices)])
+    return profile
+
+def expand_profile_to_year(profile, year, t_step):
+    if datetime(year, 1, 1, 0) < profile.index[0]:
+        missing_indices = pd.date_range(datetime(year, 1, 1, 0), profile.index[0] - timedelta(hours = t_step), freq = str(t_step) + 'H')
+        profile = pd.concat([pd.DataFrame([profile.iloc[0]], index  =missing_indices), profile])
+    if profile.index[-1] < datetime(year + 1, 1, 1, 0) - timedelta(hours = t_step):
+        missing_indices = pd.date_range(profile.index[-1] + timedelta(hours = t_step), datetime(year + 1, 1, 1, 0) - timedelta(hours = t_step), freq = str(t_step) + 'H')
+        profile = pd.concat([profile, pd.DataFrame([profile.iloc[-1]], index = missing_indices)])
+    return profile
+
+def load_profile(name, file, t_start, t_horizon, t_step):
+    t_last = t_start + pd.Timedelta(hours = t_horizon * t_step - t_step)
+    profile = pd.read_csv(file, index_col = 0)
+    try:
+        file_start = pd.to_datetime(profile.index[0], format = '%d-%m-%Y %H:%M:%S')
+        format = '%d-%m-%Y %H:%M:%S'
+    except ValueError:
+        format = '%Y-%m-%d %H:%M:%S'
+    # left_index = 0
+    # right_index = len(profile) - 1
+    # middle_index = left_index + int((right_index - left_index) / 2)
+    # middle_time = pd.to_datetime(profile.index[middle_index], format = format)
+    # while True:
+    #     if right_index - left_index == 1:
+    #         # We now know that t_start is strictly larger than the time at left_index and smaller or equal than the time at right_index
+    #         # So we have to test if t_start is equal to the time at right_index and return right_index if this is the case
+    #         # Otherwise return left_index
+    #         right_time = pd.to_datetime(profile.index[right_index], format = format)
+    #         if right_time <= t_start:
+    #             first_index = right_index
+    #         else:
+    #             first_index = left_index
+    #         break
+    #     if middle_time < t_start:
+    #         left_index = middle_index
+    #     else:
+    #         right_index = middle_index
+    #     middle_index = left_index + int((right_index - left_index) / 2)
+    #     middle_time = pd.to_datetime(profile.index[middle_index], format = format)
+    # left_index = 0
+    # right_index = len(profile) - 1
+    # middle_index = left_index + int((right_index - left_index) / 2)
+    # middle_time = pd.to_datetime(profile.index[middle_index], format = format)
+    # while True:
+    #     if right_index - left_index == 1:
+    #         # We now know that t_last is lager or equal than the time at left_index and strictly smaller than the time at right_index
+    #         # So we have to test if t_last is equal to the time at left_index and return left_index if this is the case
+    #         # Otherwise return right_index
+    #         left_time = pd.to_datetime(profile.index[left_index], format = format)
+    #         if t_last <= left_time:
+    #             last_index = left_index
+    #         else:
+    #             last_index = right_index
+    #         break
+    #     if t_last < middle_time:
+    #         right_index = middle_index
+    #     else:
+    #         left_index = middle_index
+    #     middle_index = left_index + int((right_index - left_index) / 2)
+    #     middle_time = pd.to_datetime(profile.index[middle_index], format = format)
+    first_index = 0
+    last_index = len(profile) - 1
+    profile = profile[first_index:last_index + 1]
+    profile.index = pd.to_datetime(profile.index, format = format)
+    return resample_profile(name, profile, t_start, t_horizon, t_step, t_last)
 
-def generate_profile(profile_type, parameters, input_profiles, t_start, t_horizon, t_step):
+def generate_profile(name, profile_type, parameters, input_profiles, t_start, t_horizon, t_step):
     t_last = t_start + pd.Timedelta(hours = t_horizon * t_step - t_step)
     years = range(t_start.year, t_last.year + 1)
 
     if profile_type == 'elec_demand':
         profiles = []
         for year in years:
-            profile = ElectricalDemand(year).get_profile(parameters['yearly_demand'], 'h0', True)  # True: with smoothing function for household profiles, False: no smoothing function for household profiles
-            profile = profile.resample(str(t_step) + 'H').mean().interpolate('linear')
-            profiles.append(profile)
-        timeseries = pd.concat(profiles)
-        return timeseries
+            year_profile = ElectricalDemand(year).get_profile(parameters['yearly_demand'], 'h0', True)  # True: with smoothing function for household profiles, False: no smoothing function for household profiles
+            profiles.append(year_profile)
+        profile = pd.concat(profiles)
+        return resample_profile(name, profile, t_start, t_horizon, t_step, t_last)
     elif profile_type == 'therm_demand':
         profiles = []
         for year in years:
@@ -83,7 +158,12 @@ def generate_profile(profile_type, parameters, input_profiles, t_start, t_horizo
             # Fixme: non-residential building only have building_class = 0
             # residential building could varies from 1 to 11
             # same problem for hot water demand.
-            profile = ThermalDemand(demand_df.index,
+            # temporary fix
+            if len(input_profiles[parameters['temperature']][1]) < len(demand_df.index):
+                temp = list(input_profiles[parameters['temperature']])
+                temp[1] = expand_profile_to_year(input_profiles[parameters['temperature']][1], year, t_step)
+                input_profiles[parameters['temperature']] = tuple(temp)
+            year_profile = ThermalDemand(demand_df.index,
                                          shlp_type='EFH',
                                          temperature=input_profiles[parameters['temperature']][1],
                                          building_class=1,
@@ -91,16 +171,21 @@ def generate_profile(profile_type, parameters, input_profiles, t_start, t_horizo
                                          annual_heat_demand=parameters['yearly_demand'],
                                          name='HeatDemand_EFH',
                                          ww_incl=0).get_bdew_profile()
-            profiles.append(profile)
-        timeseries = pd.concat(profiles)
-        return timeseries
+            profiles.append(year_profile)
+        profile = pd.concat(profiles)
+        return profile
     elif profile_type == 'hot_water_demand':
         profiles = []
         for year in years:
             demand_df = pd.DataFrame(index=pd.date_range(datetime(year, 1, 1, 0),
                                                          periods=8760 / t_step,
                                                          freq=str(t_step) + 'H'))
-            profile = ThermalDemand(demand_df.index,
+            # temporary fix
+            if len(input_profiles[parameters['temperature']][1]) < len(demand_df.index):
+                temp = list(input_profiles[parameters['temperature']])
+                temp[1] = expand_profile_to_year(input_profiles[parameters['temperature']][1], year, t_step)
+                input_profiles[parameters['temperature']] = tuple(temp)
+            year_profile = ThermalDemand(demand_df.index,
                                          shlp_type='EFH',
                                          temperature=input_profiles[parameters['temperature']][1],
                                          building_class=1,
@@ -108,14 +193,14 @@ def generate_profile(profile_type, parameters, input_profiles, t_start, t_horizo
                                          annual_heat_demand=parameters['yearly_demand'],
                                          name='DomesticHotWaterDemand_EFH',
                                          ww_incl=1).get_bdew_profile()
-            profiles.append(profile)
-        timeseries = pd.concat(profiles)
-        return timeseries
+            profiles.append(year_profile)
+        profile = pd.concat(profiles)
+        return profile
     else:
         raise Exception("Generator for profile type " + str(profile_type) + " is not implemented!")
 
 
-def modify_profile(mod_type: str, parameters, input_profiles, t_start, t_horizon, t_step):
+def modify_profile(name, mod_type: str, parameters, input_profiles, t_start, t_horizon, t_step):
     # Example of the parameters for use in the runme
     # ------------------------------------------------------------------------------------------------------------------
     # 'temperature_1': ['prophet', 'modify', 'input_files/data/temperature/temperature.csv', 'temperature',