Upload initial code

.gitignore

+.DS_Store
+.Git_Ordner/Optimierung/Adaptive_Rolling_Horizon/modell_LEV/results/
+Git_Ordner/GUI/utils/lwd.txt
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# exclude .vscode folder
+.vscode/
+
+# excel lock files
+*.xlsx#
+
+# PyCharm Project
+.idea/

LICENSE.txt

+Copyright © 2021 Chair for Technical Thermodynamics (Lead author: Andreas Kämper)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file

README.md

+# AutoMoG - A framework for automated data-driven modeling and optimization of multi-energy systems
+
+
+## Installation
+
+
+### from source
+
+1. Clone the AutoMoG repository to any directory of your computer.
+
+2. Open a command prompt on your computer.
+
+3. Navigate to the toplevel directory of the AutoMoG repository (where `setup.py` is located).
+
+4. Enter the command `pip install --user -e .` to install AutoMoG including all required packages.
+
+Note: To solve optimization problems with AutoMoG, you have to install at least one of the three solvers `gurobi`, `cplex`, `glpk`.
+
+## Usage
+
+`python -m automog` in any directory starts GUI. 
+
+## Additional info
+
+In the `info` folder, you can find a pdf that helps you with the first steps of using the GUI, and another pdf that describes the model we use for operational optimization with the GUI.
+
+If you use this code, please consider citing:
+
+1. Kämper, A., Leenders, L., Bahl, B., and Bardow, A. (2021). Automog: Automated Data-Driven Model Generation of Multi-Energy Systems Using Piecewise-Linear Regression. Comput. Chem. Eng. 145, 107162. doi: https://doi.org/10.1016/j.compchemeng.2020.107162 
+
+2. Kämper, A., Holtwerth, A., Leenders, L., and Bardow, A. (2021). Automog 3D: Automated Data-Driven Model Generation of Multi-Energy Systems Using Hinging Hyperplanes. Front. Energ. Res. 9, 719658. doi: https://doi.org/10.3389/fenrg.2021.719658 
+
+If you have any questions, please contact: andreas.kaemper@ltt.rwth-aachen.de
+

automog/__main__.py

+"""
+Adds necessary packages to sys.path and starts ModEst GUI.
+"""
+import sys
+import os
+
+# get absolute path of this __main__.py script 
+here = os.path.dirname(os.path.abspath(__file__))
+
+# delete 'gui' from sys.modules in order to have a clear reimport
+if 'code' in sys.path:
+    del sys.modules['gui']  
+# add all needed custom packages to sys.path
+else:
+    sys.path.append(here + '/code')
+
+# import and start gui
+from gui import gui
+gui.main()

automog/code/ModelRecreation.py

+# -*- coding: utf-8 -*-
+"""
+Created on Sun Apr 18 12:32:55 2021
+
+@author: niels
+"""
+
+from optimization.engine import OperationalOptimizationInterface
+from pathlib import Path
+
+class ModelRecreation:
+    def __init__(self,model=None,data=None, model_path=None, data_path=None, config=None):
+        if model:
+            self.model=model
+        elif model_path:
+            from optimization.interface.optimizer import model as model_opt
+            self.model=model_opt.from_file(model_path)
+        else:
+            raise AttributeError("Please provide input data either as input dictionary for pyomo or as filepath to an input file.")
+        if data:
+            self.data=data
+        elif data_path:
+            from optimization.interface.optimizer import instance as instance
+            self.data=instance.from_csv(data_path)
+        else:
+            raise AttributeError("Please provide input data either as input dictionary for pyomo or as filepath to an input file.")
+       
+        if not config:   
+            from optimization.settings.config import config
+            self.config = config
+        else:
+            self.config=config
+        
+        self.interface=OperationalOptimizationInterface(self.model,self.data,config=self.config)
+
+    def create_model(self,output_dir,abstract_model=True):
+        self.pyomo_model, self.input_dictionary=self.interface.create_pyomo_model(output_dir=output_dir, abstract=abstract_model)
+
+#model_recreator=ModelRecreation(model_path=r"C:\Users\niels\Documents\LTT\automog\automog\data\Goderbauer2016\optimization\models\transfered_model-2021_04_07_09-51-18.pkl",data_path=r"C:\Users\niels\Documents\LTT\automog\automog\data\Goderbauer2016\optimization\instances\8760.CSV")
+#model_recreator.create_model(abstract_model=True)
+#concrete_model=model_recreator.pyo_model

automog/code/gui/helpers.py

+import numpy as np
+import logging
+import pandas as pd
+    
+def get_rwth_colors():
+    """ return RWTH colors as array """
+    rwth_colors = [
+        (0,104,180),
+        (0,97,101),
+        (0,152,161),
+        (87,171,39),
+        (189,205,0),
+        (246,168,0),
+        (204,7,30),
+        (161,16,53),
+        (97,33,88),
+        (122,111,172),
+    ]
+    rwth_colors = np.divide(rwth_colors,256) # percentage of 256 spectrum
+    return rwth_colors
+
+def log_heading(message: str):
+    """
+    Takes a message and creates a nicely formatted heading log message.
+    """
+    message = "=   {0}   =".format(message)
+    length  = len(message)
+    logging.info("\n\n\n" + "=" * length + "\n" + message + "\n" + "=" * length + "\n")
\ No newline at end of file

automog/code/gui/utils/lwd.txt

+C:/Daten_Kaemper/automog/automog/data/Goderbauer2016
\ No newline at end of file

automog/code/modeling/FlowChart.py

+# -*- coding: utf-8 -*-
+"""
+Created on Fri Mar  5 13:29:35 2021
+
+@author: niels
+"""
+from graphviz import Digraph
+import copy
+
+class FlowChart(object):
+    """This class creates a Flow Chart for a given model"""
+    def __init__(self,model,directory):
+        #get all components, inputs and outputs and init stuff
+        self.components=copy.deepcopy(model.components)
+        self.products=copy.deepcopy(model.products)
+        self.storages=copy.deepcopy(model.storages)
+        self.params=copy.deepcopy(model.input_params)
+        self.inputs=list()
+        self.outputs=list()
+        self.edge_count_d=dict()
+        self.edge_count_c=dict()
+        self.edge_count_s=dict()
+        self.edge_count_f=dict()
+        self.edge_count_p=dict()
+        subcomponents=list()
+        self.directory=directory
+        self.colors=['green','black','orange','purple','cyan']
+        self.edge_colors=dict()
+       
+        #delete subcomponents and add their in and outputs to actual component 
+        for comp in self.components:
+            if "_" in comp:
+                if comp.split("_")[0] in self.components:
+                    for Input in self.components[comp].inputs:
+                        if Input not in self.components[comp.split("_")[0]].inputs:
+                            self.components[comp.split("_")[0]].inputs.append(Input)
+                    for output in self.components[comp].outputs:
+                        if output not in self.components[comp.split("_")[0]].outputs:
+                            self.components[comp.split("_")[0]].outputs.append(output)
+                    subcomponents.append(comp)
+                
+        for subcomp in subcomponents:
+            del(self.components[subcomp])
+        for comp in self.components:
+            for Input in self.components[comp].inputs:
+                self.inputs.append(Input)
+            for output in self.components[comp].outputs:
+                self.outputs.append(output)
+        
+        #predefined colors for certain products;
+        self.edges=list(set(self.inputs+self.outputs))
+    
+        for edge in self.edges:
+            if edge=='Heat':
+                self.edge_colors[edge]='red'
+            elif edge=='Gas':
+                self.edge_colors[edge]='grey'
+            elif edge=='Power':
+                self.edge_colors[edge]='yellow'
+            elif edge=='Chill':
+                self.edge_colors[edge]='blue'
+            else:
+                self.edge_colors[edge]=self.colors[0]
+                del self.colors[0]
+                
+                
+    def create_FlowChart(self, include_legend=False):
+        
+        chart=Digraph('FlowChart',format='png',directory=self.directory)
+        
+        #create all necessary nodes;subgraphs for those in same height; points as work around for joined edges
+        with chart.subgraph(name='supply') as sup_nodes:
+            sup_nodes.attr(rank='same')
+            sup_nodes.attr('node',shape='circle', width='1')
+            for Input in self.inputs:
+                if Input in self.products.keys() and self.products[Input]['price'] is not None:
+                    sup_nodes.attr('node', color=self.edge_colors[Input])
+                    sup_nodes.node(Input+"\nSupply")
+                    self.edge_count_s[Input]=0
+                elif Input in self.params:
+                    sup_nodes.attr('node', color=self.edge_colors[Input])
+                    sup_nodes.node(Input)
+                    self.edge_count_s[Input] = 0
+                else:
+                    self.edge_count_s[Input]=1
+        
+        with chart.subgraph(name='demand') as dem_nodes:
+            dem_nodes.attr(rank='same')
+            dem_nodes.attr('node', shape='doublecircle', width='1')
+            for output in self.outputs:
+                if self.products[output]['demand'] is not None:
+                    dem_nodes.attr('node', color=self.edge_colors[output])
+                    dem_nodes.node(output+"\nDemand")
+                    self.edge_count_d[output]=0
+
+        with chart.subgraph(name='Feed Option') as feed_nodes:
+            feed_nodes.attr(rank='same')
+            feed_nodes.attr('node', shape='doublecircle', width='1')
+            for output in self.outputs:
+                if self.products[output]['compensation'] is not None:
+                    feed_nodes.attr('node', color=self.edge_colors[output])
+                    feed_nodes.node(output+"\nFeed Option")
+                    self.edge_count_f[output]=0
+
+        for product in self.products:
+            chart.attr('node',shape='point', width='0.01',height='0.01',color=self.edge_colors[product])
+            chart.node(product+"point")
+        for param in self.params:
+            #chart.node(param)
+            chart.attr('node', shape='point', width='0.01', height='0.01', color=self.edge_colors[param])
+            chart.node(param + "point")
+        chart.attr("node",shape='box',color='black')
+        for comp in self.components:
+            chart.node(comp)
+            self.edge_count_c[comp]=0
+        for stor in self.storages:
+            chart.node(stor)
+
+            
+        #create all edges
+        for comp in self.components:
+            
+            for Input in self.components[comp].inputs:
+                if self.edge_count_s[Input]==0 and self.edge_count_c[comp]==0:
+                    chart.edge(Input+"\nSupply",Input+'point', color=self.edge_colors[Input], dir='none')
+                    chart.edge(Input+'point',comp, color=self.edge_colors[Input])
+                    self.edge_count_s[Input]+=1
+                    self.edge_count_c[comp]+=1
+                elif self.edge_count_c[comp]==0:
+                    chart.edge(Input+'point',comp,color=self.edge_colors[Input])
+                    self.edge_count_c[comp]+=1
+                elif Input in self.params:
+                    chart.edge(Input, Input+'point', color=self.edge_colors[Input], dir='none')
+                    chart.edge(Input+'point', comp, color=self.edge_colors[Input])
+                    self.edge_count_s[Input] += 1
+                elif self.edge_count_s[Input]==0:
+                    chart.edge(Input + "\nSupply", Input + 'point', color=self.edge_colors[Input], dir='none')
+                    chart.edge(Input + 'point', comp, color=self.edge_colors[Input])
+                    self.edge_count_s[Input]+=1
+            
+            for output in self.components[comp].outputs:
+                if self.products[output]['demand'] is not None and self.products[output]['compensation'] is not None:
+                    chart.edge(comp,output+'point',dir='none', color=self.edge_colors[output])
+                    if self.edge_count_d[output]==0:
+                        chart.edge(output+'point',output+"\nDemand", color=self.edge_colors[output])
+                        self.edge_count_d[output]+=1
+                    if self.edge_count_f[output]==0:
+                        chart.edge(output+'point',output+"\nFeed Option", color=self.edge_colors[output])
+                        self.edge_count_f[output]+=1
+                elif self.products[output]['demand'] is not None:
+                    chart.edge(comp,output+'point',dir='none', color=self.edge_colors[output])
+                    if self.edge_count_d[output]==0:
+                        chart.edge(output+'point',output+"\nDemand", color=self.edge_colors[output])
+                        self.edge_count_d[output]+=1
+                elif self.products[output]['compensation'] is not None:
+                    chart.edge(comp,output+'point',dir='none', color=self.edge_colors[output])
+                    if self.edge_count_f[output]==0:
+                        chart.edge(output+'point',output+"\nFeed Option", color=self.edge_colors[output])
+                        self.edge_count_f[output]+=1
+                elif output in self.inputs:
+                    chart.edge(comp,output+'point',dir='none', color=self.edge_colors[output])
+                else:
+                    chart.edge(comp, comp, color=self.edge_colors[output])
+
+        for stor in self.storages:
+            chart.edge(stor, self.storages[stor]['storage_product']+'point', color=self.edge_colors[self.storages[stor]['storage_product']])
+            chart.edge(self.storages[stor]['storage_product']+'point', stor, color=self.edge_colors[self.storages[stor]['storage_product']])
+        
+        #legend 
+        if include_legend==True:
+            legend_count=0
+            letters=['a','b','c','d','e','f','g','h','i','j','k']
+            with chart.subgraph(name='cluster_legend') as legend:
+                legend.attr(label='Legend')
+                legend.attr('node',shape='point', style='invis')
+                for i,edge in enumerate(self.edges):
+                    if legend_count==0:
+                        with legend.subgraph() as f:
+                            f.attr(rank='same')
+                            f.edge(str(legend_count),str(legend_count+1), label=edge, color=self.edge_colors[edge], rank='same')
+                    else:
+                        with legend.subgraph() as letters[i]:
+                            letters[i].attr(rank='same')
+                            letters[i].edge(str(legend_count),str(legend_count+1), label=edge, color=self.edge_colors[edge], rank='same')
+                        legend.edge(str(legend_count),str(legend_count-2), style='invis')
+                        legend.edge(str(legend_count+1),str(legend_count-1), style='invis')
+                    legend_count+=2
+        
+        chart.render('FlowChart',format='png',directory=self.directory)
+
+    #Not yet working method
+    def get_node_max(self,digraph):
+        import re
+        print(str(digraph))
+        heights = [height.split('=')[1] for height in re.findall('height=[0-9.]+', str(digraph))]
+        widths = [width.split('=')[1] for width in re.findall('width=[0-9.]+', str(digraph))]
+        heights.sort(key=float,reverse=True)
+        widths.sort(key=float, reverse=True)  
+        print(heights)
+        print(widths)
+        return heights[0], widths[0]
\ No newline at end of file

automog/code/modeling/Hinging_Hyperplane/prepare_constraints.py

+from scipy.spatial import ConvexHull
+
+from modeling.Hinging_Hyperplane.hinging_hyperplane import HHmodel, calculate_corner_points
+
+def calc_convex_hull(model: HHmodel):
+    for lin_element in model.milp.points:
+        conv_hull = ConvexHull(model.milp.points[lin_element]['X'])
+        corner_points = calculate_corner_points(conv_hull)
+        model.milp.hull_lin_elements[lin_element] = ConvexHull(corner_points)
+        
\ No newline at end of file

automog/code/modeling/RegressionClustering/CombiClustering.py

+# -*- coding: utf-8 -*-
+"""
+Created on Wed Jan 27 08:25:07 2021
+
+@author: niels
+"""
+
+# -*- coding: utf-8 -*-
+"""
+Created on Sat Jan 16 09:54:04 2021
+
+@author: niels
+"""
+
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jan 15 08:37:33 2021
+
+@author: niels
+"""
+
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Dec 11 10:58:32 2020
+
+@author: niels
+"""
+
+import numpy as np
+from sklearn.linear_model import LinearRegression
+import pandas as pd
+from sklearn.mixture import GaussianMixture
+import copy
+
+i=1 #number of iterations for Regression Clustering 
+patience=10 #patience for Regression Clustering; algorithm finishes after not getting better patience times
+operating_points_1=pd.read_csv(r"C:\Users\niels\Documents\LTT\RegressionClustering\GuDkomplett.CSV", delimiter=";",decimal=",")
+operating_points_2=pd.read_csv(r"C:\Users\niels\Documents\LTT\RegressionClustering\operating_points_testLarge.CSV",delimiter=";",decimal=",")
+operating_points_0=pd.read_csv(r"C:\Users\niels\Documents\LTT\RegressionClustering\GuDonetwo.CSV", delimiter=";",decimal=",")
+operating_points_3=pd.read_csv(r"C:\Users\niels\Documents\LTT\RegressionClustering\GuD_3.CSV", delimiter=";",decimal=",")
+all_components={"component_0": {"state_0": operating_points_1}, "component_1": {"state_0": operating_points_2}}
+two_states={"state_0": operating_points_0, "state_1:":operating_points_3}
+colors=['red', 'blue','green','black','orange','cyan']
+
+#CombiClustering carries out given clustering method for one state of one component of all operating points an returns the operating points with new states after clustering
+def CombiClustering(operating_points,component,state, method, number_of_clusters,i=10,patience=10,preformatted=False):
+    new_states=list()
+    new_operating_points=copy.deepcopy(operating_points)
+    if method=="gm":
+        new_states=GM(new_operating_points[component],state,number_of_clusters,preformatted)
+    if method=="rc":
+        new_states=MultiRC(new_operating_points[component],state,number_of_clusters,i,patience,preformatted)
+    for i in range(len(new_states)):
+        if i==0:    #replaces old state
+            new_operating_points[component][state]=new_states[i]
+        else:   #adds as many new states as needed
+            new_operating_points[component]["state_"+str(len(operating_points[component])+i-1)]=new_states[i]
+    return new_operating_points
+                            
+#Gaussian Mixture implementation
+def GM(component,state,number_of_clusters, preformatted=False):
+    x=[]
+    y=[]
+    if preformatted==False:
+        for p in component[state]:   #get Input as x array, Ouput as y array by header
+            if '(in)' in p:
+                for i in component[state][p]:
+                    x.append(i)
+        for p in component[state]:
+            if '(out)' in p:
+                for i in component[state][p]:
+                    y.append(i)
+        if len(x)<1:    #if header is misspelled or similar, use first column as Input and second column as Ouput data
+            for i in component[state].iloc[:,0]:
+                x.append(i)
+        if len(y)<1:
+            for i in component[state].iloc[:,1]:
+                y.append(i)
+    if preformatted==True:
+        for i in component[state].iloc[:,0]:
+            x.append(i)
+        for i in component[state].iloc[:,1]:
+            y.append(i)
+    clusters= list()
+    df_clusters=list()
+    for i in range(number_of_clusters):
+        clusters.append(list())
+    gm = GaussianMixture(n_components=number_of_clusters)
+    X=np.reshape(x,(-1,1))
+    labels=gm.fit_predict(X,y)
+    for i in range(len(x)):
+        clusters[labels[i]].append([x[i],y[i]])
+    for cluster in clusters:
+        df=pd.DataFrame(cluster, columns=[0,1])
+        df_clusters.append(df)
+    return df_clusters
+
+
+def MultiRC(component,state,number_of_clusters,i=50,patience=10,maxIterations=5000, preformatted=False): 
+    temp_TRS=0
+    TRS=0
+    clusters=list()
+    for j in range(i):
+        temp_clusters, temp_TRS= RC(component,state, number_of_clusters,patience,maxIterations,preformatted)
+        if temp_TRS>TRS:
+            TRS=temp_TRS
+            clusters=temp_clusters
+        print("Iteration",j," completed")
+    print("Clustering completed")
+    return clusters
+
+def RC (component, state, number_of_clusters,patience=10,maxIterations=5000, preformatted=False):
+    x=[]
+    y=[]
+    if preformatted==False:
+        for p in component[state]:   #get Input as x array, Ouput as y array by header
+            if '(in)' in p:
+                for i in component[state][p]:
+                    x.append(i)
+        for p in component:
+            if '(out)' in p:
+                for i in component[state][p]:
+                    y.append(i)
+        if len(x)<1:    #if header is misspelled or similar, use first column as Input and second column as Ouput data
+            for i in component[state].iloc[:,0]:
+                x.append(i)
+        if len(y)<1:
+            for i in component[state].iloc[:,1]:
+                y.append(i)
+    if preformatted==True:
+        for i in component[state].iloc[:,0]:
+            x.append(i)
+        for i in component[state].iloc[:,1]:
+            y.append(i)
+    clusters= list() #list with all clusters
+    df_clusters=[] #list with all clusters as pandas DataFrames
+    functions= list() #list with a foundational function for each cluster
+    TRS=0 #Total Regression Squared
+    CTRS=0 #Compared Total Regression Squared
+    patience_count=0
+    patientTRS=[]
+    patientClusters=[]
+    #initialize clusters with random assignment of datapoints to clusters
+    X = np.reshape(x,(-1,1)) #reshaping for regression function
+    for i in range(number_of_clusters):
+        clusters.append(list())
+    for i in range(len(X)):
+        clusters[np.random.randint(0,number_of_clusters)].append([X[i],y[i]])
+    #initial linear regression for each cluster
+    for cluster in clusters:
+        cx=list()#structure all data points in a cluster as x and y lists for regression function
+        cy=list()
+        for i in cluster:
+            cx.append(i[0])
+            cy.append(i[1])
+        function=LinearRegression().fit(cx,cy)
+        functions.append(function)
+        TRS+=function.score(cx,cy)
+    for i in range(maxIterations): #repeats until either best clusters are returned or maximum Iterations are reached to prevent running endlessly
+        #calculates minimal distanced cluster for each data point and assigns data point to it
+        for c_idx, cluster in enumerate(clusters):
+            for i in cluster:
+                minDistance=(max(y)-min(y))**2+1
+                bestfct=0
+                for f_idx,function in enumerate(functions):
+                    distance= (abs(function.predict(np.reshape(i[0],(-1,1)))-i[1]))**2
+                    if distance < minDistance :
+                        minDistance=distance
+                        bestfct=f_idx
+                if bestfct!=c_idx:
+                    cluster.remove(i)
+                    clusters[bestfct].append(i)
+        #New regression with changed clusters
+        functions.clear()
+        for cluster in clusters:
+            cx=list()
+            cy=list()
+            for i in cluster:
+                cx.append(i[0])
+                cy.append(i[1])
+            function=LinearRegression().fit(cx,cy)
+            functions.append(function)
+            CTRS+=function.score(cx,cy)
+        if CTRS<=TRS : #if Total Regression Squared is maximized, regression clustering is complete
+            patience_count+=1
+            patientTRS.append(TRS)
+            patientClusters.append(clusters)
+            if patience_count >= patience: #only finishes if TRS does not improve patience times
+                TRS=max(patientTRS)
+                clusters=patientClusters[np.argmax(patientTRS)]
+                print("TRS:", TRS)
+                for cluster in clusters: #Bringing clusters in pandas format
+                    for i in cluster:
+                        i[0]=i[0][0]
+                    df=pd.DataFrame(cluster, columns=[0,1])
+                    df_clusters.append(df)
+                return df_clusters, TRS
+        TRS=CTRS
+        CTRS=0
+    print("No solution found in maximum number of Iterations.")
+    
+
+
+new_points=CombiClustering(all_components,"component_0","state_0","gm",2,i=1,preformatted=True)
+for idx, state in enumerate(all_components["component_0"]):
+    print("old",state,":")
+    all_components["component_0"][state].plot.scatter(0,1, c=colors[idx])
+for idx, state in enumerate(new_points["component_0"]):
+    print("new",state,":")
+    new_points["component_0"][state].plot.scatter(0,1, c=colors[idx])
\ No newline at end of file

automog/code/modeling/RegressionClustering/CombiClustering_Interactive.py

+# -*- coding: utf-8 -*-
+"""
+Created on Sat Jan 16 09:54:04 2021
+
+@author: niels
+"""
+
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jan 15 08:37:33 2021
+
+@author: niels
+"""
+
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Dec 11 10:58:32 2020
+
+@author: niels
+"""
+
+import numpy as np
+from sklearn.linear_model import LinearRegression
+import matplotlib.pyplot as plt
+import pandas as pd
+from sklearn.mixture import GaussianMixture
+
+
+i=1 #number of iterations for Regression Clustering Single Use
+number_of_clusters_gm=2 #number of clusters for Gaussian Mixture Single Use
+number_of_clusters_rc=2 #number of clusters for Regression Clustering Single Use
+patience=10 #patience for Regression Clustering; algorithm finishes after not getting better patience times
+operating_points_1=pd.read_csv(r"C:\Users\niels\Documents\LTT\RegressionClustering\GuDkomplett.CSV", delimiter=";",decimal=",")
+colors=['red', 'blue','green','black','orange','cyan']
+
+#Clustering with maximum 2 steps with conscole interaction
+def CombiClustering(operating_points):
+    final_clusters=[]
+    clustering_wish="Empty"
+    method=input("Which Clustering method would you like to use first? \nType in 'gm' for Gaussian Mixture or 'rc' for Regression Clustering.")
+    if method=="gm":
+        number_of_clusters_gm=int(input("How many Clusters do you want to be used for the Gaussian Mixture?"))
+        gm_clusters=GM(operating_points,number_of_clusters_gm)
+        for i in range(number_of_clusters_gm):
+            plt.scatter(gm_clusters[i][0], gm_clusters[i][1], color=colors[i])
+            plt.show()
+        clustering_wish=input("Type in the colors of the clusters you want to cluster further")
+        for idx,color in enumerate(colors):
+            if color in clustering_wish:
+                print("\nWhich Clustering method would you like to use for the",color,"Cluster?")
+                method2=input("Type in 'gm' for Gaussian Mixture or 'rc' for Regression Clustering.")
+                if method2=="gm":
+                    number_of_clusters_gm=int(input("How many Clusters do you want to be used for the Gaussian Mixture?"))
+                    gm_clusters_2=GM(gm_clusters[idx],number_of_clusters_gm, preformatted=True)
+                    for gm_cluster in  gm_clusters_2:
+                        final_clusters.append(gm_cluster)
+                if method2=="rc":
+                    number_of_clusters_rc=int(input("How many Clusters do you want to be used for Regression Clustering?"))
+                    i=int(input("How many Iterations?"))
+                    rc_clusters, TRS=MultiRC(gm_clusters[idx],number_of_clusters_rc,i,preformatted=True)
+                    for rc_cluster in rc_clusters:
+                            final_clusters.append(rc_cluster)
+        for idx,color in enumerate(colors):
+            if color not in clustering_wish:
+                if idx in range(len(gm_clusters)):
+                    final_clusters.append(gm_clusters[idx])
+    if method=="rc":
+        number_of_clusters_rc=int(input("How many Clusters would you like to be used for Regression Clustering?"))
+        i=int(input("How many Iterations?"))
+        rc_clusters, TRS=MultiRC(operating_points,number_of_clusters_rc,i)
+        for i in range(number_of_clusters_rc):
+            plt.scatter(rc_clusters[i][0], rc_clusters[i][1], color=colors[i])
+            plt.show()
+        clustering_wish=input("Type in the colors of the clusters you want to cluster further")
+        for idx,color in enumerate(colors):
+            if color in clustering_wish:
+                print("\nWhich Clustering method would you like to use for the",color,"Cluster?")
+                method2=input("Type in 'gm' for Gaussian Mixture or 'rc' for Regression Clustering.")
+                if method2=="gm":
+                    number_of_clusters_gm=int(input("How many Clusters do you want to be used for the Gaussian Mixture?"))
+                    gm_clusters_2=GM(rc_clusters[idx],number_of_clusters_gm, preformatted=True)
+                    for gm_cluster in  gm_clusters_2:
+                        final_clusters.append(gm_cluster)
+                if method2=="rc":
+                    number_of_clusters_rc=int(input("How many Clusters do you want to be used for Regression Clustering?"))
+                    i=int(input("How many Iterations?"))
+                    rc_clusters, TRS=MultiRC(rc_clusters[idx],number_of_clusters_rc,i,preformatted=True)
+                    for rc_cluster in rc_clusters:
+                        final_clusters.append(rc_cluster)
+        for idx,color in enumerate(colors):
+            if color not in clustering_wish:
+                if idx in range(len(rc_clusters)):
+                    final_clusters.append(rc_clusters[idx])
+    for i in range(len(final_clusters)):
+        plt.scatter(final_clusters[i][0], final_clusters[i][1], color=colors[i])
+    plt.show()
+    return final_clusters
+                            
+#Gaussian Mixture implementation
+def GM(operating_points,c, preformatted=False):
+    x=[]
+    y=[]
+    if preformatted==False:
+        for p in operating_points:   #get Input as x array, Ouput as y array by header
+            if '(in)' in p:
+                for i in operating_points[p]:
+                    x.append(i)
+        for p in operating_points:
+            if '(out)' in p:
+                for i in operating_points[p]:
+                    y.append(i)
+        if len(x)<1:    #if header is misspelled or similar, use first column as Input and second column as Ouput data
+            for i in operating_points.iloc[:,0]:
+                x.append(i)
+        if len(y)<1:
+            for i in operating_points.iloc[:,1]:
+                y.append(i)
+    if preformatted==True:
+        for i in operating_points[0]:
+            x.append(i)
+        for i in operating_points[1]:
+            y.append(i)
+    clusters= list()
+    for i in range(c):
+        clusters.append(list())
+        clusters[i].append([])
+        clusters[i].append([])
+    gm = GaussianMixture(n_components=c)
+    X=np.reshape(x,(-1,1))
+    labels=gm.fit_predict(X,y)
+    for i in range(len(x)):
+        clusters[labels[i]][0].append(x[i])
+        clusters[labels[i]][1].append(y[i])
+    return clusters
+
+
+def MultiRC(operating_points,number_of_clusters,i=10,patience=10,maxIterations=5000, preformatted=False): 
+    temp_TRS=0
+    TRS=0
+    clusters=list()
+    for j in range(i):
+        temp_clusters, temp_TRS= RC(operating_points,number_of_clusters,patience,maxIterations,preformatted)
+        if temp_TRS>TRS:
+            TRS=temp_TRS
+            clusters=temp_clusters
+        print("Iteration",j," completed")
+    print("Clustering completed")
+    return clusters, TRS
+
+def RC (operating_points,number_of_clusters,patience=10,maxIterations=5000, preformatted=False):
+    x=[]
+    y=[]
+    if preformatted==False:
+        for p in operating_points:   #get Input as x array, Ouput as y array by header
+            if '(in)' in p:
+                for i in operating_points[p]:
+                    x.append(i)
+        for p in operating_points:
+            if '(out)' in p:
+                for i in operating_points[p]:
+                    y.append(i)
+        if len(x)<1:    #if header is misspelled or similar, use first column as Input and second column as Ouput data
+            for i in operating_points.iloc[:,0]:
+                x.append(i)
+        if len(y)<1:
+            for i in operating_points.iloc[:,1]:
+                y.append(i)
+    if preformatted==True:
+        for i in operating_points[0]:
+            x.append(i)
+        for i in operating_points[1]:
+            y.append(i)
+    clusters= list() #list with all clusters
+    functions= list() #list with a foundational function for each cluster
+    TRS=0 #Total Regression Squared
+    CTRS=0 #Compared Total Regression Squared
+    patience_count=0
+    patientTRS=[]
+    patientClusters=[]
+    #initialize clusters with random assignment of datapoints to clusters
+    X = np.reshape(x,(-1,1)) #reshaping for regression function
+    for i in range(number_of_clusters):
+        clusters.append(list())
+    for i in range(len(X)):
+        clusters[np.random.randint(0,number_of_clusters)].append([X[i],y[i]])
+    #initial linear regression for each cluster
+    for cluster in clusters:
+        cx=list()#structure all data points in a cluster as x and y lists for regression function
+        cy=list()
+        for i in cluster:
+            cx.append(i[0])
+            cy.append(i[1])
+        function=LinearRegression().fit(cx,cy)
+        functions.append(function)
+        TRS+=function.score(cx,cy)
+    for i in range(maxIterations): #repeats until either best clusters are returned or maximum Iterations are reached to prevent running endlessly
+        #calculates minimal distanced cluster for each data point and assigns data point to it
+        for c_idx, cluster in enumerate(clusters):
+            for i in cluster:
+                minDistance=(max(y)-min(y))**2+1
+                bestfct=0
+                for f_idx,function in enumerate(functions):
+                    distance= (abs(function.predict(np.reshape(i[0],(-1,1)))-i[1]))**2
+                    if distance < minDistance :
+                        minDistance=distance
+                        bestfct=f_idx
+                if bestfct!=c_idx:
+                    cluster.remove(i)
+                    clusters[bestfct].append(i)
+        #New regression with changed clusters
+        functions.clear()
+        for cluster in clusters:
+            cx=list()
+            cy=list()
+            for i in cluster:
+                cx.append(i[0])
+                cy.append(i[1])
+            function=LinearRegression().fit(cx,cy)
+            functions.append(function)
+            CTRS+=function.score(cx,cy)
+        if CTRS<=TRS : #if Total Regression Squared is maximized, regression clustering is complete
+            patience_count+=1
+            patientTRS.append(TRS)
+            patientClusters.append(clusters)
+            if patience_count >= patience: #only finishes if TRS does not improve patience times
+                TRS=max(patientTRS)
+                clusters=patientClusters[np.argmax(patientTRS)]
+                print("TRS:", TRS)
+                for cluster in clusters: #Bringing clusters in proper format
+                    X_Values=[]
+                    Y_Values=[]
+                    for i in cluster:
+                        X_Values.append(i[0][0])
+                        Y_Values.append(i[1])
+                    cluster.clear()
+                    cluster.append(X_Values)
+                    cluster.append(Y_Values)
+                return clusters, TRS
+        TRS=CTRS
+        CTRS=0
+    print("No solution found in maximum number of Iterations.")
+
+
+rc_clusters=CombiClustering(operating_points_1)

automog/code/modeling/RegressionClustering/RegressionClustering.py

+# -*- coding: utf-8 -*-
+"""
+Created on Wed Jan 27 08:53:02 2021
+
+@author: niels
+"""
+
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Dec 11 10:58:32 2020
+
+@author: niels
+"""
+
+import numpy as np
+from sklearn.linear_model import LinearRegression
+import pandas as pd
+
+pandapoints=pd.read_excel(r"C:\Users\niels\Documents\LTT\RegressionClustering\GuDonetwo.xlsx")
+i=3 #number of iterations for Multiple Regression Clustering
+number_of_clusters=2 #number of clusters
+patience=10 #patience; algorithm finishes after not getting better patience times
+operating_points_0=pd.read_csv(r"C:\Users\niels\Documents\LTT\RegressionClustering\GuDonetwo.CSV", delimiter=";",decimal=",")
+operating_points_3=pd.read_csv(r"C:\Users\niels\Documents\LTT\RegressionClustering\GuD_3.CSV", delimiter=";",decimal=",")
+operating_points_1=pd.read_csv(r"C:\Users\niels\Documents\LTT\RegressionClustering\GuDkomplett.CSV", delimiter=";",decimal=",")
+operating_points_2=pd.read_csv(r"C:\Users\niels\Documents\LTT\RegressionClustering\operating_points_testLarge.CSV",delimiter=";",decimal=",")
+all_components={"component_0": {"state_0": operating_points_1}, "component_1": {"state_0": operating_points_2}}
+one_state={"state_0": operating_points_1}
+two_states={"state_0": operating_points_0, "state_1:":operating_points_3}
+colors=['red', 'blue','green','black','orange','cyan']
+def MultiRC(component,state,number_of_clusters,i=50,patience=10,maxIterations=5000, preformatted=False): 
+    temp_TRS=0
+    TRS=0
+    clusters=list()
+    for j in range(i):
+        temp_clusters, temp_TRS= RC(component,state, number_of_clusters,patience,maxIterations,preformatted)
+        if temp_TRS>TRS:
+            TRS=temp_TRS
+            clusters=temp_clusters
+        print("Iteration",j," completed")
+    print("Clustering completed")
+    return clusters
+
+def RC (component, state, number_of_clusters,patience=10,maxIterations=5000, preformatted=False):
+    x=[]
+    y=[]
+    if preformatted==False:
+        for p in component[state]:   #get Input as x array, Ouput as y array by header
+            if '(in)' in p:
+                for i in component[state][p]:
+                    x.append(i)
+        for p in component:
+            if '(out)' in p:
+                for i in component[state][p]:
+                    y.append(i)
+        if len(x)<1:    #if header is misspelled or similar, use first column as Input and second column as Ouput data
+            for i in component[state].iloc[:,0]:
+                x.append(i)
+        if len(y)<1:
+            for i in component[state].iloc[:,1]:
+                y.append(i)
+    if preformatted==True:
+        for i in component[state].iloc[:,0]:
+            x.append(i)
+        for i in component[state].iloc[:,1]:
+            y.append(i)
+    clusters= list() #list with all clusters
+    df_clusters=[] #list with all clusters as pandas DataFrames
+    functions= list() #list with a foundational function for each cluster
+    TRS=0 #Total Regression Squared
+    CTRS=0 #Compared Total Regression Squared
+    patience_count=0
+    patientTRS=[]
+    patientClusters=[]
+    #initialize clusters with random assignment of datapoints to clusters
+    X = np.reshape(x,(-1,1)) #reshaping for regression function
+    for i in range(number_of_clusters):
+        clusters.append(list())
+    for i in range(len(X)):
+        clusters[np.random.randint(0,number_of_clusters)].append([X[i],y[i]])
+    #initial linear regression for each cluster
+    for cluster in clusters:
+        cx=list()#structure all data points in a cluster as x and y lists for regression function
+        cy=list()
+        for i in cluster:
+            cx.append(i[0])
+            cy.append(i[1])
+        function=LinearRegression().fit(cx,cy)
+        functions.append(function)
+        TRS+=function.score(cx,cy)
+    for i in range(maxIterations): #repeats until either best clusters are returned or maximum Iterations are reached to prevent running endlessly
+        #calculates minimal distanced cluster for each data point and assigns data point to it
+        for c_idx, cluster in enumerate(clusters):
+            for i in cluster:
+                minDistance=(max(y)-min(y))**2+1
+                bestfct=0
+                for f_idx,function in enumerate(functions):
+                    distance= (abs(function.predict(np.reshape(i[0],(-1,1)))-i[1]))**2
+                    if distance < minDistance :
+                        minDistance=distance
+                        bestfct=f_idx
+                if bestfct!=c_idx:
+                    cluster.remove(i)
+                    clusters[bestfct].append(i)
+        #New regression with changed clusters
+        functions.clear()
+        for cluster in clusters:
+            cx=list()
+            cy=list()
+            for i in cluster:
+                cx.append(i[0])
+                cy.append(i[1])
+            function=LinearRegression().fit(cx,cy)
+            functions.append(function)
+            CTRS+=function.score(cx,cy)
+        if CTRS<=TRS : #if Total Regression Squared is maximized, regression clustering is complete
+            patience_count+=1
+            patientTRS.append(TRS)
+            patientClusters.append(clusters)
+            if patience_count >= patience: #only finishes if TRS does not improve patience times
+                TRS=max(patientTRS)
+                clusters=patientClusters[np.argmax(patientTRS)]
+                print("TRS:", TRS)
+                for cluster in clusters: #Bringing clusters in pandas format
+                    for i in cluster:
+                        i[0]=i[0][0]
+                    df=pd.DataFrame(cluster, columns=[0,1])
+                    df_clusters.append(df)
+                return df_clusters, TRS
+        TRS=CTRS
+        CTRS=0
+    print("No solution found in maximum number of Iterations.")
+    
+    
+df_list, TRS=MultiRC(two_states,'state_0',number_of_clusters,i,patience)
+for idx, df in enumerate(df_list):
+    df.plot.scatter(0,1, c=colors[idx])
+
+

automog/code/modeling/__init__.py

+from .clustering import Clusterer
+from .linearization import linearizer
+from .automog import AutoMoG, MOG_Component
+from .data_preprocessing import clean_operating_points, outlier_detection, reduce_operating_points
\ No newline at end of file