merged with deep-rl

Simpy_Tutorial/processmodel.py

@@ -251,15 +251,6 @@ def main():
     print(state['event'])
     print(business_env.flatten_observation(state))
     rl.q_learning(space, activities)
-<<<<<<<< HEAD:Simpy_Tutorial/processmodel.py
-    path = r"C:\Users\nourm\OneDrive\Desktop\Nour\optis_app\out.csv"
-    eventlog.export_to_csv(business_env.process, path)
-========
-
-    el = business_env.bigeventlog
-
-    # eventlog.export_to_csv(business_env, r'D:\test\optis.csv')
->>>>>>>> origin/aleks2:simpy_tutorial/processmodel.py
     
 
 

backend/agent.py

@@ -15,15 +15,14 @@ def q_learning(space, activities):
     num_states = 1
 
     process_space = env.observation_space['process'].nvec 
-    case_space = env.observation_space['case'].nvec 
+    # case_space = env.observation_space['case'].nvec 
     event_space = env.observation_space['event'].n
     
-    """
-    for i in process_space: num_states *= (i+1)
-    for i in case_space: num_states *= (i+1)
-    num_states *= event_space + 1
-    """
-    num_states = pow(2,14)
+    for i in process_space: num_states *= i
+    # for i in case_space: num_states *= (i+1)
+    num_states *= event_space 
+    
+    # num_states = pow(2,14)
 
     """
     process_space = env.observation_space['process'] 
@@ -41,7 +40,7 @@ def q_learning(space, activities):
 
 
     # Q = np.zeros(state_shape + (num_actions,), dtype=np.int8)
-    Q = np.zeros((num_states, num_actions), dtype = int)
+    Q = np.zeros((num_states, num_actions), dtype = np.int64)
 
     # Set the hyperparameters
     alpha = 0.1   # learning rate
@@ -77,12 +76,7 @@ def q_learning(space, activities):
             old_state = state
             state = next_state
             
-            """
-            if old_state != state: 
-                print(state)
-                print(action)
-                print(Q[state][action])
-            """
+            # comment
             
 
         time = env.process.env.now - start 

backend/checkenv.py

+from stable_baselines3.common.env_checker import check_env
+import environment
+
+process = [] 
+num_s = 1
+process.append(num_s+1)
+num_ot = 5
+process.append(num_ot+1)
+num_sh_a = 3
+process.append(num_sh_a+1)
+num_sh_b = 3
+process.append(num_sh_b+1)
+num_sh_c = 3
+process.append(num_sh_c+1)
+num_m_a = 3
+process.append(num_m_a+1)
+num_m_b = 2
+process.append(num_m_b+1)
+num_p_a = 4
+process.append(num_p_a+1)
+num_p_b = 5
+process.append(num_p_b+1)
+num_p_c = 4
+process.append(num_p_c+1)
+num_ds_a = 7
+process.append(num_ds_a+1)
+num_ds_b = 7
+process.append(num_ds_b+1)
+num_ds_c = 7
+process.append(num_ds_c+1)
+
+case = []
+for i in range(15):
+    case.append(2)
+    
+space = [process, case]
+activities = 16
+
+env = environment.BusinessProcessEnv(space, activities)
+
+check_env(env)
+

backend/dqn.py

+import numpy as np
+import gymnasium as gym
+
+import environment 
+
+from stable_baselines3 import PPO, DQN
+import os
+import time
+
+def train(space, activities):
+    env = environment.BusinessProcessEnv(space, activities)
+    env.reset()
+
+    models_dir = f"models/{int(time.time())}/"
+    logdir = f"logs/{int(time.time())}/"
+
+    if not os.path.exists(models_dir):
+        os.makedirs(models_dir)
+
+    if not os.path.exists(logdir):
+        os.makedirs(logdir)
+
+    # model = PPO('MultiInputPolicy', env, verbose=1, tensorboard_log=logdir)
+    model = DQN('MultiInputPolicy', env, buffer_size = 50000000 ,verbose=1, exploration_fraction = 0.8, gamma=0.1, learning_starts = 10000, tensorboard_log=logdir)
+
+    TIMESTEPS = 1250000
+    iters = 0
+    while True:
+        iters += 1
+        model.learn(total_timesteps=TIMESTEPS, reset_num_timesteps=False, tb_log_name=f"DQN")
+        model.save(f"{models_dir}/{TIMESTEPS*iters}")
+
+
+def deploy(state):
+    model = DQN.load(r"models\other\1000000.zip")
+    action, _ = model.predict(state, deterministic=True)
+
+    return action
+
+

backend/environment.py

 import gymnasium as gym
 import numpy as np
 import simpy
-import simplesimmodel as model
+import simplesimmodel as simmodel
+from collections import OrderedDict
+import math
 
 """
 Environment for the RL agent
@@ -24,93 +26,133 @@ class BusinessProcessEnv(gym.Env):
 
         self.action_space = gym.spaces.Discrete(self.activities)
 
-        self.current_state = {
-            'process': np.array(self.ressources),
-            'case': np.zeros(len(self.case), dtype=int),
-            'event': 0
-        }
+        self.current_state = OrderedDict()
+        self.current_state['case'] = np.zeros(len(self.case), dtype=int)
+        self.current_state['event'] = 0
+        self.current_state['process'] = np.zeros(len(self.ressources), dtype=int)
+        for i in range(len(self.current_state['process'])):
+            self.current_state['process'][i] += (self.ressources[i]-1)
 
         self.model_env = simpy.Environment()
-        self.process = model.BusinessProcess(self.model_env, self.ressources)
-        self.model_env.process(model.run_process(self.model_env, self.process))
-        # self.done_cases = set([])
+        self.process = simmodel.BusinessProcess(self.model_env, self.ressources)
+        self.model_env.process(simmodel.run_process(self.model_env, self.process))
 
         self.reward = 0
 
 
     def get_current_state(self, caseid):
-        process, case, event = model.get_current_state(self.process, caseid)
-        state = {
-            'process': process,
-            'case': case,
-            'event': event
-        }
+        process, case, event = simmodel.get_current_state(self.process, caseid)
+        state = OrderedDict()
+        state['case'] = np.asarray(case) 
+        state['event'] = event
+        state['process'] = np.asarray(process)
         return state
 
     def step(self, action):
         
         self.process.next = action
         
+        '''
         if self.process.case_id in self.process.done_cases:
-            self.process.case_id = np.random.choice(self.process.active_cases)
+            # self.process.case_id = np.random.choice(self.process.active_cases)
+            self.process.case_id = self.process.case_counter
+
+            self.process.case_counter += 1
+            if self.process.case_id >= len(self.process.case_objects):
+                case_obj = simmodel.Case(self.process.case_id)
+
+                # add the case to the process's active cases list
+                self.process.case_objects.append(case_obj)
+                self.process.active_cases.append(self.process.case_id) 
+
+            self.model_env.process(simmodel.execute_case(self.model_env, self.process.case_id, self.process))
+        '''
+
+        if self.process.case_id == None or self.process.case_id in self.process.done_cases:
+            early_case = self.process.active_cases[0]
+            early = 15
+            for next_case in self.process.active_cases:
+                next_obj = self.process.case_objects[next_case]
+                if next_obj.current < early and next_obj.current > 1:
+                    early_case = next_obj
+                    early = next_obj.current
+
+            self.process.case_id = early_case.case_id
+           
         
         case_obj = self.process.case_objects[self.process.case_id]
         self.current_state = self.get_current_state(case_obj)
 
-        # print(f"{self.current_state['event']}")
-
         start = self.process.env.now
 
         self.process.flag = True
 
         if self.process.is_valid(self.current_state['event'], action, case_obj):
 
+            print(f"{self.current_state}")
+            
             while(self.process.flag):
                 self.model_env.step()
 
             stop = self.process.env.now
 
-            case_obj = self.process.case_objects[self.process.case_id]
+            #case_obj = self.process.case_objects[self.process.case_id]
 
             # print(f"Agent did case {self.process.case_id} activity {action}.")
 
             next_state = self.get_current_state(case_obj)
             self.current_state = next_state
-            next_state = self.flatten_observation_to_int(next_state)
-
+            # next_state = self.flatten_observation_to_int(next_state)
             time = stop - start
-            reward = 10000 - time
+            if time == 0:
+                reward = 4
+            else:
+                reward = 4.5 - math.log(time, 10)
             self.reward += reward
-            done = True if (len(self.process.done_cases) == 5 or len(self.process.active_cases) == 0) else False
-            return next_state, self.reward, done, None
+            done = True if (len(self.process.done_cases) == 10 or len(self.process.active_cases) == 0) else False
+            truncated = False
+            info = {}
+            return next_state, reward, done, truncated, info # either self.reward or just reward ???
         
         else: 
-            self.reward += 0
-            next_state = self.flatten_observation_to_int(self.current_state)
+            reward = -0.1
+            self.reward += reward
+            # next_state = self.flatten_observation_to_int(self.current_state)
+            next_state = self.current_state
             done = False
-            return next_state, self.reward, done, None
+            truncated = False
+            info = {}
+            return next_state, reward, done, truncated, info
     
          
     def reset(self, seed=None, options=None):
         # Reset the environment to the initial state
-        # Implement a function which extracts the current state from an event log / simulation model
+        # Implement a function which extracts the current state from an event log / simulation simmodel
         super().reset(seed=seed)
 
-        self.current_state = {
-            'process': np.array(self.ressources),
-            'case': np.zeros(len(self.case), dtype=int),
-            'event': 0
-        }
+        self.current_state = OrderedDict()
+        self.current_state['case'] = np.zeros(len(self.case), dtype=int)
+        self.current_state['event'] = 0
+        self.current_state['process'] = np.zeros(len(self.ressources), dtype=int)
+        for i in range(len(self.current_state['process'])):
+            self.current_state['process'][i] += (self.ressources[i]-1)
+
+        observation = self.current_state
+
         self.current_step = 0
 
         self.model_env = simpy.Environment()
-        self.process = model.BusinessProcess(self.model_env, self.ressources)
-        self.model_env.process(model.run_process(self.model_env, self.process))
+        self.process = simmodel.BusinessProcess(self.model_env, self.ressources)
+        self.model_env.process(simmodel.run_process(self.model_env, self.process))
+        rand_time = np.random.randint(10,2000)
+        self.model_env.run(until = rand_time)
         self.process.done_cases = set([])
 
         self.reward = 0
 
-        return self.current_state, None
+        info = {}
+
+        return observation, info
         
 
     def render(self, mode='human'):
@@ -154,45 +196,40 @@ class BusinessProcessEnv(gym.Env):
 def main():
     process = [] 
     num_s = 1
-    process.append(num_s)
+    process.append(num_s+1)
     num_ot = 5
-    process.append(num_ot)
+    process.append(num_ot+1)
     num_sh_a = 3
-    process.append(num_sh_a)
+    process.append(num_sh_a+1)
     num_sh_b = 3
-    process.append(num_sh_b)
+    process.append(num_sh_b+1)
     num_sh_c = 3
-    process.append(num_sh_c)
+    process.append(num_sh_c+1)
     num_m_a = 3
-    process.append(num_m_a)
+    process.append(num_m_a+1)
     num_m_b = 2
-    process.append(num_m_b)
+    process.append(num_m_b+1)
     num_p_a = 4
-    process.append(num_p_a)
+    process.append(num_p_a+1)
     num_p_b = 5
-    process.append(num_p_b)
+    process.append(num_p_b+1)
     num_p_c = 4
-    process.append(num_p_c)
+    process.append(num_p_c+1)
     num_ds_a = 8
-    process.append(num_ds_a)
+    process.append(num_ds_a+1)
     num_ds_b = 8
-    process.append(num_ds_b)
+    process.append(num_ds_b+1)
     num_ds_c = 8
-    process.append(num_ds_c)
+    process.append(num_ds_c+1)
 
     case = []
     for i in range(15):
-        case.append(1)
+        case.append(2)
         
     space = [process, case]
     activities = 16
 
     env = BusinessProcessEnv(space, activities)
-    state = env.current_state
-    flattened = env.flatten_observation_to_int(state)
-    print(flattened)
-    for value in range(env.action_space.n):
-        print(value)
 
 
 if __name__ == "__main__":

backend/eventlog.py

 import pandas as pd
 import simplesimmodel as model
 import numpy as np
+from collections import OrderedDict
+import simpy
 
 """
 Event log generator for our simulation model:
@@ -29,7 +31,6 @@ def export_to_csv(process, file_path):
     event_log_df = pd.DataFrame.from_dict(process.event_log)
     event_log_df.to_csv(file_path, index=False)
 
-
 def export_to_xes(process, file_path):
     # Use appropriate code to export to XES format
     pass
@@ -40,39 +41,37 @@ def get_active_cases():
     return active_cases
 
 
-def get_state(case_id):
+def get_state(case_id,):
 
-    process = [] 
+    process = np.zeros(13, dtype=int)
     num_s = 1
-    process.append(num_s)
-    num_ot = 5
-    process.append(num_ot)
-    num_sh_a = 3
-    process.append(num_sh_a)
-    num_sh_b = 3
-    process.append(num_sh_b)
-    num_sh_c = 3
-    process.append(num_sh_c)
-    num_m_a = 3
-    process.append(num_m_a)
-    num_m_b = 2
-    process.append(num_m_b)
-    num_p_a = 4
-    process.append(num_p_a)
-    num_p_b = 5
-    process.append(num_p_b)
-    num_p_c = 4
-    process.append(num_p_c)
-    num_ds_a = 8
-    process.append(num_ds_a)
-    num_ds_b = 8
-    process.append(num_ds_b)
-    num_ds_c = 8
-    process.append(num_ds_c)
-
-    case = []
-    for i in range(15):
-        case.append(0)
+    process[0] = num_s
+    num_ot = 4
+    process[1] = num_ot
+    num_sh_a = 2
+    process[2] = num_sh_a
+    num_sh_b = 2
+    process[3] = num_sh_b
+    num_sh_c = 2
+    process[4] = num_sh_c
+    num_m_a = 4
+    process[5] = num_m_a
+    num_m_b = 10
+    process[6] = num_m_b
+    num_p_a = 2
+    process[7] = num_p_a
+    num_p_b = 3
+    process[8] = num_p_b
+    num_p_c = 3
+    process[9] = num_p_c
+    num_ds_a = 25
+    process[10] = num_ds_a
+    num_ds_b = 40
+    process[11] = num_ds_b
+    num_ds_c = 45
+    process[12] = num_ds_c
+
+    case = np.zeros(15, dtype=int)
 
     activity_mapping = {
         'place order': 1,
@@ -124,50 +123,56 @@ def get_state(case_id):
     # Get the last event for the given Case ID
     event = last_activities[last_activities['CaseID'] == case_id]['Activity'].values[0]
 
+    """
     state = {
         'process': process,
         'case': case,
         'event': event
     }
-
-    print(state)
-
     """
-    flattened = []
-    for i in state['process']: flattened.append(i)
-    for j in state['case']: flattened.append(j)
-    flattened.append(state['event'])
 
+    state = OrderedDict()
+    state['case'] = case
+    state['event'] = event
+    state['process'] = process
 
-    flattened = 0
-    flattened += state['event']
-    for i in state['case']: flattened += i
-    for j in state['process']: flattened += j*process[j]
+    print(state)
 
-    print(flattened)
-    """
-    flat_state = 0
-    flat_state += state['event']*pow(2,10)
-    print(flat_state)
-    flat_state += state['case'][1]*pow(2,1)
-    flat_state += state['case'][2]*pow(2,2)
-    event = state['event']
-    if event == 0:
-        flat_state += state['process'][0]*pow(2,6)
-    elif event == 1:
-        flat_state += state['process'][1]*pow(2,6)
-    elif 1 < event <=3:
-        flat_state += state['process'][2]*pow(2,6)+state['process'][3]*pow(2,7)+state['process'][4]*pow(2,8)
-    elif 3 < event <=6:
-        flat_state += state['process'][5]*pow(2,6)+state['process'][6]*pow(2,7)
-    elif 6 < event <= 8:
-        flat_state += state['process'][7]*pow(2,6)+state['process'][8]*pow(2,7)+state['process'][9]*pow(2,8)
-    elif 8 < event <= 11:
-        flat_state += state['process'][10]*pow(2,6)+state['process'][11]*pow(2,7)+state['process'][12]*pow(2,8)
-    elif 11 < event <= 14:
-        flat_state += state['process'][0]*pow(2,6)
-    else:
-        pass
+    return state
 
-    print(flat_state)
-    return flat_state
+def generate_event_log(time):
+    ressources = [] 
+    num_s = 1
+    ressources.append(num_s+1)
+    num_ot = 4
+    ressources.append(num_ot+1)
+    num_sh_a = 2
+    ressources.append(num_sh_a+1)
+    num_sh_b = 2
+    ressources.append(num_sh_b+1)
+    num_sh_c = 2
+    ressources.append(num_sh_c+1)
+    num_m_a = 4
+    ressources.append(num_m_a+1)
+    num_m_b = 10
+    ressources.append(num_m_b+1)
+    num_p_a = 2
+    ressources.append(num_p_a+1)
+    num_p_b = 3
+    ressources.append(num_p_b+1)
+    num_p_c = 3
+    ressources.append(num_p_c+1)
+    num_ds_a = 25
+    ressources.append(num_ds_a+1)
+    num_ds_b = 40
+    ressources.append(num_ds_b+1)
+    num_ds_c = 45
+    ressources.append(num_ds_c+1)      
+    
+    # generate event log
+    env = simpy.Environment()
+    business_process = model.BusinessProcess(env, ressources)
+    business_process.event_log_flag = True
+    env.process(model.run_process(env, business_process))
+    env.run(until = time)
+    export_to_csv(business_process, r'D:\test\optis.csv')

backend/main.py

@@ -6,6 +6,10 @@ import environment
 import agent
 import eventlog as log
 import pandas as pd
+import dqn
+import test
+
+
 
 def main():
     # Setup
@@ -13,76 +17,68 @@ def main():
     # random.seed(42)
     # initialize the number of resources
 
-    process = [] 
+    ressources = [] 
     num_s = 1
-    process.append(num_s)
-    num_ot = 5
-    process.append(num_ot)
-    num_sh_a = 3
-    process.append(num_sh_a)
-    num_sh_b = 3
-    process.append(num_sh_b)
-    num_sh_c = 3
-    process.append(num_sh_c)
-    num_m_a = 3
-    process.append(num_m_a)
-    num_m_b = 2
-    process.append(num_m_b)
-    num_p_a = 4
-    process.append(num_p_a)
-    num_p_b = 5
-    process.append(num_p_b)
-    num_p_c = 4
-    process.append(num_p_c)
-    num_ds_a = 7
-    process.append(num_ds_a)
-    num_ds_b = 7
-    process.append(num_ds_b)
-    num_ds_c = 7
-    process.append(num_ds_c)
+    ressources.append(num_s+1)
+    num_ot = 4
+    ressources.append(num_ot+1)
+    num_sh_a = 2
+    ressources.append(num_sh_a+1)
+    num_sh_b = 2
+    ressources.append(num_sh_b+1)
+    num_sh_c = 2
+    ressources.append(num_sh_c+1)
+    num_m_a = 4
+    ressources.append(num_m_a+1)
+    num_m_b = 10
+    ressources.append(num_m_b+1)
+    num_p_a = 2
+    ressources.append(num_p_a+1)
+    num_p_b = 3
+    ressources.append(num_p_b+1)
+    num_p_c = 3
+    ressources.append(num_p_c+1)
+    num_ds_a = 25
+    ressources.append(num_ds_a+1)
+    num_ds_b = 40
+    ressources.append(num_ds_b+1)
+    num_ds_c = 45
+    ressources.append(num_ds_c+1)
 
     case = []
     for i in range(15):
-        case.append(1)
+        case.append(2)
         
-    space = [process, case]
+    space = [ressources, case]
     activities = 16
 
-    # q learning
-    Q = agent.q_learning(space, activities)
-    # print(Q)
+    # dqn.train(space, activities)
+    
     
     # generate event log
     env = simpy.Environment()
-    business_process = model.BusinessProcess(env, process)
+    business_process = model.BusinessProcess(env, ressources)
     business_process.event_log_flag = True
     env.process(model.run_process(env, business_process))
-    env.run(until = 10000)
+    env.run(until = 100)
     log.export_to_csv(business_process, r'D:\test\optis.csv')
     
     # extract active cases from event log
     active_cases = log.get_active_cases()
     print(active_cases)
 
-    # test agent 
+    
     for i in range(20):
-        caseid = random.choice(active_cases)
+        caseid = np.random.choice(active_cases)
+        # print(caseid)
+
         state = log.get_state(caseid)
 
-        action = np.argmax(Q[state])
-        print(action)
-        #print(Q)
-        print(Q[state])
+        print(dqn.deploy(state))
+    
 
-    state = Q[0]
-    action = np.argmax(state)
-    print(action)
-    print(state)
 
-    state = Q[64]
-    action = np.argmax(state)
-    print(action)
-    print(state)
+    test.test_agent()
     
     
 

backend/simplesimmodel.py

@@ -26,20 +26,20 @@ class BusinessProcess(object):
         self.env = env
 
         # initialize ressources
-        self.system = simpy.Resource(env, ressources[0])
-        self.order_taker = simpy.Resource(env, ressources[1])
-        self.stock_handler_a = simpy.Resource(env, ressources[2])
-        self.stock_handler_b = simpy.Resource(env, ressources[3])
-        self.stock_handler_c = simpy.Resource(env, ressources[4])
-        self.manufacturer_a = simpy.Resource(env, ressources[5])
-        self.manufacturer_b = simpy.Resource(env, ressources[6])
-        self.packer_a = simpy.Resource(env, ressources[7])
-        self.packer_b = simpy.Resource(env, ressources[8])
-        self.packer_c = simpy.Resource(env, ressources[9])
+        self.system = simpy.Resource(env, ressources[0] - 1)
+        self.order_taker = simpy.Resource(env, ressources[1] - 1)
+        self.stock_handler_a = simpy.Resource(env, ressources[2] - 1)
+        self.stock_handler_b = simpy.Resource(env, ressources[3] - 1)
+        self.stock_handler_c = simpy.Resource(env, ressources[4] - 1)
+        self.manufacturer_a = simpy.Resource(env, ressources[5] - 1)
+        self.manufacturer_b = simpy.Resource(env, ressources[6] - 1)
+        self.packer_a = simpy.Resource(env, ressources[7] - 1)
+        self.packer_b = simpy.Resource(env, ressources[8] - 1)
+        self.packer_c = simpy.Resource(env, ressources[9] - 1)
         # capacity of each delivery service instead of numbers of workers
-        self.delivery_service_a = simpy.Resource(env, ressources[10])
-        self.delivery_service_b = simpy.Resource(env, ressources[11])
-        self.delivery_service_c = simpy.Resource(env, ressources[12])
+        self.delivery_service_a = simpy.Resource(env, ressources[10] - 1)
+        self.delivery_service_b = simpy.Resource(env, ressources[11] - 1)
+        self.delivery_service_c = simpy.Resource(env, ressources[12] - 1)
     
         # initialize lists with active cases and all case objects
         self.active_cases = [0, 1, 2]
@@ -53,7 +53,7 @@ class BusinessProcess(object):
 
         self.next = 0
         
-        self.case_id = 0
+        self.case_id = None
 
         self.done_cases = set([])
 
@@ -61,10 +61,12 @@ class BusinessProcess(object):
         self.event_log = []
         self.event_counter = 0
 
+        self.case_counter = 3 # implementation where agent creates cases
+
         # random.seed(1)
 
     def place_order(self, case):
-        yield self.env.timeout(0)
+        yield self.env.timeout(2)
 
     def arrange_standard_order(self, case):
         yield self.env.timeout(random.randint(10, 15))
@@ -151,13 +153,16 @@ With that the order is completed.
 """
 def execute_case(env, case, process):
     # create a case object to keep track of case attributes if not already existing
-    if len(process.case_objects) <= case:
+    if case >= len(process.case_objects):
         case_obj = Case(case)
 
         # add the case to the process's active cases list
         process.case_objects.append(case_obj)
         process.active_cases.append(case) 
     
+
+    # case_obj = process.case_objects[case] - implementation with creation of cases by agent
+    
     # if the case is one of the first three choose it from the list
     if len(process.case_objects) <= 3:
         case_obj = process.case_objects[case]
@@ -530,6 +535,50 @@ def get_current_state(process, case):
 
     return process_state, cur_case, event
 
+def get_ressources(process):
+    process_state = []
+
+    num_system = process.system.capacity - process.system.count
+    process_state.append(num_system)
+
+    num_order_taker = process.order_taker.capacity - process.order_taker.count 
+    process_state.append(num_order_taker)
+
+    num_stock_handler_a = process.stock_handler_a.capacity - process.stock_handler_a.count 
+    process_state.append(num_stock_handler_a)
+
+    num_stock_handler_b = process.stock_handler_b.capacity - process.stock_handler_b.count 
+    process_state.append(num_stock_handler_b)
+
+    num_stock_handler_c = process.stock_handler_c.capacity - process.stock_handler_c.count 
+    process_state.append(num_stock_handler_c)
+
+    num_manufacturer_a = process.manufacturer_a.capacity - process.manufacturer_a.count 
+    process_state.append(num_manufacturer_a)
+
+    num_manufacturer_b = process.manufacturer_b.capacity - process.manufacturer_b.count 
+    process_state.append(num_manufacturer_b)
+
+    num_packer_a = process.packer_a.capacity - process.packer_a.count 
+    process_state.append(num_packer_a)
+
+    num_packer_b = process.packer_b.capacity - process.packer_b.count 
+    process_state.append(num_packer_b)
+
+    num_packer_c = process.packer_c.capacity - process.packer_c.count 
+    process_state.append(num_packer_c)
+
+    num_delivery_service_a = process.delivery_service_a.capacity - process.delivery_service_a.count 
+    process_state.append(num_delivery_service_a)
+
+    num_delivery_service_b = process.delivery_service_b.capacity - process.delivery_service_b.count
+    process_state.append(num_delivery_service_b) 
+
+    num_delivery_service_c = process.delivery_service_c.capacity - process.delivery_service_c.count 
+    process_state.append(num_delivery_service_c)
+
+    return process_state
+
 """
 Defines how often new orders (cases) come in and starts executing them
 """
@@ -540,14 +589,32 @@ def run_process(env, process):
     for case in range(3):
         env.process(execute_case(env, case, process))
 
+
     # the new incoming orders
-    while case < 1000:
-        waittime = random.randint(10,15)
+    while True:
+        while(len(process.active_cases) > 300):
+            yield env.timeout(1)
+
+        # waittime = random.randint(10,15)
+        waittime = 10
         if case % 20 == 0:
             waittime = 100
         yield env.timeout(waittime)  # Wait a bit before generating a new case
 
         case += 1
-        # process.active_cases.append(case)
         env.process(execute_case(env, case, process))
         
+        '''
+        # process.active_cases.append(case)
+        if process.case_counter >= len(process.case_objects):
+            case_obj = Case(process.case_counter)
+
+            # add the case to the process's active cases list
+            process.case_objects.append(case_obj)
+            process.active_cases.append(process.case_counter) 
+
+        env.process(execute_case(env, process.case_counter, process))
+        process.case_counter += 1
+        '''
+
+

backend/simtest.py

+import simpy
+import random
+import numpy as np
+import simplesimmodel as simmodel
+import environment 
+import agent
+import eventlog as log
+import pandas as pd
+import dqn
+
+
+
+def main():
+    # initialize the number of resources
+
+    ressources = [] 
+    num_s = 1
+    ressources.append(num_s+1)
+    num_ot = 4
+    ressources.append(num_ot+1)
+    num_sh_a = 2
+    ressources.append(num_sh_a+1)
+    num_sh_b = 2
+    ressources.append(num_sh_b+1)
+    num_sh_c = 2
+    ressources.append(num_sh_c+1)
+    num_m_a = 4
+    ressources.append(num_m_a+1)
+    num_m_b = 10
+    ressources.append(num_m_b+1)
+    num_p_a = 2
+    ressources.append(num_p_a+1)
+    num_p_b = 3
+    ressources.append(num_p_b+1)
+    num_p_c = 3
+    ressources.append(num_p_c+1)
+    num_ds_a = 25
+    ressources.append(num_ds_a+1)
+    num_ds_b = 40
+    ressources.append(num_ds_b+1)
+    num_ds_c = 40
+    ressources.append(num_ds_c+1)
+
+    model_env = simpy.Environment()
+    process = simmodel.BusinessProcess(model_env, ressources)
+    model_env.process(simmodel.run_process(model_env, process))
+
+    for i in range(2000):
+        model_env.step()
+        print(simmodel.get_ressources(process))
+
+
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

backend/test.py

+from collections import OrderedDict
+import numpy as np
+import dqn
+
+def test_agent():
+    case = [1,0,1,0,0,0,0,0,0,0,0,0,0,0,0]
+    event = 3
+    process = [1,4,2,2,2,4,0,2,3,3,20,20,20]
+
+    state = OrderedDict()
+    state['case'] = np.asarray(case) 
+    state['event'] = event
+    state['process'] = np.asarray(process)
+    print(state)
+
+    print(dqn.deploy(state))
\ No newline at end of file

keras-rl @ 216c3145

+Subproject commit 216c3145f3dc4d17877be26ca2185ce7db462bad