i

Simpy_Tutorial/eventlog.py

@@ -24,15 +24,10 @@ def add_end_event(process, event_id, end_timestamp):
 
 # add functions for adding events with their attributes to the log
 
-<<<<<<<< HEAD:simpy_tutorial/eventlog.py
-def export_to_csv(env, file_path):
-    event_log_df = pd.DataFrame.from_dict(env.bigeventlog)
-    event_log_df.to_csv(file_path)
-========
+
 def export_to_csv(process, file_path):
     event_log_df = pd.DataFrame.from_dict(process.event_log)
     event_log_df.to_csv(file_path, index=False)
->>>>>>>> 575b59827b5f928da4165070a4a1d51a85eed774:Simpy_Tutorial/eventlog.py
 
 def export_to_xes(process, file_path):
     # Use appropriate code to export to XES format

backend/agent.py

@@ -18,26 +18,44 @@ def q_learning(space, activities):
     case_space = env.observation_space['case'].nvec 
     event_space = env.observation_space['event'].n
     
-    for i in process_space: num_states *= i
-    for i in case_space: num_states *= i
-    num_states *= event_space
+    """
+    for i in process_space: num_states *= (i+1)
+    for i in case_space: num_states *= (i+1)
+    num_states *= event_space + 1
+    """
+    num_states = pow(2,14)
+
+    """
+    process_space = env.observation_space['process'] 
+    case_space = env.observation_space['case'] 
+    event_space = env.observation_space['event'] 
+
+    state_shape = []
+    for i in process_space: state_shape.append(i.n + 1)
+    for j in case_space: state_shape.append(j.n + 1)
+    state_shape.append(event_space.n)
+    state_shape = tuple(state_shape)
+    """
     
     num_actions = env.action_space.n
 
-    Q = np.zeros((num_states, num_actions), dtype=np.int16)
+
+    # Q = np.zeros(state_shape + (num_actions,), dtype=np.int8)
+    Q = np.zeros((num_states, num_actions), dtype = int)
 
     # Set the hyperparameters
     alpha = 0.1   # learning rate
-    gamma = 0.99  # discount factor
+    gamma = 0.1  # discount factor
     epsilon = 0.1 # exploration rate
 
     mean_time = 0
+    mean_reward = 0
 
     # Train the agent using Q-learning
-    num_episodes = 10
+    num_episodes = 100
     for episode in range(num_episodes):
         state, _ = env.reset()
-        state = env.flatten_observation(state)
+        state = env.flatten_observation_to_int(state)
         done = False
         start = env.process.env.now
         while not done:
@@ -52,12 +70,30 @@ def q_learning(space, activities):
             next_state, reward, done, _ = env.step(action)
 
             # Update the Q-value for the current state-action pair
-            Q[state][action] = Q[state][action] + alpha * (reward + gamma * np.max(Q[next_state]) - Q[state][action])
+            Q[state][action] = (1-alpha)*Q[state][action] + alpha * (reward + gamma * np.max(Q[next_state]) - Q[state][action])
+            #Q[state][action] = (1-alpha)*Q[state][action] + alpha*reward 
             
             # Transition to the next state
+            old_state = state
             state = next_state
             
+            """
+            if old_state != state: 
+                print(state)
+                print(action)
+                print(Q[state][action])
+            """
+            
+
         time = env.process.env.now - start 
         mean_time += time
+        mean_reward += reward
+
+
+        """
+        if (episode % 20 == 19):
+            mean_time /= 20 
+            print(f"Episode {episode-19} to episode {episode}: mean time = {mean_time}")
+        """
 
         print(f"Episode {episode}: time = {time}")
\ No newline at end of file

backend/environment.py

@@ -7,7 +7,6 @@ import simplesimmodel as model
 Environment for the RL agent
 """
 
-
 class BusinessProcessEnv(gym.Env):
 
     def __init__(self, space, activities):
@@ -73,19 +72,21 @@ class BusinessProcessEnv(gym.Env):
 
             case_obj = self.process.case_objects[self.process.case_id]
 
-            print(f"Agent did case {self.process.case_id} activity {action}.")
+            # print(f"Agent did case {self.process.case_id} activity {action}.")
 
             next_state = self.get_current_state(case_obj)
             self.current_state = next_state
-            next_state = self.flatten_observation(next_state)
+            next_state = self.flatten_observation_to_int(next_state)
 
-            self.reward += -(stop - start)
+            time = stop - start
+            reward = 10000 - time
+            self.reward += reward
             done = True if (len(self.process.done_cases) == 5 or len(self.process.active_cases) == 0) else False
             return next_state, self.reward, done, None
         
         else: 
-            self.reward += -100
-            next_state = self.flatten_observation(self.current_state)
+            self.reward += 0
+            next_state = self.flatten_observation_to_int(self.current_state)
             done = False
             return next_state, self.reward, done, None
     
@@ -125,4 +126,74 @@ class BusinessProcessEnv(gym.Env):
 
         return flattened
 
+    def flatten_observation_to_int(self, observation):
+        state = 0
+        state += observation['event']*pow(2,10)
+        state += observation['case'][1]*pow(2,2)
+        state += observation['case'][2]*pow(2,2)
+        event = observation['event']
+        if event == 0:
+            state += observation['process'][0]*pow(2,6)
+        elif event == 1:
+            state += observation['process'][1]*pow(2,6)
+        elif 1 < event <=3:
+            state += observation['process'][2]*pow(2,6)+observation['process'][3]*pow(2,7)+observation['process'][4]*pow(2,8)
+        elif 3 < event <=6:
+            state += observation['process'][5]*pow(2,6)+observation['process'][6]*pow(2,7)
+        elif 6 < event <= 8:
+            state += observation['process'][7]*pow(2,6)+observation['process'][8]*pow(2,7)+observation['process'][9]*pow(2,8)
+        elif 8 < event <= 11:
+            state += observation['process'][10]*pow(2,6)+observation['process'][11]*pow(2,7)+observation['process'][12]*pow(2,8)
+        elif 11 < event <= 14:
+            state += observation['process'][0]*pow(2,6)
+        else:
+            pass
+        
+        return state
 
+def main():
+    process = [] 
+    num_s = 1
+    process.append(num_s)
+    num_ot = 5
+    process.append(num_ot)
+    num_sh_a = 3
+    process.append(num_sh_a)
+    num_sh_b = 3
+    process.append(num_sh_b)
+    num_sh_c = 3
+    process.append(num_sh_c)
+    num_m_a = 3
+    process.append(num_m_a)
+    num_m_b = 2
+    process.append(num_m_b)
+    num_p_a = 4
+    process.append(num_p_a)
+    num_p_b = 5
+    process.append(num_p_b)
+    num_p_c = 4
+    process.append(num_p_c)
+    num_ds_a = 8
+    process.append(num_ds_a)
+    num_ds_b = 8
+    process.append(num_ds_b)
+    num_ds_c = 8
+    process.append(num_ds_c)
+
+    case = []
+    for i in range(15):
+        case.append(1)
+        
+    space = [process, case]
+    activities = 16
+
+    env = BusinessProcessEnv(space, activities)
+    state = env.current_state
+    flattened = env.flatten_observation_to_int(state)
+    print(flattened)
+    for value in range(env.action_space.n):
+        print(value)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

backend/eventlog.py

 import pandas as pd
 import simplesimmodel as model
+import numpy as np
 
 """
 Event log generator for our simulation model:
 - generate an event log
 - update an event log (adding new events)
+- export event log
+- get current state of an event log
 """
 
 def add_start_event(process, event_id, case_id, activity, start_timestamp):
+    process.event_log.append(event_id)
     process.event_log[event_id] = {
         'CaseID': case_id,
         'Activity': activity,
         'StartTimestamp': float(start_timestamp),
         'EndTimestamp': None
     }
+    process.event_counter += 1
 
 def add_end_event(process, event_id, end_timestamp):
-    # if event_id in process.event_log:
     event = process.event_log[event_id]
     event['EndTimestamp'] = end_timestamp
-    # process.event_log.append(event)
-    # del process.event_log[event_id]
-
-# add functions for adding events with their attributes to the log
 
 
 def export_to_csv(process, file_path):
     event_log_df = pd.DataFrame.from_dict(process.event_log)
     event_log_df.to_csv(file_path, index=False)
 
-def export_to_csv(env, file_path):
-    event_log_df = pd.DataFrame.from_dict(env.bigeventlog)
-    event_log_df.to_csv(file_path)
-
 
 def export_to_xes(process, file_path):
     # Use appropriate code to export to XES format
     pass
+
+def get_active_cases():
+    event_log = pd.read_csv(r'D:\test\optis.csv')
+    active_cases = event_log.groupby('CaseID').filter(lambda x: 'order completed' not in x['Activity'].values)['CaseID'].unique().tolist()
+    return active_cases
+
+
+def get_state(case_id):
+
+    process = [] 
+    num_s = 1
+    process.append(num_s)
+    num_ot = 5
+    process.append(num_ot)
+    num_sh_a = 3
+    process.append(num_sh_a)
+    num_sh_b = 3
+    process.append(num_sh_b)
+    num_sh_c = 3
+    process.append(num_sh_c)
+    num_m_a = 3
+    process.append(num_m_a)
+    num_m_b = 2
+    process.append(num_m_b)
+    num_p_a = 4
+    process.append(num_p_a)
+    num_p_b = 5
+    process.append(num_p_b)
+    num_p_c = 4
+    process.append(num_p_c)
+    num_ds_a = 8
+    process.append(num_ds_a)
+    num_ds_b = 8
+    process.append(num_ds_b)
+    num_ds_c = 8
+    process.append(num_ds_c)
+
+    case = []
+    for i in range(15):
+        case.append(0)
+
+    activity_mapping = {
+        'place order': 1,
+        'arrange standard order': 2,
+        'arrange custom order': 3,
+        'pick from stock A': 4, 
+        'pick from stock B': 5, 
+        'pick from stock C': 6, 
+        'manufacture A': 7, 
+        'manufacture B': 8, 
+        'pack A': 9,
+        'pack B': 10,
+        'pack C': 11,
+        'attempt delivery A': 12,
+        'attempt delivery B': 13,
+        'attempt delivery C': 14,
+        'order completed': 15,
+    }
+
+    event_log = pd.read_csv(r'D:\test\optis.csv')
+    # Sort the event log by case ID and start timestamp
+    event_log.sort_values(by=['CaseID', 'StartTimestamp'], inplace=True)
+
+    # Group the event log by case ID and get the last activity for each case
+    last_activities = event_log.groupby('CaseID').tail(1).reset_index()
+   
+    # Remap the activity names to numbers using the mapping dictionary
+    last_activities['Activity'] = last_activities['Activity'].map(activity_mapping)
+
+    # Filter the cases where the end timestamp of the last activity is None or empty
+    unfinished_cases = last_activities[last_activities['EndTimestamp'].isnull()]['CaseID'].tolist()
+
+    # Update the state of the ressources given all unfinished cases
+    for i in unfinished_cases:
+        activity = last_activities[last_activities['CaseID'] == i]['Activity'].values[0]
+        if activity == 1 or activity == 15:
+            process[0] -= 1
+        elif activity == 2 or activity == 3:
+            process[1] -= 1
+        else:
+            process[activity-2] -= 1
+
+    # Get the state of the case for the given Case ID
+    filtered_log = event_log[event_log['CaseID'] == case_id]
+    activities = filtered_log['Activity'].map(activity_mapping).tolist()
+    for i in activities:
+        case[i-1] += 1
+
+    # Get the last event for the given Case ID
+    event = last_activities[last_activities['CaseID'] == case_id]['Activity'].values[0]
+
+    state = {
+        'process': process,
+        'case': case,
+        'event': event
+    }
+
+    print(state)
+
+    """
+    flattened = []
+    for i in state['process']: flattened.append(i)
+    for j in state['case']: flattened.append(j)
+    flattened.append(state['event'])
+    
+
+    flattened = 0
+    flattened += state['event']
+    for i in state['case']: flattened += i
+    for j in state['process']: flattened += j*process[j]
+
+    print(flattened)
+    """
+    flat_state = 0
+    flat_state += state['event']*pow(2,10)
+    print(flat_state)
+    flat_state += state['case'][1]*pow(2,1)
+    flat_state += state['case'][2]*pow(2,2)
+    event = state['event']
+    if event == 0:
+        flat_state += state['process'][0]*pow(2,6)
+    elif event == 1:
+        flat_state += state['process'][1]*pow(2,6)
+    elif 1 < event <=3:
+        flat_state += state['process'][2]*pow(2,6)+state['process'][3]*pow(2,7)+state['process'][4]*pow(2,8)
+    elif 3 < event <=6:
+        flat_state += state['process'][5]*pow(2,6)+state['process'][6]*pow(2,7)
+    elif 6 < event <= 8:
+        flat_state += state['process'][7]*pow(2,6)+state['process'][8]*pow(2,7)+state['process'][9]*pow(2,8)
+    elif 8 < event <= 11:
+        flat_state += state['process'][10]*pow(2,6)+state['process'][11]*pow(2,7)+state['process'][12]*pow(2,8)
+    elif 11 < event <= 14:
+        flat_state += state['process'][0]*pow(2,6)
+    else:
+        pass
+    
+    print(flat_state)
+    return flat_state

backend/input.py

+import pandas as pd
+

backend/main.py

+import simpy
+import random
+import numpy as np
+import simplesimmodel as model
+import environment 
+import agent
+import eventlog as log
+import pandas as pd
+
+def main():
+    # Setup
+    # we can use a random seed if we want to generate the same results every time (maybe useful later for the training)
+    # random.seed(42)
+    # initialize the number of resources
+
+    process = [] 
+    num_s = 1
+    process.append(num_s)
+    num_ot = 5
+    process.append(num_ot)
+    num_sh_a = 3
+    process.append(num_sh_a)
+    num_sh_b = 3
+    process.append(num_sh_b)
+    num_sh_c = 3
+    process.append(num_sh_c)
+    num_m_a = 3
+    process.append(num_m_a)
+    num_m_b = 2
+    process.append(num_m_b)
+    num_p_a = 4
+    process.append(num_p_a)
+    num_p_b = 5
+    process.append(num_p_b)
+    num_p_c = 4
+    process.append(num_p_c)
+    num_ds_a = 7
+    process.append(num_ds_a)
+    num_ds_b = 7
+    process.append(num_ds_b)
+    num_ds_c = 7
+    process.append(num_ds_c)
+
+    case = []
+    for i in range(15):
+        case.append(1)
+        
+    space = [process, case]
+    activities = 16
+
+    # q learning
+    Q = agent.q_learning(space, activities)
+    # print(Q)
+
+    # generate event log
+    env = simpy.Environment()
+    business_process = model.BusinessProcess(env, process)
+    business_process.event_log_flag = True
+    env.process(model.run_process(env, business_process))
+    env.run(until = 10000)
+    log.export_to_csv(business_process, r'D:\test\optis.csv')
+
+    # extract active cases from event log
+    active_cases = log.get_active_cases()
+    print(active_cases)
+
+    # test agent 
+    for i in range(20):
+        caseid = random.choice(active_cases)
+        state = log.get_state(caseid)
+
+        action = np.argmax(Q[state])
+        print(action)
+        #print(Q)
+        print(Q[state])
+
+    state = Q[0]
+    action = np.argmax(state)
+    print(action)
+    print(state)
+
+    state = Q[64]
+    action = np.argmax(state)
+    print(action)
+    print(state)
+    
+
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file