Skip to content
Snippets Groups Projects
Commit 99f04ac8 authored by Aleksandra Dimitrova's avatar Aleksandra Dimitrova
Browse files

backend

parent 43fdb271
No related branches found
No related tags found
No related merge requests found
No preview for this file type
No preview for this file type
File added
No preview for this file type
......@@ -18,26 +18,44 @@ def q_learning(space, activities):
case_space = env.observation_space['case'].nvec
event_space = env.observation_space['event'].n
for i in process_space: num_states *= i
for i in case_space: num_states *= i
num_states *= event_space
"""
for i in process_space: num_states *= (i+1)
for i in case_space: num_states *= (i+1)
num_states *= event_space + 1
"""
num_states = pow(2,14)
"""
process_space = env.observation_space['process']
case_space = env.observation_space['case']
event_space = env.observation_space['event']
state_shape = []
for i in process_space: state_shape.append(i.n + 1)
for j in case_space: state_shape.append(j.n + 1)
state_shape.append(event_space.n)
state_shape = tuple(state_shape)
"""
num_actions = env.action_space.n
Q = np.zeros((num_states, num_actions), dtype=np.int16)
# Q = np.zeros(state_shape + (num_actions,), dtype=np.int8)
Q = np.zeros((num_states, num_actions), dtype = int)
# Set the hyperparameters
alpha = 0.1 # learning rate
gamma = 0.99 # discount factor
gamma = 0.1 # discount factor
epsilon = 0.1 # exploration rate
mean_time = 0
mean_reward = 0
# Train the agent using Q-learning
num_episodes = 100
num_episodes = 1000
for episode in range(num_episodes):
state, _ = env.reset()
state = env.flatten_observation(state)
state = env.flatten_observation_to_int(state)
done = False
start = env.process.env.now
while not done:
......@@ -52,18 +70,39 @@ def q_learning(space, activities):
next_state, reward, done, _ = env.step(action)
# Update the Q-value for the current state-action pair
Q[state][action] = Q[state][action] + alpha * (reward + gamma * np.max(Q[next_state]) - Q[state][action])
Q[state][action] = (1-alpha)*Q[state][action] + alpha * (reward + gamma * np.max(Q[next_state]) - Q[state][action])
#Q[state][action] = (1-alpha)*Q[state][action] + alpha*reward
# Transition to the next state
old_state = state
state = next_state
"""
if old_state != state:
print(state)
print(action)
print(Q[state][action])
"""
time = env.process.env.now - start
mean_time += time
mean_reward += reward
"""
if (episode % 20 == 19):
mean_reward /= 20
mean_time /= 20
print(f"Episode {episode-19} to episode {episode}: mean time = {mean_time}")
"""
print(f"Episode {episode-19} to episode {episode}: mean time = {mean_time}, mean reward: {mean_reward}")
if episode == 19:
start_reward = mean_reward
# print(f"Episode {episode}: time = {time}, reward = {reward}")
if episode == 999:
end_reward = mean_reward
improvement = end_reward - start_reward
print(f"Reward improved by {improvement}")
print(f"Episode {episode}: time = {time}")
\ No newline at end of file
return Q
......@@ -7,7 +7,6 @@ import simplesimmodel as model
Environment for the RL agent
"""
class BusinessProcessEnv(gym.Env):
def __init__(self, space, activities):
......@@ -73,19 +72,21 @@ class BusinessProcessEnv(gym.Env):
case_obj = self.process.case_objects[self.process.case_id]
print(f"Agent did case {self.process.case_id} activity {action}.")
# print(f"Agent did case {self.process.case_id} activity {action}.")
next_state = self.get_current_state(case_obj)
self.current_state = next_state
next_state = self.flatten_observation(next_state)
next_state = self.flatten_observation_to_int(next_state)
self.reward += -(stop - start)
time = stop - start
reward = 10000 - time
self.reward += reward
done = True if (len(self.process.done_cases) == 5 or len(self.process.active_cases) == 0) else False
return next_state, self.reward, done, None
else:
self.reward += -100
next_state = self.flatten_observation(self.current_state)
self.reward += 0
next_state = self.flatten_observation_to_int(self.current_state)
done = False
return next_state, self.reward, done, None
......@@ -125,4 +126,74 @@ class BusinessProcessEnv(gym.Env):
return flattened
def flatten_observation_to_int(self, observation):
state = 0
state += observation['event']*pow(2,10)
state += observation['case'][1]*pow(2,2)
state += observation['case'][2]*pow(2,2)
event = observation['event']
if event == 0:
state += observation['process'][0]*pow(2,6)
elif event == 1:
state += observation['process'][1]*pow(2,6)
elif 1 < event <=3:
state += observation['process'][2]*pow(2,6)+observation['process'][3]*pow(2,7)+observation['process'][4]*pow(2,8)
elif 3 < event <=6:
state += observation['process'][5]*pow(2,6)+observation['process'][6]*pow(2,7)
elif 6 < event <= 8:
state += observation['process'][7]*pow(2,6)+observation['process'][8]*pow(2,7)+observation['process'][9]*pow(2,8)
elif 8 < event <= 11:
state += observation['process'][10]*pow(2,6)+observation['process'][11]*pow(2,7)+observation['process'][12]*pow(2,8)
elif 11 < event <= 14:
state += observation['process'][0]*pow(2,6)
else:
pass
return state
def main():
process = []
num_s = 1
process.append(num_s)
num_ot = 5
process.append(num_ot)
num_sh_a = 3
process.append(num_sh_a)
num_sh_b = 3
process.append(num_sh_b)
num_sh_c = 3
process.append(num_sh_c)
num_m_a = 3
process.append(num_m_a)
num_m_b = 2
process.append(num_m_b)
num_p_a = 4
process.append(num_p_a)
num_p_b = 5
process.append(num_p_b)
num_p_c = 4
process.append(num_p_c)
num_ds_a = 8
process.append(num_ds_a)
num_ds_b = 8
process.append(num_ds_b)
num_ds_c = 8
process.append(num_ds_c)
case = []
for i in range(15):
case.append(1)
space = [process, case]
activities = 16
env = BusinessProcessEnv(space, activities)
state = env.current_state
flattened = env.flatten_observation_to_int(state)
print(flattened)
for value in range(env.action_space.n):
print(value)
if __name__ == "__main__":
main()
\ No newline at end of file
import pandas as pd
import simplesimmodel as model
import numpy as np
"""
Event log generator for our simulation model:
- generate an event log
- update an event log (adding new events)
- export event log
- get current state of an event log
"""
def add_start_event(process, event_id, case_id, activity, start_timestamp):
process.event_log.append(event_id)
process.event_log[event_id] = {
'CaseID': case_id,
'Activity': activity,
'StartTimestamp': float(start_timestamp),
'EndTimestamp': None
}
process.event_counter += 1
def add_end_event(process, event_id, end_timestamp):
# if event_id in process.event_log:
event = process.event_log[event_id]
event['EndTimestamp'] = end_timestamp
# process.event_log.append(event)
# del process.event_log[event_id]
# add functions for adding events with their attributes to the log
def export_to_csv(env, file_path):
event_log_df = pd.DataFrame.from_dict(env.bigeventlog)
event_log_df.to_csv(file_path)
def export_to_csv(process, file_path):
event_log_df = pd.DataFrame.from_dict(process.event_log)
event_log_df.to_csv(file_path, index=False)
def export_to_xes(process, file_path):
# Use appropriate code to export to XES format
pass
def get_active_cases():
event_log = pd.read_csv(r'D:\test\optis.csv')
active_cases = event_log.groupby('CaseID').filter(lambda x: 'order completed' not in x['Activity'].values)['CaseID'].unique().tolist()
return active_cases
def get_state(case_id):
process = []
num_s = 1
process.append(num_s)
num_ot = 5
process.append(num_ot)
num_sh_a = 3
process.append(num_sh_a)
num_sh_b = 3
process.append(num_sh_b)
num_sh_c = 3
process.append(num_sh_c)
num_m_a = 3
process.append(num_m_a)
num_m_b = 2
process.append(num_m_b)
num_p_a = 4
process.append(num_p_a)
num_p_b = 5
process.append(num_p_b)
num_p_c = 4
process.append(num_p_c)
num_ds_a = 8
process.append(num_ds_a)
num_ds_b = 8
process.append(num_ds_b)
num_ds_c = 8
process.append(num_ds_c)
case = []
for i in range(15):
case.append(0)
activity_mapping = {
'place order': 1,
'arrange standard order': 2,
'arrange custom order': 3,
'pick from stock A': 4,
'pick from stock B': 5,
'pick from stock C': 6,
'manufacture A': 7,
'manufacture B': 8,
'pack A': 9,
'pack B': 10,
'pack C': 11,
'attempt delivery A': 12,
'attempt delivery B': 13,
'attempt delivery C': 14,
'order completed': 15,
}
event_log = pd.read_csv(r'D:\test\optis.csv')
# Sort the event log by case ID and start timestamp
event_log.sort_values(by=['CaseID', 'StartTimestamp'], inplace=True)
# Group the event log by case ID and get the last activity for each case
last_activities = event_log.groupby('CaseID').tail(1).reset_index()
# Remap the activity names to numbers using the mapping dictionary
last_activities['Activity'] = last_activities['Activity'].map(activity_mapping)
# Filter the cases where the end timestamp of the last activity is None or empty
unfinished_cases = last_activities[last_activities['EndTimestamp'].isnull()]['CaseID'].tolist()
# Update the state of the ressources given all unfinished cases
for i in unfinished_cases:
activity = last_activities[last_activities['CaseID'] == i]['Activity'].values[0]
if activity == 1 or activity == 15:
process[0] -= 1
elif activity == 2 or activity == 3:
process[1] -= 1
else:
process[activity-2] -= 1
# Get the state of the case for the given Case ID
filtered_log = event_log[event_log['CaseID'] == case_id]
activities = filtered_log['Activity'].map(activity_mapping).tolist()
for i in activities:
case[i-1] += 1
# Get the last event for the given Case ID
event = last_activities[last_activities['CaseID'] == case_id]['Activity'].values[0]
state = {
'process': process,
'case': case,
'event': event
}
print(state)
"""
flattened = []
for i in state['process']: flattened.append(i)
for j in state['case']: flattened.append(j)
flattened.append(state['event'])
flattened = 0
flattened += state['event']
for i in state['case']: flattened += i
for j in state['process']: flattened += j*process[j]
print(flattened)
"""
flat_state = 0
flat_state += state['event']*pow(2,10)
print(flat_state)
flat_state += state['case'][1]*pow(2,1)
flat_state += state['case'][2]*pow(2,2)
event = state['event']
if event == 0:
flat_state += state['process'][0]*pow(2,6)
elif event == 1:
flat_state += state['process'][1]*pow(2,6)
elif 1 < event <=3:
flat_state += state['process'][2]*pow(2,6)+state['process'][3]*pow(2,7)+state['process'][4]*pow(2,8)
elif 3 < event <=6:
flat_state += state['process'][5]*pow(2,6)+state['process'][6]*pow(2,7)
elif 6 < event <= 8:
flat_state += state['process'][7]*pow(2,6)+state['process'][8]*pow(2,7)+state['process'][9]*pow(2,8)
elif 8 < event <= 11:
flat_state += state['process'][10]*pow(2,6)+state['process'][11]*pow(2,7)+state['process'][12]*pow(2,8)
elif 11 < event <= 14:
flat_state += state['process'][0]*pow(2,6)
else:
pass
print(flat_state)
return flat_state
import pandas as pd
import simpy
import random
import numpy as np
import simplesimmodel as model
import environment
import agent
import eventlog as log
import pandas as pd
def main():
# Setup
# we can use a random seed if we want to generate the same results every time (maybe useful later for the training)
# random.seed(42)
# initialize the number of resources
process = []
num_s = 1
process.append(num_s)
num_ot = 5
process.append(num_ot)
num_sh_a = 3
process.append(num_sh_a)
num_sh_b = 3
process.append(num_sh_b)
num_sh_c = 3
process.append(num_sh_c)
num_m_a = 3
process.append(num_m_a)
num_m_b = 2
process.append(num_m_b)
num_p_a = 4
process.append(num_p_a)
num_p_b = 5
process.append(num_p_b)
num_p_c = 4
process.append(num_p_c)
num_ds_a = 7
process.append(num_ds_a)
num_ds_b = 7
process.append(num_ds_b)
num_ds_c = 7
process.append(num_ds_c)
case = []
for i in range(15):
case.append(1)
space = [process, case]
activities = 16
# q learning
Q = agent.q_learning(space, activities)
# print(Q)
# generate event log
env = simpy.Environment()
business_process = model.BusinessProcess(env, process)
business_process.event_log_flag = True
env.process(model.run_process(env, business_process))
env.run(until = 10000)
log.export_to_csv(business_process, r'D:\test\optis.csv')
# extract active cases from event log
active_cases = log.get_active_cases()
print(active_cases)
# test agent
for i in range(20):
caseid = random.choice(active_cases)
state = log.get_state(caseid)
action = np.argmax(Q[state])
print(action)
#print(Q)
print(Q[state])
state = Q[0]
action = np.argmax(state)
print(action)
print(state)
state = Q[64]
action = np.argmax(state)
print(action)
print(state)
if __name__ == "__main__":
main()
\ No newline at end of file
This diff is collapsed.
......@@ -24,15 +24,10 @@ def add_end_event(process, event_id, end_timestamp):
# add functions for adding events with their attributes to the log
<<<<<<<< HEAD:simpy_tutorial/eventlog.py
def export_to_csv(env, file_path):
event_log_df = pd.DataFrame.from_dict(env.bigeventlog)
event_log_df.to_csv(file_path)
========
def export_to_csv(process, file_path):
event_log_df = pd.DataFrame.from_dict(process.event_log)
event_log_df.to_csv(file_path, index=False)
>>>>>>>> 575b59827b5f928da4165070a4a1d51a85eed774:Simpy_Tutorial/eventlog.py
def export_to_xes(process, file_path):
# Use appropriate code to export to XES format
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment