Skip to content
Snippets Groups Projects
Commit 5d73867a authored by Aleksandra Dimitrova's avatar Aleksandra Dimitrova
Browse files

first draft

parent d2a9235e
No related branches found
No related tags found
No related merge requests found
No preview for this file type
No preview for this file type
File added
No preview for this file type
...@@ -15,29 +15,46 @@ def q_learning(space, activities): ...@@ -15,29 +15,46 @@ def q_learning(space, activities):
num_states = 1 num_states = 1
process_space = env.observation_space['process'].nvec process_space = env.observation_space['process'].nvec
case_space = env.observation_space['case'].nvec # case_space = env.observation_space['case'].nvec
event_space = env.observation_space['event'].n event_space = env.observation_space['event'].n
for i in process_space: num_states *= i for i in process_space: num_states *= i
for i in case_space: num_states *= i # for i in case_space: num_states *= (i+1)
num_states *= event_space num_states *= event_space
# num_states = pow(2,14)
"""
process_space = env.observation_space['process']
case_space = env.observation_space['case']
event_space = env.observation_space['event']
state_shape = []
for i in process_space: state_shape.append(i.n + 1)
for j in case_space: state_shape.append(j.n + 1)
state_shape.append(event_space.n)
state_shape = tuple(state_shape)
"""
num_actions = env.action_space.n num_actions = env.action_space.n
Q = np.zeros((num_states, num_actions), dtype=np.int16)
# Q = np.zeros(state_shape + (num_actions,), dtype=np.int8)
Q = np.zeros((num_states, num_actions), dtype = np.int64)
# Set the hyperparameters # Set the hyperparameters
alpha = 0.1 # learning rate alpha = 0.1 # learning rate
gamma = 0.99 # discount factor gamma = 0.1 # discount factor
epsilon = 0.1 # exploration rate epsilon = 0.1 # exploration rate
mean_time = 0 mean_time = 0
mean_reward = 0
# Train the agent using Q-learning # Train the agent using Q-learning
num_episodes = 10 num_episodes = 1000
for episode in range(num_episodes): for episode in range(num_episodes):
state, _ = env.reset() state, _ = env.reset()
state = env.flatten_observation(state) state = env.flatten_observation_to_int(state)
done = False done = False
start = env.process.env.now start = env.process.env.now
while not done: while not done:
...@@ -52,18 +69,34 @@ def q_learning(space, activities): ...@@ -52,18 +69,34 @@ def q_learning(space, activities):
next_state, reward, done, _ = env.step(action) next_state, reward, done, _ = env.step(action)
# Update the Q-value for the current state-action pair # Update the Q-value for the current state-action pair
Q[state][action] = Q[state][action] + alpha * (reward + gamma * np.max(Q[next_state]) - Q[state][action]) Q[state][action] = (1-alpha)*Q[state][action] + alpha * (reward + gamma * np.max(Q[next_state]) - Q[state][action])
#Q[state][action] = (1-alpha)*Q[state][action] + alpha*reward
# Transition to the next state # Transition to the next state
old_state = state
state = next_state state = next_state
# comment
time = env.process.env.now - start time = env.process.env.now - start
mean_time += time mean_time += time
mean_reward += reward
"""
if (episode % 20 == 19): if (episode % 20 == 19):
mean_reward /= 20
mean_time /= 20 mean_time /= 20
print(f"Episode {episode-19} to episode {episode}: mean time = {mean_time}") print(f"Episode {episode-19} to episode {episode}: mean time = {mean_time}, mean reward: {mean_reward}")
"""
if episode == 19:
start_reward = mean_reward
# print(f"Episode {episode}: time = {time}, reward = {reward}")
if episode == 999:
end_reward = mean_reward
improvement = end_reward - start_reward
print(f"Reward improved by {improvement}")
print(f"Episode {episode}: time = {time}") return Q
\ No newline at end of file
from stable_baselines3.common.env_checker import check_env
import environment
process = []
num_s = 1
process.append(num_s+1)
num_ot = 5
process.append(num_ot+1)
num_sh_a = 3
process.append(num_sh_a+1)
num_sh_b = 3
process.append(num_sh_b+1)
num_sh_c = 3
process.append(num_sh_c+1)
num_m_a = 3
process.append(num_m_a+1)
num_m_b = 2
process.append(num_m_b+1)
num_p_a = 4
process.append(num_p_a+1)
num_p_b = 5
process.append(num_p_b+1)
num_p_c = 4
process.append(num_p_c+1)
num_ds_a = 7
process.append(num_ds_a+1)
num_ds_b = 7
process.append(num_ds_b+1)
num_ds_c = 7
process.append(num_ds_c+1)
case = []
for i in range(15):
case.append(2)
space = [process, case]
activities = 16
env = environment.BusinessProcessEnv(space, activities)
check_env(env)
import numpy as np
import gymnasium as gym
import environment
from stable_baselines3 import PPO, DQN
import os
import time
def main():
process = []
num_s = 1
process.append(num_s+1)
num_ot = 5
process.append(num_ot+1)
num_sh_a = 3
process.append(num_sh_a+1)
num_sh_b = 3
process.append(num_sh_b+1)
num_sh_c = 3
process.append(num_sh_c+1)
num_m_a = 3
process.append(num_m_a+1)
num_m_b = 2
process.append(num_m_b+1)
num_p_a = 4
process.append(num_p_a+1)
num_p_b = 5
process.append(num_p_b+1)
num_p_c = 4
process.append(num_p_c+1)
num_ds_a = 7
process.append(num_ds_a+1)
num_ds_b = 7
process.append(num_ds_b+1)
num_ds_c = 7
process.append(num_ds_c+1)
case = []
for i in range(15):
case.append(2)
space = [process, case]
activities = 16
env = environment.BusinessProcessEnv(space, activities)
env.reset()
models_dir = f"models/{int(time.time())}/"
logdir = f"logs/{int(time.time())}/"
if not os.path.exists(models_dir):
os.makedirs(models_dir)
if not os.path.exists(logdir):
os.makedirs(logdir)
# model = PPO('MultiInputPolicy', env, verbose=1, tensorboard_log=logdir)
model = DQN('MultiInputPolicy', env, verbose=1, tensorboard_log=logdir)
TIMESTEPS = 10000000
iters = 0
while True:
iters += 1
model.learn(total_timesteps=TIMESTEPS, reset_num_timesteps=False, tb_log_name=f"PPO")
model.save(f"{models_dir}/{TIMESTEPS*iters}")
if __name__ == "__main__":
main()
import gymnasium as gym import gymnasium as gym
import numpy as np import numpy as np
import simpy import simpy
import simplesimmodel as model import simplesimmodel as simmodel
from collections import OrderedDict
""" """
Environment for the RL agent Environment for the RL agent
""" """
class BusinessProcessEnv(gym.Env): class BusinessProcessEnv(gym.Env):
def __init__(self, space, activities): def __init__(self, space, activities):
...@@ -25,27 +25,26 @@ class BusinessProcessEnv(gym.Env): ...@@ -25,27 +25,26 @@ class BusinessProcessEnv(gym.Env):
self.action_space = gym.spaces.Discrete(self.activities) self.action_space = gym.spaces.Discrete(self.activities)
self.current_state = { self.current_state = OrderedDict()
'process': np.array(self.ressources), self.current_state['case'] = np.zeros(len(self.case), dtype=int)
'case': np.zeros(len(self.case), dtype=int), self.current_state['event'] = 0
'event': 0 self.current_state['process'] = np.zeros(len(self.ressources), dtype=int)
} for i in range(len(self.current_state['process'])):
self.current_state['process'][i] += (self.ressources[i]-1)
self.model_env = simpy.Environment() self.model_env = simpy.Environment()
self.process = model.BusinessProcess(self.model_env, self.ressources) self.process = simmodel.BusinessProcess(self.model_env, self.ressources)
self.model_env.process(model.run_process(self.model_env, self.process)) self.model_env.process(simmodel.run_process(self.model_env, self.process))
# self.done_cases = set([])
self.reward = 0 self.reward = 0
def get_current_state(self, caseid): def get_current_state(self, caseid):
process, case, event = model.get_current_state(self.process, caseid) process, case, event = simmodel.get_current_state(self.process, caseid)
state = { state = OrderedDict()
'process': process, state['case'] = np.asarray(case)
'case': case, state['event'] = event
'event': event state['process'] = np.asarray(process)
}
return state return state
def step(self, action): def step(self, action):
...@@ -77,39 +76,51 @@ class BusinessProcessEnv(gym.Env): ...@@ -77,39 +76,51 @@ class BusinessProcessEnv(gym.Env):
next_state = self.get_current_state(case_obj) next_state = self.get_current_state(case_obj)
self.current_state = next_state self.current_state = next_state
next_state = self.flatten_observation(next_state) # next_state = self.flatten_observation_to_int(next_state)
time = stop - start
self.reward += -(stop - start) reward = 10000 - time
done = True if (len(self.process.done_cases) == 5 or len(self.process.active_cases) == 0) else False self.reward += reward
return next_state, self.reward, done, None done = True if (len(self.process.done_cases) == 1 or len(self.process.active_cases) == 0) else False
truncated = False
info = {}
return next_state, self.reward, done, truncated, info
else: else:
self.reward += -100 self.reward += 0
next_state = self.flatten_observation(self.current_state) # next_state = self.flatten_observation_to_int(self.current_state)
next_state = self.current_state
done = False done = False
return next_state, self.reward, done, None truncated = False
info = {}
return next_state, self.reward, done, truncated, info
def reset(self, seed=None, options=None): def reset(self, seed=None, options=None):
# Reset the environment to the initial state # Reset the environment to the initial state
# Implement a function which extracts the current state from an event log / simulation model # Implement a function which extracts the current state from an event log / simulation simmodel
super().reset(seed=seed) super().reset(seed=seed)
self.current_state = { self.current_state = OrderedDict()
'process': np.array(self.ressources), self.current_state['case'] = np.zeros(len(self.case), dtype=int)
'case': np.zeros(len(self.case), dtype=int), self.current_state['event'] = 0
'event': 0 self.current_state['process'] = np.zeros(len(self.ressources), dtype=int)
} for i in range(len(self.current_state['process'])):
self.current_state['process'][i] += (self.ressources[i]-1)
observation = self.current_state
self.current_step = 0 self.current_step = 0
self.model_env = simpy.Environment() self.model_env = simpy.Environment()
self.process = model.BusinessProcess(self.model_env, self.ressources) self.process = simmodel.BusinessProcess(self.model_env, self.ressources)
self.model_env.process(model.run_process(self.model_env, self.process)) self.model_env.process(simmodel.run_process(self.model_env, self.process))
self.process.done_cases = set([]) self.process.done_cases = set([])
self.reward = 0 self.reward = 0
return self.current_state, None info = {}
return observation, info
def render(self, mode='human'): def render(self, mode='human'):
...@@ -125,4 +136,74 @@ class BusinessProcessEnv(gym.Env): ...@@ -125,4 +136,74 @@ class BusinessProcessEnv(gym.Env):
return flattened return flattened
def flatten_observation_to_int(self, observation):
state = 0
state += observation['event']*pow(2,10)
state += observation['case'][1]*pow(2,2)
state += observation['case'][2]*pow(2,2)
event = observation['event']
if event == 0:
state += observation['process'][0]*pow(2,6)
elif event == 1:
state += observation['process'][1]*pow(2,6)
elif 1 < event <=3:
state += observation['process'][2]*pow(2,6)+observation['process'][3]*pow(2,7)+observation['process'][4]*pow(2,8)
elif 3 < event <=6:
state += observation['process'][5]*pow(2,6)+observation['process'][6]*pow(2,7)
elif 6 < event <= 8:
state += observation['process'][7]*pow(2,6)+observation['process'][8]*pow(2,7)+observation['process'][9]*pow(2,8)
elif 8 < event <= 11:
state += observation['process'][10]*pow(2,6)+observation['process'][11]*pow(2,7)+observation['process'][12]*pow(2,8)
elif 11 < event <= 14:
state += observation['process'][0]*pow(2,6)
else:
pass
return state
def main():
process = []
num_s = 1
process.append(num_s+1)
num_ot = 5
process.append(num_ot+1)
num_sh_a = 3
process.append(num_sh_a+1)
num_sh_b = 3
process.append(num_sh_b+1)
num_sh_c = 3
process.append(num_sh_c+1)
num_m_a = 3
process.append(num_m_a+1)
num_m_b = 2
process.append(num_m_b+1)
num_p_a = 4
process.append(num_p_a+1)
num_p_b = 5
process.append(num_p_b+1)
num_p_c = 4
process.append(num_p_c+1)
num_ds_a = 8
process.append(num_ds_a+1)
num_ds_b = 8
process.append(num_ds_b+1)
num_ds_c = 8
process.append(num_ds_c+1)
case = []
for i in range(15):
case.append(2)
space = [process, case]
activities = 16
env = BusinessProcessEnv(space, activities)
state = env.current_state
sample = env.observation_space.sample()
print(sample)
print(state)
if __name__ == "__main__":
main()
\ No newline at end of file
import pandas as pd import pandas as pd
import simplesimmodel as model import simplesimmodel as model
import numpy as np
""" """
Event log generator for our simulation model: Event log generator for our simulation model:
- generate an event log - generate an event log
- update an event log (adding new events) - update an event log (adding new events)
- export event log
- get current state of an event log
""" """
def add_start_event(process, event_id, case_id, activity, start_timestamp): def add_start_event(process, event_id, case_id, activity, start_timestamp):
process.event_log.append(event_id)
process.event_log[event_id] = { process.event_log[event_id] = {
'CaseID': case_id, 'CaseID': case_id,
'Activity': activity, 'Activity': activity,
'StartTimestamp': float(start_timestamp), 'StartTimestamp': float(start_timestamp),
'EndTimestamp': None 'EndTimestamp': None
} }
process.event_counter += 1
def add_end_event(process, event_id, end_timestamp): def add_end_event(process, event_id, end_timestamp):
# if event_id in process.event_log:
event = process.event_log[event_id] event = process.event_log[event_id]
event['EndTimestamp'] = end_timestamp event['EndTimestamp'] = end_timestamp
# process.event_log.append(event)
# del process.event_log[event_id]
# add functions for adding events with their attributes to the log
def export_to_csv(process, file_path): def export_to_csv(process, file_path):
event_log_df = pd.DataFrame.from_dict(process.event_log) event_log_df = pd.DataFrame.from_dict(process.event_log)
event_log_df.to_csv(file_path, index=False) event_log_df.to_csv(file_path, index=False)
def export_to_csv(env, file_path):
event_log_df = pd.DataFrame.from_dict(env.bigeventlog)
event_log_df.to_csv(file_path)
def export_to_xes(process, file_path): def export_to_xes(process, file_path):
# Use appropriate code to export to XES format # Use appropriate code to export to XES format
pass pass
def get_active_cases():
event_log = pd.read_csv(r'D:\test\optis.csv')
active_cases = event_log.groupby('CaseID').filter(lambda x: 'order completed' not in x['Activity'].values)['CaseID'].unique().tolist()
return active_cases
def get_state(case_id):
process = []
num_s = 1
process.append(num_s)
num_ot = 5
process.append(num_ot)
num_sh_a = 3
process.append(num_sh_a)
num_sh_b = 3
process.append(num_sh_b)
num_sh_c = 3
process.append(num_sh_c)
num_m_a = 3
process.append(num_m_a)
num_m_b = 2
process.append(num_m_b)
num_p_a = 4
process.append(num_p_a)
num_p_b = 5
process.append(num_p_b)
num_p_c = 4
process.append(num_p_c)
num_ds_a = 8
process.append(num_ds_a)
num_ds_b = 8
process.append(num_ds_b)
num_ds_c = 8
process.append(num_ds_c)
case = []
for i in range(15):
case.append(0)
activity_mapping = {
'place order': 1,
'arrange standard order': 2,
'arrange custom order': 3,
'pick from stock A': 4,
'pick from stock B': 5,
'pick from stock C': 6,
'manufacture A': 7,
'manufacture B': 8,
'pack A': 9,
'pack B': 10,
'pack C': 11,
'attempt delivery A': 12,
'attempt delivery B': 13,
'attempt delivery C': 14,
'order completed': 15,
}
event_log = pd.read_csv(r'D:\test\optis.csv')
# Sort the event log by case ID and start timestamp
event_log.sort_values(by=['CaseID', 'StartTimestamp'], inplace=True)
# Group the event log by case ID and get the last activity for each case
last_activities = event_log.groupby('CaseID').tail(1).reset_index()
# Remap the activity names to numbers using the mapping dictionary
last_activities['Activity'] = last_activities['Activity'].map(activity_mapping)
# Filter the cases where the end timestamp of the last activity is None or empty
unfinished_cases = last_activities[last_activities['EndTimestamp'].isnull()]['CaseID'].tolist()
# Update the state of the ressources given all unfinished cases
for i in unfinished_cases:
activity = last_activities[last_activities['CaseID'] == i]['Activity'].values[0]
if activity == 1 or activity == 15:
process[0] -= 1
elif activity == 2 or activity == 3:
process[1] -= 1
else:
process[activity-2] -= 1
# Get the state of the case for the given Case ID
filtered_log = event_log[event_log['CaseID'] == case_id]
activities = filtered_log['Activity'].map(activity_mapping).tolist()
for i in activities:
case[i-1] += 1
# Get the last event for the given Case ID
event = last_activities[last_activities['CaseID'] == case_id]['Activity'].values[0]
state = {
'process': process,
'case': case,
'event': event
}
print(state)
return state
import pandas as pd
import simpy
import random
import numpy as np
import simplesimmodel as model
import environment
import agent
import eventlog as log
import pandas as pd
def main():
# Setup
# we can use a random seed if we want to generate the same results every time (maybe useful later for the training)
# random.seed(42)
# initialize the number of resources
process = []
num_s = 1
process.append(num_s+1)
num_ot = 5
process.append(num_ot+1)
num_sh_a = 3
process.append(num_sh_a+1)
num_sh_b = 3
process.append(num_sh_b+1)
num_sh_c = 3
process.append(num_sh_c+1)
num_m_a = 3
process.append(num_m_a+1)
num_m_b = 2
process.append(num_m_b+1)
num_p_a = 4
process.append(num_p_a+1)
num_p_b = 5
process.append(num_p_b+1)
num_p_c = 4
process.append(num_p_c+1)
num_ds_a = 7
process.append(num_ds_a+1)
num_ds_b = 7
process.append(num_ds_b+1)
num_ds_c = 7
process.append(num_ds_c+1)
case = []
for i in range(15):
case.append(2)
space = [process, case]
activities = 16
# q learning
Q = agent.q_learning(space, activities)
# print(Q)
# generate event log
env = simpy.Environment()
business_process = model.BusinessProcess(env, process)
business_process.event_log_flag = True
env.process(model.run_process(env, business_process))
env.run(until = 10000)
log.export_to_csv(business_process, r'D:\test\optis.csv')
# extract active cases from event log
active_cases = log.get_active_cases()
print(active_cases)
# test agent
for i in range(20):
caseid = random.choice(active_cases)
state = log.get_state(caseid)
action = np.argmax(Q[state])
print(action)
#print(Q)
print(Q[state])
state = Q[0]
action = np.argmax(state)
print(action)
print(state)
state = Q[64]
action = np.argmax(state)
print(action)
print(state)
if __name__ == "__main__":
main()
\ No newline at end of file
...@@ -3,6 +3,7 @@ import random ...@@ -3,6 +3,7 @@ import random
import numpy as np import numpy as np
import environment import environment
import agent import agent
import eventlog as log
""" """
Simulation model for a simpler business process, including: Simulation model for a simpler business process, including:
...@@ -25,20 +26,20 @@ class BusinessProcess(object): ...@@ -25,20 +26,20 @@ class BusinessProcess(object):
self.env = env self.env = env
# initialize ressources # initialize ressources
self.system = simpy.Resource(env, ressources[0]) self.system = simpy.Resource(env, ressources[0] - 1)
self.order_taker = simpy.Resource(env, ressources[1]) self.order_taker = simpy.Resource(env, ressources[1] - 1)
self.stock_handler_a = simpy.Resource(env, ressources[2]) self.stock_handler_a = simpy.Resource(env, ressources[2] - 1)
self.stock_handler_b = simpy.Resource(env, ressources[3]) self.stock_handler_b = simpy.Resource(env, ressources[3] - 1)
self.stock_handler_c = simpy.Resource(env, ressources[4]) self.stock_handler_c = simpy.Resource(env, ressources[4] - 1)
self.manufacturer_a = simpy.Resource(env, ressources[5]) self.manufacturer_a = simpy.Resource(env, ressources[5] - 1)
self.manufacturer_b = simpy.Resource(env, ressources[6]) self.manufacturer_b = simpy.Resource(env, ressources[6] - 1)
self.packer_a = simpy.Resource(env, ressources[7]) self.packer_a = simpy.Resource(env, ressources[7] - 1)
self.packer_b = simpy.Resource(env, ressources[8]) self.packer_b = simpy.Resource(env, ressources[8] - 1)
self.packer_c = simpy.Resource(env, ressources[9]) self.packer_c = simpy.Resource(env, ressources[9] - 1)
# capacity of each delivery service instead of numbers of workers # capacity of each delivery service instead of numbers of workers
self.delivery_service_a = simpy.Resource(env, ressources[10]) self.delivery_service_a = simpy.Resource(env, ressources[10] - 1)
self.delivery_service_b = simpy.Resource(env, ressources[11]) self.delivery_service_b = simpy.Resource(env, ressources[11] - 1)
self.delivery_service_c = simpy.Resource(env, ressources[12]) self.delivery_service_c = simpy.Resource(env, ressources[12] - 1)
# initialize lists with active cases and all case objects # initialize lists with active cases and all case objects
self.active_cases = [0, 1, 2] self.active_cases = [0, 1, 2]
...@@ -56,7 +57,11 @@ class BusinessProcess(object): ...@@ -56,7 +57,11 @@ class BusinessProcess(object):
self.done_cases = set([]) self.done_cases = set([])
random.seed(1) self.event_log_flag = False
self.event_log = []
self.event_counter = 0
# random.seed(1)
def place_order(self, case): def place_order(self, case):
yield self.env.timeout(0) yield self.env.timeout(0)
...@@ -162,12 +167,17 @@ def execute_case(env, case, process): ...@@ -162,12 +167,17 @@ def execute_case(env, case, process):
case_obj.agent = True case_obj.agent = True
# place order # place order
with process.system.request() as request:
yield request
case_obj.state[0] += 1 case_obj.state[0] += 1
case_obj.current = 1 case_obj.current = 1
with process.system.request() as request:
yield request
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "place order", env.now)
yield env.process(process.place_order(case)) yield env.process(process.place_order(case))
if case_obj.agent: print(f"Case {case}: 'placed order' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: 'placed order' at {env.now:.2f}")
# if the last action was made from the agent set the process flag to be able to return to the environment's step function # if the last action was made from the agent set the process flag to be able to return to the environment's step function
...@@ -184,30 +194,38 @@ def execute_case(env, case, process): ...@@ -184,30 +194,38 @@ def execute_case(env, case, process):
choice = random.randint(2,3) if not case_obj.agent else process.next choice = random.randint(2,3) if not case_obj.agent else process.next
if choice == 2: if choice == 2:
case_obj.standard_order = True case_obj.standard_order = True
with process.order_taker.request() as request:
yield request
case_obj.state[1] += 1 case_obj.state[1] += 1
case_obj.current = 2 case_obj.current = 2
if case_obj.agent: print(f"Case {case}: started 'arrange standard order' at {env.now:.2f}") with process.order_taker.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'arrange standard order' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "arrange standard order", env.now)
yield env.process(process.arrange_standard_order(case)) yield env.process(process.arrange_standard_order(case))
if case_obj.agent: print(f"Case {case}: finished 'arrange standard order' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'arrange standard order' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
else: else:
case_obj.state[2] += 1
case_obj.current = 3
case_obj.standard_order = False case_obj.standard_order = False
with process.order_taker.request() as request: with process.order_taker.request() as request:
yield request yield request
case_obj.state[2] += 1 # if case_obj.agent: print(f"Case {case}: started 'arrange custom order' at {env.now:.2f}")
case_obj.current = 3 if process.event_log_flag:
if case_obj.agent: print(f"Case {case}: started 'arrange custom order' at {env.now:.2f}") event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "arrange custom order", env.now)
yield env.process(process.arrange_custom_order(case)) yield env.process(process.arrange_custom_order(case))
if case_obj.agent: print(f"Case {case}: finished 'arrange custom order' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'arrange custom order' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
...@@ -223,68 +241,90 @@ def execute_case(env, case, process): ...@@ -223,68 +241,90 @@ def execute_case(env, case, process):
# choose stock or manufacturer # choose stock or manufacturer
if choice == 4: if choice == 4:
with process.stock_handler_a.request() as request:
yield request
case_obj.state[3] += 1 case_obj.state[3] += 1
case_obj.current = 4 case_obj.current = 4
if case_obj.agent: print(f"Case {case}: started 'pick from stock A' at {env.now:.2f}") with process.stock_handler_a.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'pick from stock A' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "pick from stock A", env.now)
yield env.process(process.pick_from_stock_a(case)) yield env.process(process.pick_from_stock_a(case))
if case_obj.agent: print(f"Case {case}: finished 'pick from stock A' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'pick from stock A' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
elif choice == 5: elif choice == 5:
with process.stock_handler_b.request() as request:
yield request
case_obj.state[4] += 1 case_obj.state[4] += 1
case_obj.current = 5 case_obj.current = 5
if case_obj.agent: print(f"Case {case}: started 'pick from stock B' at {env.now:.2f}") with process.stock_handler_b.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'pick from stock B' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "pick from stock B", env.now)
yield env.process(process.pick_from_stock_b(case)) yield env.process(process.pick_from_stock_b(case))
if case_obj.agent: print(f"Case {case}: finished 'pick from stock B' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'pick from stock B' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
elif choice == 6: elif choice == 6:
with process.stock_handler_c.request() as request:
yield request
case_obj.state[5] += 1 case_obj.state[5] += 1
case_obj.current = 6 case_obj.current = 6
if case_obj.agent: print(f"Case {case}: started 'pick from stock C' at {env.now:.2f}") with process.stock_handler_c.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'pick from stock C' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "pick from stock C", env.now)
yield env.process(process.pick_from_stock_c(case)) yield env.process(process.pick_from_stock_c(case))
if case_obj.agent: print(f"Case {case}: finished 'pick from stock C' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'pick from stock C' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
elif choice == 7: elif choice == 7:
with process.manufacturer_a.request() as request:
yield request
case_obj.state[6] += 1 case_obj.state[6] += 1
case_obj.current = 7 case_obj.current = 7
if case_obj.agent: print(f"Case {case}: started 'manufacture A' at {env.now:.2f}") with process.manufacturer_a.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'manufacture A' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "manufacture A", env.now)
yield env.process(process.manufacture_a(case)) yield env.process(process.manufacture_a(case))
if case_obj.agent: print(f"Case {case}: finished 'manufacture A' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'manufacture A' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
else: else:
with process.manufacturer_b.request() as request:
yield request
case_obj.state[7] += 1 case_obj.state[7] += 1
case_obj.current = 8 case_obj.current = 8
if case_obj.agent: print(f"Case {case}: started 'manufacture B' at {env.now:.2f}") with process.manufacturer_b.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'manufacture B' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "manufacture B", env.now)
yield env.process(process.manufacture_b(case)) yield env.process(process.manufacture_b(case))
if case_obj.agent: print(f"Case {case}: finished 'manufacture B' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'manufacture B' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
...@@ -294,42 +334,58 @@ def execute_case(env, case, process): ...@@ -294,42 +334,58 @@ def execute_case(env, case, process):
choice = random.randint(9,11) if not case_obj.agent else process.next choice = random.randint(9,11) if not case_obj.agent else process.next
if choice == 9: if choice == 9:
with process.packer_a.request() as request:
yield request
case_obj.state[8] += 1 case_obj.state[8] += 1
case_obj.current = 9 case_obj.current = 9
if case_obj.agent: print(f"Case {case}: started 'pack A' at {env.now:.2f}") with process.packer_a.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'pack A' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "pack A", env.now)
yield env.process(process.pack_a(case)) yield env.process(process.pack_a(case))
if case_obj.agent: print(f"Case {case}: finished 'pack A' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'pack A' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
elif choice == 10: elif choice == 10:
with process.packer_b.request() as request:
yield request
case_obj.state[9] += 1 case_obj.state[9] += 1
case_obj.current = 10 case_obj.current = 10
if case_obj.agent: print(f"Case {case}: started 'pack B' at {env.now:.2f}") with process.packer_b.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'pack B' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "pack B", env.now)
yield env.process(process.pack_b(case)) yield env.process(process.pack_b(case))
if case_obj.agent: print(f"Case {case}: finished 'pack B' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'pack B' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
else: else:
with process.packer_c.request() as request:
yield request
case_obj.state[10] += 1 case_obj.state[10] += 1
case_obj.current = 11 case_obj.current = 11
if case_obj.agent: print(f"Case {case}: started 'pack C' at {env.now:.2f}") with process.packer_c.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'pack C' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "pack C", env.now)
yield env.process(process.pack_c(case)) yield env.process(process.pack_c(case))
if case_obj.agent: print(f"Case {case}: finished 'pack C' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'pack C' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
...@@ -338,42 +394,57 @@ def execute_case(env, case, process): ...@@ -338,42 +394,57 @@ def execute_case(env, case, process):
# choose delivery # choose delivery
choice = random.randint(12,14) if not case_obj.agent else process.next choice = random.randint(12,14) if not case_obj.agent else process.next
if choice == 12: if choice == 12:
with process.delivery_service_a.request() as request:
yield request
case_obj.state[11] += 1 case_obj.state[11] += 1
case_obj.current = 12 case_obj.current = 12
if case_obj.agent: print(f"Case {case}: started 'attempt delivery A' at {env.now:.2f}") with process.delivery_service_a.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'attempt delivery A' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "attempt delivery A", env.now)
yield env.process(process.attempt_delivery_a(case)) yield env.process(process.attempt_delivery_a(case))
if case_obj.agent: print(f"Case {case}: finished 'attempt delivery A' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'attempt delivery A' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
elif choice == 13: elif choice == 13:
with process.delivery_service_b.request() as request:
yield request
case_obj.state[12] += 1 case_obj.state[12] += 1
case_obj.current = 13 case_obj.current = 13
if case_obj.agent: print(f"Case {case}: started 'attempt delivery B' at {env.now:.2f}") with process.delivery_service_b.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'attempt delivery B' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "attempt delivery B", env.now)
yield env.process(process.attempt_delivery_b(case)) yield env.process(process.attempt_delivery_b(case))
if case_obj.agent: print(f"Case {case}: finished 'attempt delivery B' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'attempt delivery B' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
else: else:
with process.delivery_service_c.request() as request:
yield request
case_obj.state[13] += 1 case_obj.state[13] += 1
case_obj.current = 14 case_obj.current = 14
if case_obj.agent: print(f"Case {case}: started 'attempt delivery C' at {env.now:.2f}") with process.delivery_service_c.request() as request:
yield request
# if case_obj.agent: print(f"Case {case}: started 'attempt delivery C' at {env.now:.2f}")
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "attempt delivery C", env.now)
yield env.process(process.attempt_delivery_c(case)) yield env.process(process.attempt_delivery_c(case))
if case_obj.agent: print(f"Case {case}: finished 'attempt delivery C' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: finished 'attempt delivery C' at {env.now:.2f}")
if case_obj.agent: if case_obj.agent:
process.flag = False process.flag = False
with process.system.request() as request: with process.system.request() as request:
yield request yield request
...@@ -381,12 +452,17 @@ def execute_case(env, case, process): ...@@ -381,12 +452,17 @@ def execute_case(env, case, process):
if process.case_id == case: case_obj.agent = True if process.case_id == case: case_obj.agent = True
# case completed # case completed
with process.system.request() as request:
yield request
case_obj.state[14] += 1 case_obj.state[14] += 1
case_obj.current = 15 case_obj.current = 15
with process.system.request() as request:
yield request
if process.event_log_flag:
event_counter = process.event_counter
log.add_start_event(process, event_counter, case, "order completed", env.now)
yield env.process(process.order_completed(case)) yield env.process(process.order_completed(case))
if case_obj.agent: print(f"Case {case}: 'completed' at {env.now:.2f}") if process.event_log_flag:
log.add_end_event(process, event_counter, env.now)
# if case_obj.agent: print(f"Case {case}: 'completed' at {env.now:.2f}")
if case in process.active_cases: if case in process.active_cases:
...@@ -399,6 +475,13 @@ def execute_case(env, case, process): ...@@ -399,6 +475,13 @@ def execute_case(env, case, process):
process.done_cases.add(process.case_id) process.done_cases.add(process.case_id)
process.flag = False process.flag = False
"""
Get the cureent state of the process and a specific case
- available ressouces
- events which already happened in the case
- current event of the case
"""
def get_current_state(process, case): def get_current_state(process, case):
process_state = [] process_state = []
...@@ -447,6 +530,9 @@ def get_current_state(process, case): ...@@ -447,6 +530,9 @@ def get_current_state(process, case):
return process_state, cur_case, event return process_state, cur_case, event
"""
Defines how often new orders (cases) come in and starts executing them
"""
def run_process(env, process): def run_process(env, process):
# process = Process(env, num_ot, num_m, num_sh_a, num_sh_b, num_sh_c, num_m_a, num_m_b, num_ds_a, num_ds_b, num_ds_c) # process = Process(env, num_ot, num_m, num_sh_a, num_sh_b, num_sh_c, num_m_a, num_m_b, num_ds_a, num_ds_b, num_ds_c)
...@@ -457,63 +543,11 @@ def run_process(env, process): ...@@ -457,63 +543,11 @@ def run_process(env, process):
# the new incoming orders # the new incoming orders
while case < 1000: while case < 1000:
waittime = random.randint(10,15) waittime = random.randint(10,15)
if case % 20 == 0:
waittime = 100
yield env.timeout(waittime) # Wait a bit before generating a new case yield env.timeout(waittime) # Wait a bit before generating a new case
case += 1 case += 1
# process.active_cases.append(case) # process.active_cases.append(case)
env.process(execute_case(env, case, process)) env.process(execute_case(env, case, process))
def main():
# Setup
# we can use a random seed if we want to generate the same results every time (maybe useful later for the training)
# random.seed(42)
# initialize the number of resources
process = []
num_s = 1
process.append(num_s)
num_ot = 5
process.append(num_ot)
num_sh_a = 3
process.append(num_sh_a)
num_sh_b = 3
process.append(num_sh_b)
num_sh_c = 3
process.append(num_sh_c)
num_m_a = 3
process.append(num_m_a)
num_m_b = 2
process.append(num_m_b)
num_p_a = 4
process.append(num_p_a)
num_p_b = 5
process.append(num_p_b)
num_p_c = 4
process.append(num_p_c)
num_ds_a = 8
process.append(num_ds_a)
num_ds_b = 8
process.append(num_ds_b)
num_ds_c = 8
process.append(num_ds_c)
case = []
for i in range(15):
case.append(1)
space = [process, case]
activities = 16
business_env = environment.BusinessProcessEnv(space, activities)
print(business_env.observation_space.shape)
print(business_env.observation_space.sample())
state, _ = business_env.reset()
print(state)
print(business_env.current_state)
print(state['event'])
print(business_env.flatten_observation(state))
agent.q_learning(space, activities)
if __name__ == "__main__":
main()
keras-rl @ 216c3145
Subproject commit 216c3145f3dc4d17877be26ca2185ce7db462bad
File added
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment