dqn解简易迷宫

参考用keras搭建DQN和莫凡的git

#RL_brain_diy.py
from tensorflow import keras
import numpy as np,time
class DeepQNetwork:
def __init__(
self,
n_actions,
n_features,
is_train=True,
load_model=True,
learning_rate = .01,
reward_decay = 0.9,
e_greedy = 0.9,
replace_target_iter = 300,
memory_size = 500,
batch_size = 32,
# 第一次运行设置1e-3，之后设置None
e_greedy_increment = None,
# e_greedy_increment = 1e-3,
output_graph = False,
first_layer_neurno = 8,
second_layer_neurno = 1):
self.n_actions=n_actions
self.n_features=n_features
self.lr = learning_rate
self.gamma = reward_decay
self.replace_target_iter=replace_target_iter
self.learn_step_counter=1
self.memory_size = memory_size
self.memory=np.zeros((memory_size,2+2*n_features))
self.memory_count=0
self.batch_size = batch_size
self.epsilon_max = e_greedy
self.epsilon = 0 if e_greedy_increment is not None else e_greedy
self.epsilon_increment=e_greedy_increment
print(self.epsilon,self.epsilon_increment)
self.output_graph = output_graph
self.output_1=first_layer_neurno
self.output_2=second_layer_neurno
self.is_train = is_train
self._build_net()
if load_model:self.load_model()
def _build_net(self):
self.model_eval=keras.Sequential()
d1=keras.layers.Dense(self.output_1,
input_shape=(self.n_features,),
activation='relu')
self.model_eval.add(d1)
d2 = keras.layers.Dense(self.n_actions)
self.model_eval.add(d2)
optimizer=keras.optimizers.Adam(learning_rate=3e-3)
# optimizer =keras.optimizers.RMSprop(lr=self.lr, rho=0.9, epsilon=1e-08, decay=0.0)
# optimizer =keras.optimizers.RMSprop(lr=self.lr)
self.model_eval.compile(loss='mse',optimizer=optimizer)
self.model_target = keras.Sequential()
d1=keras.layers.Dense(self.output_1,
input_shape=(self.n_features,),
activation='relu')
self.model_target.add(d1)
d2 = keras.layers.Dense(self.n_actions)
self.model_target.add(d2)
def store_transition(self,observation, action, reward, observation_):
index=self.memory_count%self.memory_size
data=np.hstack((observation, action, reward, observation_))
self.memory[index]=data
self.memory_count += 1
def choose_action(self,observation):
if np.random.uniform()<self.epsilon:
observation = observation[np.newaxis, :]
actions_value=self.model_eval.predict(observation)
# action=np.argmax(actions_value)
# 如果最大值有多个，从中随机取
action=np.random.choice(np.where(actions_value == np.max(actions_value))[1])
print(actions_value, action, observation)
else:
action = np.random.randint(0,self.n_actions)
print('random>>', action, observation)
return action
def learn(self):
if self.learn_step_counter%self.replace_target_iter==0:
self.model_target.set_weights(self.model_eval.get_weights())
# print('t target_params_replacedn')
sample_index=np.random.choice(min(self.memory_count,self.memory_size),self.batch_size)
batch=self.memory[sample_index]
o_s=batch[:,:self.n_features]
a_s=batch[:,self.n_features].astype(int)
r_s=batch[:,1+self.n_features]
o_s_=batch[:,-self.n_features:]
q_next=self.model_target.predict(o_s_ ,batch_size=self.batch_size)
q_eval = self.model_eval.predict(o_s,batch_size=self.batch_size)
# q_target = q_eval.copy()
q_target = q_eval
target_part=r_s+self.gamma * np.max(q_next,axis=1)
q_target[range(self.batch_size),a_s]=target_part
self.model_eval.train_on_batch(o_s,q_target)
if self.epsilon < self.epsilon_max:
self.epsilon +=self.epsilon_increment
self.learn_step_counter+=1
def plot_cost(self):
pass
def save_model(self):
self.model_eval.save_weights('model_eval.h5')
self.model_target.save_weights('model_target.h5')
def load_model(self):
try:
self.model_eval.load_weights('model_eval.h5')
self.model_target.load_weights('model_target.h5')
except Exception:
print('没找到模型，不加载')

#run_this.py
from dqn.day2.maze_env import Maze
# from dqn.day2.RL_brain import DeepQNetwork
# from dqn.day2.RL_brain_Keras import DeepQNetwork
from dqn.day2.RL_brain_diy import DeepQNetwork
import numpy as np
def run_maze():
step = 0
success,acc=0,.01
for episode in range(100):
# initial observation
observation = env.reset()
if episode>0:
acc=success/episode
print('episode:',episode,'success:',acc)
while True:
# fresh env
env.render()
# RL choose action based on observation
action = RL.choose_action(observation)
# RL take action and get next observation and reward
observation_, reward, done = env.step(action)
if reward==1:success+=1
RL.store_transition(observation, action, reward, observation_)
if (step > 20) and (step % 5 == 0):
RL.learn()
# swap observation
observation = observation_
# break while loop when end of this episode
if done:
break
step += 1
# end of game
print('game over')
env.destroy()
if __name__ == "__main__":
# maze game
env = Maze()
RL = DeepQNetwork(env.n_actions, env.n_features,
learning_rate=0.01,
reward_decay=0.9,
e_greedy=0.9,
replace_target_iter=200,
memory_size=2000,
# output_graph=True
)
env.after(100, run_maze)
env.mainloop()
RL.save_model()
RL.plot_cost()
for i in range(4):
for j in range(4):
x=.25*(i-2)
y=.25*(j-2)
z=RL.model_eval.predict(np.array((x,y)).reshape(1,-1))
print(x,y,z)

#maze_env.py
import numpy as np
import time
import sys
if sys.version_info.major == 2:
import Tkinter as tk
else:
import tkinter as tk
UNIT = 40
# pixels
MAZE_H = 4
# grid height
MAZE_W = 4
# grid width
class Maze(tk.Tk, object):
def __init__(self):
super(Maze, self).__init__()
self.action_space = ['u', 'd', 'l', 'r']
self.n_actions = len(self.action_space)
self.n_features = 2
self.title('maze')
self.geometry('{0}x{1}'.format(MAZE_H * UNIT, MAZE_H * UNIT))
self._build_maze()
def _build_maze(self):
self.canvas = tk.Canvas(self, bg='white',
height=MAZE_H * UNIT,
width=MAZE_W * UNIT)
# create grids
for c in range(0, MAZE_W * UNIT, UNIT):
x0, y0, x1, y1 = c, 0, c, MAZE_H * UNIT
self.canvas.create_line(x0, y0, x1, y1)
for r in range(0, MAZE_H * UNIT, UNIT):
x0, y0, x1, y1 = 0, r, MAZE_W * UNIT, r
self.canvas.create_line(x0, y0, x1, y1)
# create origin
origin = np.array([20, 20])
# hell
hell1_center = origin + np.array([UNIT * 2, UNIT])
self.hell1 = self.canvas.create_rectangle(
hell1_center[0] - 15, hell1_center[1] - 15,
hell1_center[0] + 15, hell1_center[1] + 15,
fill='black')
# hell
# hell2_center = origin + np.array([UNIT, UNIT * 2])
# self.hell2 = self.canvas.create_rectangle(
#
hell2_center[0] - 15, hell2_center[1] - 15,
#
hell2_center[0] + 15, hell2_center[1] + 15,
#
fill='black')
# create oval
oval_center = origin + UNIT * 2
self.oval = self.canvas.create_oval(
oval_center[0] - 15, oval_center[1] - 15,
oval_center[0] + 15, oval_center[1] + 15,
fill='yellow')
# create red rect
self.rect = self.canvas.create_rectangle(
origin[0] - 15, origin[1] - 15,
origin[0] + 15, origin[1] + 15,
fill='red')
# pack all
self.canvas.pack()
def reset(self):
self.update()
time.sleep(0.1)
self.canvas.delete(self.rect)
origin = np.array([20, 20])
self.rect = self.canvas.create_rectangle(
origin[0] - 15, origin[1] - 15,
origin[0] + 15, origin[1] + 15,
fill='red')
# return observation
return (np.array(self.canvas.coords(self.rect)[:2]) - np.array(self.canvas.coords(self.oval)[:2]))/(MAZE_H*UNIT)
def step(self, action):
s = self.canvas.coords(self.rect)
base_action = np.array([0, 0])
if action == 0:
# up
if s[1] > UNIT:
base_action[1] -= UNIT
elif action == 1:
# down
if s[1] < (MAZE_H - 1) * UNIT:
base_action[1] += UNIT
elif action == 3:
# right
if s[0] < (MAZE_W - 1) * UNIT:
base_action[0] += UNIT
elif action == 2:
# left
if s[0] > UNIT:
base_action[0] -= UNIT
self.canvas.move(self.rect, base_action[0], base_action[1])
# move agent
next_coords = self.canvas.coords(self.rect)
# next state
# reward function
if next_coords == self.canvas.coords(self.oval):
reward = 1
done = True
elif next_coords in [self.canvas.coords(self.hell1)]:
reward = -1
done = True
else:
reward = 0
done = False
s_ = (np.array(next_coords[:2]) - np.array(self.canvas.coords(self.oval)[:2]))/(MAZE_H*UNIT)
return s_, reward, done
def render(self):
# time.sleep(0.01)
self.update()

最后

以上就是优雅荷花为你收集整理的dqn解简易迷宫的全部内容，希望文章能够帮你解决dqn解简易迷宫所遇到的程序开发问题。

如果觉得靠谱客网站的内容还不错，欢迎将靠谱客网站推荐给程序员好友。