，不过就是建立一个从状态空间

到动作空间

的映射，而映射的具体形式是什么，完全可以自己定，只要能够接近真实的最优

，在原论文中与上图所示稍有出入，卷积结构如下：

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import cv2
import gym
import matplotlib.pyplot as pltclass QNetwork(nn.Module):"""Actor (Policy) Model."""def __init__(self, state_size, action_size, seed):"""Initialize parameters and build model.Params======state_size (int): Dimension of each stateaction_size (int): Dimension of each actionseed (int): Random seed"""super(QNetwork, self).__init__()self.seed = torch.manual_seed(seed)"*** YOUR CODE HERE ***"self.conv = nn.Sequential(nn.Conv2d(state_size[1], 32, kernel_size=8, stride=4),nn.ReLU(),nn.Conv2d(32, 64, kernel_size=4, stride=2),nn.ReLU(),nn.Conv2d(64, 64, kernel_size=3, stride=1),nn.ReLU())self.fc = nn.Sequential(nn.Linear(64*7*7, 512),nn.ReLU(),nn.Linear(512, action_size))def forward(self, state):"""Build a network that maps state -> action values."""conv_out = self.conv(state).view(state.size()[0], -1)return self.fc(conv_out)def pre_process(observation):"""Process (210, 160, 3) picture into (1, 84, 84)"""x_t = cv2.cvtColor(cv2.resize(observation, (84, 84)), cv2.COLOR_BGR2GRAY)ret, x_t = cv2.threshold(x_t, 1, 255, cv2.THRESH_BINARY)return np.reshape(x_t, (1, 84, 84)), x_tdef stack_state(processed_obs):"""Four frames as a state"""return np.stack((processed_obs, processed_obs, processed_obs, processed_obs), axis=0)if __name__ == '__main__':env = gym.make('Breakout-v0')print('State shape: ', env.observation_space.shape)print('Number of actions: ', env.action_space.n)obs = env.reset()x_t, img = pre_process(obs)state = stack_state(img)print(np.shape(state[0]))# plt.imshow(img, cmap='gray')# 用cv2模块显示# cv2.imshow('Breakout', img)# cv2.waitKey(0)state = torch.randn(32, 4, 84, 84)  # (batch_size, color_channel, img_height,img_width)state_size = state.size()cnn_model = QNetwork(state_size, action_size=4, seed=1)outputs = cnn_model(state)print(outputs)

class ReplayBuffer:"""Fixed-size buffer to store experience tuples."""def __init__(self, action_size, buffer_size, batch_size, seed):"""Initialize a ReplayBuffer object.Params======action_size (int): dimension of each actionbuffer_size (int): maximum size of bufferbatch_size (int): size of each training batchseed (int): random seed"""self.action_size = action_sizeself.memory = deque(maxlen=buffer_size)  self.batch_size = batch_sizeself.experience = namedtuple("Experience", field_names=["state", "action", "reward", "next_state", "done"])self.seed = random.seed(seed)def add(self, state, action, reward, next_state, done):"""Add a new experience to memory."""e = self.experience(state, action, reward, next_state, done)self.memory.append(e)def sample(self):"""Randomly sample a batch of experiences from memory."""experiences = random.sample(self.memory, k=self.batch_size)states = torch.from_numpy(np.stack([e.state for e in experiences if e is not None])).float().to(device)actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(device)rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(device)next_states = torch.from_numpy(np.stack([e.next_state for e in experiences if e is not None])).float().to(device)dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)return (states, actions, rewards, next_states, dones)def __len__(self):"""Return the current size of internal memory."""return len(self.memory)

，我们的训练目标是训练一个CNN使

能够逼近它，经过对梯度的求导，感觉上我们希望TD目标值

与

应该是一回事，但实际上由于

是不依赖于参数w的，而TD目标值依赖于参数w，这在数学上是不合理的。也就是说TD目标值本来就是估计值，与旧的估计值作差再去更新

，就感觉像追着一个移动的目标却一直够不着。论文作者提出设置一个target network，而之前与环境交互产生动作的网络称为behavior network，训练开始时二者使用一样的架构和参数，训练过程中每完成一定数目的迭代，behavior network的参数就同步给target network。这是原始论文中提出更新方法，而在笔者的代码中，借鉴了DDPG中soft update法。

import numpy as np
import random
from collections import namedtuple, dequefrom q_model import QNetworkimport torch
import torch.nn.functional as F
import torch.optim as optimBUFFER_SIZE = int(1e6)  # replay buffer size
BATCH_SIZE = 32         # minibatch size
GAMMA = 0.99            # discount factor
TAU = 1e-3              # for soft update of target parameters
LR = 1e-5               # learning rate
UPDATE_EVERY = 4        # how often to update the networkdevice = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)class Agent():"""Interacts with and learns from the environment."""def __init__(self, state_size, action_size, seed):"""Initialize an Agent object.Params======state_size (int): dimension of each stateaction_size (int): dimension of each actionseed (int): random seed"""self.state_size = state_sizeself.action_size = action_sizeself.seed = random.seed(seed)# Q-Networkself.qnetwork_local = QNetwork(state_size, action_size, seed).to(device)  # behavior networkself.qnetwork_target = QNetwork(state_size, action_size, seed).to(device)  # target networkself.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)# Replay memoryself.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)# Initialize time step (for updating every UPDATE_EVERY steps)self.t_step = 0def step(self, state, action, reward, next_state, done):# Save experience in replay memoryself.memory.add(state, action, reward, next_state, done)# Learn every UPDATE_EVERY time steps.self.t_step = (self.t_step + 1) % UPDATE_EVERYif self.t_step == 0:# If enough samples are available in memory, get random subset and learnif len(self.memory) > BATCH_SIZE:experiences = self.memory.sample()self.learn(experiences, GAMMA)def act(self, state, eps=0.):"""Returns actions for given state as per current policy.Params======state (array_like): current stateeps (float): epsilon, for epsilon-greedy action selection"""state = torch.from_numpy(state).float().unsqueeze(0).to(device)self.qnetwork_local.eval()with torch.no_grad():action_values = self.qnetwork_local(state)self.qnetwork_local.train()# Epsilon-greedy action selectionif random.random() > eps:return np.argmax(action_values.cpu().data.numpy())else:return random.choice(np.arange(self.action_size))def learn(self, experiences, gamma):"""Update value parameters using given batch of experience tuples.Params======experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples gamma (float): discount factor"""states, actions, rewards, next_states, dones = experiences## TODO: compute and minimize the loss# Get max predicted Q values (for next states) from target modelQ_targets_next = self.qnetwork_target(next_states).detach().max(1)[0].unsqueeze(1)# Compute Q targets for current statesQ_targets = rewards + (gamma * Q_targets_next * (1 - dones))Q_expected = self.qnetwork_local(states).gather(1, actions)  # 固定行号，确认列号# Compute lossloss = F.mse_loss(Q_expected, Q_targets)# Minimize the lossself.optimizer.zero_grad()loss.backward()self.optimizer.step()# ------------------- update target network ------------------- #self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)                     def soft_update(self, local_model, target_model, tau):"""Soft update model parameters.θ_target = τ*θ_local + (1 - τ)*θ_targetParams======local_model (PyTorch model): weights will be copied fromtarget_model (PyTorch model): weights will be copied totau (float): interpolation parameter """for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)

衰减率。replay buffer大小至少设置为100万，迭代次数也是越多越好。

import gym
import random
import torch
import numpy as np
from collections import deque
from dqn_agent import Agent
import matplotlib.pyplot as plt
import cv2
import timeenv = gym.make('Breakout-v0')
state_size = env.observation_space.shape
action_size = env.action_space.n
print('Original state shape: ', state_size)
print('Number of actions: ', env.action_space.n)agent = Agent((32, 4, 84, 84), action_size, seed=1)  # state size (batch_size, 4 frames, img_height, img_width)
TRAIN = False  # train or test flag def pre_process(observation):"""Process (210, 160, 3) picture into (1, 84, 84)"""x_t = cv2.cvtColor(cv2.resize(observation, (84, 84)), cv2.COLOR_BGR2GRAY)ret, x_t = cv2.threshold(x_t, 1, 255, cv2.THRESH_BINARY)return x_tdef init_state(processed_obs):return np.stack((processed_obs, processed_obs, processed_obs, processed_obs), axis=0)def dqn(n_episodes=30000, max_t=40000, eps_start=1.0, eps_end=0.01, eps_decay=0.9995):"""Deep Q-Learning.Params======n_episodes (int): maximum number of training episodesmax_t (int): maximum number of timesteps per episode, maximum frameseps_start (float): starting value of epsilon, for epsilon-greedy action selectioneps_end (float): minimum value of epsiloneps_decay (float): multiplicative factor (per episode) for decreasing epsilon"""scores = []  # list containing scores from each episodescores_window = deque(maxlen=100)  # last 100 scoreseps = eps_start  # initialize epsilonfor i_episode in range(1, n_episodes + 1):obs = env.reset()obs = pre_process(obs)state = init_state(obs)score = 0for t in range(max_t):action = agent.act(state, eps)next_state, reward, done, _ = env.step(action)# last three frames and current frame as the next statenext_state = np.stack((state[1], state[2], state[3], pre_process(next_state)), axis=0)agent.step(state, action, reward, next_state, done)state = next_statescore += rewardif done:breakscores_window.append(score)  # save most recent scorescores.append(score)  # save most recent scoreeps = max(eps_end, eps_decay * eps)  # decrease epsilonprint('tEpsilon now : {:.2f}'.format(eps))print('rEpisode {}tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end="")if i_episode % 1000 == 0:print('rEpisode {}tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))print('rEpisode {}tThe length of replay buffer now: {}'.format(i_episode, len(agent.memory)))if np.mean(scores_window) >= 50.0:print('nEnvironment solved in {:d} episodes!tAverage Score: {:.2f}'.format(i_episode - 100,np.mean(scores_window)))torch.save(agent.qnetwork_local.state_dict(), 'checkpoint/dqn_checkpoint_solved.pth')breaktorch.save(agent.qnetwork_local.state_dict(), 'checkpoint/dqn_checkpoint_8.pth')return scoresif __name__ == '__main__':if TRAIN:start_time = time.time()scores = dqn()print('COST: {} min'.format((time.time() - start_time)/60))print("Max score:", np.max(scores))# plot the scoresfig = plt.figure()ax = fig.add_subplot(111)plt.plot(np.arange(len(scores)), scores)plt.ylabel('Score')plt.xlabel('Episode #')plt.show()else:# load the weights from fileagent.qnetwork_local.load_state_dict(torch.load('checkpoint/dqn_checkpoint_8.pth'))rewards = []for i in range(10):  # episodes, play ten timestotal_reward = 0obs = env.reset()obs = pre_process(obs)state = init_state(obs)for j in range(10000):  # frames, in case stuck in one frameaction = agent.act(state)env.render()next_state, reward, done, _ = env.step(action)state = np.stack((state[1], state[2], state[3], pre_process(next_state)), axis=0)total_reward += reward# time.sleep(0.01)if done:rewards.append(total_reward)breakprint("Test rewards are:", *rewards)print("Average reward:", np.mean(rewards))env.close()