【强化学习】Q-Learning原理及代码实现

最近工作是在太忙了，无奈，也没空更新博客，职业上也从研发变成了产品，有小半年没写代码了，怕自己手生的不行，给自己两天时间，写了点东西，之前做搞机器学习，搞深度学习，但一直对依赖全场景数据喂模型的方向有点感冒，因为数据又贵又难搞全，企业靠这个发家有点难，且本身需要企业具有很大的体量，另收集数据-训练-部署三板斧就当做AI的自进化说法感觉有点勉强，不谈特定场景妄图一个AI模型解决通用问题的都是大忽悠，咱们还是等待学术界的大佬们能真正提出新的理论来造福业内吧。

一直在想，有没有一种更好适应的算法方向可以不那么依赖数据的，或者说能依赖数据少一些，这样对于一个产品也许有更好的成本控制和生命力加成，因此想了想是不是强化学习有可能，顺势就了解了下，花了几天时间简单学习了下入门皮毛，对最简单的Q-Learning先进行了复现。

设定环境为一个5x5的棋盘，墙壁四周无法通行，碰到炸弹直接死亡，找到宝石就算过关。

关于理论知识和排版我下次总结补充哈，先把我前两天写的代码放在这，先上传个效果：

训练刚开始随机探索，：

训练几十轮之后，基本直奔终点：

代码部分：

环境Env(为啥不使用我之前写的pycharm加pyqt5.0进行环境开发，我表示，忘光了，赶时间，选个最简单tk来用~)：

import tkinter as tk
from PIL import ImageTk
from PIL import Image
import timeclass Env:def __init__(self):self.grid_size = 100self.win = tk.Tk()self.pic_player, self.pic_diamond, self.pic_boom1, self.pic_boom2, self.pic_boom3, self.pic_boom4 = self.__load_img()self.__init_win()self.canvas = self.__init_rc()self.texts= self.__produce_text()self.canvas.pack()# self._init_test_case()# self.win.mainloop()def __init_win(self):self.win.title('Grid World')# self.win.geometry("500x300")def __init_rc(self):canvas = tk.Canvas(self.win, width=500, height=720, bg='white')for h in range(5):for v in range(5):canvas.create_rectangle(self.grid_size * v, self.grid_size * h, self.grid_size * (v + 1), self.grid_size * (h + 1))trans_pixel = int(self.grid_size / 2)self.player = canvas.create_image(trans_pixel + self.grid_size * 0, trans_pixel + self.grid_size * 0, image = self.pic_player)self.diamond = canvas.create_image(trans_pixel + self.grid_size * 4, trans_pixel + self.grid_size * 4, image=self.pic_diamond)self.boom1 = canvas.create_image(trans_pixel + self.grid_size * 1, trans_pixel + self.grid_size * 1, image=self.pic_boom1)self.boom2 = canvas.create_image(trans_pixel + self.grid_size * 3, trans_pixel + self.grid_size * 1, image=self.pic_boom2)self.boom3 = canvas.create_image(trans_pixel + self.grid_size * 1, trans_pixel + self.grid_size * 3, image=self.pic_boom3)self.boom4 = canvas.create_image(trans_pixel + self.grid_size * 3, trans_pixel + self.grid_size * 3, image=self.pic_boom4)return canvasdef __load_img(self):pic_resize = int (self.grid_size / 2)player = ImageTk.PhotoImage(Image.open("player.png").resize((pic_resize, pic_resize)))diamond = ImageTk.PhotoImage(Image.open("diamond.png").resize((pic_resize, pic_resize)))boom1 = ImageTk.PhotoImage(Image.open('boom.png').resize((pic_resize, pic_resize)))boom2 = ImageTk.PhotoImage(Image.open('boom.png').resize((pic_resize, pic_resize)))boom3 = ImageTk.PhotoImage(Image.open('boom.png').resize((pic_resize, pic_resize)))boom4 = ImageTk.PhotoImage(Image.open('boom.png').resize((pic_resize, pic_resize)))return player, diamond, boom1, boom2, boom3, boom4def __produce_text(self):texts = []x = self.grid_size / 2y = self.grid_size / 6for h in range(5):for v in range(5):up = self.canvas.create_text(x + h * self.grid_size, y + v * self.grid_size,text=0)down = self.canvas.create_text(x + h *self.grid_size, self.grid_size - y + v * self.grid_size, text = 0)left = self.canvas.create_text(y + h*self.grid_size, x + v*self.grid_size,text = 0)right = self.canvas.create_text(self.grid_size-y + h*self.grid_size, x + v*self.grid_size,text = 0)texts.append({"up": up, "down": down, "left": left, "right": right})return textsdef _win_d_update(self):self.win.update()time.sleep(0.1)class GridWorld(Env):def __init__(self):super().__init__()self._win_d_update()def player_move(self, x, y):# x横向移动向右,y纵向移动向下self.canvas.move(self.player, x * self.grid_size, y*self.grid_size)self._win_d_update()def reset(self):# 重置为起始位置x, y = self.canvas.coords(self.player)self.canvas.move(self.player, -x + self.grid_size/2, -y + self.grid_size/2)self._win_d_update()return self.get_state(self.player)def get_state(self, who):x, y = self.canvas.coords(who)state = [int(x/self.grid_size), int(y/self.grid_size)]return statedef update_val(self, num, arrow, val):pos = num[0] * 5 + num[1]x, y = self.canvas.coords(self.texts[pos][arrow])self.canvas.delete(self.texts[pos][arrow])self.texts[pos][arrow] = self.canvas.create_text(x, y, text=val)# self._win_d_update()def exec_calc(self, action):# 执行一次决策feedback = 'alive'  # alive, stop, dead 分别对应通过，撞墙，炸死next_state = []next_h, next_v, reward = 0.0, 0.0, 0.0h, v = self.get_state(self.player)if action == 0:     # upnext_h = hnext_v = v - 1# self.player_move(0, -1)elif action == 1:   # downnext_h = hnext_v = v + 1# self.player_move(0, 1)elif action == 2:   # leftnext_h = h - 1next_v = v# self.player_move(-1, 0)elif action == 3:   # rightnext_h = h + 1next_v = v# self.player_move(1, 0)else:print('programmer bug ...')next_state = [next_h, next_v]boom1, boom2, boom3, boom4 = self.get_state(self.boom1), self.get_state(self.boom2), self.get_state(self.boom3), self.get_state(self.boom4)diamond = self.get_state(self.diamond)if next_h < 0 or next_v < 0 or next_h > 4 or next_v >4:  # 超过边界reward = -1feedback = 'stop'elif next_state == boom1 or next_state == boom2 or next_state == boom3 or next_state == boom4:    # 炸弹区域reward = -100feedback = 'dead'elif next_state == diamond:   # 获得的通关物品reward = 500else:reward = 0return feedback, next_state, rewarddef update_view(self, state, action, next_state, q_val):action_list = ['up', 'down', 'left', 'right']self.player_move(next_state[0]-state[0], next_state[1]-state[1])self.update_val(state, action_list[action], round(q_val, 2))def attach(self):# 到达终点，返回True , 未到达，返回Falsereturn str(self.get_state(self.player)) == str(self.get_state(self.diamond))

智能体Agent代码：

import numpy as np
import envclass Agent:def __init__(self):self.actions = [0, 1, 2, 3]  # up down left rightself.q_table = dict()self.__init_q_table()self.epsilon = 0.1self.learning_rate = 0.1self.gamma = 0.8# print(self.q_table)def __init_q_table(self):for v in range(5):for h in range(5):self.q_table[str([h, v])] = [0.0, 0.0, 0.0, 0.0]def get_action(self, state):# 根据状态选取下一个动作，但不对无法通过的区域进行选取action_list = self.q_table[str(state)]pass_action_index = []for index, val in enumerate(action_list):if val >= 0:pass_action_index.append(index)# 使用epsilon greedy来进行动作选取if np.random.rand() <= self.epsilon:# 进行探索return np.random.choice(pass_action_index)else:# 直接选取q最大值max_val = action_list[pass_action_index[0]]max_list = []for i in pass_action_index:# 最大值相同且不止一个则随机选个最大值if max_val < action_list[i]:max_list.clear()max_val = action_list[i]max_list.append(i)elif max_val == action_list[i]:max_list.append(i)return np.random.choice(max_list)def update_q_table(self, feedback, state, action, reward, next_state):# Q(s,a) = Q(s,a) + lr * { reward + gamma * max[Q(s`,a`)] - Q(s,a) }q_s_a = self.q_table[str(state)][action] # 取出对应当前状态动作的q值if feedback == 'stop':q_ns_a = 0  # 撞墙时不存在下一状态，属于原地不变else :q_ns_a = np.max(self.q_table[str(next_state)])# 贝尔曼方程更新# self.q_table[str(state)][action] = q_s_a + self.learning_rate * (#     reward + self.gamma * q_ns_a - q_s_a# )self.q_table[str(state)][action] = (1 - self.learning_rate) * q_s_a + self.learning_rate * (reward + self.gamma * q_ns_a)# print(self.q_table)return self.q_table[str(state)][action]if __name__ == '__main__':np.random.seed(0)env = env.GridWorld()agent = Agent()for ep in range(2000):if ep < 100 :agent.epsilon = 0.2else:agent.epsilon = 0.1state = env.reset()print('第{}轮训练开始 ... '.format(ep + 1))while not env.attach():action = agent.get_action(state)    # 产生动作# print(action)feedback, next_state, reward = env.exec_calc(action)     # 计算状态q_val = agent.update_q_table(feedback, state, action, reward, next_state)   # 更新Q表if feedback == 'stop':env.update_view(state, action, state, q_val)continueelif feedback == 'dead':env.update_view(state, action, next_state, q_val)breakelse:env.update_view(state, action, next_state, q_val)state = next_state   # 状态改变

代码里出现的细节点和博客要讲的理论知识，我后续有空补充哈！

【强化学习】Q-Learning原理及代码实现相关推荐

深度强化学习-DQN算法原理与代码
DQN算法是DeepMind团队提出的一种深度强化学习算法,在许多电动游戏中达到人类玩家甚至超越人类玩家的水准,本文就带领大家了解一下这个算法,论文和代码的链接见下方. 论文:Human-level ...
深度强化学习-D3QN算法原理与代码
Dueling Double Deep Q Network(D3QN)算法结合了Double DQN和Dueling DQN算法的思想,进一步提升了算法的性能.如果对Doubel DQN和Duelin ...
初学者的强化学习q learning和sarsa
Reinforcement learning is a fast-moving field. Many companies are realizing the potential of RL. Rec ...
强化学习q学习求最值_通过Q学习更深入地学习强化学习
强化学习q学习求最值 by Thomas Simonini 通过托马斯·西蒙尼(Thomas Simonini) 通过Q学习更深入地学习强化学习 (Diving deeper into Reinfor ...
深度强化学习-DDPG算法原理和实现
全文共3077个字,8张图,预计阅读时间15分钟. 基于值的强化学习算法的基本思想是根据当前的状态,计算采取每个动作的价值,然后根据价值贪心的选择动作.如果我们省略中间的步骤,即直接根据当前的状态来选 ...
强化学习q学习求最值_Q学习简介：强化学习
强化学习q学习求最值 by ADL 通过ADL Q学习简介:强化学习 (An introduction to Q-Learning: reinforcement learning) This arti ...
强化学习 Reinforcement Learning（三）——是时候用 PARL 框架玩会儿 DOOM 了！！！（下）
强化学习 Reinforcement Learning(三)-- 是时候用 PARL 框架玩会儿 DOOM 了!!!(下) 本文目录强化学习 Reinforcement Learning(三)-- ...
深度强化学习——DQN算法原理
DQN算法原理一.DQN算法是什么二.DQN训练过程三.经验回放 (Experience Replay) 四.目标网络(Target Network) 1.自举(Bootstrapping) 2 ...
强化学习 (Reinforcement Learning)
强化学习: 强化学习是机器学习中的一个领域,强调如何基于环境而行动,以取得最大化的预期利益.其灵感来源于心理学中的行为主义理论,即有机体如何在环境给予的奖励或惩罚的刺激下,逐步形成对刺激的预期,产生能 ...

【强化学习】Q-Learning原理及代码实现

【强化学习】Q-Learning原理及代码实现相关推荐

最新文章

热门文章