二十一点 Blackjack-v0

%matplotlib inline
import numpy as np
np.random.seed(0)
import matplotlib.pyplot as plt
import gym

环境使用

env = gym.make("Blackjack-v0")
env.seed(0)
print('观察空间 = {}'.format(env.observation_space))
print('动作空间 = {}'.format(env.action_space))
print('动作数量 = {}'.format(env.action_space.n))

观察空间 = Tuple(Discrete(32), Discrete(11), Discrete(2))
动作空间 = Discrete(2)
动作数量 = 2

在线回合更新

回合更新预测

def ob2state(observation):return observation[0], observation[1], int(observation[2])def evaluate_action_monte_carlo(env, policy, episode_num=500000):q = np.zeros_like(policy)c = np.zeros_like(policy)for _ in range(episode_num):# 玩一回合state_actions = []observation = env.reset()while True:state = ob2state(observation)action = np.random.choice(env.action_space.n, p=policy[state])state_actions.append((state, action))observation, reward, done, _ = env.step(action)if done:break # 回合结束g = reward # 回报for state, action in state_actions:c[state][action] += 1.q[state][action] += (g - q[state][action]) / c[state][action]return q

policy = np.zeros((22, 11, 2, 2))
policy[20:, :, :, 0] = 1 # >= 20 时收手
policy[:20, :, :, 1] = 1 # < 20 时继续q = evaluate_action_monte_carlo(env, policy) # 动作价值
v = (q * policy).sum(axis=-1) # 状态价值

def plot(data):fig, axes = plt.subplots(1, 2, figsize=(9, 4))titles = ['without ace', 'with ace']have_aces = [0, 1]extent = [12, 22, 1, 11]for title, have_ace, axis in zip(titles, have_aces, axes):dat = data[extent[0]:extent[1], extent[2]:extent[3], have_ace].Taxis.imshow(dat, extent=extent, origin='lower')axis.set_xlabel('player sum')axis.set_ylabel('dealer showing')axis.set_title(title)

def play_once(env, policy=None):total_reward = 0observation = env.reset()print('观测 = {}'.format(observation))while True:print('玩家 = {}, 庄家 = {}'.format(env.player, env.dealer))if policy is None:action = np.random.choice(env.action_space.n)print('动作 = {}'.format(action))observation, reward, done, _ = env.step(action)print('观测 = {}, 奖励 = {}, 结束指示 = {}'.format(observation, reward, done))total_reward += rewardif done:return total_reward # 回合结束print("随机策略 奖励：{}".format(play_once(env)))

观测 = (11, 10, False)
玩家 = [8, 3], 庄家 = [10, 9]
动作 = 1
观测 = (12, 10, False), 奖励 = 0, 结束指示 = False
玩家 = [8, 3, 1], 庄家 = [10, 9]
动作 = 0
观测 = (12, 10, False), 奖励 = -1.0, 结束指示 = True
随机策略 奖励：-1.0

plot(v)

带起始探索的回合更新

def monte_carlo_with_exploring_start(env, episode_num=500000):policy = np.zeros((22, 11, 2, 2))policy[:, :, :, 1] = 1.q = np.zeros_like(policy)c = np.zeros_like(policy)for _ in range(episode_num):# 随机选择起始状态和起始动作state = (np.random.randint(12, 22),np.random.randint(1, 11),np.random.randint(2))action = np.random.randint(2)# 玩一回合env.reset()if state[2]: # 有Aenv.player = [1, state[0] - 11]else: # 没有Aif state[0] == 21:env.player = [10, 9, 2]else:env.player = [10, state[0] - 10]env.dealer[0] = state[1]state_actions = []while True:state_actions.append((state, action))observation, reward, done, _ = env.step(action)if done:break # 回合结束state = ob2state(observation)action = np.random.choice(env.action_space.n, p=policy[state])g = reward # 回报for state, action in state_actions:c[state][action] += 1.q[state][action] += (g - q[state][action]) / c[state][action]a = q[state].argmax()policy[state] = 0.policy[state][a] = 1.return policy, q

policy, q = monte_carlo_with_exploring_start(env)
v = q.max(axis=-1)

plot(policy.argmax(-1))
plot(v)

基于柔性策略的回合更新

def monte_carlo_with_soft(env, episode_num=500000, epsilon=0.1):policy = np.ones((22, 11, 2, 2)) * 0.5 # 柔性策略q = np.zeros_like(policy)c = np.zeros_like(policy)for _ in range(episode_num):# 玩一回合state_actions = []observation = env.reset()while True:state = ob2state(observation)action = np.random.choice(env.action_space.n, p=policy[state])state_actions.append((state, action))observation, reward, done, _ = env.step(action)if done:break # 回合结束g = reward # 回报for state, action in state_actions:c[state][action] += 1.q[state][action] += (g - q[state][action]) / c[state][action]# 更新策略为柔性策略a = q[state].argmax()policy[state] = epsilon / 2.policy[state][a] += (1. - epsilon)return policy, q

policy, q = monte_carlo_with_soft(env)
v = q.max(axis=-1)

plot(policy.argmax(-1))
plot(v)

离线回合更新

重要性采样策略评估

def evaluate_monte_carlo_importance_resample(env, policy, behavior_policy,episode_num=500000):q = np.zeros_like(policy)c = np.zeros_like(policy)for _ in range(episode_num):# 用行为策略玩一回合state_actions = []observation = env.reset()while True:state = ob2state(observation)action = np.random.choice(env.action_space.n,p=behavior_policy[state])state_actions.append((state, action))observation, reward, done, _ = env.step(action)if done:break # 玩好了g = reward # 回报rho = 1. # 重要性采样比率for state, action in reversed(state_actions):c[state][action] += rhoq[state][action] += (rho / c[state][action] * (g - q[state][action]))rho *= (policy[state][action] / behavior_policy[state][action])if rho == 0:break # 提前终止return q

policy = np.zeros((22, 11, 2, 2))
policy[20:, :, :, 0] = 1 # >= 20 时收手
policy[:20, :, :, 1] = 1 # < 20 时继续
behavior_policy = np.ones_like(policy) * 0.5
q = evaluate_monte_carlo_importance_resample(env, policy, behavior_policy)
v = (q * policy).sum(axis=-1)

plot(v)

重要性采样回合更新

def monte_carlo_importance_resample(env, episode_num=500000):policy = np.zeros((22, 11, 2, 2))policy[:, :, :, 0] = 1.behavior_policy = np.ones_like(policy) * 0.5 # 柔性策略q = np.zeros_like(policy)c = np.zeros_like(policy)for _ in range(episode_num):# 用行为策略玩一回合state_actions = []observation = env.reset()while True:state = ob2state(observation)action = np.random.choice(env.action_space.n,p=behavior_policy[state])state_actions.append((state, action))observation, reward, done, _ = env.step(action)if done:break # 玩好了g = reward # 回报rho = 1. # 重要性采样比率for state, action in reversed(state_actions):c[state][action] += rhoq[state][action] += (rho / c[state][action] * (g - q[state][action]))# 策略改进a = q[state].argmax()policy[state] = 0.policy[state][a] = 1.if a != action: # 提前终止breakrho /= behavior_policy[state][action]return policy, q

policy, q = monte_carlo_importance_resample(env)
v = q.max(axis=-1)

plot(policy.argmax(-1))
plot(v)

强化学习原理与Python实现（四）相关推荐

强化学习原理与python实现原理pdf_纯Python实现！Facebook发布PyTorch分布式强化学习库...
图灵TOPIA来源:Facebook编译:刘静图灵联邦编辑部出品Facebook于近日发布了PyTorch中用于强化学习(RL)研究的平台:TorchBeast.TorchBeast实现了流行的IMP ...
强化学习(十九) AlphaGo Zero强化学习原理
在强化学习(十八) 基于模拟的搜索与蒙特卡罗树搜索(MCTS)中,我们讨论了MCTS的原理和在棋类中的基本应用.这里我们在前一节MCTS的基础上,讨论下DeepMind的AlphaGo Zero强化学 ...
强化学习原理及应用作业之动态规划算法【SYSU_2023SpringRL】
强化学习原理及应用作业之动态规划算法[SYSU_2023SpringRL] 题目描述: 任务一:动态规划方法一.策略迭代算法 1.代码 2.结果 3.思路讲解策略评估策略提升二.价值迭代算法 ...
深度强化学习：基于Python的理论及实践（英文版）
深度强化学习结合了深度学习和强化学习,使人工智能体能够在没有监督的情况下学习如何解决顺序决策问题.在过去的十年中,深度强化学习在一系列问题上取得了显著的成果,涵盖从单机游戏和多人游戏到机器人技术等方方 ...
强化学习经典算法笔记(十四)：双延迟深度确定性策略梯度算法TD3的PyTorch实现
强化学习经典算法笔记(十四):双延迟深度确定性策略梯度算法TD3的PyTorch实现 TD3算法简介 TD3是Twin Delayed Deep Deterministic policy gradie ...
系统学习深度学习（四十一）--AlphaGo Zero强化学习原理
转自:https://www.cnblogs.com/pinard/p/10609228.html 本篇主要参考了AlphaGo Zero的论文, AlphaGo Zero综述和AlphaGo Zer ...
独家 | 浅谈强化学习原理（附代码链接）
作者:Michel Kana 翻译:王琦校对:王雨桐本文约4900字,建议阅读15分钟. 本文介绍了强化学习的基本原理,并通过代码实例来讲解如何找到最优策略. Google在2017年年底发布了A ...
图解强化学习原理超详解（一）
强化学习一.背景机器学习是人工智能的一个分支,在近30多年已发展为一门多领域交叉学科,涉及概率论.统计学.逼近论.凸分析.计算复杂性理论等的学科.强化学习(RL)作为机器学习的一个子领域,其灵感来 ...
【强化学习论文合集】四十一.2022机器人与自动化国际会议论文(ICRA2022)
欢迎订阅本专栏:<强化学习论文合集> 专栏介绍: 本专栏整理了2017~2022年(后面会持续更新)强化学习领域国际顶级会议已录用的论文,会议包括但不限于:ICML.NeurIPS.AAA ...
[强化学习代码笔记]Python复习
文章目录 Python复习 1. 介绍 2. 编写规范 3. 基本语法一切都是对象查看帮助空/否定保留字 as assert del try...except...finally global ...

强化学习原理与Python实现（四）

二十一点 Blackjack-v0

环境使用

在线回合更新

回合更新预测

带起始探索的回合更新

基于柔性策略的回合更新

离线回合更新

重要性采样策略评估

重要性采样回合更新

强化学习原理与Python实现（四）相关推荐

最新文章

热门文章

强化学习 原理与Python实现（四）

二十一点 Blackjack-v0

环境使用

在线回合更新

回合更新预测

带起始探索的回合更新

基于柔性策略的回合更新

离线回合更新

重要性采样策略评估

重要性采样回合更新

强化学习 原理与Python实现（四）相关推荐

最新文章

热门文章

强化学习原理与Python实现（四）

强化学习原理与Python实现（四）相关推荐