【强化学习】用pandas 与 numpy 分别实现 q-learning, saras, saras(lambda)算法

本文作者：hhh5460

本文地址：https://www.cnblogs.com/hhh5460/p/10159331.html

特别感谢：本文的三幅图皆来自莫凡的教程 https://morvanzhou.github.io/

pandas是基于numpy的，但是两者之间的操作有区别，故在实现上述算法时的细节有出入。故记录之

几点说明：

1). 为了更好的说明问题，采用最简单的例一。

2). 分离了环境与个体，采用类编程的形式。

3). 调整了环境与个体的变量、函数的位置，使得Agent完全不需要改动！

4). 个体与环境的互动逻辑更符合实际

〇、效果图

一、pandas实现

1.q-learning

class RLQLearning(Agent):
    '''Agent的子类'''
    def __init__(self, env):
        super().__init__(env)

    def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.4):
        '''学习'''
        print('q-learning算法')
        for _ in range(episode):
            s = self.env.reset()
            is_win = False
            while not is_win:
                a = self.observe(s, epsilon)
                r, s1, is_win = self.env.step(a)
                self.Q.ix[s, a] += alpha * (r + gamma * self.Q.ix[s1, self.env.get_valid_actions(s1)].max() - self.Q.ix[s, a])
                s = s1

2.saras

class RLSaras(Agent):
    '''Agent的子类'''
    def __init__(self, env):
        super().__init__(env)

    def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.4):
        '''学习'''
        print('saras算法')
        for _ in range(episode):
            s = self.env.reset()
            a = self.observe(s, epsilon)
            is_win = False
            while not is_win:
                r, s1, is_win = self.env.step(a)
                a1 = self.observe(s1, epsilon)
                self.Q.ix[s, a] += alpha * (r + gamma * self.Q.ix[s1, a1] - self.Q.ix[s, a])
                s, a = s1, a1

3.saras(lambda)

class RLSarasLambda(Agent):
    '''Agent的子类'''
    def __init__(self, env):
        super().__init__(env)
        self.E = self.Q.copy() # 复制Q table

    def learn(self, alpha=0.01, gamma=0.9, lambda_=0.9, episode=100, epsilon=0.4):
        '''学习'''
        print('saras(lambda)算法，lambda_为衰减值')
        for _ in range(episode):
            self.E *= 0
            s = self.env.reset()
            a = self.observe(s, epsilon)
            is_win = False
            while not is_win:
                r, s1, is_win = self.env.step(a)
                a1 = self.observe(s1, epsilon)
                delta = r + gamma * self.Q.ix[s1, a1] - self.Q.ix[s, a]
                #self.E.ix[s, a] += 1 # 效果不如下两句
                self.E.ix[s] *= 0
                self.E.ix[s, a] = 1
                for s_ in self.env.states:
                    for a_ in self.env.actions:
                        self.Q.ix[s_, a_] += alpha * delta * self.E.ix[s_, a_]
                        self.E.ix[s_, a_] *= gamma * lambda_
                s, a = s1, a1

4.完整代码

 import pandas as pd
 import random
 import time

 '''
 -o---T
 # T 就是宝藏的位置, o 是探索者的位置
 '''

 # 作者：hhh5460
 # 时间：20181221
 # 地点：Tai Zi Miao

 class Env(object):
     '''环境'''
     def __init__(self):
         '''初始化'''
         self.board = list('-----T')
         self.states = range(6)
         self.actions = ['left', 'right']
         self.rewards = [0,0,0,0,0,1]

     def get_valid_actions(self, state):
         '''取当前状态下所有的合法动作'''
         valid_actions = []
         if state != 5:              # 除末状态（位置），皆可向右
             valid_actions.append('right')
         if state != 0:              # 除首状态（位置），皆可向左
             valid_actions.append('left')
         return valid_actions

     def _step(self, action):
         '''执行动作，到达新状态'''
         if action == 'right' and self.state != self.states[-1]: # 除末状态（位置），向右+1
             self.state += 1
         elif action == 'left' and self.state != self.states[0]: # 除首状态（位置），向左-1
             self.state -= 1

     def reset(self):
         '''重置环境，返回状态0'''
         self.board = list('-----T')
         self.state = 0
         self.board[self.state] = 'o'
         print('\r                  ', end='')
         print('\r{}'.format(''.join(self.board)), end='')
         return self.state

     def step(self, action, step_time=0.1):
         '''执行动作 返回奖励、新状态、胜利标志'''
         self.board[self.state] = '-' # 擦除旧位置'o'
         self._step(action)           # 到达新位置
         self.board[self.state] = 'o' # 改变新位置

         reward = self.rewards[self.state] # 奖励
         is_win = [False, True][self.state == self.states[-1]] # 胜利标志
         if is_win == True:
             print('\r{}  WIN!'.format(''.join(self.board)), end='') # 胜利，则加特写镜头
         else:
             print('\r{}'.format(''.join(self.board)), end='')
         time.sleep(step_time)

         return reward, self.state, is_win

 class Agent(object):
     '''智能体'''
     def __init__(self, env):
         '''初始化'''
         # 环境
         self.env = env
         # 大脑
         self.Q = pd.DataFrame(data=[[0 for _ in self.env.actions] for _ in self.env.states],
                                     index=self.env.states,
                                     columns=self.env.actions)

     def observe(self, state, epsilon=0.4):
         '''观察'''
         # 根据自身所处状态，按某种策略选择相应的动作
         if random.uniform(0,1) < epsilon:   # 贪婪
             s = self.Q.ix[state].filter(items=self.env.get_valid_actions(state))
             action = random.choice(s[s==s.max()].index) # 可能多个最大值！
         else:                               # 探索
             action = random.choice(self.env.get_valid_actions(state))
         return action

     def learn(self,*args, **kw):
         '''学习'''
         pass

     def play(self, step_time=0.5):
         '''玩耍'''
         # 学有所成
         s = self.env.reset()
         is_win = False
         while not is_win:
             a = self.observe(s, epsilon=1.) # 1.，100%贪婪，即利用
             _, s1, is_win = self.env.step(a, step_time)
             s = s1
         print()

 class RLQLearning(Agent):
     '''Agent的子类'''
     def __init__(self, env):
         super().__init__(env)

     def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.4):
         '''学习'''
         print('q-learning算法')
         for _ in range(episode):
             s = self.env.reset()
             is_win = False
             while not is_win:
                 a = self.observe(s, epsilon)
                 r, s1, is_win = self.env.step(a)
                 self.Q.ix[s, a] += alpha * (r + gamma * self.Q.ix[s1, self.env.get_valid_actions(s1)].max() - self.Q.ix[s, a])
                 s = s1

 class RLSaras(Agent):
     '''Agent的子类'''
     def __init__(self, env):
         super().__init__(env)

     def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.4):
         '''学习'''
         print('saras算法')
         for _ in range(episode):
             s = self.env.reset()
             a = self.observe(s, epsilon)
             is_win = False
             while not is_win:
                 r, s1, is_win = self.env.step(a)
                 a1 = self.observe(s1, epsilon)
                 self.Q.ix[s, a] += alpha * (r + gamma * self.Q.ix[s1, a1] - self.Q.ix[s, a])
                 s, a = s1, a1

 class RLSarasLambda(Agent):
     '''Agent的子类'''
     def __init__(self, env):
         super().__init__(env)
         self.E = self.Q.copy() # 复制Q table

     def learn(self, alpha=0.01, gamma=0.9, lambda_=0.9, episode=100, epsilon=0.4):
         '''学习'''
         print('saras(lambda)算法，lambda_为衰减值')
         for _ in range(episode):
             self.E *= 0
             s = self.env.reset()
             a = self.observe(s, epsilon)
             is_win = False
             while not is_win:
                 r, s1, is_win = self.env.step(a)
                 a1 = self.observe(s1, epsilon)
                 delta = r + gamma * self.Q.ix[s1, a1] - self.Q.ix[s, a]
                 #self.E.ix[s, a] += 1 # 效果不如下两句
                 self.E.ix[s] *= 0
                 self.E.ix[s, a] = 1
                 for s_ in self.env.states:
                     for a_ in self.env.actions:
                         self.Q.ix[s_, a_] += alpha * delta * self.E.ix[s_, a_]
                         self.E.ix[s_, a_] *= gamma * lambda_
                 s, a = s1, a1

 if __name__ == '__main__':
     env = Env()         # 环境

     agent = RLQLearning(env)  # 个体
     agent.learn(episode=13) # 先学
     agent.play()            # 再玩

     agent2 = RLSaras(env)  # 个体2
     agent2.learn(episode=13) # 先学
     agent2.play()            # 再玩

     agent3 = RLSarasLambda(env)  # 个体3
     agent3.learn(episode=13) # 先学
     agent3.play()            # 再玩

二、numpy实现

1.q-learning

2.saras

3.saras(lambda)

4.完整代码

 import numpy as np
 import time

 '''
 -o---T
 # T 就是宝藏的位置, o 是探索者的位置
 '''

 # 作者：hhh5460
 # 时间：20181221
 # 地点：Tai Zi Miao

 class Env(object):
     '''环境'''
     def __init__(self):
         '''初始化'''
         self.board = list('-----T')
         self.states = range(6)
         self.actions = ['left', 'right'] # 索引[0,1]
         self.rewards = [0,0,0,0,0,1]

     def get_valid_actions(self, state):
         '''取当前状态下所有的合法动作（索引）'''
         valid_actions = []
         if state != self.states[0]:     # 除首状态（位置），皆可向左
             valid_actions.append(self.actions.index('left'))
         if state != self.states[-1]:    # 除末状态（位置），皆可向右
             valid_actions.append(self.actions.index('right'))
         return valid_actions

     def _step(self, action):
         '''执行动作（索引），到达新状态'''
         if self.actions[action] == 'left' and self.state > self.states[0]:     # 除首状态（位置），向左-1
             self.state = self.state - 1
         elif self.actions[action] == 'right' and self.state < self.states[-1]: # 除末状态（位置），向右+1
             self.state = self.state + 1

     def reset(self):
         '''重置环境，返回状态0'''
         self.board = list('-----T')
         self.state = 0
         self.board[self.state] = 'o'
         print('\r                  ', end='')
         print('\r{}'.format(''.join(self.board)), end='')
         return self.state

     def step(self, action, step_time=0.1):
         '''执行动作 返回奖励、新状态、胜利标志'''
         self.board[self.state] = '-' # 擦除旧位置'o'
         self._step(action) # 到达新位置
         self.board[self.state] = 'o' # 改变新位置

         reward = self.rewards[self.state] # 奖励
         is_win = [False, True][self.state == self.states[-1]] # 胜利标志
         if is_win == True:
             print('\r{}  WIN!'.format(''.join(self.board)), end='') # 胜利，则加特写镜头
         else:
             print('\r{}'.format(''.join(self.board)), end='')
         time.sleep(step_time)

         return reward, self.state, is_win

 class Agent(object):
     '''智能体'''
     def __init__(self, env):
         '''初始化'''
         # 环境
         self.env = env
         # 大脑
         self.Q = np.zeros((len(self.env.states), len(self.env.actions)), dtype=np.float32)

     def observe(self, state, epsilon=0.8):
         '''观察'''
         # 根据自身所处状态，按某种策略选择相应的动作（索引）
         valid_actions = self.env.get_valid_actions(state)
         arr = self.Q[state, valid_actions]
         if (np.random.uniform() > epsilon
             or arr.max() == 0
             or len(arr[arr==arr.max()]) > 1):
             action = np.random.choice(valid_actions) # 探索
         else:
             action = self.Q[state].argmax()          # 利用
         return action

     def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.8):
         '''学习'''
         pass

     def play(self, step_time=0.5):
         '''玩耍'''
         # 学有所成
         s = self.env.reset()
         is_win = False
         while not is_win:
             a = self.observe(s, epsilon=1.) # 1.，100%贪婪，即利用
             _, s1, is_win = self.env.step(a, step_time)
             s = s1
         print()

 class RLQLearning(Agent):
     '''智能体'''
     def __init__(self, env):
         '''初始化'''
         super().__init__(env)

     def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.8):
         '''学习'''
         print('q-learning算法')
         for _ in range(episode):
             s = self.env.reset()
             is_win = False
             while not is_win:
                 a = self.observe(s, epsilon)
                 r, s1, is_win = self.env.step(a)
                 self.Q[s, a] += alpha * (r + gamma * self.Q[s1, self.env.get_valid_actions(s1)].max() - self.Q[s, a])
                 s = s1

 class RLSaras(Agent):
     '''Agent的子类'''
     def __init__(self, env):
         super().__init__(env)

     def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.4):
         '''学习'''
         print('saras算法')
         for _ in range(episode):
             s = self.env.reset()
             a = self.observe(s, epsilon)
             is_win = False
             while not is_win:
                 r, s1, is_win = self.env.step(a)
                 a1 = self.observe(s1, epsilon)
                 self.Q[s, a] += alpha * (r + gamma * self.Q[s1, a1] - self.Q[s, a])
                 s, a = s1, a1

 class RLSarasLambda(Agent):
     '''Agent的子类'''
     def __init__(self, env):
         super().__init__(env)
         self.E = self.Q.copy() # 复制Q table

     def learn(self, alpha=0.01, gamma=0.9, lambda_=0.9, episode=100, epsilon=0.4):
         '''学习'''
         print('saras(lambda)算法，lambda_为衰减值')
         for _ in range(episode):
             self.E *= 0
             s = self.env.reset()
             a = self.observe(s, epsilon)
             is_win = False
             while not is_win:
                 r, s1, is_win = self.env.step(a)
                 a1 = self.observe(s1, epsilon)
                 delta = r + gamma * self.Q[s1, a1] - self.Q[s, a]
                 #self.E.ix[s, a] += 1 # 效果不如下两句
                 self.E[s] *= 0
                 self.E[s, a] = 1
                 for s_ in self.env.states:
                     for a_ in range(len(self.env.actions)): # 遍历动作索引！！
                         self.Q[s_, a_] += alpha * delta * self.E[s_, a_]
                         self.E[s_, a_] *= gamma * lambda_
                 s, a = s1, a1

 if __name__ == '__main__':
     env = Env()         # 环境
     agent = RLQLearning(env)  # 个体
     agent.learn(episode=13) # 先学
     agent.play()            # 再玩

     agent2 = RLSaras(env)  # 个体2
     agent2.learn(episode=13) # 先学
     agent2.play()            # 再玩

     agent3 = RLSarasLambda(env)  # 个体3
     agent3.learn(episode=13) # 先学
     agent3.play()            # 再玩