[Python知识库]强化学习-莫烦强化学习2.1视频中的代码复现

该代码为可以正常运行的代码

import numpy as np
import pandas as pd
import time
np.random.seed(2) #reproducible

N_STATES = 6 #the length of the 1 dimensional world
ACTIONS = ['left','right'] #available actions
EPSILON = 0.9 #greedy police 决策上的策略
ALPHA = 0.1 #learning rate
LAMBDA = 0.9 # discount factor 对于未来奖励的衰减值
MAX_EPISODES =13 #maxinum  episodes  13回合
#FRESH_TIME = 0.3 #fresh time for one move 间隔时间
FRESH_TIME = 0.001
def build_q_table(n_states,actions):
    table = pd.DataFrame(
        np.zeros((n_states,len(actions))), #q_table initial values
        columns= actions, #action's name
    )
    print(table)
    return table

#print(build_q_table(N_STATE,ACTIONs))

def choose_action(state,q_table):
    #This is how to choose an action
    state_actions = q_table.iloc[state,:]
    if (np.random.uniform() > EPSILON) or (state_actions.all()==0): #act non-greedy or state-action high
        action_name = np.random.choice(ACTIONS)
    else: #act greedy
        #action_name = state_actions.argmax()
        action_name = state_actions.idxmax()
    return action_name

def get_env_feedback(S,A):
    #This is how agent will interact with the environment
    if A == 'right': #move right
        if S == N_STATES - 2:#terminate
            S_ = 'terminal'
            R = 1
        else:
            S_ = S + 1
            R = 0
    else:
        R = 0
        if S == 0:
            S_ = S #reach the wall
        else:
            S_ = S - 1
    return S_, R


# environment
def update_env(S,episode,step_counter):
    #This is how environment be updated
    env_list = ['_']*(N_STATES-1) + ['T']  #'____________T' our environment
    if S == 'terminal':
        interaction = 'Episode %s: total_steps = %s' % (episode+1,step_counter)
        print('\r{}'.format(interaction),end='')
        time.sleep(2)
        print('\r                ',end='')
    else:
        env_list[S] = 'o'
        interaction = ''.join(env_list)
        print('\r{}'.format(interaction),end='')
        time.sleep(FRESH_TIME)

def rl():
    # main part of RL loop
    q_table = build_q_table(N_STATES,ACTIONS)
    for episode in range(MAX_EPISODES):
        step_counter = 0
        S = 0
        is_terminated = False
        update_env(S,episode,step_counter)
        while not is_terminated:
            A = choose_action(S,q_table)
            S_,R = get_env_feedback(S,A)
            q_predict = q_table.loc[S, A]
            if S_ != 'terminal':
                q_target = R + LAMBDA * q_table.iloc[S_, :].max()
            else:
                q_target = R # next state is terminal
                is_terminated = True #terminate this episode


            q_table.loc[S, A] += ALPHA * (q_target - q_predict)  # update
            S = S_ #move to next state

            update_env(S,episode,step_counter+1)
            step_counter +=1
    return q_table

if __name__ == "__main__":
    q_table = rl()
    print('\r\nQ-table:\n')
    print(q_table)

运行程序时遇到的问题：
当直接按照视频中的代码运行时，q_predict = q_table.ix[S, A]，该行代码报错： ‘DataFrame’ object has no attribute ‘ix’ 。
于是，到网上查到是因为新库对名称变化，将ix改为iloc，变为q_predict = q_table.iloc[S, A]。

但是，运行后又开始报错ValueError: Location based indexing can only have [integer, integer slice (START point is INCLUDED, END point is EXCLUDED), listlike of integers, boolean array] types。
到网上查找原因：说使用iloc方法的时候，两个关键变量都要是下标位置而不是列的名称，iloc[下标位置，下标位置]。然后将其修改为

q_predict = q_table.ix[S,ACTIONS.index(A)]

第一轮可以运行，后面又开始报错，因为后面会出现A为1的情况，于是，又去网上搜，发现：

`loc：通过行标签索引数据

iloc：通过行号索引行数据

ix：通过行标签或行号索引数据（基于loc和iloc的混合）`

于是试图采用try,except的方法来跳过这个问题，即修改为

try：
	q_predict = q_table.iloc[S, A]
except：
	q_predict = q_table.loc[S, A]

之后，代码确实能够运行，并且不报错，但是速度奇慢，当把EPSILON ，也就是策略调成0.1的时候才能看到最终结果，就去翻评论，然后再看代码，表示代码逻辑上是没有问题的，可能是try，except导致的速度变慢，后来经过多次查找，终于找到一位大佬修改的pytorch版本的代码，比对之后，发现

def choose_action(state,q_table):
    #This is how to choose an action
    state_actions = q_table.iloc[state,:]
    if (np.random.uniform() > EPSILON) or (state_actions.all()==0): #act non-greedy or state-action high
        action_name = np.random.choice(ACTIONS)
    else: #act greedy
        #action_name = state_actions.argmax()
        action_name = state_actions.idxmax()
    return action_name

之前该函数中的最后一条语句 action_name = state_actions.argmax()返回的是最大数的索引.argmax有一个参数axis,默认是0,表示第几维的最大值。因本实验中行索引为数字，列索引为字符串，所以会出现之后单独使用q_table.loc或q_table.iloc报错的情况。于是将其改为action_name = state_actions.idxmax()，后面凡是用到.ix的地方改成.loc就解决该问题了。