基于值函数的强化学习 小例子（策略退化）

#encoding:UTF-8
#!/usr/bin/env python3
import random

#动作
actions=[0, 1]

#状态
states=[1, 2]

#构建环境   s, a    r s a
def next_state_reward(state, action):
    alfa=random.random()

    if state==1:
        if action==0:
            if alfa<=1.0/3:
                new_state=1
                reward=0
            else:
                new_state=2
                reward=1
        else:
            if alfa>1.0/3:
                new_state=1
                reward=0
            else:
                new_state=2
                reward=1
    else:
         if action==0:
            if alfa<=1.0/3:
                new_state=1
                reward=0
            else:
                new_state=2
                reward=1
         else:
            if alfa>1.0/3:
                new_state=1
                reward=0
            else:
                new_state=2
                reward=1
    return new_state, reward


import torch.nn as nn
import torch.optim


# 构建模型
# 最小二乘法
# 输入状态1维  输出action_q值2维(两个动作)
model = nn.Sequential(
        nn.Linear(1, 2),
        )

#反传优化器
optimizer = torch.optim.Adam(model.parameters())

epsilon=0.4
def act(model, state, act_type="greedy", epsilon=epsilon):
    if (act_type!="greedy") or (random.random() > epsilon):
        # 选最大的
        q_value = model.forward(state)
        action = q_value.max(1)[1].item()
    else: # 随便选
        action = random.randrange(len(actions))
    return action


gama=0.99
learning_rate=0.1


def q_learning():
    state=random.choice(states)
    # 单个状态变为序列型数据
    state=(state, )
    state = torch.FloatTensor(state).unsqueeze(0)

    action=act(model, state, act_type="greedy")
    # q 值
    qs = model(state)
    # 单个数据  位置为0
    q_value = qs[0][action]

    
    # 下一状态 和 奖励
    next_state, reward=next_state_reward(state, action)
    # 单个状态变为序列型数据
    next_state=(next_state, )
    next_state = torch.FloatTensor(next_state).unsqueeze(0)


    # 选择的下一动作
    next_action=act(model, next_state, act_type="max")
    # q 值
    next_q_value, _ = model(next_state).max(1)


    q_estimate = reward + gama*next_q_value 

    td_error=q_estimate - q_value 
   
    # 计算 MSE 损失
    loss = td_error.pow(2).mean()

    # 根据损失改进网络
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()


if __name__=="__main__":
    for episode in range(10**5):
            q_learning()

    state=((0,), (1,)) 
    state = torch.FloatTensor(state) #.unsqueeze(0)
    # q 值
    qs = model(state)
    print(qs)