强化学习
基于
- 概率最大
- 价值最大 决定下一个行动取舍
机械手臂、无人驾驶等运动相关的算法都和强化学习有关
python安装gym
pip install gym或者pip install openai gym找不到make函数，通过gym.file 查看模块文件路径，避免因为文件名命名错误导致加载错误的模块
强化学习列表: http://gym.openai.com/envs/#classic_control
gym（CartPole-v0）游戏用于强化学习训练
openai
agent智能体（代码）、environment游戏环境（openai中gym）
DQN
代码语言：javascript复制import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from torch.autograd import Variable
import gym
import matplotlib.pyplot as plt
# 超参数
BATCH_SIZE = 32
LR = 0.01 # learning rate
# 强化学习的参数
EPSILON = 0.9 # greedy policy
GAMMA = 0.9 # reward discount
TARGET_REPLACE_ITER = 100 # target update frequency
MEMORY_CAPACITY = 2000
# 导入实验环境
env = gym.make('CartPole-v0')
env = env.unwrapped
N_ACTIONS = env.action_space.n
N_STATES = env.observation_space.shape[0]
class Net(nn.Module):
def __init__(self, ):
super(Net, self).__init__()
self.fc1 = nn.Linear(N_STATES, 10)
self.fc1.weight.data.normal_(0, 0.1) # 初始化
self.out = nn.Linear(10, N_ACTIONS)
self.out.weight.data.normal_(0, 0.1) # 初始化
def forward(self, x):
x = self.fc1(x)
x = F.relu(x)
actions_value = self.out(x)
return actions_value
class DQN(object):
def __init__(self):
self.eval_net, self.target_net = Net(), Net()
# 记录学习到多少步
self.learn_step_counter = 0 # for target update
self.memory_counter = 0 # for storing memory
# 初始化memory
self.memory = np.zeros((MEMORY_CAPACITY, N_STATES * 2 2))
self.optimizer = torch.optim.Adam(self.eval_net.parameters(), lr=LR)
self.loss_func = nn.MSELoss()
def choose_action(self, x):
x = Variable(torch.unsqueeze(torch.FloatTensor(x), 0))
if np.random.uniform() < EPSILON:
action_value = self.eval_net(x)
#action = torch.max(action_value, 1)[1].data.numpy()[0, 0]
action = torch.max(action_value, 1)[1].data.numpy()[0]
else: # random
action = np.random.randint(0, N_ACTIONS)
return action
# s:当前状态， a:动作, r:reward奖励, s_:下一步状态
def store_transaction(self, s, a, r, s_):
transaction = np.hstack((s, [a, r], s_))
# replace the old memory with new memory
index = self.memory_counter % MEMORY_CAPACITY
self.memory[index, :] = transaction
self.memory_counter = 1
def learn(self):
# target net update
if self.learn_step_counter % TARGET_REPLACE_ITER == 0:
self.target_net.load_state_dict(self.eval_net.state_dict())
sample_index = np.random.choice(MEMORY_CAPACITY, BATCH_SIZE)
b_memory = self.memory[sample_index, :]
b_s = Variable(torch.FloatTensor(b_memory[:, :N_STATES]))
b_a = Variable(torch.LongTensor(b_memory[:, N_STATES: N_STATES 1].astype(int)))
b_r = Variable(torch.FloatTensor(b_memory[:, N_STATES 1: N_STATES 2]))
b_s_ = Variable(torch.FloatTensor(b_memory[:, -N_STATES: ]))
q_eval = self.eval_net(b_s).gather(1, b_a)
q_next = self.target_net(b_s_).detach()
q_target = b_r GAMMA * q_next.max(1)[0]
loss = self.loss_func(q_eval, q_target)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
plt.ion()
dqn = DQN()
last_time_steps = torch.FloatTensor([])
print('nCollecting experience...')
for i_episode in range(4000):
s = env.reset()
episode_reward,t = 0,0 # 初始化本场游戏的得分
while True:
env.render()
a = dqn.choose_action(s)
# take action
s_, r, done, info = env.step(a)
episode_reward = r
# modify the reward
x, x_dot, theta, theta_dot = s_
r1 = (env.x_threshold - abs(x)) / env.x_threshold - 0.8
r2 = (env.theta_threshold_radians - abs(theta)) / env.theta_threshold_radians - 0.5
r = r1 r2
t =1
dqn.store_transaction(s, a, r, s_)
if dqn.memory_counter > MEMORY_CAPACITY:
dqn.learn()
if done:
plt.figure(1)
plt.clf()
plt.title('Training...')
plt.xlabel('Episode')
plt.ylabel('Duration')
plt.plot(last_time_steps.numpy())
if len(last_time_steps) >= 100:
means = last_time_steps.unfold(0, 100, 1).mean(1).view(-1)
plt.plot(means.numpy())
plt.pause(0.001)
last_time_steps = torch.FloatTensor(np.hstack((last_time_steps.numpy(),[episode_reward]))) # 更新最近100场游戏的得分stack
break
s = s_
plt.ioff()
plt.show()参考：https://blog.csdn.net/qq_41352018/article/details/80274425
q-learning
- 建立Q-table（m动作数*n状态数）
- Q(s 1,a 1) = R(s,a) y*Max{Q(s,a)} 0<y<1 使用贪心算法策略（局部最优解）
决策树
决策树用数学方法分类数据，准确度不高
代码语言：javascript复制import pandas as pd
filename = 'C:pycharmDataMiningdatasales_data.xls'
data = pd.read_excel(filename, index_col='No')
data[data == u'好'] = 1
data[data == u'是'] = 1
data[data == u'高'] = 1
data[data != 1] = -1
#print(data)
x = pd.DataFrame(data.iloc[:, :3].astype(int))
#x = data.iloc[:, :3].values.astype(int)
print(x)
y = pd.DataFrame(data.iloc[:, 3].astype(int))
print(y)
from sklearn.tree import DecisionTreeClassifier as DTC
dtc = DTC(criterion='entropy') #建立决策树模型，基于信息熵
dtc.fit(x, y)
from sklearn.tree import export_graphviz #可视化决策树
name = 'C:pycharmDataMiningdata\tree.dot'
with open(name, 'w') as f:
f = export_graphviz(dtc, feature_names=x.columns, out_file=f)命令行dot -Tpdf jueceshu.dot -o output.pdf，转换dot为pdf文件 或vscode Graphviz (dot) language support for Visual Studio Code插件 参考：https://blog.csdn.net/weixin_36372879/article/details/80981691
