目录
4.5 实践：基于前馈神经网络完成鸢尾花分类
继续使用第三章中的鸢尾花分类任务，将Softmax分类器替换为前馈神经网络。
损失函数：交叉熵损失；
优化器：随机梯度下降法；
评价指标：准确率。
4.5.1 小批量梯度下降法
4.5.2 数据处理
import numpy as np
import torch
import torch.utils.data as io
from nndl.dataset import load_data
class IrisDataset(io.Dataset):
def __init__(self, mode='train', num_train=120, num_dev=15):
super(IrisDataset, self).__init__()
# 调用第三章中的数据读取函数，其中不需要将标签转成one-hot类型
X, y = load_data(shuffle=True)
if mode == 'train':
self.X, self.y = X[:num_train], y[:num_train]
elif mode == 'dev':
self.X, self.y = X[num_train:num_train + num_dev], y[num_train:num_train + num_dev]
else:
self.X, self.y = X[num_train + num_dev:], y[num_train + num_dev:]
def __getitem__(self, idx):
return self.X[idx], self.y[idx]
def __len__(self):
return len(self.y)
torch.seed()
train_dataset = IrisDataset(mode='train')
dev_dataset = IrisDataset(mode='dev')
test_dataset = IrisDataset(mode='test')
# 打印训练集长度
print ("length of train set: ", len(train_dataset))
length of train set: 120????????4.5.2.2 用DataLoader进行封装
# 批量大小
batch_size = 16
# 加载数据
train_loader = io.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
dev_loader = io.DataLoader(dev_dataset, batch_size=batch_size)
test_loader = io.DataLoader(test_dataset, batch_size=batch_size)
4.5.3 模型构建
from torch import nn
# 定义前馈神经网络
class Model_MLP_L2_V3(nn.Module):
def __init__(self, input_size, output_size, hidden_size):
super(Model_MLP_L2_V3, self).__init__()
# 构建第一个全连接层
self.fc1 = nn.Linear(
input_size,
hidden_size,
)
nn.init.normal_(self.fc1.weight, mean=0, std=0.01)
nn.init.constant_(self.fc1.bias,1.0)
# 构建第二全连接层
self.fc2 = nn.Linear(
hidden_size,
output_size,
)
nn.init.normal_(self.fc2.weight, mean=0, std=0.01)
nn.init.constant_(self.fc2.bias, 1.0)
# 定义网络使用的激活函数
self.act = nn.Sigmoid()
def forward(self, inputs):
outputs = self.fc1(inputs)
outputs = self.act(outputs)
outputs = self.fc2(outputs)
return outputs
fnn_model = Model_MLP_L2_V3(input_size=4, output_size=3, hidden_size=6)
4.5.4 完善Runner类
class RunnerV3(object):
def __init__(self, model, optimizer, loss_fn, metric, **kwargs):
self.model = model
self.optimizer = optimizer
self.loss_fn = loss_fn
self.metric = metric # 只用于计算评价指标
# 记录训练过程中的评价指标变化情况
self.dev_scores = []
# 记录训练过程中的损失函数变化情况
self.train_epoch_losses = [] # 一个epoch记录一次loss
self.train_step_losses = [] # 一个step记录一次loss
self.dev_losses = []
# 记录全局最优指标
self.best_score = 0
def train(self, train_loader, dev_loader=None, **kwargs):
# 将模型切换为训练模式
self.model.train()
# 传入训练轮数，如果没有传入值则默认为0
num_epochs = kwargs.get("num_epochs", 0)
# 传入log打印频率，如果没有传入值则默认为100
log_steps = kwargs.get("log_steps", 100)
# 评价频率
eval_steps = kwargs.get("eval_steps", 0)
# 传入模型保存路径，如果没有传入值则默认为"best_model.pdparams"
save_path = kwargs.get("save_path", "best_model.pdparams")
custom_print_log = kwargs.get("custom_print_log", None)
# 训练总的步数
num_training_steps = num_epochs * len(train_loader)
if eval_steps:
if self.metric is None:
raise RuntimeError('Error: Metric can not be None!')
if dev_loader is None:
raise RuntimeError('Error: dev_loader can not be None!')
# 运行的step数目
global_step = 0
# 进行num_epochs轮训练
for epoch in range(num_epochs):
# 用于统计训练集的损失
total_loss = 0
for step, data in enumerate(train_loader):
X, y = data
# 获取模型预测
logits = self.model(X)
loss = self.loss_fn(logits, y) # 默认求mean
total_loss += loss
# 训练过程中，每个step的loss进行保存
self.train_step_losses.append((global_step, loss.item()))
if log_steps and global_step % log_steps == 0:
print(
f"[Train] epoch: {epoch}/{num_epochs}, step: {global_step}/{num_training_steps}, loss: {loss.item():.5f}")
# 梯度反向传播，计算每个参数的梯度值
loss.backward()
if custom_print_log:
custom_print_log(self)
# 小批量梯度下降进行参数更新
self.optimizer.step()
# 梯度归零
optimizer.zero_grad()
# 判断是否需要评价
if eval_steps > 0 and global_step > 0 and \
(global_step % eval_steps == 0 or global_step == (num_training_steps - 1)):
dev_score, dev_loss = self.evaluate(dev_loader, global_step=global_step)
print(f"[Evaluate] dev score: {dev_score:.5f}, dev loss: {dev_loss:.5f}")
# 将模型切换为训练模式
self.model.train()
# 如果当前指标为最优指标，保存该模型
if dev_score > self.best_score:
self.save_model(save_path)
print(
f"[Evaluate] best accuracy performence has been updated: {self.best_score:.5f} --> {dev_score:.5f}")
self.best_score = dev_score
global_step += 1
# 当前epoch 训练loss累计值
trn_loss = (total_loss / len(train_loader)).item()
# epoch粒度的训练loss保存
self.train_epoch_losses.append(trn_loss)
print("[Train] Training done!")
# 模型评估阶段，使用'paddle.no_grad()'控制不计算和存储梯度
@torch.no_grad()
def evaluate(self, dev_loader, **kwargs):
assert self.metric is not None
# 将模型设置为评估模式
self.model.eval()
global_step = kwargs.get("global_step", -1)
# 用于统计训练集的损失
total_loss = 0
# 重置评价
self.metric.reset()
# 遍历验证集每个批次
for batch_id, data in enumerate(dev_loader):
X, y = data
# 计算模型输出
logits = self.model(X)
# 计算损失函数
loss = self.loss_fn(logits, y).item()
# 累积损失
total_loss += loss
# 累积评价
self.metric.update(logits, y)
dev_loss = (total_loss / len(dev_loader))
dev_score = self.metric.compute()
# 记录验证集loss
if global_step != -1:
self.dev_losses.append((global_step, dev_loss))
self.dev_scores.append(dev_score)
return dev_score, dev_loss
# 模型评估阶段，使用'paddle.no_grad()'控制不计算和存储梯度
@torch.no_grad()
def predict(self, x, **kwargs):
# 将模型设置为评估模式
self.model.eval()
# 运行模型前向计算，得到预测值
logits = self.model(x)
return logits
def save_model(self, save_path):
torch.save(self.model.state_dict(), save_path)
def load_model(self, model_path):
state_dict = torch.load(model_path)
self.model.load_state_dict(state_dict)
4.5.5 模型训练
lr = 0.2
# 定义网络
model = fnn_model
# 定义优化器
optimizer = opt.SGD(model.parameters(),lr, )
# 定义损失函数。softmax+交叉熵
loss_fn = F.cross_entropy
metric = Accuracy(is_logist=True)
runner = RunnerV3(model, optimizer, loss_fn, metric)
# 启动训练
log_steps = 100
eval_steps = 50
runner.train(train_loader, dev_loader,
num_epochs=150, log_steps=log_steps, eval_steps = eval_steps,
save_path="best_model.pdparams")
[Train] epoch: 0/150, step: 0/1200, loss: 1.09752
[Evaluate] dev score: 0.26667, dev loss: 1.09940
[Evaluate] best accuracy performence has been updated: 0.00000 --> 0.26667
[Train] epoch: 12/150, step: 100/1200, loss: 1.11704
[Evaluate] dev score: 0.26667, dev loss: 1.13060
[Evaluate] dev score: 0.26667, dev loss: 1.12613
[Train] epoch: 25/150, step: 200/1200, loss: 1.08698
[Evaluate] dev score: 0.26667, dev loss: 1.16977
[Evaluate] dev score: 0.26667, dev loss: 1.09303
[Train] epoch: 37/150, step: 300/1200, loss: 1.11398
[Evaluate] dev score: 0.26667, dev loss: 1.13442
[Evaluate] dev score: 0.26667, dev loss: 1.12057
[Train] epoch: 50/150, step: 400/1200, loss: 1.06980
[Evaluate] dev score: 0.33333, dev loss: 1.06150
[Evaluate] best accuracy performence has been updated: 0.26667 --> 0.33333
[Evaluate] dev score: 0.53333, dev loss: 0.97077
[Evaluate] best accuracy performence has been updated: 0.33333 --> 0.53333
[Train] epoch: 62/150, step: 500/1200, loss: 0.77592
[Evaluate] dev score: 0.86667, dev loss: 0.77120
[Evaluate] best accuracy performence has been updated: 0.53333 --> 0.86667
[Evaluate] dev score: 0.66667, dev loss: 0.67145
[Train] epoch: 75/150, step: 600/1200, loss: 0.54504
[Evaluate] dev score: 0.86667, dev loss: 0.58758
[Evaluate] dev score: 1.00000, dev loss: 0.51056
[Evaluate] best accuracy performence has been updated: 0.86667 --> 1.00000
[Train] epoch: 87/150, step: 700/1200, loss: 0.43032
[Evaluate] dev score: 1.00000, dev loss: 0.47934
[Evaluate] dev score: 0.93333, dev loss: 0.45488
[Train] epoch: 100/150, step: 800/1200, loss: 0.39755
[Evaluate] dev score: 1.00000, dev loss: 0.40774
[Evaluate] dev score: 1.00000, dev loss: 0.39165
[Train] epoch: 112/150, step: 900/1200, loss: 0.40829
[Evaluate] dev score: 1.00000, dev loss: 0.36444
[Evaluate] dev score: 0.93333, dev loss: 0.36047
[Train] epoch: 125/150, step: 1000/1200, loss: 0.26360
[Evaluate] dev score: 1.00000, dev loss: 0.33027
[Evaluate] dev score: 1.00000, dev loss: 0.31560
[Train] epoch: 137/150, step: 1100/1200, loss: 0.36213
[Evaluate] dev score: 1.00000, dev loss: 0.29368
[Evaluate] dev score: 1.00000, dev loss: 0.26658
[Evaluate] dev score: 1.00000, dev loss: 0.25821
[Train] Training done!
# 绘制训练集和验证集的损失变化以及验证集上的准确率变化曲线
def plot_training_loss_acc(runner, fig_name,
fig_size=(16, 6),
sample_step=20,
loss_legend_loc="upper right",
acc_legend_loc="lower right",
train_color="#e4007f",
dev_color='#f19ec2',
fontsize='large',
train_linestyle="-",
dev_linestyle='--'):
plt.figure(figsize=fig_size)
plt.subplot(1, 2, 1)
train_items = runner.train_step_losses[::sample_step]
train_steps = [x[0] for x in train_items]
train_losses = [x[1] for x in train_items]
plt.plot(train_steps, train_losses, color=train_color, linestyle=train_linestyle, label="Train loss")
if len(runner.dev_losses) > 0:
dev_steps = [x[0] for x in runner.dev_losses]
dev_losses = [x[1] for x in runner.dev_losses]
plt.plot(dev_steps, dev_losses, color=dev_color, linestyle=dev_linestyle, label="Dev loss")
# 绘制坐标轴和图例
plt.ylabel("loss", fontsize=fontsize)
plt.xlabel("step", fontsize=fontsize)
plt.legend(loc=loss_legend_loc, fontsize='x-large')
# 绘制评价准确率变化曲线
if len(runner.dev_scores) > 0:
plt.subplot(1, 2, 2)
plt.plot(dev_steps, runner.dev_scores,
color=dev_color, linestyle=dev_linestyle, label="Dev accuracy")
# 绘制坐标轴和图例
plt.ylabel("score", fontsize=fontsize)
plt.xlabel("step", fontsize=fontsize)
plt.legend(loc=acc_legend_loc, fontsize='x-large')
plt.savefig(fig_name)
plt.show()
4.5.6 模型评价
# 加载最优模型
runner.load_model('best_model.pdparams')
# 模型评价
score, loss = runner.evaluate(test_loader)
print("[Test] accuracy/loss: {:.4f}/{:.4f}".format(score, loss))[Test] accuracy/loss: 1.0000/0.5525
4.5.7 模型预测
# 获取测试集中第一条数据
#print(test_loader)
for i_batch, batch_data in enumerate(test_loader):
print(i_batch) # 打印batch编号
print(batch_data[0].size()) # 打印该batch里面src
print(batch_data[1].size()) # 打印该batch里面trg
X, label = batch_data[0],batch_data[1],
logits = runner.predict(X)
pred_class = torch.argmax(logits[0])
label = label[0]
# 输出真实类别与预测类别
print("The true category is {} and the predicted category is {}".format(label, pred_class))The true category is 2 and the predicted category is 2思考题
1. 对比Softmax分类和前馈神经网络分类。（必做）
import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn.linear_model import LogisticRegression
iris = datasets.load_iris() # 加载数据
list(iris.keys()) # ['data', 'target', 'frame', 'target_names', 'DESCR', 'feature_names', 'filename', 'data_module']
X = iris["data"][:, 0:]
y = (iris["target"] == 2).astype(np.int32) # 标签，是维吉尼亚鸢尾花y就是1，否则为0
log_reg = LogisticRegression(solver="lbfgs", random_state=42)
log_reg.fit(X, y) # 训练模型
X = iris["data"][:, (0, 1)]
y = iris["target"]
# 设置超参数multi_class为"multinomial"，指定一个支持Softmax回归的求解器，默认使用l2正则化，可以通过超参数C进行控制
softmax_reg = LogisticRegression(multi_class="multinomial", solver="lbfgs", C=10, random_state=42)
softmax_reg.fit(X, y)
softmax_reg.predict([[5, 2]]) # 输出：array([2])
softmax_reg.predict_proba([[5, 2]])
# 输出：array([[6.38014896e-07, 5.74929995e-02, 9.42506362e-01]])
# 分别对应：山鸢尾、变色鸢尾和维吉尼亚鸢尾
x0, x1 = np.meshgrid(
np.linspace(4, 8, 500).reshape(-1, 1),
np.linspace(1.5, 4.5, 200).reshape(-1, 1),
)
X_new = np.c_[x0.ravel(), x1.ravel()]
y_proba = softmax_reg.predict_proba(X_new)
y_predict = softmax_reg.predict(X_new)
zz1 = y_proba[:, 1].reshape(x0.shape)
zz = y_predict.reshape(x0.shape)
plt.figure(figsize=(10, 4))
plt.axis([4, 8, 1.5, 4.5])
plt.plot(X[y == 2, 0], X[y == 2, 1], "g^", label="Iris virginica")
plt.plot(X[y == 1, 0], X[y == 1, 1], "bs", label="Iris versicolor")
plt.plot(X[y == 0, 0], X[y == 0, 1], "yo", label="Iris setosa")
from matplotlib.colors import ListedColormap
custom_cmap = ListedColormap(['#FF0000', '#9898ff', '#a0faa0'])
plt.contourf(x0, x1, zz, cmap=custom_cmap)
plt.xlabel("SepalLengthCm", fontsize=14)
plt.ylabel("SepalWidthCm", fontsize=14)
plt.legend(loc="center left", fontsize=14)
plt.show()
?
前馈神经网络
softmax：
?
发现无论是哪种方法，对鸢尾花的分类都是比较好用的。
Softmax是一种激活函数，它可以将一个数值向量归一化为一个概率分布向量，且各个概率之和为1。Softmax可以用来作为神经网络的最后一层，用于多分类问题的输出。Softmax层常常和交叉熵损失函数一起结合使用。
Softmax函数可以将上一层的原始数据进行归一化，转化为一个(0,1)之间的数值，这些数值可以被当做概率分布，用来作为多分类的目标预测值。Softmax函数一般作为神经网络的最后一层，接受来自上一层网络的输入值，然后将其转化为概率。?
?总结来说：Softmax训练的深度特征，会把整个超空间或者超球，按照分类个数进行划分，保证类别是可分的，这一点对多分类任务如MNIST和ImageNet非常合适，因为测试类别必定在训练类别中。
但Softmax并不要求类内紧凑和类间分离，这一点非常不适合人脸识别任务，因为训练集的1W人数，相对测试集整个世界70亿人类来说，非常微不足道，而我们不可能拿到所有人的训练样本，更过分的是，一般我们还要求训练集和测试集不重叠。
所以需要改造Softmax，除了保证可分性外，还要做到特征向量类内尽可能紧凑，类间尽可能分离。
神经网络的优点是分类的准确度高；并行分布处理能力强,分布存储及学习能力强，对噪声神经有较强的鲁棒性和容错能力，能充分逼近复杂的非线性关系；具备联想记忆的功能。
而它的缺点是神经网络需要大量的参数，如网络拓扑结构、权值和阈值的初始值；不能观察之间的学习过程，输出结果难以解释，会影响到结果的可信度和可接受程度；学习时间过长,甚至可能达不到学习的目的。
总结：
一：通过这次实验我基本掌握前馈神经网络框架，以及参数优化问题，对比了不同分类器的效果，这几次前馈神经网络的实验课，我对BP算法又有了跟深刻的理解，并且对于paddle，pytorch之间的区别有了更多的了解。
二思维导图：
?