[人工智能]【学习笔记】PyTorch Learning

QUICKSTART

首先展示一个完整的pytorch代码：

1. load data
2. create the model
3. train the model
4. save & load the model

import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor, Lambda, Compose
import matplotlib.pyplot as plt

### Working with data ###
### step 1: get datasets.
### step 2: get dataloaders. Pass the 'Dataset' as an argument to 'DataLoader'. This supports automatic batching, sampling, shuffling and multiprocess data loading.

# Download training data from open datasets.
training_data = datasets.FashionMNIST(root="data", train=True, download=True, transform=ToTensor())
test_data = datasets.FashionMNIST(root="data", train=False, download=True, transform=ToTensor()）

batch_size = 64

# Create data loaders.
train_dataloader = DataLoader(training_data, batch_size=batch_size)
test_dataloader = DataLoader(test_data, batch_size=batch_size)

for X, y in test_dataloader:
    print("Shape of X [N, C, H, W]: ", X.shape)
    print("Shape of y: ", y.shape, y.dtype)
    break

### Creating Models ###
### step 1: create a class that inherits from nn.Module
### step 2: define the layers in '__init__' function
### step 3: define specify the data flow in 'forward' function

# Get cpu or gpu device for training.
device = "cuda" if torch.cuda.is_available() else "cpu"
print("Using {} device".format(device))

# Define model
class NeuralNetwork(nn.Module):
    def __init__(self):
        super(NeuralNetwork, self).__init__()
        self.flatten = nn.Flatten()
        self.linear_relu_stack = nn.Sequential(
            nn.Linear(28*28, 512),
            nn.ReLU(),
            nn.Linear(512, 512),
            nn.ReLU(),
            nn.Linear(512, 10),
            nn.ReLU()
        )

    def forward(self, x):
        x = self.flatten(x)
        logits = self.linear_relu_stack(x)
        return logits

model = NeuralNetwork().to(device)
print(model)

### Optimizing the Model Parameters ###
### step 1: define loss function
### step 2: define optimizer

loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)

### Traing the Model ###
### step 1: define training process
### -- sub step 1: get data size, and for each data do the following:
### -- sub step 2: go through the model and compute prediction error
### -- sub step 3: do backpropagation
### step 2: define testing process

def train(dataloader, model, loss_fn, optimizer):
    size = len(dataloader.dataset)
    for batch, (X, y) in enumerate(dataloader):
        X, y = X.to(device), y.to(device)

        # Compute prediction error
        pred = model(X)
        loss = loss_fn(pred, y)

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if batch % 100 == 0:
            loss, current = loss.item(), batch * len(X)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

def test(dataloader, model, loss_fn):
    size = len(dataloader.dataset)
    num_batches = len(dataloader)
    model.eval()
    test_loss, correct = 0, 0
    with torch.no_grad():
        for X, y in dataloader:
            X, y = X.to(device), y.to(device)
            pred = model(X)
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()
    test_loss /= num_batches
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")

epochs = 5
for t in range(epochs):
    print(f"Epoch {t+1}\n-------------------------------")
    train(train_dataloader, model, loss_fn, optimizer)
    test(test_dataloader, model, loss_fn)
print("Done!")

### Saving the Model ###
### A common way to save a model is to serialize the internal state dictionary (containing the model parameters).

torch.save(model.state_dict(), "model.pth")
print("Saved PyTorch Model State to model.pth")

### Loading the Model ###
### step 1: re-create the model structure
### step 2: load the state dictionary into the odel
model = NeuralNetwork()
model.load_state_dict(torch.load("model.pth"))

# This model can now be used to make predictions.
classes = ["T-shirt/top", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot"]

model.eval()
x, y = test_data[0][0], test_data[0][1]
with torch.no_grad():
    pred = model(x)
    predicted, actual = classes[pred[0].argmax(0)], classes[y]
    print(f'Predicted: "{predicted}", Actual: "{actual}"')

Preliminary

Tensor用法

• tensor初始化及属性

rand_tensor = torch.rand(2,3)
ones_tensor = torch.ones(2,3)
zeros_tensor = torch.zeros(2,3)
print(tensor.shape, tensor.dtype, tensor.device)

p.s.tensor默认是创建在CPU上的，需要显式地移动到GPU。

# We move our tensor to the GPU if available
if torch.cuda.is_available():
  tensor = tensor.to('cuda')

• tensor操作
  1.索引和切片

tensor = torch.ones(4, 4)
print('First row: ',tensor[0])
print('First column: ', tensor[:, 0])
print('Last column:', tensor[..., -1])

2.拼接

t1 = torch.cat([tensor, tensor, tensor], dim=1)
# 还有torch.stack

3.算术运算

# 矩阵乘法. y1, y2, y3 will have the same value
y1 = tensor @ tensor.T
y2 = tensor.matmul(tensor.T)

y3 = torch.rand_like(tensor)
torch.matmul(tensor, tensor.T, out=y3)

# element-wise乘法. z1, z2, z3 will have the same value
z1 = tensor * tensor
z2 = tensor.mul(tensor)

z3 = torch.rand_like(tensor)
torch.mul(tensor, tensor, out=z3)

# in-place operations: 带下划线
tensor.add_(5)
print(tensor)

# 常见操作：sum()和item()!
agg = tensor.sum() # 所有位置上的值求和后aggregate为单值tensor
agg_item = agg.item() # 获取单值tensor的数值
print(agg_item, type(agg_item))

Datasets & DataLoaders

torch.utils.data.DataLoader和torch.utils.data.Dataset两个模块负责数据相关操作， Dataset存储数据和标签，DataLoader把Dataset变成可迭代对象。

官方数据集使用方法：

1.使用Dataset类得到data
2.使用DataLoader类创建可迭代对象dataloader
3.使用dataloader遍历数据集：每一轮迭代返回一批也就是batch_size个样本，如果shuffle=True则每当所有数据被迭代一遍，就会shuffle一次

import torch
from torch.utils.data import Dataset, DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor

# Loading a Dataset
training_data = datasets.FashionMNIST(
    root="data", # the path where the train/test data is stored
    train=True, # specifies training or test dataset
    download=True, # downloads the data from the internet if it’s not available at root
    transform=ToTensor() # specify the feature and label transformations
)

# Preparing your data for training with DataLoaders
train_dataloader = DataLoader(training_data, batch_size=64, shuffle=True)

# Iterate through the DataLoader
# iter()函数获取可迭代对象的迭代器，本质是调用__iter__方法
# next()函数获取该对象的下一条数据
train_features, train_labels = next(iter(train_dataloader))
print(f"Feature batch shape: {train_features.size()}")
print(f"Labels batch shape: {train_labels.size()}")
img = train_features[0].squeeze()
label = train_labels[0]

自定义数据集使用方法：

定义自己的数据集类（继承Dataset类）并实现三个接口：
（1）__init__, 实例化类的时候运行一次
（2）__len__, 返回数据集大小
（3）__getitem__, 返回一个指定index处的样本

# 前提：图片存在img_dir文件夹下，标签在annotations_file表格中
import os
import pandas as pd
from torchvision.io import read_image

class CustomImageDataset(Dataset):
    def __init__(self, annotations_file, img_dir, transform=None, target_transform=None):
        self.img_labels = pd.read_csv(annotations_file)
        self.img_dir = img_dir
        self.transform = transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.img_labels)

    def __getitem__(self, idx):
        img_path = os.path.join(self.img_dir, self.img_labels.iloc[idx, 0])
        image = read_image(img_path)
        label = self.img_labels.iloc[idx, 1]
        if self.transform:
            image = self.transform(image)
        if self.target_transform:
            label = self.target_transform(label)
        return image, label

Transforms

包含一些转换数据格式的操作。例如，FashionMNIST特征是PIL image格式，标签是整数；但模型训练需要特征是标准化的tensor，标签是one-hot tensor，所以读取数据集后我们用transform参数转化下数据格式。

所有的torchvision数据集都有这两个参数：
（1）transform：用来modify特征。transform=ToTensor()把PIL image或Numpy ndarray转化为FloatTensor标准化tensor，并把数值归一化至[0., 1.]
（2）target_transform：用来modify标签。Lambda()可以自定义lambda函数

import torch
from torchvision import datasets
from torchvision.transforms import ToTensor, Lambda

ds = datasets.FashionMNIST(
    root="data",
    train=True,
    download=True,
    transform=ToTensor(), 
    target_transform=Lambda(lambda y: torch.zeros(10, dtype=torch.float).scatter_(0, torch.tensor(y), value=1))
)

相关官方文档Links:
pytorch内置的文本数据集及用法：链接
torch.utils.data API docs: 链接
torchvision.transforms API docs: 链接
scatter()函数：链接

Build Model

模型继承nn.Module类。forward()函数是当数据被作为参数传递给模型时自动调用的。

查看模型结构：

print("Model structure: ", model, "\n\n")
for name, param in model.named_parameters():
    print(f"Layer: {name} | Size: {param.size()} | Values : {param[:2]} \n")

Train Model

Autograd

• requires_grad=True标志一个tensor的梯度需要被记录
• .grad_fn属性存储了这个tensor的反向传播函数的引用
• .backward()函数计算这个tensor对各个参数的偏导数。在一个计算图上如果需要多次重复计算，则需要先设置retain_graph=True
• 具体参数的.grad属性存储了函数对该参数的偏导数

import torch

x = torch.ones(5)  # input tensor
y = torch.zeros(3)  # expected output
w = torch.randn(5, 3, requires_grad=True)
b = torch.randn(3, requires_grad=True)
z = torch.matmul(x, w)+b
loss = torch.nn.functional.binary_cross_entropy_with_logits(z, y)

print('Gradient function for z =',z.grad_fn)
print('Gradient function for loss =', loss.grad_fn)

# compute the gradients
loss.backward()
print(w.grad)
print(b.grad)

Optimization

Save & Load Model

pytorch模型把学到的参数存在内部状态字典internal state dictionary中， 叫做state_dict。

model = models.vgg16(pretrained=True)
torch.save(model.state_dict(), 'model_weights.pth')

加载模型时要先实例化一个相同的model:

model = model.vgg16() # we do not specify pretrained=True, i.e. do not load default weights
model.load_state_dict(torch.load('model_weights.pth'))
model.eval()
# 做inference前要先model.eval()来把内部的dropout和batch norm层设置为evaluation模式

如果想把整体的模型结构也存储下来：

torch.save(model, 'model.pth')
model = torch.load('model.pth') # relies on module pickle

(FOR CHECK ONLY) PyTorch代码模板

待看的问题

1. super(Net, self).__init__()到底有什么用：直接的来说，就是在子类也有自己的__init__()初始化函数是，也初始化父类的构造函数，从而继承父类的方法和属性。
   【待看】最好的文章, 简单明了的对比，简单的文章