torch.utils.Dataset PyTorch数据集配置
class Dataset(object):
r"""An abstract class representing a :class:`Dataset`.
All datasets that represent a map from keys to data samples should subclass
it. All subclasses should overwrite :meth:`__getitem__`, supporting fetching a
data sample for a given key. Subclasses could also optionally overwrite
:meth:`__len__`, which is expected to return the size of the dataset by many
:class:`~torch.utils.data.Sampler` implementations and the default options
of :class:`~torch.utils.data.DataLoader`.
.. note::
:class:`~torch.utils.data.DataLoader` by default constructs a index
sampler that yields integral indices. To make it work with a map-style
dataset with non-integral indices/keys, a custom sampler must be provided.
"""
def __getitem__(self, index):
raise NotImplementedError
def __add__(self, other):
return ConcatDataset([self, other])
# No `def __len__(self)` default?
# See NOTE [ Lack of Default `__len__` in Python Abstract Base Classes ]
# in pytorch/torch/utils/data/sampler.py
根据api描述,Dataset的子类必须overwrite两个方法:
__getitem__ : 支持获取给定key/index的数据样本。__len__ : 支持获取数据size
再加上init方法,就是最基本的三个方法。
以sklearn中breast_cancer数据为例,做一个dataset的Demo [github source]:
import torch
from torch.utils.data import Dataset, DataLoader
from sklearn.datasets import load_breast_cancer
from sklearn.preprocessing import StandardScaler
import numpy as np
class MyDataSet(Dataset):
def __init__(self) -> None:
super().__init__()
self._data = load_breast_cancer()
self._X = self._data.data.astype(np.float32)
self._y = self._data.target.astype(np.float32)
self._sc = StandardScaler()
self._X = self._sc.fit_transform(self._X)
self._X = torch.from_numpy(self._X).type(torch.float32)
self._y = torch.from_numpy(self._y).type(torch.float32)
self._len = len(self._X)
def __getitem__(self, index: int):
return self._X[index], self._y[index]
def __len__(self) -> int:
return self._len
data = MyDataSet()
dataLoader = DataLoader(dataset=data, batch_size=10, shuffle=True, drop_last=False, num_workers=1)
n = 0
for data_val, label_val in dataLoader:
print(f'iter: {n}')
print(f'data_val: {len(data_val)}, label_val: {len(label_val)}')
n += 1
# iter: 0
# data_val: 10, label_val: 10
# iter: 1
# data_val: 10, label_val: 10
# iter: 2
# data_val: 10, label_val: 10
# iter: 3
# data_val: 10, label_val: 10
# iter: 4
# data_val: 10, label_val: 10
...
|