|
目录
1. tf feature_column的使用
1.1 构造训练数据集
1.2 对训练数据集的每个特征有一个大致的了解
1.3?使用feature_columns将离散特征进行编码，生成数据集
1.4 直接调用model.fit训练模型
2. 将model转换成estimator
3. pre_estimator
3.1 BaselineClassifier
3.2?LinearClassifier
3.3 使用DNNClassifier
4. cross_feature
1. tf feature_column的使用
1.1 构造训练数据集
import tensorflow as tf
from tensorflow import keras
import pandas as pd
import pprint
train_file = '/content/drive/MyDrive/data/titanic/train.csv'
eval_file = '/content/drive/MyDrive/data/titanic/eval.csv'
train_df = pd.read_csv(train_file)
eval_df = pd.read_csv(eval_file)
# 将features 和 label 分开
y_train = train_df.pop('survived')
y_eval = eval_df.pop('survived')
1.2 对训练数据集的每个特征有一个大致的了解
1.3?使用feature_columns将离散特征进行编码，生成数据集
# 离散特征
categorical_columns = ['sex', 'n_siblings_spouses', 'parch', 'class', 'deck',
'embark_town', 'alone']
# 连续特征
numeric_columns = ['age', 'fare']
feature_columns = []
for categorical_column in categorical_columns:
vocab = train_df[categorical_column].unique()
print(categorical_column, vocab)
feature_columns.append(
tf.feature_column.indicator_column(
tf.feature_column.categorical_column_with_vocabulary_list(
categorical_column, vocab)))
for numeric_column in numeric_columns:
feature_columns.append(
tf.feature_column.numeric_column(numeric_column, dtype=tf.float32))
pp.pprint(feature_columns)
根据epoch和batch_size 生成 dataset
def make_dataset(data_df, label_df, epochs=10, shuffle=True, batch_size=32):
dataset = tf.data.Dataset.from_tensor_slices(
(dict(data_df), label_df))
if shuffle:
dataset = dataset.shuffle(10000)
dataset = dataset.repeat(epochs).batch(batch_size)
return dataset
将feature_column 和 dataset结合使用。
keras.layers.DenseFeature 可以把feature_column应用到dataset中去，
feature_column本质上来说是一组对feature变换的规则，然后DenseFeature可以把这个规则应用到每个数据集中去。
1.4 直接调用model.fit训练模型
batch_size=32
train_dataset = make_dataset(train_df, y_train, epochs=100,
batch_size=batch_size)
eval_dataset = make_dataset(eval_df, y_eval, epochs=1,
batch_size=batch_size, shuffle=False)
history = model.fit(train_dataset,
validation_data = eval_dataset,
steps_per_epoch = train_df.shape[0] // batch_size,
validation_steps = eval_df.shape[0] // batch_size,
epochs = 100)
2. 将model转换成estimator
note：estimator.train的入参，
input_fn必须是一个函数，可以是一个具体的函数，也可以是一个lambda函数，但该函数没有入参
该函数的返回结果必须是
a:(features, labels)——> features, labels 组成的元组，features，labels可以是列表，也可以是numpy数组。
b:Dataset, dataset中的元素为（feature, label）.
estimator = keras.estimator.model_to_estimator(model)
estimator.train(input_fn = lambda : make_dataset(train_df, y_train, epochs=100))
3. pre_estimator
3.1 BaselineClassifier
base_output_dir = '/content/drive/MyDrive/code/tf_estimator/baseline_model'
if not os.path.exists(base_output_dir):
os.mkdir(base_output_dir)
baseline_estimator = tf.estimator.BaselineClassifier(model_dir = base_output_dir,
n_classes = 2)
baseline_estimator.train(input_fn = lambda: make_dataset(
train_df, y_train, epochs = 100))
baseline_estimator.evaluate(input_fn = lambda: make_dataset(
eval_df, y_eval, epochs = 20, shuffle = False, batch_size = 20))
baseline_estimator并没有在数据中学习到什么规律，而是进行随机的猜测，把每个类别出现的样本都统计一遍，计算这个类别出现的比例是多少，利用这个比例去对某个样本进行猜测。
比如泰坦尼克号数据集中，被拯救的概率是60%，没被拯救的概率是40%。出现一个新的样本，则被推理为拯救的概率是60%
3.2?LinearClassifier
linear_output_dir = '/content/drive/MyDrive/code/tf_estimator/linear_model'
if not os.path.exists(linear_output_dir):
os.mkdir(linear_output_dir)
linear_estimator = tf.estimator.LinearClassifier(model_dir = linear_output_dir,
n_classes = 2,
feature_columns = feature_columns)
linear_estimator.train(input_fn = lambda: make_dataset(
train_df, y_train, epochs = 100))
linear_estimator.evaluate(input_fn = lambda: make_dataset(
eval_df, y_eval, epochs = 1, shuffle = False))
3.3 使用DNNClassifier
dnn_output_dir = '/content/drive/MyDrive/code/tf_estimator/dnn_model'
if not os.path.exists(dnn_output_dir):
os.mkdir(dnn_output_dir)
dnn_estimator = tf.estimator.DNNClassifier(model_dir = dnn_output_dir,
n_classes = 2,
feature_columns = feature_columns,
hidden_units = [128, 128],
activation_fn = tf.nn.relu,
optimizer = 'Adam')
dnn_estimator.train(input_fn = lambda: make_dataset(
train_df, y_train, epochs = 100))
dnn_estimator.evaluate(input_fn = lambda: make_dataset(
eval_df, y_eval, epochs = 1, shuffle = False))
4. cross_feature
# 交叉特征
# cross feature: age: [1, 2, 3, 4, 5], gender:[male, female]
# age_x_gender:[(1, male), (2, male), ......(5, male), ......(5, female)]
# 100000: 100 ——> hash(100000 values) % 100
# 离散特征
categorical_columns = ['sex', 'n_siblings_spouses', 'parch', 'class', 'deck',
'embark_town', 'alone']
# 连续特征
numeric_columns = ['age', 'fare']
feature_columns = []
for categorical_column in categorical_columns:
vocab = train_df[categorical_column].unique()
print(categorical_column, vocab)
feature_columns.append(
tf.feature_column.indicator_column(
tf.feature_column.categorical_column_with_vocabulary_list(
categorical_column, vocab)))
for numeric_column in numeric_columns:
feature_columns.append(
tf.feature_column.numeric_column(numeric_column, dtype=tf.float32))
# 交叉特征
# cross feature: age: [1, 2, 3, 4, 5], gender:[male, female]
# age_x_gender:[(1, male), (2, male), ......(5, male), ......(5, female)]
# 100000: 100 ——> hash(100000 values) % 100
feature_columns.append(
tf.feature_column.indicator_column(
tf.feature_column.crossed_column(
['age', 'sex'], hash_bucket_size = 100)))
|