喜欢0次
第5讲讲解了keras制作mnist数据集的流程，进一步的，有时候我们需要构建自己的数据集。
以flower分类为例，见参考3（这里直接用别人的数据，也可以自己整理，只是比较耗时间，脚本也做了改写，方便理解）
step1：下载flower数据集
数据下载地址见参考3，下载完成后得到5个文件夹，分别存放5种不同花的jpg图像：
flower_photos# tree -L 1
.
├── LICENSE.txt
├── daisy # 雏菊，编号0
├── dandelion # 蒲公英，编号1
├── roses # 玫瑰，编号2
├── sunflowers # 向日葵，编号3
└── tulips # 郁金香，编号4step2：制作图像-标签字典
我们需要给图像打上标签，比如0代表daisy，1代表dandelion，等，神经网络训练完成后，输入一个上述5种花的一种，经过模型处理后，得到标签[0-4]，这个标签表示花的种类。
# create_image2label.py
import os, glob, json, random
def create_image2label(root, jsonname):
'''
root:images文件夹路径
jsonname：写入的json文件名
'''
name2label = {}
# step1: 将flower_photos文件夹依次编号
for name in sorted(os.listdir(root)):
# 如果flower_photos/name不是文件夹，跳过
if not os.path.isdir(os.path.join(root, name)):
continue
else:
# 依次编号
name2label[name] = len(name2label.keys())
# 这样得到 name2label = {'daisy': 0, 'dandelion': 1, 'roses': 2, 'sunflowers': 3, 'tulips': 4}
# step2: 将image-label对应关系写入到json中
if not os.path.exists(os.path.join(root, jsonname)):
images = []
for name in name2label.keys():
# 返回图片路径：flower_photos/xxx/xxx, 支持jpg/png等格式
images += glob.glob(os.path.join(root, name, "*.*g"))
# 图片按类别写入列表，需要打乱顺序，原位打乱
random.shuffle(images)
# 用来保存图片路径-label对应关系
path_label = {}
for img in images:
# 将路径按/或\分离[flower_photos, daisy, **.jpg]取出类别名
name = img.split(os.sep)[-2]
# 根据字典key，对应value找出对应标签
label = name2label[name]
path_label[img] = label
# 将路径,标签信息写入json文件
with open(os.path.join(root, jsonname), 'w', newline="") as js:
json.dump(path_label, js)
if __name__ == '__main__':
cwd = './flower_photos'
jsonname = 'images.json'
create_image2label(cwd, jsonname)在当前路径下得到images.json文件，里面记录了图片路径与标签，如下（截取部分）。
{ "./flower_photos/daisy/7133935763_82b17c8e1b_n.jpg": 0,
"./flower_photos/sunflowers/5020805135_1219d7523d.jpg": 3,
"./flower_photos/roses/10090824183_d02c613f10_m.jpg": 2,
"./flower_photos/daisy/9346508462_f0af3163f4.jpg": 0,
"./flower_photos/sunflowers/9655029591_7a77f87500.jpg": 3,
"./flower_photos/sunflowers/22255608949_172d7c8d22_m.jpg": 3,
"./flower_photos/tulips/113291410_1bdc718ed8_n.jpg": 4,
}step3：load数据以及数据的预处理
import os, glob, json, random
import tensorflow as tf
import numpy as np
def load_json(root, jsonname):
images, labels = [], []
with open(os.path.join(root, jsonname)) as js:
jscontent = json.load(js)
for key, value in jscontent.items():
label = int(value)
images.append(key)
labels.append(label)
assert len(images) == len(labels)
return images, labels
def load_data(root, mode='train'):
jsonname = 'images.json'
images, labels = load_json(root, jsonname)
# 训练集
if mode == 'train': # 60%
images = images[:int(0.6 * len(images))]
labels = labels[:int(0.6 * len(labels))]
# 验证集
elif mode == 'val': # 20% = 60%->80%
images = images[int(0.6 * len(images)):int(0.8 * len(images))]
labels = labels[int(0.6 * len(labels)):int(0.8 * len(labels))]
# 测试集
else: # 20% = 80%->100%
images = images[int(0.8 * len(images)):]
labels = labels[int(0.8 * len(labels)):]
return images, labels
def preprocess(x, y):
x = tf.io.read_file(x)
x = tf.image.decode_jpeg(x, channels=3)
x = tf.image.resize(x, [244, 244])
x = tf.cast(x, dtype=tf.float32) / 255.0
y = tf.convert_to_tensor(y)
y = tf.one_hot(y, depth=5)
return x, y
if __name__ == '__main__':
cwd = './flower_photos'
batchsz = 32
# 创建训练集Datset对象
images, labels = load_data(cwd, mode='train')
db_train = tf.data.Dataset.from_tensor_slices((images, labels))
db_train = db_train.shuffle(1000).map(preprocess).batch(batchsz)
# 创建验证集Datset对象
images2, labels2 = load_data(cwd, mode='val')
db_val = tf.data.Dataset.from_tensor_slices((images2, labels2))
db_val = db_val.map(preprocess).batch(batchsz)
# 创建测试集Datset对象
images3, labels3 = load_data(cwd, mode='test')
db_test = tf.data.Dataset.from_tensor_slices((images3, labels3))
db_test = db_test.map(preprocess).batch(batchsz)这样得到训练数据集db_train，验证数据集db_val，测试数据集db_test
step4：训练验证
# train.py
import os
import tensorflow as tf
import numpy as np
from tensorflow import keras
from tensorflow.keras import layers,optimizers,losses
from tensorflow.keras.callbacks import EarlyStopping
from load_data import load_data, preprocess
if __name__ == '__main__':
cwd = './flower_photos'
batchsz = 32
# 创建训练集Datset对象
images, labels = load_data(cwd, mode='train')
db_train = tf.data.Dataset.from_tensor_slices((images, labels))
db_train = db_train.shuffle(1000).map(preprocess).batch(batchsz)
# 创建验证集Datset对象
images2, labels2 = load_data(cwd, mode='val')
db_val = tf.data.Dataset.from_tensor_slices((images2, labels2))
db_val = db_val.map(preprocess).batch(batchsz)
# 创建测试集Datset对象
images3, labels3 = load_data(cwd, mode='test')
db_test = tf.data.Dataset.from_tensor_slices((images3, labels3))
db_test = db_test.map(preprocess).batch(batchsz)
#创建网络模型
net = keras.applications.VGG16(weights='imagenet',include_top=False,pooling='max')
net.trainable = False
newnet = keras.Sequential([
net,
layers.Dense(128),
layers.Dense(64),
layers.Dense(5)
])
newnet.build(input_shape=(4, 224, 224, 3))
newnet.summary()
early_stopping = EarlyStopping(
monitor='val_accuracy',
min_delta=0.001,
patience=5
)
newnet.compile(optimizer=optimizers.Adam(lr=1e-3),
loss=losses.CategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
# 为了节省时间，epochs取5，为了提高识别率，这里可以增大epochs
newnet.fit(db_train, validation_data=db_val, validation_freq=1, epochs=5,
callbacks=[early_stopping])
# 用测试集测试模型准确率
newnet.evaluate(db_test)测试log如下：
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
vgg16 (Functional) (None, 512) 14714688
dense (Dense) (None, 128) 65664
dense_1 (Dense) (None, 64) 8256
dense_2 (Dense) (None, 5) 325
=================================================================
Total params: 14,788,933
Trainable params: 74,245
Non-trainable params: 14,714,688
_________________________________________________________________
/usr/local/lib/python3.8/dist-packages/keras/optimizers/optimizer_v2/adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
super(Adam, self).__init__(name, **kwargs)
Epoch 1/5
69/69 [==============================] - 181s 3s/step - loss: 1.0423 - accuracy: 0.6190 - val_loss: 0.6405 - val_accuracy: 0.7684
Epoch 2/5
69/69 [==============================] - 188s 3s/step - loss: 0.5819 - accuracy: 0.7929 - val_loss: 0.5232 - val_accuracy: 0.8147
Epoch 3/5
69/69 [==============================] - 169s 2s/step - loss: 0.4582 - accuracy: 0.8361 - val_loss: 0.5095 - val_accuracy: 0.8256
Epoch 4/5
69/69 [==============================] - 187s 3s/step - loss: 0.3972 - accuracy: 0.8615 - val_loss: 0.5235 - val_accuracy: 0.8134
Epoch 5/5
69/69 [==============================] - 190s 3s/step - loss: 0.3540 - accuracy: 0.8747 - val_loss: 0.5412 - val_accuracy: 0.8202
23/23 [==============================] - 51s 2s/step - loss: 0.5609 - accuracy: 0.8202可见识别效率为0.82，也可以调参搭建更加复杂的模型。
参考：
