python - Tensorflow model.fit() 使用数据集生成器

Question

我正在使用数据集 API 生成训练数据并将其分类为神经网络的批处理。

这是我的代码的最小工作示例:

import tensorflow as tf
import numpy as np
import random


def my_generator():
    while True:
        x = np.random.rand(4, 20)
        y = random.randint(0, 11)
        label = tf.one_hot(y, depth=12)
        yield x.reshape(4, 20, 1), label

def my_input_fn():
    dataset = tf.data.Dataset.from_generator(lambda: my_generator(),
                                             output_types=(tf.float64, tf.int32))

    dataset = dataset.batch(32)
    iterator = dataset.make_one_shot_iterator()
    batch_features, batch_labels = iterator.get_next()

    return batch_features, batch_labels


if __name__ == "__main__":
    tf.enable_eager_execution()

    model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
                                 tf.keras.layers.Dense(128, activation=tf.nn.relu),
                                 tf.keras.layers.Dense(12, activation=tf.nn.softmax)])

    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    data_generator = my_input_fn()
    model.fit(data_generator)

代码在调用 model.fit() 时使用 TensorFlow 1.13.1 失败，出现以下错误:

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit(data_generator)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py", line 880, in fit
    validation_steps=validation_steps)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 310, in model_iteration
    ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 526, in slice_arrays
    return [None if x is None else x[start] for x in arrays]
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 526, in <listcomp>
    return [None if x is None else x[start] for x in arrays]
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 654, in _slice_helper
    name=name)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 820, in strided_slice
    shrink_axis_mask=shrink_axis_mask)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 9334, in strided_slice
    _six.raise_from(_core._status_to_exception(e.code, message), None)
  File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: Attr shrink_axis_mask has value 4294967295 out of range for an int32 [Op:StridedSlice] name: strided_slice/

我尝试使用 TensorFlow 2.0 在另一台机器上运行相同的代码(在删除 tf.enable_eager_execution() 行之后，因为它默认急切地运行)并且我收到以下错误:

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit(data_generator)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 873, in fit
    steps_name='steps_per_epoch')
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 352, in model_iteration
    batch_outs = f(ins_batch)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/backend.py", line 3217, in __call__
    outputs = self._graph_fn(*converted_inputs)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 558, in __call__
    return self._call_flat(args)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 627, in _call_flat
    outputs = self._inference_function.call(ctx, args)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 397, in call
    (len(args), len(list(self.signature.input_arg))))
ValueError: Arguments and signature arguments do not match: 21 23 

我尝试将 model.fit() 更改为 model.fit_generator() 但这在两个 TensorFlow 版本上都失败了。在 TF 1.13.1 上，我收到以下错误:

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit_generator(data_generator)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py", line 1426, in fit_generator
    initial_epoch=initial_epoch)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_generator.py", line 115, in model_iteration
    shuffle=shuffle)
  File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_generator.py", line 377, in convert_to_generator_like
    num_samples = int(nest.flatten(data)[0].shape[0])
TypeError: __int__ returned non-int (type NoneType)

在 TF 2.0 上我得到以下错误:

Traceback (most recent call last):
  File "scripts/min_working_example.py", line 37, in <module>
    model.fit_generator(data_generator)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 1515, in fit_generator
    steps_name='steps_per_epoch')
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_generator.py", line 140, in model_iteration
    shuffle=shuffle)
  File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_generator.py", line 477, in convert_to_generator_like
    raise ValueError('You must specify `batch_size`')
ValueError: You must specify `batch_size`

然而 batch_size 不是 fit_generator() 的可识别关键字。

我对这些错误消息感到困惑，如果有人能阐明它们，或指出我做错了什么，我将不胜感激。

Answer 1

虽然错误的来源仍然模糊不清，但我已经找到了使代码正常工作的解决方案。我会把它张贴在这里，以防它对处于类似情况的任何人有用。

基本上，我更改了 my_input_fn()进入发电机并使用model.fit_generator()如下:

import tensorflow as tf
import numpy as np
import random


def my_generator(total_items):
    i = 0
    while i < total_items:
        x = np.random.rand(4, 20)
        y = random.randint(0, 11)
        label = tf.one_hot(y, depth=12)
        yield x.reshape(4, 20, 1), label
        i += 1

def my_input_fn(total_items, epochs):
    dataset = tf.data.Dataset.from_generator(lambda: my_generator(total_items),
                                             output_types=(tf.float64, tf.int64))

    dataset = dataset.repeat(epochs)
    dataset = dataset.batch(32)


    iterator = dataset.make_one_shot_iterator()
    while True:
        batch_features, batch_labels = iterator.get_next()
        yield batch_features, batch_labels

if __name__ == "__main__":
    tf.enable_eager_execution()

    model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
                                 tf.keras.layers.Dense(64, activation=tf.nn.relu),
                                 tf.keras.layers.Dense(12, activation=tf.nn.softmax)])

    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    total_items = 200
    batch_size = 32
    epochs = 10
    num_batches = int(total_items/batch_size)
    train_data_generator = my_input_fn(total_items, epochs)
    model.fit_generator(generator=train_data_generator, steps_per_epoch=num_batches, epochs=epochs, verbose=1)

编辑

正如 giser_yugang 在评论中暗示的那样，也可以使用 my_input_fn() 来完成。作为返回 dataset 的函数而不是单独的批处理。

def my_input_fn(total_items, epochs):
    dataset = tf.data.Dataset.from_generator(lambda: my_generator(total_items),
                                             output_types=(tf.float64, tf.int64))

    dataset = dataset.repeat(epochs)
    dataset = dataset.batch(32)
    return dataset

if __name__ == "__main__":
    tf.enable_eager_execution()

    model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
                                 tf.keras.layers.Dense(64, activation=tf.nn.relu),
                                 tf.keras.layers.Dense(12, activation=tf.nn.softmax)])

    model.compile(optimizer='adam',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    total_items = 100
    batch_size = 32
    epochs = 10
    num_batches = int(total_items/batch_size)
    dataset = my_input_fn(total_items, epochs)
    model.fit_generator(dataset, epochs=epochs, steps_per_epoch=num_batches)

这些方法之间似乎没有任何平均性能差异。