python - 将 Tensorflow Graph 转换为使用 Estimator，使用 'TypeError: data type not understood' 或 `sampled_softmax_loss` 获得损失函数 `nce_loss`

Question

我正在尝试将 Tensorflow 的官方基本 word2vec 实现转换为使用 tf.Estimator。问题是损失函数(sampled_softmax_loss 或 nce_loss)在使用 Tensorflow Estimator 时会出错。它在原始实现中工作得很好。

这是 Tensorflow 的官方基本 word2vec 实现:

https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/word2vec/word2vec_basic.py

这是我在其中实现此代码的 Google Colab 笔记本，它正在运行。

https://colab.research.google.com/drive/1nTX77dRBHmXx6PEF5pmYpkIVxj_TqT5I

这是 Google Colab notebook，我在其中更改了代码，以便它使用 Tensorflow Estimator，但它无法正常工作。

https://colab.research.google.com/drive/1IVDqGwMx6BK5-Bgrw190jqHU6tt3ZR3e

为方便起见，这里是我定义 model_fn

上面 Estimator 版本的确切代码

batch_size = 128
embedding_size = 128  # Dimension of the embedding vector.
skip_window = 1  # How many words to consider left and right.
num_skips = 2  # How many times to reuse an input to generate a label.
num_sampled = 64  # Number of negative examples to sample.

def my_model( features, labels, mode, params):

    with tf.name_scope('inputs'):
        train_inputs = features
        train_labels = labels

    with tf.name_scope('embeddings'):
        embeddings = tf.Variable(
          tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        embed = tf.nn.embedding_lookup(embeddings, train_inputs)

    with tf.name_scope('weights'):
        nce_weights = tf.Variable(
          tf.truncated_normal(
              [vocabulary_size, embedding_size],
              stddev=1.0 / math.sqrt(embedding_size)))
    with tf.name_scope('biases'):
        nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

    with tf.name_scope('loss'):
        loss = tf.reduce_mean(
            tf.nn.nce_loss(
                weights=nce_weights,
                biases=nce_biases,
                labels=train_labels,
                inputs=embed,
                num_sampled=num_sampled,
                num_classes=vocabulary_size))

    tf.summary.scalar('loss', loss)

    if mode == "train":
        with tf.name_scope('optimizer'):
            optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)

        return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=optimizer)

这里是我调用估算器和训练的地方

word2vecEstimator = tf.estimator.Estimator(
        model_fn=my_model,
        params={
            'batch_size': 16,
            'embedding_size': 10,
            'num_inputs': 3,
            'num_sampled': 128,
            'batch_size': 16
        })

word2vecEstimator.train(
    input_fn=generate_batch,
    steps=10)

这是调用 Estimator 训练时收到的错误消息:

INFO:tensorflow:Calling model_fn.
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-22-955f44867ee5> in <module>()
      1 word2vecEstimator.train(
      2     input_fn=generate_batch,
----> 3     steps=10)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in train(self, input_fn, hooks, steps, max_steps, saving_listeners)
    352 
    353       saving_listeners = _check_listeners_type(saving_listeners)
--> 354       loss = self._train_model(input_fn, hooks, saving_listeners)
    355       logging.info('Loss for final step: %s.', loss)
    356       return self

/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _train_model(self, input_fn, hooks, saving_listeners)
   1205       return self._train_model_distributed(input_fn, hooks, saving_listeners)
   1206     else:
-> 1207       return self._train_model_default(input_fn, hooks, saving_listeners)
   1208 
   1209   def _train_model_default(self, input_fn, hooks, saving_listeners):

/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _train_model_default(self, input_fn, hooks, saving_listeners)
   1235       worker_hooks.extend(input_hooks)
   1236       estimator_spec = self._call_model_fn(
-> 1237           features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
   1238       global_step_tensor = training_util.get_global_step(g)
   1239       return self._train_with_estimator_spec(estimator_spec, worker_hooks,

/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _call_model_fn(self, features, labels, mode, config)
   1193 
   1194     logging.info('Calling model_fn.')
-> 1195     model_fn_results = self._model_fn(features=features, **kwargs)
   1196     logging.info('Done calling model_fn.')
   1197 

<ipython-input-20-9d389437162a> in my_model(features, labels, mode, params)
     33                 inputs=embed,
     34                 num_sampled=num_sampled,
---> 35                 num_classes=vocabulary_size))
     36 
     37     # Add the loss value as a scalar to summary.

/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_impl.py in nce_loss(weights, biases, labels, inputs, num_sampled, num_classes, num_true, sampled_values, remove_accidental_hits, partition_strategy, name)
   1246       remove_accidental_hits=remove_accidental_hits,
   1247       partition_strategy=partition_strategy,
-> 1248       name=name)
   1249   sampled_losses = sigmoid_cross_entropy_with_logits(
   1250       labels=labels, logits=logits, name="sampled_losses")

/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_impl.py in _compute_sampled_logits(weights, biases, labels, inputs, num_sampled, num_classes, num_true, sampled_values, subtract_log_q, remove_accidental_hits, partition_strategy, name, seed)
   1029   with ops.name_scope(name, "compute_sampled_logits",
   1030                       weights + [biases, inputs, labels]):
-> 1031     if labels.dtype != dtypes.int64:
   1032       labels = math_ops.cast(labels, dtypes.int64)
   1033     labels_flat = array_ops.reshape(labels, [-1])

TypeError: data type not understood

编辑:应要求，input_fn 的典型输出如下所示

print(generate_batch(batch_size=8, num_skips=2, skip_window=1))

(array([3081, 3081,   12,   12,    6,    6,  195,  195], dtype=int32), array([[5234],
       [  12],
       [   6],
       [3081],
       [  12],
       [ 195],
       [   6],
       [   2]], dtype=int32))

Answer 1

您在这里像使用变量一样使用generate_batch:

word2vecEstimator.train(
    input_fn=generate_batch,
    steps=10)

使用 generate_batch() 调用函数。但我认为您必须向函数传递一些值。