tensorflow - 渴望 tf.GradientTape() 只返回无

Question

我尝试在 Eager 模式下使用 Tensorflow 计算梯度，但是tf.GradientTape () 仅返回 None 值。我不明白为什么。梯度在 update_policy() 函数中计算。

该行的输出:

grads = tape.gradient(loss, self.model.trainable_variables)

是

{list}<class 'list'>:[None, None, ... ,None]

这里是代码。

import tensorflow as tf
from keras.backend.tensorflow_backend import set_session

import numpy as np

tf.enable_eager_execution()
print(tf.executing_eagerly())

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
set_session(sess)


class PGEagerAtariNetwork:
    def __init__(self, state_space, action_space, lr, gamma):
        self.state_space = state_space
        self.action_space = action_space
        self.gamma = gamma

        self.model = tf.keras.Sequential()
        # Conv
        self.model.add(
            tf.keras.layers.Conv2D(filters=32, kernel_size=[8, 8], strides=[4, 4], activation='relu',
                                   input_shape=(84, 84, 4,),
                                   name='conv1'))
        self.model.add(
            tf.keras.layers.Conv2D(filters=64, kernel_size=[4, 4], strides=[2, 2], activation='relu', name='conv2'))
        self.model.add(
            tf.keras.layers.Conv2D(filters=128, kernel_size=[4, 4], strides=[2, 2], activation='relu', name='conv3'))
        self.model.add(tf.keras.layers.Flatten(name='flatten'))

        # Fully connected
        self.model.add(tf.keras.layers.Dense(units=512, activation='relu', name='fc1'))
        self.model.add(tf.keras.layers.Dropout(rate=0.4, name='dr1'))
        self.model.add(tf.keras.layers.Dense(units=256, activation='relu', name='fc2'))
        self.model.add(tf.keras.layers.Dropout(rate=0.3, name='dr2'))
        self.model.add(tf.keras.layers.Dense(units=128, activation='relu', name='fc3'))
        self.model.add(tf.keras.layers.Dropout(rate=0.1, name='dr3'))

        # Logits
        self.model.add(tf.keras.layers.Dense(units=self.action_space, activation=None, name='logits'))

        self.model.summary()

        # Optimizer
        self.optimizer = tf.train.AdamOptimizer(learning_rate=lr)

    def get_probs(self, s):
        s = s[np.newaxis, :]
        logits = self.model.predict(s)
        probs = tf.nn.softmax(logits).numpy()
        return probs

    def update_policy(self, s, r, a):
        with tf.GradientTape() as tape:
            logits = self.model.predict(s)
            policy_loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=a, logits=logits)
            policy_loss = policy_loss * tf.stop_gradient(r)
            loss = tf.reduce_mean(policy_loss)
        grads = tape.gradient(loss, self.model.trainable_variables)
        self.optimizer.apply_gradients(zip(grads, self.model.trainable_variables))

Answer 1

您的模型中没有前向传递。 Model.predict() 方法返回 numpy() 数组，而不使用前向传递。看看这个例子:

给定以下数据和模型:

import tensorflow as tf
import numpy as np

x_train = tf.convert_to_tensor(np.ones((1, 2), np.float32), dtype=tf.float32)
y_train = tf.convert_to_tensor([[0, 1]])

model = tf.keras.models.Sequential([tf.keras.layers.Dense(2, input_shape=(2, ))])

首先我们使用predict():

with tf.GradientTape() as tape:
    logits = model.predict(x_train)
    print('`logits` has type {0}'.format(type(logits)))
    # `logits` has type <class 'numpy.ndarray'>
    xentropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_train, logits=logits)
    reduced = tf.reduce_mean(xentropy)
    grads = tape.gradient(reduced, model.trainable_variables)
    print('grads are: {0}'.format(grads))
    # grads are: [None, None]

现在我们使用模型的输入:

with tf.GradientTape() as tape:
    logits = model(x_train)
    print('`logits` has type {0}'.format(type(logits)))
    # `logits` has type <class 'tensorflow.python.framework.ops.EagerTensor'>
    xentropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_train, logits=logits)
    reduced = tf.reduce_mean(xentropy)
    grads = tape.gradient(reduced, model.trainable_variables)
    print('grads are: {0}'.format(grads))
    # grads are: [<tf.Tensor: id=2044, shape=(2, 2), dtype=float32, numpy=
    # array([[ 0.77717704, -0.777177  ],
    #        [ 0.77717704, -0.777177  ]], dtype=float32)>, <tf.Tensor: id=2042, 
    # shape=(2,), dtype=float32, numpy=array([ 0.77717704, -0.777177  ], dtype=float32)>]

所以使用模型的 __call__()(即 model(x))进行前向传递，而不是 predict()。