python - tf.global_variables_initializer() 的位置

Question

我是深度学习的初学者，一直困扰着这个问题。

import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.utils import  shuffle
from sklearn.model_selection import train_test_split

#define the one hot encode function
def one_hot_encode(labels):
  n_labels = len(labels)
  n_unique_labels = len(np.unique(labels))
  one_hot_encode = np.zeros((n_labels,n_unique_labels))
  one_hot_encode[np.arange(n_labels), labels] = 1
  return one_hot_encode

#Read the sonar dataset
df = pd.read_csv('sonar.csv')
print(len(df.columns))
X = df[df.columns[0:60]].values
y=df[df.columns[60]]
#encode the dependent variable containing categorical values
encoder = LabelEncoder()
encoder.fit(y)
y = encoder.transform(y)
Y = one_hot_encode(y)

#Transform the data in training and testing
X,Y = shuffle(X,Y,random_state=1)
train_x,test_x,train_y,test_y = train_test_split(X,Y,test_size=0.20,       random_state=42)


#define and initialize the variables to work with the tensors
learning_rate = 0.1
training_epochs = 1000

 #Array to store cost obtained in each epoch
 cost_history = np.empty(shape=[1],dtype=float)

 n_dim = X.shape[1]
 n_class = 2

x = tf.placeholder(tf.float32,[None,n_dim])
W = tf.Variable(tf.zeros([n_dim,n_class]))
b = tf.Variable(tf.zeros([n_class]))

#initialize all variables.


#define the cost function
y_ = tf.placeholder(tf.float32,[None,n_class])
y = tf.matmul(x, W)+ b
 init = tf.global_variables_initializer()#wrong position
cost_function =       tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y,labels=y_))

training_step = tf.train.AdamOptimizer(learning_rate).minimize(cost_function)
 init = tf.global_variables_initializer()#correct position
 #initialize the session

 sess = tf.Session()

  sess.run(init)
  mse_history = []

  #calculate the cost for each epoch
 for epoch in range(training_epochs):
sess.run(training_step,feed_dict={x:train_x,y_:train_y})
cost = sess.run(cost_function,feed_dict={x: train_x,y_: train_y})
cost_history = np.append(cost_history,cost)
print('epoch : ', epoch,  ' - ', 'cost: ', cost)

 pred_y = sess.run(y, feed_dict={x: test_x})
 print(pred_y) 
#Calculate Accuracy
 correct_prediction = tf.equal(tf.argmax(pred_y,1), tf.argmax(test_y,1))
 accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
 print(sess.run(accuracy))
 sess.close()

在上面的代码中，如果我使用 init = tf.global_variables_initializer()高于 AdamOptimizer 那么它会给出错误，但如果我在之后使用它AdamOptimizer 那么它就可以正常工作了。是什么原因？尽管它在两个位置都可以与 GradientDescentOptimizer 一起正常工作。

Answer 1

查看documentation init = tf.global_variables_initializer()与 init = tf.variables_initializer(tf.global_variables()) 相同

tf.train.AdamOptimizer 需要初始化一些内部变量(平均值等统计数据)

<tf.Variable 'beta1_power:0' shape=() dtype=float32_ref>
<tf.Variable 'beta2_power:0' shape=() dtype=float32_ref>
<tf.Variable 'x/Adam:0' shape=(2, 1) dtype=float32_ref>    # 1st moment vector
<tf.Variable 'x/Adam_1:0' shape=(2, 1) dtype=float32_ref>  # 2nd moment vector

documentation告诉您如何应用更新。

相反，普通梯度下降优化器 tf.train.GradientDescentOptimizer 不依赖于任何变量。这是有区别的。现在，在 tf.train.AdamOptimizer 之前可以使用它的变量，这些变量需要在某个时刻进行初始化。

创建操作 init它初始化所有需要的变量，这个操作 init需要知道运行程序需要哪个变量。因此，它需要放在 tf.train.AdamOptimizer 之后 .

如果您将 init = tf.global_variables_initializer() 在 tf.train.AdamOptimizer 之前就像

init_op = tf.variables_initializer(tf.global_variables())
optimize_op = tf.train.AdamOptimizer(0.1).minimize(cost_function)

你会得到

Attempting to use uninitialized value beta1_power

它告诉你， tf.train.AdamOptimizer 尝试访问<tf.Variable 'beta1_power:0' shape=() dtype=float32_ref> ，尚未初始化。

所以

# ...
... = tf.train.AdamOptimizer(0.1).minimize(cost_function)
# ...
init = tf.global_variables_initializer()

是唯一正确的方法。您可以检查哪些变量可以通过放置来初始化

for variable in tf.global_variables():
    print(variable)

在源代码中。

考虑最小化二次形式的示例 0.5x'Ax + bx + c 。在 TensorFlow 中，这将是

import tensorflow as tf
import numpy as np

x = tf.Variable(np.random.rand(2, 1), dtype=tf.float32, name="x")
# we already make clear, that we are not going to optimize these variables
b = tf.constant([[5], [6]], dtype=tf.float32, name="b")
A = tf.constant([[9, 2], [2, 10]], dtype=tf.float32, name="A")

cost_function = 0.5 * tf.matmul(tf.matmul(tf.transpose(x), A), x) - tf.matmul(tf.transpose(b), x) + 42

for variable in tf.global_variables():
    print('before ADAM: global_variables_initializer would init {}'.format(variable))

optimize_op = tf.train.AdamOptimizer(0.1).minimize(cost_function)

for variable in tf.global_variables():
    print('after ADAM: global_variables_initializer would init 

{}'.format(变量))

init_op = tf.variables_initializer(tf.global_variables())
with tf.Session() as sess:
    sess.run(init_op)

    for i in range(5):
        loss, _ = sess.run([cost_function, optimize_op])
        print(loss)

输出为

before ADAM global_variables_initializer would init <tf.Variable 'x:0' shape=(2, 1) dtype=float32_ref>
after ADAM global_variables_initializer would init <tf.Variable 'x:0' shape=(2, 1) dtype=float32_ref>
after ADAM global_variables_initializer would init <tf.Variable 'beta1_power:0' shape=() dtype=float32_ref>
after ADAM global_variables_initializer would init <tf.Variable 'beta2_power:0' shape=() dtype=float32_ref>
after ADAM global_variables_initializer would init <tf.Variable 'x/Adam:0' shape=(2, 1) dtype=float32_ref>
after ADAM global_variables_initializer would init <tf.Variable 'x/Adam_1:0' shape=(2, 1) dtype=float32_ref>

所以tf.global_variables_initializer()当放置 init = tf.global_variables_initializer() 时，没有看到 ADAM 所需的变量ADAM 定义之前tf.train.AdamOptimizer 。使用 GradientDescentOptimizer 时，值为

before ADAM global_variables_initializer would init <tf.Variable 'x:0' shape=(2, 1) dtype=float32_ref>
after ADAM global_variables_initializer would init <tf.Variable 'x:0' shape=(2, 1) dtype=float32_ref>

所以优化器前后没有任何变化。