python - Pandas/Keras : use data from DataFrame to train Keras model, 输入形状错误

Question

我有一个包含 n 行和 23 列(不包括索引)的 DataFrame。
首先，我将它们分成 X 和 Y:

Y = df.pop("Target").values
X = df.values # now X has 22 columns

然后我使用train_test_split来分割它们:

X_tr, X_val, y_tr, y_val = train_test_split(X, Y)

我将它们转换为数据集:

dataset = tf.data.Dataset.from_tensor_slices((X_tr, y_tr))
dataset = dataset.batch(32)
valid_ds = tf.data.Dataset.from_tensor_slices((X_val, y_val))

问题是，当我创建模型时，我把input_shape放错了，如下所示:

def create_model():
    tfkl = tf.keras.layers
    inp = tf.keras.Input(shape=(None, 22))
    x = tfkl.Dense(128, activation="linear")(inp)
    x = tfkl.Dense(64, activation="linear")(x)
    x = tfkl.Dense(1, activation="linear")(x)
    
    model = tf.keras.models.Model(inp, x)
    model.compile(loss="mae", optimizer="adam", metrics=["mae"])
    return model

当我运行 fit 时，在纪元结束时它会抛出错误:

ValueError: Input 0 of layer dense is incompatible with the layer: expected axis  
-1 of input shape to have value 22 but received input with shape [22, 1]

我将其更改为 (None, None, 22) 和许多其他内容，但它不起作用。如有任何帮助，我们将不胜感激。

Answer 1

我能够复制您的问题。 X 有 1000 条记录和 22 特征，y 有 1 特征和 >1000 条记录。请引用下面的示例代码

import tensorflow as tf
import numpy as np
from sklearn.model_selection import train_test_split

X = np.random.random((1000,22))
y = np.random.random((1000,1))
 
X_train,X_test, y_train,y_test = train_test_split(X,y)
 
dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))
train_data = dataset.shuffle(len(X_train)).batch(32)
train_data = train_data.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
 
valid_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test))

def create_model():
    tfkl = tf.keras.layers
    inp = tf.keras.Input(shape=(None,22))
    x = tfkl.Dense(128, activation="linear")(inp)
    x = tfkl.Dense(64, activation="linear")(x)
    x = tfkl.Dense(1, activation="linear")(x)
    
    model = tf.keras.models.Model(inp, x)
    model.compile(loss="mae", optimizer="adam", metrics=["mae"])
    return model

model=create_model()
model.summary()

model.fit(train_data, epochs=3, validation_data=valid_ds) 

输出:

Model: "functional_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         [(None, None, 22)]        0         
_________________________________________________________________
dense (Dense)                (None, None, 128)         2944      
_________________________________________________________________
dense_1 (Dense)              (None, None, 64)          8256      
_________________________________________________________________
dense_2 (Dense)              (None, None, 1)           65        
=================================================================
Total params: 11,265
Trainable params: 11,265
Non-trainable params: 0
_________________________________________________________________
Epoch 1/3
WARNING:tensorflow:Model was constructed with shape (None, None, 22) for input Tensor("input_1:0", shape=(None, None, 22), dtype=float32), but it was called on an input with incompatible shape (None, 22).
WARNING:tensorflow:Model was constructed with shape (None, None, 22) for input Tensor("input_1:0", shape=(None, None, 22), dtype=float32), but it was called on an input with incompatible shape (None, 22).
 1/24 [>.............................] - ETA: 0s - loss: 0.3535 - mae: 0.3535WARNING:tensorflow:Model was constructed with shape (None, None, 22) for input Tensor("input_1:0", shape=(None, None, 22), dtype=float32), but it was called on an input with incompatible shape (22, 1).
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-1-9499c98e2515> in <module>()
     28 model.summary()
     29 
---> 30 model.fit(train_data, epochs=3, validation_data=valid_ds)

12 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
    971           except Exception as e:  # pylint:disable=broad-except
    972             if hasattr(e, "ag_error_metadata"):
--> 973               raise e.ag_error_metadata.to_exception(e)
    974             else:
    975               raise

ValueError: in user code:

    /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1224 test_function  *
        return step_function(self, iterator)
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1215 step_function  **
        outputs = model.distribute_strategy.run(run_step, args=(data,))
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py:1211 run
        return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py:2585 call_for_each_replica
        return self._call_for_each_replica(fn, args, kwargs)
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py:2945 _call_for_each_replica
        return fn(*args, **kwargs)
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1208 run_step  **
        outputs = model.test_step(data)
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1174 test_step
        y_pred = self(x, training=False)
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py:985 __call__
        outputs = call_fn(inputs, *args, **kwargs)
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/functional.py:386 call
        inputs, training=training, mask=mask)
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/functional.py:508 _run_internal_graph
        outputs = node.layer(*args, **kwargs)
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py:976 __call__
        self.name)
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/input_spec.py:216 assert_input_compatibility
        ' but received input with shape ' + str(shape))

    ValueError: Input 0 of layer dense is incompatible with the layer: expected axis -1 of input shape to have value 22 but received input with shape [22, 1]

固定代码:

我已将输入形状从 (None, 22) 更改为 (22,) 和 验证数据 批量 32 为 valid_data = valid_ds.batch(32)

请引用如下所示的工作代码

import tensorflow as tf
import numpy as np
from sklearn.model_selection import train_test_split

X = np.random.random((1000,22))
y = np.random.random((1000,1))
 
X_train,X_test, y_train,y_test = train_test_split(X,y)
 
dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))
train_data = dataset.shuffle(len(X_train)).batch(32)
train_data = train_data.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
 
valid_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test))
valid_data = valid_ds.batch(32) 

def create_model():
    tfkl = tf.keras.layers
    inp = tf.keras.Input(shape=(22,))
    x = tfkl.Dense(128, activation="linear")(inp)
    x = tfkl.Dense(64, activation="linear")(x)
    x = tfkl.Dense(1, activation="linear")(x)
    
    model = tf.keras.models.Model(inp, x)
    model.compile(loss="mae", optimizer="adam", metrics=["mae"])
    return model

model=create_model()
model.summary()

model.fit(train_data, epochs=3, validation_data=valid_data) 

输出:

Model: "functional_3"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_2 (InputLayer)         [(None, 22)]              0         
_________________________________________________________________
dense_3 (Dense)              (None, 128)               2944      
_________________________________________________________________
dense_4 (Dense)              (None, 64)                8256      
_________________________________________________________________
dense_5 (Dense)              (None, 1)                 65        
=================================================================
Total params: 11,265
Trainable params: 11,265
Non-trainable params: 0
_________________________________________________________________
Epoch 1/3
24/24 [==============================] - 0s 4ms/step - loss: 0.2807 - mae: 0.2807 - val_loss: 0.2773 - val_mae: 0.2773
Epoch 2/3
24/24 [==============================] - 0s 2ms/step - loss: 0.2630 - mae: 0.2630 - val_loss: 0.2600 - val_mae: 0.2600
Epoch 3/3
24/24 [==============================] - 0s 2ms/step - loss: 0.2575 - mae: 0.2575 - val_loss: 0.2616 - val_mae: 0.2616
<tensorflow.python.keras.callbacks.History at 0x7ff6fb1ad358>