tensorflow 错误 "raise ValueError("形状 %s 和 %s 不兼容"% (self, other)) ValueError : Shapes (? , 5) and (5,) are not compatible"

Question

我尝试使用 tensorflow 1.4.0 对我的原始记录进行分类。过程如下。

拳头:读取图片和标签，输出“tfrecord”格式的文件。第二:读取tf记录和训练

编写tfrecord脚本是

!/usr/bin/env python3
#coding:utf-8

import argparse
import os
import random

import numpy as np
from PIL import Image
import tensorflow as tf

def make_example(label_index, image):
    return tf.train.Example(features = tf.train.Features(feature={
        'label_index': tf.train.Feature(int64_list=tf.train.Int64List(value=[label_index])),
        'image': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image]))
        }))

def write_tfrecord(dataset, outputfilepath):
    writer = tf.python_io.TFRecordWriter(outputfilepath)
    for label_of_one_hot, image in dataset:
        ex = make_example(label_of_one_hot, image)
        writer.write(ex.SerializeToString())
    writer.close()

def importingargs():
    parser = argparse.ArgumentParser("tensorflow exampe")
    parser.add_argument("--datafolderpath", "-df", help="datafolderpath")
    parser.add_argument("--filepath", "-f", help="filepath", required=True)
    parser.add_argument("--labelfilepath", "-lf", help="label filepath")
    parser.add_argument("--outputfolderpath", "-of", help="outputfolderpath of tf records")
    parser.add_argument("--seed", "-s", type=int, required=False, default=0)
    args = parser.parse_args()

    return args.filepath, args.datafolderpath, args.labelfilepath, args.outputfolderpath, args.seed


def load_data(filepath, datafolderpath, labelfilepath):
    with open(labelfilepath, "r") as rf:
        labellist = [ line.strip() for line in rf.readlines() ]

    with open(filepath,  "r") as rf:
        filepathlist = [ line.strip() for line in rf.readlines() ]


    alldatasets = list()
    for filepath in filepathlist:
        imagefilepath = os.path.join(datafolderpath, filepath)
        # image = open(imagefilepath).read()
        img_obj = Image.open(imagefilepath).convert("L")
        img = np.array(img_obj)
        w, h = img.shape
        print(w, h)
        print(w*h)
        img = img.reshape(w*h).tostring()
        print(type(img))
        filename = filepath.split(os.path.sep)[-1]
        label = filename.split(".")[0].split("_")[1]
        index = labellist.index(label) +1
        print(index)
        alldatasets.append([ index, img ])
    return alldatasets

def splitdata(datasets):
    random.shuffle(datasets)
    train_indexes = [ 0, int(len(datasets) * 0.8 ) ]
    valid_indexes = [ train_indexes[-1], int(len(datasets) * 0.9 ) ]
    test_indexes = [ valid_indexes[-1], int(len(datasets)) ]

    train_data = datasets[train_indexes[0]:train_indexes[1]]
    valid_data = datasets[valid_indexes[0]:valid_indexes[1]]
    test_data = datasets[test_indexes[0]:test_indexes[1]]

    print("train num: %d" % len(train_data))
    print("test  num: %d" % len(test_data))
    print("valid num: %d" % len(valid_data))

    return train_data, valid_data, test_data

def main():
    filepath, datafolderpath, labelfilepath, outputfolderpath, seed = importingargs()
    random.seed(seed)
    alldatasets = load_data(filepath, datafolderpath, labelfilepath)
    train_data, valid_data, test_data = splitdata(alldatasets)
    train_outputfilepath = os.path.join(outputfolderpath, "train.tfrecord")
    valid_outputfilepath = os.path.join(outputfolderpath, "valid.tfrecord")
    test_outptufilepath = os.path.join(outputfolderpath, "test.tfrecord")

    write_tfrecord(train_data, train_outputfilepath)
    write_tfrecord(valid_data, valid_outputfilepath)
    write_tfrecord(test_data, test_outptufilepath)

if __name__ == "__main__":
    main()

导入train.py的load_dataset文件

#!/usr/bin/env python3
#coding:utf-8

import argparse
import os

import numpy as np
from PIL import Image
import tensorflow as tf

def read_tfrecord(inputfilepath):
    print("read record")
    reader = tf.TFRecordReader()
    filename_que = tf.train.string_input_producer([inputfilepath])
    key, value = reader.read(filename_que)
    features  = tf.parse_single_example(value,features = {
            'label_index': tf.FixedLenFeature([], tf.string),
            'image': tf.FixedLenFeature([], tf.string)
            })

    images = tf.decode_raw(features['image'], tf.float32)
    images.set_shape([32*32])
    images = tf.cast(images, tf.float32) * (1. / 255)
    # images = tf.reshape(images, [-1])
    labels = tf.decode_raw(features['label_index'], tf.int32)
    # labels = tf.cast(features['label_index'], tf.int64)
    # labels.set_shape([5])
    print("call one hot")
    label_index_one_hot = tf.one_hot(labels, 5)
    label_index_one_hot.set_shape([5])
    return images, label_index_one_hot

训练脚本是

import os
import random

import tensorflow as tf

import load_datasets
import datasets
import make_datasets

print("def input and output")
images = tf.placeholder(tf.float32, shape=[None, 32*32])
labels = tf.placeholder(tf.int32, shape=[None, 5])


print("def layers")
x = tf.placeholder(tf.float32, [ None, 32*32 ])
y_ = tf.placeholder(tf.float32, [None, 5 ])

# W1 = tf.Variable(tf.zeros([ 32*32, 500 ]))
# b1 = tf.Variable(tf.zeros([ 500 ]))

# W2 = tf.Variable(tf.zeros([ 500, 5 ]))
# b2 = tf.Variable(tf.zeros([ 5 ]))

print("def function")
# h1 = tf.matmul(x, W1) + b1
# y = tf.matmul(h1, W2) + b2

W = tf.Variable(tf.zeros([ 32*32, 5 ]))
b = tf.Variable(tf.zeros([ 5 ]))
y = tf.matmul(x, W) + b

print("def leraning model")
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=y))
train_step = tf.train.AdamOptimizer(0.001).minimize(cross_entropy)

correct_prediction= tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))


print("load train dataset")
trainfilepath = "../03tfrecords/train.tfrecord"
images, labels = load_datasets.read_tfrecord(trainfilepath)
input_queue = tf.train.slice_input_producer( [images, labels ], num_epochs=10, shuffle=False )
image_batch, label_batch = tf.train.batch( [images, labels], batch_size=10)

print("load test dataset")
testfilepath = "../03tfrecords/test.tfrecord"
test_image, test_label = load_datasets.read_tfrecord(testfilepath)
img_test_batch, label_test_batch = tf.train.batch([test_image,test_label],batch_size=16)

with tf.Session() as sess:
    print("init layer value")
    sess.run(tf.global_variables_initializer())
    print("start training")
    tf.train.start_queue_runners(sess)
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    try:
        while not coord.should_stop():
            for i in range(0, 10):
                print("train num %d" % (i+1))
                imgs, labels = sess.run([image_batch, label_batch])
                sess.run(train_step, feed_dict={x:imgs, y_: labels})

                imgs_test, labels_text = sess.run([img_test_batch, label_test_batch])
                print(sess.run(accuracy, feed_dict={x:imgs_test, y_:labels_text}))


    finally:
        coord.request_stop()
        coord.join(threads)

制作tfrecords效果很好，但是在训练脚本中，出现错误。

Traceback (most recent call last):
  File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 576, in merge_with
    self.assert_same_rank(other)
  File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 621, in assert_same_rank
    other))
ValueError: Shapes (?, 5) and (5,) must have the same rank

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "train.py", line 45, in <module>
    images, labels = load_datasets.read_tfrecord(trainfilepath)
  File "/home/omori/tensorflow_example/01src/load_datasets.py", line 30, in read_tfrecord
    label_index_one_hot.set_shape([5])
  File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 407, in set_shape
    self._shape = self._shape.merge_with(shape)
  File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 582, in merge_with
    raise ValueError("Shapes %s and %s are not compatible" % (self, other))
ValueError: Shapes (?, 5) and (5,) are not compatible

我搜索了很多网站，但找不到解决方案。我该如何解决？

Answer 1

decode_raw :

Returns:

A Tensor of type out_type. A Tensor with one more dimension than the input bytes. The added 
dimension will have size equal to the length of the elements of bytes divided by the number 
of bytes to represent out_type.

所以在你的 read_tfrecord 函数中有一行

labels = tf.decode_raw(features['label_index'], tf.int32)

给 labels 一个额外的尺寸。您可以使用

解决此问题

label_index_one_hot = tf.one_hot(labels[0], 5)

(注意添加的[0])

我不得不承认，我不明白添加的维度是干什么用的。