tensorflow - 将 Pytorch 程序翻译成 Keras : different results

Question

我已经将一个 pytorch 程序翻译成 keras。

一个有效的 Pytorch 程序:

import numpy as np
import cv2
import torch
import torch.nn as nn
from skimage import segmentation

np.random.seed(1)
torch.manual_seed(1)


fi = "in.jpg"


class MyNet(nn.Module):
    def __init__(self, n_inChannel, n_outChannel):
        super(MyNet, self).__init__()

        self.seq = nn.Sequential(
            nn.Conv2d(n_inChannel, n_outChannel, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(n_outChannel),

            nn.Conv2d(n_outChannel, n_outChannel, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(n_outChannel),

            nn.Conv2d(n_outChannel, n_outChannel, kernel_size=1, stride=1, padding=0),
            nn.BatchNorm2d(n_outChannel)
        )

    def forward(self, x):
        return self.seq(x)


im = cv2.imread(fi)
data = torch.from_numpy(np.array([im.transpose((2, 0, 1)).astype('float32')/255.]))
data = data.cuda()

labels = segmentation.slic(im, compactness=100, n_segments=10000)
labels = labels.flatten()
u_labels = np.unique(labels)
label_indexes = np.array([np.where(labels == u_label)[0] for u_label in u_labels])

n_inChannel = 3
n_outChannel = 100

model = MyNet(n_inChannel, n_outChannel)
model.cuda()
model.train()
loss_fn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)

label_colours = np.random.randint(255,size=(100,3))

for batch_idx in range(100):
    optimizer.zero_grad()
    output = model( data )[ 0 ]
    output = output.permute( 1, 2, 0 ).view(-1, n_outChannel)
    ignore, target = torch.max( output, 1 )

    im_target = target.data.cpu().numpy()
    nLabels = len(np.unique(im_target))

    im_target_rgb = np.array([label_colours[ c % 100 ] for c in im_target]) # correct position of "im_target"
    im_target_rgb = im_target_rgb.reshape( im.shape ).astype( np.uint8 )

    for inds in label_indexes:
        u_labels_, hist = np.unique(im_target[inds], return_counts=True) 
        im_target[inds] = u_labels_[np.argmax(hist, 0)]

    target = torch.from_numpy(im_target)
    target = target.cuda()    

    loss = loss_fn(output, target)

    loss.backward()
    optimizer.step()

    print (batch_idx, '/', 100, ':', nLabels, loss.item())

    if nLabels <= 3:
        break         
fo = "out.jpg"
cv2.imwrite(fo, im_target_rgb)

(来源: https://github.com/kanezaki/pytorch-unsupervised-segmentation/blob/master/demo.py)

我对 Keras 的翻译:

import cv2
import numpy as np
from skimage import segmentation
from keras.layers import Conv2D, BatchNormalization, Input, Reshape
from keras.models import Model
import keras.backend as k
from keras.optimizers import SGD, Adam
from skimage.util import img_as_float
from skimage import io
from keras.models import Sequential

np.random.seed(0)

fi = "in.jpg"
im = cv2.imread(fi).astype(float)/255.

labels = segmentation.slic(im, compactness=100, n_segments=10000)
labels = labels.flatten()
print (labels.shape)
u_labels = np.unique(labels)

label_indexes = [np.where(labels == u_label)[0] for u_label in np.unique(labels)]

n_channels = 100

model = Sequential()

model.add (  Conv2D(n_channels, kernel_size=3, activation='relu', input_shape=im.shape, padding='same'))    
model.add( BatchNormalization())

model.add( Conv2D(n_channels, kernel_size=3, activation='relu', padding='same'))    
model.add( BatchNormalization())

model.add( Conv2D(n_channels, kernel_size=1, padding='same'))
model.add( BatchNormalization())

model.add( Reshape((im.shape[0] * im.shape[1], n_channels)))

img = np.expand_dims(im,0)
print (img.shape)
output = model.predict(img)

print (output.shape)

im_target = np.argmax(output[0], 1)
print (im_target.shape)

for inds in label_indexes:
    u_labels_, hist = np.unique(im_target[inds], return_counts=True)
    im_target[inds] = u_labels_[np.argmax(hist, 0)]

def custom_loss(loss_target, loss_output):
  return k.categorical_crossentropy(target=k.stack(loss_target), output=k.stack(loss_output), from_logits=True)


model.compile(optimizer=SGD(lr=0.1, momentum=0.9), loss=custom_loss)

model.fit(img, output, epochs=100, batch_size=1, verbose=1)

pred_result = model.predict(x=[img])[0]
print (pred_result.shape)

target = np.argmax(pred_result, 1)
print (target.shape)

nLabels = len(np.unique(target))

label_colours = np.random.randint(255, size=(100, 3))

im_target_rgb = np.array([label_colours[c % 100] for c in im_target])

im_target_rgb = im_target_rgb.reshape(im.shape).astype(np.uint8)

cv2.imwrite("out.jpg", im_target_rgb)

但是，Keras 的输出与 pytorch 的输出确实不同

输入图像:



火炬结果:



凯拉斯结果:



有人可以帮我翻译一下吗？

编辑 1:

I corrected two errors as advised by @sebrockm
1. removed `relu` from last conv layer
2. added `from_logits = True` in the loss function

另外，改了号。从 4 到 3 的卷积层与原始代码相匹配。

However, output image did not improve than before and the `loss` are resulted in negative:

Epoch 99/100
1/1 [==============================] - 0s 92ms/step - loss: -22.8380

Epoch 100/100    
1/1 [==============================] - 0s 99ms/step - loss: -23.039

我认为 Keras 代码在模型和输出之间缺乏联系。但是，无法弄清楚建立这种联系。

Answer 1

我看到的两个主要错误(可能相关):

原始模型中的最后一个卷积层没有激活函数，而您的翻译使用 relu .

原模型使用CrossEntropyLoss作为损失函数，而您的模型使用 categorical_crossentropy与 logits=False (默认参数)。如果没有数学背景，这种差异很难解释，但简而言之:CrossEntropyLoss有一个 softmax内置，这就是模型在最后一层没有的原因。要在 keras 中执行相同操作，请使用 k.categorical_crossentropy(..., logits=True) . “logits”表示预期的输入值不是 要“softmaxed”，即所有值都可以是任意的。目前，您的损失函数期望输出值是“softmaxed”，即所有值必须在 0 和 1 之间(并且总和为 1)。

更新:

另一个错误，可能是一个很大的错误:在 Keras 中，您计算​​ output一开始就一次，从那以后永远不要改变它。然后你训练你的模型以适应这个最初生成的输出。

在原始 pytorch 代码中， target (这是被训练的变量)在每个训练循环中都会更新。
所以，你不能使用 Keras 的 fit旨在为您进行整个培训的方法(给定固定的培训数据)。您将不得不手动复制训练循环，就像在 pytorch 代码中所做的那样。我不确定这是否可以通过 API 轻松实现Keras 提供。 train_on_batch是您在手动循环中肯定需要的一种方法。你将不得不做更多的工作，我担心......