python - Tensorflow 自定义过滤层定义，如 glcm 或 gabor

Question

我想应用各种过滤器，如 GLCM或 Gabor filter bank作为 Tensorflow 中的自定义层，但我找不到足够的自定义层示例。如何将这些类型的过滤器应用为图层？
生成GLCM的过程在scikit-image库中定义如下:

from skimage.feature import greycomatrix, greycoprops
from skimage import data
#load image
img = data.brick()
#result glcm
glcm = greycomatrix(img, distances=[5], angles=[0], levels=256, symmetric=True, normed=True)

Gabor滤波器组的使用如下:

import matplotlib.pyplot as plt
import numpy as np
from scipy import ndimage as ndi
from skimage import data
from skimage.util import img_as_float
from skimage.filters import gabor_kernel

shrink = (slice(0, None, 3), slice(0, None, 3))
brick = img_as_float(data.brick())[shrink]
grass = img_as_float(data.grass())[shrink]
gravel = img_as_float(data.gravel())[shrink]
image_names = ('brick', 'grass', 'gravel')
images = (brick, grass, gravel)

def power(image, kernel):
    # Normalize images for better comparison.
    image = (image - image.mean()) / image.std()
    return np.sqrt(ndi.convolve(image, np.real(kernel), mode='wrap')**2 +
                   ndi.convolve(image, np.imag(kernel), mode='wrap')**2)

# Plot a selection of the filter bank kernels and their responses.
results = []
kernel_params = []
for theta in (0, 1):
    theta = theta / 4. * np.pi
    for sigmax in (1, 3):
        for sigmay in (1, 3):
            for frequency in (0.1, 0.4):
                kernel = gabor_kernel(frequency, theta=theta,sigma_x=sigmax, sigma_y=sigmay)
                params = 'theta=%d,f=%.2f\nsx=%.2f sy=%.2f' % (theta * 180 / np.pi, frequency,sigmax, sigmay)
                kernel_params.append(params)
                # Save kernel and the power image for each image
                results.append((kernel, [power(img, kernel) for img in images]))

fig, axes = plt.subplots(nrows=6, ncols=4, figsize=(5, 6))
plt.gray()
fig.suptitle('Image responses for Gabor filter kernels', fontsize=12)
axes[0][0].axis('off')
# Plot original images
for label, img, ax in zip(image_names, images, axes[0][1:]):
    ax.imshow(img)
    ax.set_title(label, fontsize=9)
    ax.axis('off')
for label, (kernel, powers), ax_row in zip(kernel_params, results, axes[1:]):
    # Plot Gabor kernel
    ax = ax_row[0]
    ax.imshow(np.real(kernel))
    ax.set_ylabel(label, fontsize=7)
    ax.set_xticks([])
    ax.set_yticks([])
    # Plot Gabor responses with the contrast normalized for each filter
    vmin = np.min(powers)
    vmax = np.max(powers)
    for patch, ax in zip(powers, ax_row[1:]):
        ax.imshow(patch, vmin=vmin, vmax=vmax)
        ax.axis('off')
plt.show()

我如何在 tensorflow 中定义这些和类似的过滤器。
我尝试了上面的代码，但没有给出相同的结果: https://scikit-image.org/docs/dev/auto_examples/features_detection/plot_gabor.html

我得到了这个:

import numpy as np
import matplotlib.pyplot as plt
import tensorflow.keras.backend as K
from tensorflow.keras import Input, layers
from tensorflow.keras.models import Model
from scipy import ndimage as ndi

from skimage import data
from skimage.util import img_as_float
from skimage.filters import gabor_kernel

import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'


def gfb_filter(shape,size=3, tlist=[1,2,3], slist=[2,5],flist=[0.01,0.25],dtype=None):
    print(shape)
    fsize=np.ones([size,size])
    kernels = []
    for theta in tlist:
        theta = theta / 4. * np.pi
        for sigma in slist:
            for frequency in flist:
                kernel = np.real(gabor_kernel(frequency, theta=theta,sigma_x=sigma, sigma_y=sigma))
                kernels.append(kernel)
    gfblist = []
    for k, kernel in enumerate(kernels):
        ck=ndi.convolve(fsize, kernel, mode='wrap')
        gfblist.append(ck)
    
    gfblist=np.asarray(gfblist).reshape(size,size,1,len(gfblist))
    print(gfblist.shape)
    return K.variable(gfblist, dtype='float32')


dimg=img_as_float(data.brick())
input_mat = dimg.reshape((1, 512, 512, 1))

def build_model():
    input_tensor = Input(shape=(512,512,1))
    x = layers.Conv2D(filters=12, 
                      kernel_size = 3,
                      kernel_initializer=gfb_filter,
                      strides=1, 
                      padding='valid') (input_tensor)

    model = Model(inputs=input_tensor, outputs=x)
    return model

model = build_model()
out = model.predict(input_mat)
print(out)

o1=out.reshape(12,510,510)
plt.subplot(2,2,1)
plt.imshow(dimg)

plt.subplot(2,2,2)
plt.imshow(o1[0,:,:])

plt.subplot(2,2,3)
plt.imshow(o1[6,:,:])

plt.subplot(2,2,4)
plt.imshow(o1[10,:,:])

Answer 1

您可以阅读有关编写 custom layer 的文档。 ，以及关于 Making new Layers and Models via subclassing
这是基于您的代码的 Gabor 滤波器组的简单实现:

import numpy as np
import tensorflow as tf
from tensorflow.keras import layers
from skimage.filters import gabor_kernel
class GaborFilterBank(layers.Layer):
    def __init__(self):
        super().__init__()
    
    def build(self, input_shape):
        # assumption: shape is NHWC 
        self.n_channel = input_shape[-1]
        self.kernels = []
        for theta in range(4):
            theta = theta / 4.0 * np.pi
            for sigma in (1, 3):
                for frequency in (0.05, 0.25):
                    kernel = np.real(
                        gabor_kernel(
                            frequency, theta=theta, sigma_x=sigma, sigma_y=sigma
                        )
                    ).astype(np.float32)
                    # tf.nn.conv2d does crosscorrelation, not convolution, so flipping 
                    # the kernel is needed
                    kernel = np.flip(kernel)
                    # we stack the kernel on itself to match the number of channel of
                    # the input
                    kernel = np.stack((kernel,)*self.n_channel, axis=-1)
                    # print(kernel.shape)
                    # adding the number of out channel, here 1.
                    kernel = kernel[:, :, : , np.newaxis] 
                    # because the kernel shapes are different, we can't do the conv op
                    # in one go, so we stack the kernels in a list
                    self.kernels.append(tf.Variable(kernel, trainable=False))

    def call(self, x):
        out_list = []
        for kernel in self.kernels:
            out_list.append(tf.nn.conv2d(x, kernel, strides=1, padding="SAME"))
        # output is [batch_size, H, W, 16] where 16 is the number of filters
        # 16 = n_theta * n_sigma * n_freq = 4 * 2 * 2 
        return tf.concat(out_list,axis=-1)

但是有一些区别:

tensorflow 没有卷积的“包裹”模式。我使用了类似于“常量”的“SAME”，在 scipy 中填充值为 0 .可以提供您自己的填充，因此绝对可以模仿“包装”模式，我将其作为练习留给读者。

tf.nn.conv2d期望 4D 输入，因此我将批处理维度和 channel 维度作为输入添加到 img。

tf.nn.conv2d 的过滤器必须遵循形状[filter_height, filter_width, in_channels, out_channels] .在那种情况下，我使用输入的 channel 数作为 in_channels . out_channels可以等于滤波器组中滤波器的数量，但是因为它们的形状不是恒定的，后面更容易将它们连接起来，所以我将其设置为1。这意味着该层的输出为[N,H,W,C]其中 C 是银行中的过滤器数量(在您的示例中为 16)。

tf.nn.conv2d不是真正的卷积，而是互相关(请参阅 doc )，因此需要事先翻转过滤器才能获得实际的卷积。

我正在添加一个关于如何使用它的快速示例:

# defining the model 
inp = tf.keras.Input(shape=(512,512,1))
conv = tf.keras.layers.Conv2D(4, (3,3), padding="SAME")(inp)
g = GaborFilterBank()(conv)
model = tf.keras.Model(inputs=inp, outputs=g)

# calling the model with an example Image
img = img_as_float(data.brick())
img_nhwc = img[np.newaxis, :, :, np.newaxis]
out = model(img_nhwc)
# out shape is [1,512,512,16]