python - 如何在神经网络的隐藏层中对权重矩阵的列实现正交约束？

Question

我想在隐藏层的权重矩阵的列上训练一个具有正交约束的神经网络，即学习到的权重矩阵应该具有正交列。如何在 keras 中做到这一点？

Answer 1

class WeightsOrthogonalityConstraint(tf.keras.constraints.Constraint):
    def __init__(self, encoding_dim, weightage = 1.0, axis = 0):
        self.encoding_dim = encoding_dim
        self.weightage = weightage
        self.axis = axis

    def weights_orthogonality(self, w):
        if(self.axis==1):
            w = tf.keras.backend.transpose(w)
        if(self.encoding_dim > 1):
            m = tf.keras.backend.dot(tf.keras.backend.transpose(w), tf.Variable(w)) - tf.keras.backend.eye(self.encoding_dim)
            return self.weightage * tf.keras.backend.sqrt(tf.keras.backend.sum(tf.keras.backend.square(m)))
        else:
            m = tf.keras.backend.sum(w ** 2) - 1.
            return m

    def __call__(self, w):
        return self.weights_orthogonality(w)

用法:

x = tf.keras.layers.Dense(2, 'relu', input_shape=(4,), kernel_regularizer=WeightsOrthogonalityConstraint(2))