python - 用梯度下降拟合直线

Question

我正在尝试使用梯度下降将一条线拟合到几个点。我不是这方面的专家，并试图用 python 写下它的数学算法。它运行了几次迭代，但我的预测似乎在某个时候爆炸了。这是代码:

import numpy as np
import matplotlib.pyplot as plt

def mean_squared_error(n, A, b, m, c):
    e = 0
    for i in range(n):
        e += (b[i] - (m*A[i] + c)) ** 2   
    return e/n

def der_wrt_m(n,A,b,m,c):
    d = 0
    for i in range(n):
        d += (2 * (b[i] - (m*A[i] + c)) * (-A[i]))
    return d/n

def der_wrt_c(n,A,b,m,c):
    d = 0
    for i in range(n):
        d += (2 * (b[i] - (m*A[i] + c)))
    return d/n

def update(n,A,b,m,c,descent_rate):
    return descent_rate * der_wrt_m(n,A,b,m,c)), descent_rate * der_wrt_c(n,A,b,m,c))

A = np.array(((0,1),
             (1,1),
             (2,1),
             (3,1)))
x = A.T[0]
b = np.array((1,2,0,3), ndmin=2 ).T
y = b.reshape(4)

def descent(x,y):
    m = 0
    c = 0

    descent_rate = 0.00001
    iterations = 100

    n = len(x)
    plt.scatter(x, y)
    u = np.linspace(0,3,100)
    prediction = 0
    for itr in range(iterations):
        print(m,c)
        prediction = prediction + m * x + c
        m,c = update(n,x,y,m,c,descent_rate)

    plt.plot(u, u * m + c, '-')   


descent(x,y)

这是我的输出:

0 0
19.25 -10.5
-71335.1953125 24625.9453125
5593771382944640.0 -2166081169939480.2
-2.542705027685638e+48 9.692684648057364e+47
2.40856742196228e+146 -9.202614421953049e+145
-inf inf
nan nan
nan nan
nan nan
nan nan
nan nan
nan nan
etc...

---

更新:值不再爆炸，但仍然没有很好地收敛:

# We could also solve it using gradient descent
import numpy as np
import matplotlib.pyplot as plt

def mean_squared_error(n, A, b, m, c):
    e = 0
    for i in range(n):
        e += ((b[i] - (m * A[i] + c)) ** 2)   
    #print("mse:",e/n)
    return e/n

def der_wrt_m(n,A,b,m,c):
    d = 0
    for i in range(n):
        # d += (2 * (b[i] - (m*A[i] + c)) * (-A[i]))
        d += (A[i] * (b[i] - (m*A[i] + c)))
    #print("Dm",-2 * d/n)
    return (-2 * d/n)

def der_wrt_c(n,A,b,m,c):
    d = 0
    for i in range(n):
        d += (2 * (b[i] - (m*A[i] + c)))
    #print("Dc",d/n)
    return d/n

def update(n,A,b,m,c, descent_rate):
    return (m - descent_rate * der_wrt_m(n,A,b,m,c)),(c - descent_rate * der_wrt_c(n,A,b,m,c))

A = np.array(((0,1),
             (1,1),
             (2,1),
             (3,1)))
x = A.T[0]
b = np.array((1,2,0,3), ndmin=2 ).T
y = b.reshape(4)

def descent(x,y):
    m = 0
    c = 0

    descent_rate = 0.0001
    iterations = 10000

    n = len(x)
    plt.scatter(x, y)
    u = np.linspace(0,3,100)
    prediction = 0
    for itr in range(iterations):
        prediction = prediction + m * x + c
        m,c = update(n,x,y,m,c,descent_rate)
        loss = mean_squared_error(n, A, b, m, c)

    print(loss)
    print(m,c)
    plt.plot(u, u * m + c, '-')    

descent(x,y)

在学习率为 0.0001 的大约 10000 次迭代之后，现在该图看起来像这样:

[4.10833186 5.21468937]
1.503547594304175 -1.9947003678083184

而最小二乘拟合显示如下:

Answer 1

在你的更新函数中，你应该从当前的 m 和 c 中减去计算出的梯度

def update(n,A,b,m,c,descent_rate):
    return m - (descent_rate * der_wrt_m(n,A,b,m,c)), c - (descent_rate * der_wrt_c(n,A,b,m,c))

更新:这是工作版本。我在获得 x,y 后摆脱了 A 矩阵，因为它让我感到困惑 =)。例如在你的梯度计算中你有一个表达式 d += (A[i] * (b[i] - (m*A[i] + c))) 但它应该是 d += (x[i] * (b[i] - (m*x[i] + c))) 因为 x[i] 给你一个元素，而 A[i] 给你一个列表。

在计算关于 c 的导数时，您还忘记了减号。如果你的表达式是 (y - (m*x + c))^2) 那么关于 c 的导数应该是 2 * (-1) * (y - (m*x + c)) 因为c前面有个减号。

# We could also solve it using gradient descent
import numpy as np
import matplotlib.pyplot as plt

def mean_squared_error(n, x, y, m, c):
    e = 0
    for i in range(n):
        e += (m*x[i]+c - y[i])**2
    e = e/n
    return e/n

def der_wrt_m(n, x, y, m, c):
    d = 0
    for i in range(n):
        d += x[i] * (y[i] - (m*x[i] + c))
    d = -2 * d/n
    return d

def der_wrt_c(n, x, y, m, c):
    d = 0
    for i in range(n):
        d += (y[i] - (m*x[i] + c))
    d = -2 * d/n
    return d


def update(n,x,y,m,c, descent_rate):
    return (m - descent_rate * der_wrt_m(n,x,y,m,c)),(c - descent_rate * der_wrt_c(n,x,y,m,c))


A = np.array(((0,1),
             (1,1),
             (2,1),
             (3,1)))
x = A.T[0]
b = np.array((1,2,0,3), ndmin=2 ).T
y = b.reshape(4)

print(x)
print(y)

def descent(x,y):
    m = 0.0
    c = 0.0

    descent_rate = 0.01
    iterations = 10000

    n = len(x)
    plt.scatter(x, y)
    u = np.linspace(0,3,100)
    prediction = 0
    for itr in range(iterations):
        prediction = prediction + m * x + c
        m,c = update(n,x,y,m,c,descent_rate)
        loss = mean_squared_error(n, x, y, m, c)
        print(loss)

    print(loss)
    print(m,c)
    plt.plot(u, u * m + c, '-')    
    plt.show()

descent(x,y)