python - 逻辑回归成本变化变为恒定

Question

经过几次梯度下降迭代后，成本函数变化变得恒定，这绝对是它不应该执行的方式:

梯度下降函数的初始结果似乎是正确的，成本函数的结果以及假设函数的结果也是正确的，所以我认为问题不在那里。抱歉，如果问题太不确定，但我自己无法再缩小问题范围。如果您能解释我的程序出了什么问题，我将不胜感激。

这就是我正在使用的数据的样子:

34.62365962451697,78.0246928153624,0

30.28671076822607,43.89499752400101,0

35.84740876993872,72.90219802708364,0

60.18259938620976,86.30855209546826,1

79.0327360507101,75.3443764369103,1

45.08327747668339,56.3163717815305,0

这是代码:

import numpy as np
from matplotlib import pyplot as plt


data = np.genfromtxt("ex2data1.txt", delimiter=",")

X = data[:,0:-1]
X = np.array(X)
m = len(X)
ones = np.ones((m,1))
X = np.hstack((ones,X))

Y = data[:,-1]
Y = np.array(Y)
Y = Y.reshape((m,1))

Cost_History = [[],[]]

def Sigmoid(z):
    G = float(1/float(1+np.exp(-1.0*z)))
    return G

def Hypothesis(theta, x):
    z = np.dot(x,theta)
    return Sigmoid(z)

def Cost_Function(X, Y, theta, m):
    sumOfErrors = 0
    for i in range(m):
        xi = X[i]
        yi = Y[i]
        hi = Hypothesis(theta, xi)
        sumOfErrors += yi*np.log(hi) + (1-yi)*np.log(1-hi)
    const = -(1/m)
    J = const * sumOfErrors
    return J

def Cost_Function_Derivative(X, Y, theta, feature, alpha, m):
    sumErrors = 0
    for i in range(m):
        xi = X[i]
        yi = Y[i]
        hi = Hypothesis(theta, xi)
        error = (hi - yi)*xi[feature]
        sumErrors += error
    constant = float(alpha)/float(m)
    J = constant * sumErrors
    return J

def Gradient_Descent(X, Y, theta, alpha, m):
    new_theta = np.zeros((len(theta),1))
    for feature in range(len(theta)):
        CFDerivative = Cost_Function_Derivative(X, Y, theta, feature, alpha, m)
        new_theta[feature] = theta[feature] - CFDerivative
    return new_theta


def Logistic_Regression(X,Y,alpha, theta, iterations, m):
    for iter in range(iterations):
        theta = Gradient_Descent(X, Y, theta, alpha, m)
        cost = Cost_Function(X, Y, theta, m)
        Cost_History[0].append(cost)
        Cost_History[1].append(iter)
        if iter % 100 == 0:
            print(theta, cost, iter)
    return theta

alpha = 0.001
iterations = 1500

theta = np.zeros((len(X[0]),1))
theta = Logistic_Regression(X, Y, alpha, theta, iterations, m)
print(theta)
test = np.array((1,85,45))
print(Hypothesis(theta, test))

wrong = 0
for i in range(m):
    xi = X[i]
    yi = Y[i]
    hi = Hypothesis(theta, xi)
    if yi != round(hi):
        wrong+=1
print(wrong/m)

plt.plot(Cost_History[1], Cost_History[0], "b")
plt.show()

Answer 1

从给定的图中可以看出，成本实际上仍在下降。快速搜索一下就会发现您的数据当前可以是 found here ，并通过运行您的代码 500000 次迭代，我得到的是以下内容，它看起来正如您所期望的:

大约 20000000 步后，theta 的值变为 [-25.15510086, 0.20618186, 0.20142117]。为了确保这是我们所期望的，我们可以将该值与使用 sci-kit learn's implementation 获得的参数进行比较具有较大的 C 值:

from sklearn.linear_model import LogisticRegression
model = LogisticRegression(C=1e10, tol=1e-6).fit(X, Y.ravel())
model.coef_[0] + np.array([model.intercept_[0], 0, 0])
# array([-25.16127356,   0.20623123,   0.20147112])

或者 statsmodels 中的相同内容:

from statsmodels.discrete.discrete_model import Logit
model = Logit(Y.ravel(), X)
model.fit().params
# array([-25.16133357,   0.20623171,   0.2014716 ])

当然，运行这么多步骤的算法肯定需要一段时间。在实践中，您通常会转向二阶优化例程或随机梯度下降，但事实证明，大多数代码都可以用向量化运算(例如矩阵乘法)来表达，这将增加足够的性能以使其相对收敛快速地。特别是，您的方法可以重写如下，唯一的区别是 Y 不再需要重新整形:

def hypothesis(theta, X):
    return 1/(1+np.exp(-np.dot(X, theta)))

def cost_function(X, Y, theta):
    h = hypothesis(theta, X)
    return -np.mean(Y*np.log(h) + (1-Y)*np.log(1-h))

def gradient_descent(X, Y, theta, alpha):
    h = hypothesis(theta, X)
    return theta - alpha*np.dot(X.T, h - Y)/len(X)

def logistic_regression(X, Y, alpha, theta, iterations):
    for iter in range(iterations):
        theta = gradient_descent(X, Y, theta, alpha)     
        if iter % 100000 == 0:
            cost = cost_function(X, Y, theta)
            cost_history[0].append(cost)
            cost_history[1].append(iter)
            print(theta, cost, iter)
    return theta