python - 如何使用 Python 在 Keras 中绘制 ANN 的 ROC 曲线以进行 10 折交叉验证？

Question

我只是想为 Keras 中的 ANN 的 10 折交叉验证的所有 10 个实验找到 ROC 图。我坚持了一个星期，找不到解决方案。有人可以帮忙吗？我已经尝试了来自 sklearn 的以下链接(https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc_crossval.html)中的代码，并想使用包装器在 sklearn 中使用 Keras 模型，但它显示错误。我在 python 中的代码:

    ## Creating NN in Keras
# Load libraries
import numpy as np
from keras import models
from keras import layers
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score
from sklearn.datasets import make_classification

# Set random seed
np.random.seed(7)
#Create Function That Constructs Neural Network
# Create function returning a compiled network
def create_network():
    
    # Start neural network
    network = models.Sequential()

    # Add fully connected layer with a ReLU activation function
    network.add(layers.Dense(units=25, activation='relu', input_shape=(X.shape[1],)))

    # Add fully connected layer with a ReLU activation function
    network.add(layers.Dense(units=X.shape[1], activation='relu'))

    # Add fully connected layer with a sigmoid activation function
    network.add(layers.Dense(units=1, activation='sigmoid'))

    # Compile neural network
    network.compile(loss='binary_crossentropy', # Cross-entropy
                    optimizer='adam', # Root Mean Square Propagation
                    metrics=['accuracy']) # Accuracy performance metric
    
    # Return compiled network
    return network

###
#Wrap Function In KerasClassifier
# Wrap Keras model so it can be used by scikit-learn
neural_network = KerasClassifier(build_fn=create_network, 
                                 epochs=150, 
                                 batch_size=10, 
                                 verbose=0)


    import numpy as np
    import matplotlib.pyplot as plt
    
    from sklearn import svm, datasets
    from sklearn.metrics import auc
    from sklearn.metrics import plot_roc_curve
    from sklearn.model_selection import StratifiedKFold
    
    n_samples, n_features = X.shape
    
    # Add noisy features
    random_state = np.random.RandomState(0)
    X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]
    
    # #############################################################################
    # Classification and ROC analysis
    
    # Run classifier with cross-validation and plot ROC curves
    cv = StratifiedKFold(n_splits=10)
    classifier = neural_network
    
    tprs = []
    aucs = []
    mean_fpr = np.linspace(0, 1, 100)
    
    fig, ax = plt.subplots()
    for i, (train, test) in enumerate(cv.split(X, y)):
        classifier.fit(X[train], y[train])
        viz = plot_roc_curve(classifier, X[test], y[test],
                             name='ROC fold {}'.format(i),
                             alpha=0.3, lw=1, ax=ax)
        interp_tpr = np.interp(mean_fpr, viz.fpr, viz.tpr)
        interp_tpr[0] = 0.0
        tprs.append(interp_tpr)
        aucs.append(viz.roc_auc)
    
    ax.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r',
            label='Chance', alpha=.8)
    
    mean_tpr = np.mean(tprs, axis=0)
    mean_tpr[-1] = 1.0
    mean_auc = auc(mean_fpr, mean_tpr)
    std_auc = np.std(aucs)
    ax.plot(mean_fpr, mean_tpr, color='b',
            label=r'Mean ROC (AUC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc),
            lw=2, alpha=.8)
    
    std_tpr = np.std(tprs, axis=0)
    tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
    tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
    ax.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2,
                    label=r'$\pm$ 1 std. dev.')
    
    ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05],
           title="Receiver operating characteristic example")
    ax.legend(loc="lower right")
    plt.show()


    **It shows the following error:**



    ---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-29-f10078491154> in <module>()
     40     viz = plot_roc_curve(classifier, X[test], y[test],
     41                          name='ROC fold {}'.format(i),
---> 42                          alpha=0.3, lw=1, ax=ax)
     43     interp_tpr = np.interp(mean_fpr, viz.fpr, viz.tpr)
     44     interp_tpr[0] = 0.0

/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_plot/roc_curve.py in plot_roc_curve(estimator, X, y, sample_weight, drop_intermediate, response_method, name, ax, **kwargs)
    170     )
    171     if not is_classifier(estimator):
--> 172         raise ValueError(classification_error)
    173 
    174     prediction_method = _check_classifer_response_method(estimator,

ValueError: KerasClassifier should be a binary classifier
    

Answer 1

我也有同样的疑问。我发现这个链接非常有用。 https://www.kaggle.com/kanncaa1/roc-curve-with-k-fold-cv .我已针对我的情况对其进行了如下修改:

seed = 7
np.random.seed(seed)
tprs = []

aucs = []
mean_fpr = np.linspace(0, 1, 100)
i = 1
fig, ax = plt.subplots()
kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=seed)
# for i, (train, test) in enumerate(cv.split(X_13 , target)):
for train, test in kfold.split(X_train, y_train):
  # create model
    model= Sequential()
    
    model.add(Dense(100, input_dim=X_train.shape[1], activation= 'relu',kernel_constraint=maxnorm(3)))
    model.add(Dropout(0.2))
    model.add(Dense(80, activation = 'relu',kernel_constraint=maxnorm(3)))
    model.add(Dropout(0.2))
    model.add(Dense(1, activation = 'sigmoid'))

##- compile model
    sgd = SGD(lr=0.1, momentum=0.8)
    model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])
    model.fit(X_train[train], y_train[train], epochs=100, batch_size=15,verbose=0)
# evaluate the model
    
    y_pred_keras = model.predict_proba(X_train[test]).ravel()
    
    fpr, tpr, thresholds = roc_curve(y_train[test], y_pred_keras)
    tprs.append(interp(mean_fpr, fpr, tpr))
    roc_auc = auc(fpr, tpr)
    aucs.append(roc_auc)
    plt.plot(fpr, tpr, lw=2, alpha=0.3, label='ROC fold %d (AUC = %0.2f)' % (i, roc_auc))
    i= i+1


plt.plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'black')
mean_tpr = np.mean(tprs, axis=0)
mean_auc = auc(mean_fpr, mean_tpr)
plt.plot(mean_fpr, mean_tpr, color='blue',
         label=r'Mean ROC (AUC = %0.2f )' % (mean_auc),lw=2, alpha=1)

plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC')
plt.legend(loc="lower right")

plt.show()

希望对您有所帮助!