python - PyTorch:为什么在训练时期循环内部或外部调用它后验证准确性会发生变化？

Question

我正在学习深度学习，我正在尝试将 RNN 与时间序列金融数据的训练、测试和验证集结合使用。下面是我的代码:

def get_lr(optimizer):
    for param_group in optimizer.param_groups:
        return param_group['lr']


# In[63]:


def train_model(epoch, model, optimizer, train_loader):
    model.train()
    t0 = time.time()
    correct = 0
    total = 0
    final_loss = 0
    for batch_idx, (X,labels) in enumerate(train_loader):
        data,labels = map(lambda x: Variable(x), [X,labels])
        optimizer.zero_grad()
        output = model(data)
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        ##print('device : ', device)
        final_output=output.to(device)
        loss = F.cross_entropy(final_output, labels)
        final_loss += loss.item()
        loss.backward()
        optimizer.step()
        print('predicted labels',final_output.squeeze())
        #print('Actual labels',labels.squeeze())
        print('Train Epoch: {} Batch: {} [{}/{} ({:.2f}%, time:{:.2f}s)]\tBatch Loss: {:.6f}'.format(
                epoch, batch_idx, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), time.time() - t0,
                final_loss))
                ##avg_loss))
        _, predicted = torch.max(output.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()
        t0 = time.time()
    final_loss /= (batch_idx+1)
    accuracy =  100*correct/total
    lr = get_lr(optimizer)
    learning_rates.append(lr)
    print('Training Accuracy : ',accuracy)
    print('Training Loss : ',final_loss)
    print('Learning Rate : ',lr)
    if epoch%epoch_interval == 0 or epoch ==1 or epoch == epochs: 
        path = base_path + 'models/RNN/rnn_'
        torch.save(model,path+str(epoch)+'.pth')
        ##torch.save(model,path)
        print('model saved')
    if epoch%plot_epoch_interval == 0 or epoch ==1 or epoch == epochs: 
        epochs_list.append(epoch)
        train_loss.append(final_loss)
        train_accuracies.append(accuracy)
    return lr,final_loss,accuracy


# In[166]:


def validate(epoch,model, val_loader,optimizer):
    model.eval()
    val_loss = 0
    correct = 0
    total = 0
    loss = 0
    ypred,ytrue,scores = [],[],[]
    for batch_idx,(X,labels) in enumerate(val_loader):
        data,labels = map(lambda x: Variable(x), [X,labels])
        output = model(data)
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        final_val_output=output.to(device)
        val_loss += F.cross_entropy(final_val_output, labels)  # sum up batch loss
        _, predicted = torch.max(output.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()
        ypred.extend(predicted.tolist())
        ytrue.extend(labels.tolist())
        scores.extend(output.tolist())
    val_loss /= (batch_idx+1)
    accuracy =  100*correct/total
    if epoch%plot_epoch_interval == 0 or epoch ==1 or epoch == epochs: 
        validation_loss.append(val_loss.item())
        val_accuracies.append(accuracy)

    print('Accuracy : ',accuracy)
    print('\nVal set: Average loss: {:.4f}, Accuracy: {}/{} ({:.4f}%)\n'.format(
        val_loss, correct,total,accuracy))
    print("==============================================")
    return "{:.4f}%".format(100.* correct / total), accuracy,loss,ypred,ytrue,scores


# In[276]:

def test(data_loader,model):
    torch.manual_seed(1)
    np.random.seed(1)
    #data_loader = DataLoader(FinancialData(xtest,ytest), batch_size = batch_size, shuffle = False)
    model = torch.load(path)
    model.eval()
    for params in model.parameters():
         print(params)
    val_loss = 0
    correct = 0
    total = 0
    loss = 0
    ypred,ytrue,scores = [],[],[]
    with torch.no_grad():
        for batch_idx,(X,labels) in enumerate(data_loader):
            data,labels = map(lambda x: Variable(x), [X,labels])
            output = model(data)
            _, predicted = torch.max(output.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
            ypred.extend(predicted.tolist())
            ytrue.extend(labels.tolist())
            scores.extend(output.tolist())
            accuracy =  100*correct/total

    print('Test Accuracy : ',accuracy)


# In[288]:


def train_on_batch(lr,epochs,momentum,X_train,Y_train,X_val,Y_val,batch_size):
    cuda=False
    seed=1
    torch.manual_seed(seed)

    train_loader = DataLoader(FinancialData(X_train,Y_train),batch_size=batch_size,shuffle=True) 
    val_loader = DataLoader(FinancialData(X_val,Y_val),batch_size=batch_size,shuffle=False) 
    test_loader = DataLoader(FinancialData(X_test_new,Y_test), batch_size = batch_size, shuffle = False)
    input_size = 1
    hid_size = 10
    num_layers = 2
    num_classes = len(np.unique(Y_train))
    dropRate = 0.0
    bidirection = True
    model = Network(input_size=input_size,hid_size =hid_size,window_size = window_size,num_layers=num_layers,
                num_classes=num_classes,dropRate = dropRate,bidirection=bidirection)
    ypred,ytrue, scores = [],[],[]
    for params in model.parameters():
        print(params)
    optimizer = torch.optim.Adam(model.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=1e-4, amsgrad=False)

    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,'max', factor=0.25, patience=6, verbose=True,
                                                            threshold_mode='abs', threshold=0.01, min_lr=1e-6)

    #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'max', factor=0.5, patience=5,
    #                                                       verbose=True,threshold_mode='abs', threshold=0.01, 
    #                                                       min_lr=1e-6)
    path = base_path + 'models/RNN/rnn_best_model.pth'
    best_val_loss = 0
    best_val_acc = 0
    best_epoch = 0
    best_lr = lr
    best_tr_acc = 0
    for epoch in range(1, epochs + 1):
        tuned_lr,tr_loss,tr_acc = train_model(epoch, model, optimizer, train_loader)
        acc_str, val_acc, val_loss, ypred, ytrue, scores = validate(epoch,model,val_loader,optimizer)
        if val_acc >= best_val_acc:
                torch.save(model,path)
                #best_val_loss = val_loss
                best_val_acc = val_acc
                best_epoch = epoch
                best_lr = tuned_lr
                best_tr_acc = tr_acc
        scheduler.step(val_acc)
        #scheduler.step(tr_acc)
        print('='*100)
#         for params in model.parameters():
#             print(params)
#         print('='*100)
    test(val_loader,model)
    test(test_loader,model)
    #validate(epoch,model,val_loader,optimizer)
    #validate(epoch,model,test_loader,optimizer)
    print('best epoch : {}, best_lr : {}, best_tr_acc : {}, best val_acc : {:.4f}\n'.format(best_epoch,best_lr,best_tr_acc,best_val_acc))    
    scores = np.asarray(scores)
    return tr_acc,val_acc, ypred, ytrue, scores


# In[289]:


cuda=torch.cuda.is_available()


X_train,Y_train,X_val,Y_val,X_test,Y_test = splitDataWithVal(feat_wise_data,labels_new,test_size=0.2,val_size=0.25)

X_train_new, X_val_new, X_test_new = standardizeDataVal(X_train, X_test, X_val, mode = 'Normalizer') 


# # Check for Class Imbalance

# In[292]:


Ytrain_df= pd.DataFrame(Y_train,columns=[0])
print(Ytrain_df.shape)
print(Ytrain_df.columns)
print(Ytrain_df.groupby(0).size())




train_loss = []
validation_loss = []
epochs_list = []
train_accuracies = []
val_accuracies = []
learning_rates = []
epoch_interval = 1#10
plot_epoch_interval = 5



lr = 0.01
momentum = 0.9
epochs = 3
batch_size = 4
print('batch_size : ',batch_size)
tr_acc,val_acc, ypred, ytrue, scores = train_on_batch(lr,epochs,momentum,X_train_new,Y_train,X_val_new,Y_val,batch_size)

我对其进行了 3 个 epoch 的测试，并在每个 epoch 之后保存了模型。然而，在第 3 个时期之后，即完成 3 个时期的训练后，当我通过调用我的代码的 test() 函数来测试我的模型时，它给出了 49.7% 的验证准确度和 59.3% 的测试准确度。

而如果我在代码中使用 validate() 函数，则在训练循环中的第 3 个训练纪元之后调用时，它会提供 51.146% 的验证准确率。在完成 3 个时期的训练后使用 validate() 函数，即。在 for 循环外，我得到 49.12% 的验证准确率和 54.0697% 的测试准确率。

为什么两次调用相同的验证函数时验证准确性会发生变化，即一次在训练时期循环内，另一次在训练时期循环之后？另外，哪个函数是测试和验证的正确方法，validate() 还是 test()？

我什至加载了我在每个时期后保存的所有模型，并检查了它们的权重，这与它们在训练期间看到的相同。我是这个领域的新手。

Answer 1

1. 是not unusual每次运行代码时都会获得不同的准确度，因为参数是在训练开始时随机初始化的。算法 AI 算法是 stochastic是自然的，即模型是 naturally dependent关于随机性。
2. 即使我是一名新手，也曾遇到过这样的疑惑，甚至对Validation 和Test 数据集感到困惑。要测试训练模型的准确性，请使用 test() 函数。 Use the validation()在最终模型之间进行比较或选择时，可以对最终调整模型的技能进行无偏估计。

==========EDIT-1==========

1. 因此，当您在训练循环中调用 validate() 函数时，它会返回为仅 third(即最后一个纪元)计算的准确度，当您在训练循环后调用相同的 validate() 函数时，准确性是使用它在所有 时期看到的数据计算的.尝试打印您的正确变量，这样您就会注意到准确性背后的原因! :)

希望我的解释清楚并注意验证不学习数据集但only sees (即微调)它。对于问题的第二部分，请参阅我的第 2 点和第 2 点中的链接。