c# - 神经网络 : why does my function return different outputs to the in-built one?

Question

我正在使用 NeuronDotNet用于 C# 中的神经网络。为了测试网络(以及训练网络)，我编写了自己的函数来获取误差平方和。然而，当我通过在训练数据上运行它来测试这个函数并将它与反向传播网络的 MeanSquaredError 进行比较时，结果是不同的。

我发现出现不同错误的原因是当我在学习阶段运行时网络返回不同的输出。我使用以下方法为每个 TrainingSample 运行它:

double[] output = xorNetwork.Run(sample.InputVector);

在学习阶段使用:

xorNetwork.Learn(trainingSet, cycles);

...使用委托(delegate)来捕获结束示例事件:

xorNetwork.EndSampleEvent +=
    delegate(object network, TrainingSampleEventArgs args)
    {
        double[] test = xorNetwork.OutputLayer.GetOutput();
        debug.addSampleOutput(test);
    };

为了简单起见，我尝试使用 XOR 问题来执行此操作，但输出仍然不同。例如，在第一个纪元结束时，EndSampleEvent 委托(delegate)的输出与我的函数的输出是:

• 输入:01，预期:1，my_function:0.703332，EndSampleEvent 0.734385
• 输入:00，预期:0，my_function:0.632568，EndSampleEvent 0.649198
• 输入:10，预期:1，my_function:0.650141，EndSampleEvent 0.710484
• 输入:11，预期:0，my_function:0.715175，EndSampleEvent 0.647102
• 错误:my_function:0.280508，EndSampleEvent 0.291236

它不像在 epoch 的不同阶段捕获那么简单，输出与下一个/上一个 epoch 的输出不相同。

我试过调试，但我不是 Visual Studio 的专家，我对此有点吃力。我的项目引用了 NeuronDotNet DLL。当我在我的代码中放置断点时，它不会从 DLL 进入代码。我在别处寻找过这方面的建议，并尝试了几种解决方案，但一无所获。

我不认为这是由于“观察者效应”，即我函数中的 Run 方法导致网络发生变化。我检查了代码(在生成 DLL 的项目中)，我认为 Run 不会改变任何权重。我的函数的错误往往比 EndSampleEvent 的错误低一个因子，该因子超过了典型纪元的错误减少，即它好像网络在我的代码期间暂时领先于自身(就训练而言)。

神经网络在训练期间调整其功能的意义上是随机的。但是，输出应该是确定性的。为什么两组输出不同？

编辑:这是我正在使用的代码。

/***********************************************************************************************
COPYRIGHT 2008 Vijeth D

This file is part of NeuronDotNet XOR Sample.
(Project Website : http://neurondotnet.freehostia.com)

NeuronDotNet is a free software. You can redistribute it and/or modify it under the terms of
the GNU General Public License as published by the Free Software Foundation, either version 3
of the License, or (at your option) any later version.

NeuronDotNet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License along with NeuronDotNet.
If not, see <http://www.gnu.org/licenses/>.

***********************************************************************************************/

using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Text;
using System.Windows.Forms;
using NeuronDotNet.Core;
using NeuronDotNet.Core.Backpropagation;
using ZedGraph;

namespace NeuronDotNet.Samples.XorSample
{
    public partial class MainForm : Form
    {
        private BackpropagationNetwork xorNetwork;
        private double[] errorList;
        private int cycles = 5000;
        private int neuronCount = 3;
        private double learningRate = 0.25d;

        public MainForm()
        {
            InitializeComponent();
        }

        private void Train(object sender, EventArgs e)
        {
            EnableControls(false);
            if (!int.TryParse(txtCycles.Text.Trim(), out cycles)) { cycles = 5000; }
            if (!double.TryParse(txtLearningRate.Text.Trim(), out learningRate)) { learningRate = 0.25d; }
            if (!int.TryParse(txtNeuronCount.Text.Trim(), out neuronCount)) { neuronCount = 3; }

            if (cycles < 1) { cycles = 1; }
            if (learningRate < 0.01) { learningRate = 0.01; }
            if (neuronCount < 1) { neuronCount = 1; }

            txtNeuronCount.Text = neuronCount.ToString();
            txtCycles.Text = cycles.ToString();
            txtLearningRate.Text = learningRate.ToString();

            errorList = new double[cycles];
            InitGraph();

            LinearLayer inputLayer = new LinearLayer(2);
            SigmoidLayer hiddenLayer = new SigmoidLayer(neuronCount);
            SigmoidLayer outputLayer = new SigmoidLayer(1);
            new BackpropagationConnector(inputLayer, hiddenLayer);
            new BackpropagationConnector(hiddenLayer, outputLayer);
            xorNetwork = new BackpropagationNetwork(inputLayer, outputLayer);
            xorNetwork.SetLearningRate(learningRate);

            TrainingSet trainingSet = new TrainingSet(2, 1);
            trainingSet.Add(new TrainingSample(new double[2] { 0d, 0d }, new double[1] { 0d }));
            trainingSet.Add(new TrainingSample(new double[2] { 0d, 1d }, new double[1] { 1d }));
            trainingSet.Add(new TrainingSample(new double[2] { 1d, 0d }, new double[1] { 1d }));
            trainingSet.Add(new TrainingSample(new double[2] { 1d, 1d }, new double[1] { 0d }));
           Console.WriteLine("mse_begin,mse_end,output,outputs,myerror");
            double max = 0d;
         Console.WriteLine(NNDebug.Header);
           List < NNDebug > debugList = new List<NNDebug>();
           NNDebug debug = null;
         xorNetwork.BeginEpochEvent +=
              delegate(object network, TrainingEpochEventArgs args)
                 {
                  debug = new NNDebug(trainingSet);
                 };

           xorNetwork.EndSampleEvent +=
            delegate(object network, TrainingSampleEventArgs args)
                 {                                                  
                  double[] test = xorNetwork.OutputLayer.GetOutput();

                  debug.addSampleOutput(args.TrainingSample, test);
                 };

         xorNetwork.EndEpochEvent +=
            delegate(object network, TrainingEpochEventArgs args)
            {    
               errorList[args.TrainingIteration] = xorNetwork.MeanSquaredError;
               debug.setMSE(xorNetwork.MeanSquaredError);
               double[] test = xorNetwork.OutputLayer.GetOutput();
               GetError(trainingSet, debug);
               max = Math.Max(max, xorNetwork.MeanSquaredError);
               progressBar.Value = (int)(args.TrainingIteration * 100d / cycles);
               //Console.WriteLine(debug);
               debugList.Add(debug);
            };

            xorNetwork.Learn(trainingSet, cycles);
            double[] indices = new double[cycles];
            for (int i = 0; i < cycles; i++) { indices[i] = i; }

            lblTrainErrorVal.Text = xorNetwork.MeanSquaredError.ToString("0.000000");

            LineItem errorCurve = new LineItem("Error Dynamics", indices, errorList, Color.Tomato, SymbolType.None, 1.5f);
            errorGraph.GraphPane.YAxis.Scale.Max = max;
            errorGraph.GraphPane.CurveList.Add(errorCurve);
            errorGraph.Invalidate();
         writeOut(debugList);
            EnableControls(true);
        }

       private const String pathFileName = "C:\\Temp\\NDN_Debug_Output.txt";

      private void writeOut(IEnumerable<NNDebug> data)
      {
         using (StreamWriter streamWriter = new StreamWriter(pathFileName))
         {
            streamWriter.WriteLine(NNDebug.Header);

            //write results to a file for each load combination
            foreach (NNDebug debug in data)
            {
               streamWriter.WriteLine(debug);
            }
         } 
      }

      private void GetError(TrainingSet trainingSet, NNDebug debug)
      {
         double total = 0;
         foreach (TrainingSample sample in trainingSet.TrainingSamples)
         {
            double[] output = xorNetwork.Run(sample.InputVector);

            double[] expected = sample.OutputVector;
            debug.addOutput(sample, output);
            int len = output.Length;
            for (int i = 0; i < len; i++)
            {
               double error = output[i] - expected[i];
               total += (error * error);
            }
         }
         total = total / trainingSet.TrainingSampleCount;
         debug.setMyError(total);
      }

      private class NNDebug
      {
         public const String Header = "output(00->0),output(01->1),output(10->1),output(11->0),mse,my_output(00->0),my_output(01->1),my_output(10->1),my_output(11->0),my_error";

         public double MyErrorAtEndOfEpoch;
         public double MeanSquaredError;
         public double[][] OutputAtEndOfEpoch;
         public double[][] SampleOutput;
         private readonly List<TrainingSample> samples;

         public NNDebug(TrainingSet trainingSet)
         {
            samples =new List<TrainingSample>(trainingSet.TrainingSamples);
            SampleOutput = new double[samples.Count][];
            OutputAtEndOfEpoch = new double[samples.Count][];
         } 

         public void addSampleOutput(TrainingSample mySample, double[] output)
         {
            int index = samples.IndexOf(mySample);
            SampleOutput[index] = output;
         }

         public void addOutput(TrainingSample mySample, double[] output)
         {
            int index = samples.IndexOf(mySample);
            OutputAtEndOfEpoch[index] = output;
         }

         public void setMyError(double error)
         {
            MyErrorAtEndOfEpoch = error;
         }

         public void setMSE(double mse)
         {
            this.MeanSquaredError = mse;
         }

         public override string ToString()
         {
            StringBuilder sb = new StringBuilder();
            foreach (double[] arr in SampleOutput)
            {
               writeOut(arr, sb);
               sb.Append(',');
            }
            sb.Append(Math.Round(MeanSquaredError,6));
            sb.Append(',');
            foreach (double[] arr in OutputAtEndOfEpoch)
            {
               writeOut(arr, sb);
               sb.Append(',');
            }
            sb.Append(Math.Round(MyErrorAtEndOfEpoch,6));
            return sb.ToString();
         }
      }

      private static void writeOut(double[] arr, StringBuilder sb)
      {
         bool first = true;
         foreach (double d in arr)
         {
            if (first)
            {
               first = false;
            }
            else
            {
               sb.Append(',');
            }
            sb.Append(Math.Round(d, 6));
         }  
      }   

        private void EnableControls(bool enabled)
        {
            btnTrain.Enabled = enabled;
            txtCycles.Enabled = enabled;
            txtNeuronCount.Enabled = enabled;
            txtLearningRate.Enabled = enabled;
            progressBar.Value = 0;
            btnTest.Enabled = enabled;
            txtTestInput.Enabled = enabled;
        }

        private void LoadForm(object sender, EventArgs e)
        {
            InitGraph();
            txtCycles.Text = cycles.ToString();
            txtLearningRate.Text = learningRate.ToString();
            txtNeuronCount.Text = neuronCount.ToString();
        }

        private void InitGraph()
        {
            GraphPane pane = errorGraph.GraphPane;
            pane.Chart.Fill = new Fill(Color.AntiqueWhite, Color.Honeydew, -45F);
            pane.Title.Text = "Back Propagation Training - Error Graph";
            pane.XAxis.Title.Text = "Training Iteration";
            pane.YAxis.Title.Text = "Sum Squared Error";
            pane.XAxis.MajorGrid.IsVisible = true;
            pane.YAxis.MajorGrid.IsVisible = true;
            pane.YAxis.MajorGrid.Color = Color.LightGray;
            pane.XAxis.MajorGrid.Color = Color.LightGray;
            pane.XAxis.Scale.Max = cycles;
            pane.XAxis.Scale.Min = 0;
            pane.YAxis.Scale.Min = 0;
            pane.CurveList.Clear();
            pane.Legend.IsVisible = false;
            pane.AxisChange();
            errorGraph.Invalidate();
        }

        private void Test(object sender, EventArgs e)
        {
            if (xorNetwork != null)
            {
                lblTestOutput.Text = xorNetwork.Run(
                new double[] {double.Parse(txtTestInput.Text.Substring(2,4)),
                    double.Parse(txtTestInput.Text.Substring(8,4))})[0].ToString("0.000000");
            }
        }
    }
}

这与归一化无关，因为两组输出之间的映射不是单调的。例如，{0,1} 中的输出在 EndSampleEvent 中较高，但在 {1,1} 中较低。归一化将是一个简单的线性函数。

这也与抖动无关，因为我已经尝试将其关闭，但结果仍然不同。

Answer 1

我已经收到教授的答复。问题在于 BackpropagationNetwork 类中的 LearnSample 方法，每次迭代都会为每个训练样本调用该方法。

此方法中相关事件的顺序是……。1) 添加到仅使用输出层和所需输出计算的 MeanSquaredError2)将错误反向传播到所有较早的层；这对网络没有影响。3)最后重新计算每一层的偏差；这会影响网络。

(3) 是 LearnSample 方法中发生的最后一件事，发生在计算每个训练实例的输出误差之后。对于 XOR 示例，这意味着网络从进行 MSE 计算时的状态改变了 4 次。

理论上，如果您想比较训练和测试错误，那么您应该进行手动计算(例如我的 GetError 函数)并运行两次:每个数据集一次。然而，在现实中可能没有必要去解决所有这些麻烦，因为值(value)观并没有那么不同。