AForge.NETで遊んでいる。Back Propagation Learning事態は20年以上前からあるのだが、当時は少ないメモリをやり繰りしながら一から実装するほかなかった。お手軽にライブラリでできるって素晴らしいよね。ただ残念なことにAForge.NETのBack Propagation LearningはMulti Threadに対応しないので、Parallel.Forを使って修正してみた。
マルチスレッドに対応させるための書き換えも.Net Framework 4.xではとてもお手軽にできる。今時マルチスレッドに動作しないのってすごく罪だよね。
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using AForge.Neuro;
using AForge.Neuro.Learning;
namespace RemoveRule
{
class MultiThreadBackPropagationLearning : BackPropagationLearning
{
// network to teach
private ActivationNetwork network;
// learning rate
private double learningRate = 0.1;
// momentum
private double momentum = 0.0;
// neuron's errors
private double[][] neuronErrors = null;
// weight's updates
private double[][][] weightsUpdates = null;
// threshold's updates
private double[][] thresholdsUpdates = null;
public new double LearningRate
{
get { return learningRate; }
set
{
learningRate = Math.Max(0.0, Math.Min(1.0, value));
}
}
public new double Momentum
{
get { return momentum; }
set
{
momentum = Math.Max(0.0, Math.Min(1.0, value));
}
}
public MultiThreadBackPropagationLearning(ActivationNetwork network) : base(network)
{
this.network = network;
// create error and deltas arrays
neuronErrors = new double[network.Layers.Length][];
weightsUpdates = new double[network.Layers.Length][][];
thresholdsUpdates = new double[network.Layers.Length][];
// initialize errors and deltas arrays for each layer
for (int i = 0; i < network.Layers.Length; i++)
{
Layer layer = network.Layers[i];
neuronErrors[i] = new double[layer.Neurons.Length];
weightsUpdates[i] = new double[layer.Neurons.Length][];
thresholdsUpdates[i] = new double[layer.Neurons.Length];
// for each neuron
for (int j = 0; j < weightsUpdates[i].Length; j++)
{
weightsUpdates[i][j] = new double[layer.InputsCount];
}
}
}
public new double Run(double[] input, double[] output)
{
// compute the network's output
network.Compute(input);
// calculate network error
double error = CalculateError(output);
// calculate weights updates
CalculateUpdates(input);
// update the network
UpdateNetwork();
return error;
}
public new double RunEpoch(double[][] input, double[][] output)
{
double error = 0.0;
// run learning procedure for all samples
for (int i = 0; i < input.Length; i++)
{
error += Run(input[i], output[i]);
}
// return summary error
return error;
}
private double CalculateError(double[] desiredOutput)
{
// current and the next layers
Layer layer, layerNext;
// current and the next errors arrays
double[] errors, errorsNext;
// error values
double error = 0;
// neuron's output value
double output;
// layers count
int layersCount = network.Layers.Length;
// assume, that all neurons of the network have the same activation function
IActivationFunction function = (network.Layers[0].Neurons[0] as ActivationNeuron).ActivationFunction;
// calculate error values for the last layer first
layer = network.Layers[layersCount - 1];
errors = neuronErrors[layersCount - 1];
Parallel.For(0, layer.Neurons.Length, i =>
{
output = layer.Neurons[i].Output;
// error of the neuron
double e = desiredOutput[i] - output;
// error multiplied with activation function's derivative
double derivative;
lock (function)
{
derivative = function.Derivative2(output);
}
errors[i] = e * derivative;
// squre the error and sum it
Monitor.Enter(this);
error += (e * e);
Monitor.Exit(this);
});
// calculate error values for other layers
for (int j = layersCount - 2; j >= 0; j--)
{
layer = network.Layers[j];
layerNext = network.Layers[j + 1];
errors = neuronErrors[j];
errorsNext = neuronErrors[j + 1];
// for all neurons of the layer
Parallel.For(0, layer.Neurons.Length, i =>
{
double sum = 0.0;
// for all neurons of the next layer
for (int k = 0; k < layerNext.Neurons.Length; k++)
{
sum += errorsNext[k] * layerNext.Neurons[k].Weights[i];
}
double derivative;
lock (function)
{
derivative = function.Derivative2(layer.Neurons[i].Output);
}
errors[i] = sum * derivative;
});
}
// return squared error of the last layer divided by 2
return error / 2.0;
}
private void CalculateUpdates(double[] input)
{
// current neuron
Neuron neuron;
// current and previous layers
Layer layer, layerPrev;
// layer's weights updates
double[][] layerWeightsUpdates;
// layer's thresholds updates
double[] layerThresholdUpdates;
// layer's error
double[] errors;
// error value
// double error;
// 1 - calculate updates for the first layer
layer = network.Layers[0];
errors = neuronErrors[0];
layerWeightsUpdates = weightsUpdates[0];
layerThresholdUpdates = thresholdsUpdates[0];
// cache for frequently used values
double cachedMomentum = learningRate * momentum;
double cached1mMomentum = learningRate * (1 - momentum);
// for each neuron of the layer
Parallel.For(0, layer.Neurons.Length, i =>
{
neuron = layer.Neurons[i];
double cachedError = errors[i] * cached1mMomentum;
double[] neuronWeightUpdates = layerWeightsUpdates[i];
// for each weight of the neuron
for (int j = 0; j < neuronWeightUpdates.Length; j++)
{
// calculate weight update
neuronWeightUpdates[j] = cachedMomentum * neuronWeightUpdates[j] + cachedError * input[j];
}
// calculate treshold update
layerThresholdUpdates[i] = cachedMomentum * layerThresholdUpdates[i] + cachedError;
});
// 2 - for all other layers
for (int k = 1; k < network.Layers.Length; k++)
{
layerPrev = network.Layers[k - 1];
layer = network.Layers[k];
errors = neuronErrors[k];
layerWeightsUpdates = weightsUpdates[k];
layerThresholdUpdates = thresholdsUpdates[k];
// for each neuron of the layer
Parallel.For(0, layer.Neurons.Length, i =>
{
neuron = layer.Neurons[i];
double cachedError = errors[i] * cached1mMomentum;
double[] neuronWeightUpdates = layerWeightsUpdates[i];
// for each synapse of the neuron
for (int j = 0; j < neuronWeightUpdates.Length; j++)
{
// calculate weight update
neuronWeightUpdates[j] = cachedMomentum * neuronWeightUpdates[j] + cachedError * layerPrev.Neurons[j].Output;
}
// calculate treshold update
layerThresholdUpdates[i] = cachedMomentum * layerThresholdUpdates[i] + cachedError;
});
}
}
private void UpdateNetwork()
{
// current layer
Layer layer;
// layer's weights updates
double[][] layerWeightsUpdates;
// layer's thresholds updates
double[] layerThresholdUpdates;
// for each layer of the network
for (int i = 0; i < network.Layers.Length; i++)
{
layer = network.Layers[i];
layerWeightsUpdates = weightsUpdates[i];
layerThresholdUpdates = thresholdsUpdates[i];
// for each neuron of the layer
Parallel.For(0, layer.Neurons.Length, j =>
{
ActivationNeuron neuron = layer.Neurons[j] as ActivationNeuron;
double[] neuronWeightUpdates = layerWeightsUpdates[j];
// for each weight of the neuron
for (int k = 0; k < neuron.Weights.Length; k++)
{
// update weight
neuron.Weights[k] += neuronWeightUpdates[k];
}
// update treshold
neuron.Threshold += layerThresholdUpdates[j];
});
}
}
}
}
元々のBackPropagationLearningクラスでメンバ変数がprivate宣言されていたのがかなり残念。派生したクラスで挙動をオーバーライト出来ないので、丸ごと同じクラスを実装する他に無いのだ。こうなると、そもそもBackPropagationLearningを継承する必要すらなかったかもしれない。
Download MultiThreadBackPropagationLearning.zip