shin/NeuralNetworkLib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

learning: naming in bp changed and qp modified

Browse Source
This commit is contained in:
Shin
2016-05-07 20:41:52 +02:00
parent c03a13c0f8
commit 36ce3f6463
4 changed files with 82 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
include/NeuralNetwork/Learning/BackPropagation.h
View File

@@ -16,7 +16,7 @@ namespace Learning {
public:
inline BackPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()):
network(feedForwardNetwork), correctionFunction(correction),learningCoefficient(0.4), deltas() {
network(feedForwardNetwork), correctionFunction(correction),learningCoefficient(0.4), slopes() {
resize();
}
@@ -34,24 +34,26 @@ namespace Learning {
protected:
virtual inline void resize() {
if(deltas.size()!=network.size())
deltas.resize(network.size());
if(slopes.size()!=network.size())
slopes.resize(network.size());
for(std::size_t i=0; i < network.size(); i++) {
if(deltas[i].size()!=network[i].size())
deltas[i].resize(network[i].size());
if(slopes[i].size()!=network[i].size())
slopes[i].resize(network[i].size());
}
}
virtual void updateWeights(const std::vector<float> &input);
virtual void computeDeltas(const std::vector<float> &expectation);
FeedForward::Network &network;
CorrectionFunction::CorrectionFunction *correctionFunction;
float learningCoefficient;
std::vector<std::vector<float>> deltas;
std::vector<std::vector<float>> slopes;
};
}
}

41
include/NeuralNetwork/Learning/QuickPropagation.h
View File

@@ -16,7 +16,7 @@ namespace NeuralNetwork {
public:
inline QuickPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()):
BackPropagation(feedForwardNetwork,correction),deltasPrev() {
BackPropagation(feedForwardNetwork,correction),previousSlopes() {
resize();
}
@@ -24,32 +24,49 @@ namespace NeuralNetwork {
}
protected:
float _maxChange=1.75;
float _epsilon=0.5;
virtual inline void resize() override {
if(deltas.size()!=network.size())
deltas.resize(network.size());
if(slopes.size()!=network.size())
slopes.resize(network.size());
for(std::size_t i=0; i < network.size(); i++) {
if(deltas[i].size()!=network[i].size())
deltas[i].resize(network[i].size());
if(slopes[i].size()!=network[i].size())
slopes[i].resize(network[i].size());
}
if(deltasPrev.size()!=network.size())
deltasPrev.resize(network.size());
if(previousSlopes.size()!=network.size())
previousSlopes.resize(network.size());
for(std::size_t i=0; i < network.size(); i++) {
if(deltasPrev[i].size()!=network[i].size())
deltasPrev[i].resize(network[i].size());
if(previousSlopes[i].size()!=network[i].size())
previousSlopes[i].resize(network[i].size());
for(std::size_t j=0; j < deltasPrev[i].size(); j++) {
deltasPrev[i][j]=1.0;
for(std::size_t j=0; j < previousSlopes[i].size(); j++) {
previousSlopes[i][j]=1.0;
}
}
if(lastWeightChange.size()!=network.size())
lastWeightChange.resize(network.size());
for(std::size_t i=0; i < network.size(); i++) {
if(lastWeightChange[i].size()!=network[i].size())
lastWeightChange[i].resize(network[i].size());
for(std::size_t j=0; j < previousSlopes[i].size(); j++) {
lastWeightChange[i][j]=1.0;
}
}
weightChange= lastWeightChange;
}
virtual void updateWeights(const std::vector<float> &input) override;
std::vector<std::vector<float>> deltasPrev;
std::vector<std::vector<float>> previousSlopes ={};
std::vector<std::vector<float>> lastWeightChange ={};
std::vector<std::vector<float>> weightChange ={};
};
}
}

55
src/NeuralNetwork/Learning/BackPropagation.cpp
View File

@@ -9,30 +9,12 @@ void NeuralNetwork::Learning::BackPropagation::teach(const std::vector<float> &i
resize();
auto& outputLayer=network[network.size()-1];
for(std::size_t j=1;j<outputLayer.size();j++) {
auto& neuron = outputLayer[j];
deltas[network.size()-1][j]=correctionFunction->operator()( expectation[j-1], neuron.output())*
neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output());
}
for(int layerIndex=static_cast<int>(network.size()-2);layerIndex>0;layerIndex--) {
auto &layer=network[layerIndex];
for(std::size_t j=1;j<layer.size();j++) {
float deltasWeight = 0;
for(std::size_t k=1;k<network[layerIndex+1].size();k++) {
deltasWeight+=deltas[layerIndex+1][k]* network[layerIndex+1][k].weight(j);
}
deltas[layerIndex][j]=deltasWeight*layer[j].getActivationFunction().derivatedOutput(layer[j].value(),layer[j].output());
}
}
computeDeltas(expectation);
updateWeights(input);
}
void NeuralNetwork::Learning::BackPropagation::updateWeights(const std::vector<float> &input) {
for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
@@ -44,18 +26,43 @@ void NeuralNetwork::Learning::BackPropagation::updateWeights(const std::vector<f
for(std::size_t j=1;j<layerSize;j++) {
deltas[layerIndex][j]*=learningCoefficient;
float delta =slopes[layerIndex][j]*learningCoefficient;
layer[j].weight(0)+=deltas[layerIndex][j];
layer[j].weight(0)+=delta;
for(std::size_t k=1;k<prevLayerSize;k++) {
if(layerIndex==1) {
layer[j].weight(k)+=deltas[layerIndex][j]*input[k-1];
layer[j].weight(k)+=delta*input[k-1];
} else {
layer[j].weight(k)+=deltas[layerIndex][j]*prevLayer[k].output();
layer[j].weight(k)+=delta*prevLayer[k].output();
}
}
}
}
}
void NeuralNetwork::Learning::BackPropagation::computeDeltas(const std::vector<float> &expectation) {
auto& outputLayer=network[network.size()-1];
for(std::size_t j=1;j<outputLayer.size();j++) {
auto& neuron = outputLayer[j];
slopes[network.size()-1][j]=correctionFunction->operator()( expectation[j-1], neuron.output())*
neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output());
}
for(int layerIndex=static_cast<int>(network.size()-2);layerIndex>0;layerIndex--) {
auto &layer=network[layerIndex];
for(std::size_t j=1;j<layer.size();j++) {
float deltasWeight = 0;
for(std::size_t k=1;k<network[layerIndex+1].size();k++) {
deltasWeight+=slopes[layerIndex+1][k]* network[layerIndex+1][k].weight(j);
}
slopes[layerIndex][j]=deltasWeight*layer[j].getActivationFunction().derivatedOutput(layer[j].value(),layer[j].output());
}
}
}

21
src/NeuralNetwork/Learning/QuickPropagation.cpp
View File

@@ -14,22 +14,29 @@ void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector<
for(std::size_t j=1;j<layerSize;j++) {
//TODO: is this correct??
float delta=deltas[layerIndex][j]/(deltasPrev[layerIndex][j]-deltas[layerIndex][j]);
float newChange=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * lastWeightChange[layerIndex][j];
deltas[layerIndex][j]=delta;
// according to original paper
newChange+= slopes[layerIndex][j]*_epsilon;
layer[j].weight(0)+=delta;
if(newChange > lastWeightChange[layerIndex][j]*_maxChange) {
newChange=lastWeightChange[layerIndex][j];
}
weightChange[layerIndex][j]=newChange;
layer[j].weight(0)+=newChange;
for(std::size_t k=1;k<prevLayerSize;k++) {
if(layerIndex==1) {
layer[j].weight(k)+=delta*input[k-1];
layer[j].weight(k)+=newChange*(input[k-1]);
} else {
layer[j].weight(k)+=delta*prevLayer[k].output();
layer[j].weight(k)+=newChange*(prevLayer[k].output());
}
}
}
}
deltas.swap(deltasPrev);
slopes.swap(previousSlopes);
weightChange.swap(lastWeightChange);
}