shin/NeuralNetworkLib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactored propagation

Browse Source
This commit is contained in:
Shin
2016-10-31 15:03:27 +01:00
parent 8749b3eb03
commit 77b38dec65
19 changed files with 285 additions and 548 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@@ -61,8 +61,9 @@ endif(USE_SSE)
set (LIBRARY_SOURCES
src/sse_mathfun.cpp
src/NeuralNetwork/Learning/BatchPropagation.cpp
src/NeuralNetwork/Learning/BackPropagation.cpp
src/NeuralNetwork/Learning/QuickPropagation.cpp
#src/NeuralNetwork/Learning/QuickPropagation.cpp
src/NeuralNetwork/Learning/PerceptronLearning.cpp
src/NeuralNetwork/Learning/RProp.cpp

9
include/NeuralNetwork/FeedForward/Layer.h
View File

@@ -67,6 +67,15 @@ namespace FeedForward {
return *neurons[neuron];
}
/**
* @brief This is a virtual function for selecting neuron
* @param neuron is position in layer
* @returns Specific neuron
*/
const NeuronInterface& operator[](const std::size_t& neuron) const {
return *neurons[neuron];
}
void solve(const std::vector<float> &input, std::vector<float> &output);
/**

83
include/NeuralNetwork/Learning/BackPropagation.h
View File

@@ -1,10 +1,6 @@
#pragma once
#include <vector>
#include <cmath>
#include <NeuralNetwork/FeedForward/Network.h>
#include "CorrectionFunction/Linear.h"
#include "BatchPropagation.h"
namespace NeuralNetwork {
namespace Learning {
@@ -12,23 +8,17 @@ namespace Learning {
/** @class BackPropagation
* @brief
*/
class BackPropagation {
class BackPropagation : public BatchPropagation {
public:
inline BackPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()):
network(feedForwardNetwork), correctionFunction(correction),learningCoefficient(0.4), slopes() {
BackPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr<CorrectionFunction::CorrectionFunction> correction = std::make_shared<CorrectionFunction::Linear>()):
BatchPropagation(feedForwardNetwork,correction), learningCoefficient(0.4) {
resize();
}
virtual ~BackPropagation() {
delete correctionFunction;
}
BackPropagation(const BackPropagation&)=delete;
BackPropagation& operator=(const NeuralNetwork::Learning::BackPropagation&) = delete;
void teach(const std::vector<float> &input, const std::vector<float> &output);
inline virtual void setLearningCoefficient (const float& coefficient) { learningCoefficient=coefficient; }
float getMomentumWeight() const {
@@ -48,75 +38,22 @@ namespace Learning {
weightDecay=wd;
}
std::size_t getBatchSize() const {
return batchSize;
}
void setBatchSize(std::size_t size) {
batchSize = size;
}
protected:
virtual inline void resize() {
if(slopes.size()!=network.size())
slopes.resize(network.size());
for(std::size_t i=0; i < network.size(); i++) {
if(slopes[i].size()!=network[i].size())
slopes[i].resize(network[i].size());
}
if(deltas.size() != network.size())
deltas.resize(network.size());
bool resized = false;
for(std::size_t i = 0; i < network.size(); i++) {
if(deltas[i].size() != network[i].size()) {
deltas[i].resize(network[i].size());
resized = true;
if(i > 0) {
for(std::size_t j = 0; j < deltas[i].size(); j++) {
deltas[i][j].resize(network[i - 1].size());
std::fill(deltas[i][j].begin(),deltas[i][j].end(),0.0);
}
}
virtual inline void resize() override {
BatchPropagation::resize();
if(momentumWeight > 0.0) {
_lastDeltas = _gradients;
}
}
if(momentumWeight > 0.0 && (resized || lastDeltas.size() != deltas.size())) {
lastDeltas = deltas;
}
}
virtual void computeDeltas(const std::vector<float> &input);
void updateWeights();
virtual void computeSlopes(const std::vector<float> &expectation);
virtual void endBatch() {
}
FeedForward::Network &network;
CorrectionFunction::CorrectionFunction *correctionFunction;
virtual void updateWeightsAndEndBatch() override;
float learningCoefficient;
float momentumWeight = 0.0;
float weightDecay = 0.0;
std::size_t batchSize = 1;
std::size_t currentBatchSize = 0;
std::vector<std::vector<float>> slopes;
std::vector<std::vector<std::vector<float>>> deltas = {};
std::vector<std::vector<std::vector<float>>> lastDeltas = {};
std::vector<std::vector<std::vector<float>>> _lastDeltas = {};
};
}

52
include/NeuralNetwork/Learning/BatchPropagation.h Normal file
View File

@@ -0,0 +1,52 @@
#pragma once
#include <NeuralNetwork/FeedForward/Network.h>
#include "CorrectionFunction/Linear.h"
#include <vector>
#include <memory>
namespace NeuralNetwork {
namespace Learning {
class BatchPropagation {
public:
BatchPropagation(FeedForward::Network &ffn, std::shared_ptr<CorrectionFunction::CorrectionFunction> correction) : _network(ffn), _correctionFunction(correction) {
}
virtual ~BatchPropagation() {
}
void teach(const std::vector<float> &input, const std::vector<float> &output);
void finishTeaching();
std::size_t getBatchSize() const {
return _batchSize;
}
void setBatchSize(std::size_t size) {
_batchSize = size;
}
protected:
virtual void updateWeightsAndEndBatch() = 0;
virtual void resize();
FeedForward::Network &_network;
std::shared_ptr<CorrectionFunction::CorrectionFunction> _correctionFunction;
std::size_t _batchSize = 1;
std::size_t _currentBatchSize = 0;
std::vector<std::vector<float>> _slopes = {};
std::vector<std::vector<std::vector<float>>> _gradients = {};
bool init = false;
private:
void computeSlopes(const std::vector<float> &expectation);
void computeDeltas(const std::vector<float> &input);
};
}
}

23
include/NeuralNetwork/Learning/OpticalBackPropagation.h
View File

@@ -1,23 +0,0 @@
#pragma once
#include "./BackPropagation.h"
#include "./CorrectionFunction/Optical.h"
namespace NeuralNetwork {
namespace Learning {
/** @class OpticalBackPropagation
* @brief
*/
class OpticalBackPropagation : public BackPropagation {
public:
OpticalBackPropagation(FeedForward::Network &feedForwardNetwork): BackPropagation(feedForwardNetwork,new CorrectionFunction::Optical()) {
}
virtual ~OpticalBackPropagation() {
}
};
}
}

34
include/NeuralNetwork/Learning/QuickPropagation.h
View File

@@ -15,46 +15,24 @@ namespace NeuralNetwork {
class QuickPropagation : public BackPropagation {
public:
inline QuickPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()):
BackPropagation(feedForwardNetwork,correction),previousSlopes() {
resize();
inline QuickPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr<CorrectionFunction::CorrectionFunction> correction = std::make_shared<CorrectionFunction::Linear>()):
BackPropagation(feedForwardNetwork,correction) {
}
virtual ~QuickPropagation() {
}
protected:
float _maxChange=1.75;
float _epsilon=0.5;
virtual inline void resize() override {
if(slopes.size()!=network.size())
slopes.resize(network.size());
for(std::size_t i=0; i < network.size(); i++) {
if(slopes[i].size()!=network[i].size())
slopes[i].resize(network[i].size());
BackPropagation::resize();
_previousSlopes = _slopes;
}
if(deltas.size()!=network.size())
deltas.resize(network.size());
for(std::size_t i=0; i < network.size(); i++) {
if(deltas[i].size()!=network[i].size())
deltas[i].resize(network[i].size());
for(std::size_t j=0; j < previousSlopes[i].size(); j++) {
deltas[i][j]=1.0;
}
}
weightChange= deltas;
}
virtual void updateWeights(const std::vector<float> &input) override;
std::vector<std::vector<float>> previousSlopes ={};
std::vector<std::vector<float>> deltas ={};
std::vector<std::vector<float>> weightChange ={};
std::vector<std::vector<float>> _previousSlopes ={};
};
}
}

121
include/NeuralNetwork/Learning/RProp.h
View File

@@ -1,10 +1,7 @@
#pragma once
#include <vector>
#include <cmath>
#include <NeuralNetwork/FeedForward/Network.h>
#include "CorrectionFunction/Linear.h"
#include "BatchPropagation.h"
namespace NeuralNetwork {
namespace Learning {
@@ -12,122 +9,48 @@ namespace NeuralNetwork {
/** @class Resilient Propagation
* @brief
*/
class RProp {
class RProp : public BatchPropagation {
public:
RProp(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()):
network(feedForwardNetwork), correctionFunction(correction) {
resize();
}
virtual ~RProp() {
delete correctionFunction;
RProp(FeedForward::Network &feedForwardNetwork, std::shared_ptr<CorrectionFunction::CorrectionFunction> correction = std::make_shared<CorrectionFunction::Linear>()):
BatchPropagation(feedForwardNetwork, correction) {
}
RProp(const RProp&)=delete;
RProp& operator=(const NeuralNetwork::Learning::RProp&) = delete;
void teach(const std::vector<float> &input, const std::vector<float> &output);
std::size_t getBatchSize() const {
return batchSize;
void setInitialWeightChange(float initVal) {
initialWeightChange=initVal;
}
void setLearningCoefficient(float) {
void setBatchSize(std::size_t size) {
batchSize = size;
}
void setInitialWeightChange(float init) {
initialWeightChange=init;
}
protected:
virtual inline void resize() {
if(slopes.size()!=network.size())
slopes.resize(network.size());
virtual inline void resize() override {
BatchPropagation::resize();
for(std::size_t i=0; i < network.size(); i++) {
if(slopes[i].size()!=network[i].size())
slopes[i].resize(network[i].size());
}
_lastGradients =_gradients;
if(gradients.size() != network.size())
gradients.resize(network.size());
bool resized = false;
for(std::size_t i = 0; i < network.size(); i++) {
if(gradients[i].size() != network[i].size()) {
gradients[i].resize(network[i].size());
resized = true;
if(i > 0) {
for(std::size_t j = 0; j < gradients[i].size(); j++) {
gradients[i][j].resize(network[i - 1].size());
std::fill(gradients[i][j].begin(),gradients[i][j].end(),0.0);
}
}
}
}
if(resized) {
lastGradients = gradients;
if(changesOfWeightChanges.size() != network.size())
changesOfWeightChanges.resize(network.size());
for(std::size_t i = 0; i < network.size(); i++) {
if(changesOfWeightChanges[i].size() != network[i].size()) {
changesOfWeightChanges[i].resize(network[i].size());
if(i > 0) {
for(std::size_t j = 0; j < changesOfWeightChanges[i].size(); j++) {
changesOfWeightChanges[i][j].resize(network[i - 1].size());
std::fill(changesOfWeightChanges[i][j].begin(),changesOfWeightChanges[i][j].end(),initialWeightChange);
}
}
}
}
}
if(resized) {
if(lastWeightChanges.size() != network.size())
lastWeightChanges.resize(network.size());
for(std::size_t i = 0; i < network.size(); i++) {
if(lastWeightChanges[i].size() != network[i].size()) {
lastWeightChanges[i].resize(network[i].size());
if(i > 0) {
for(std::size_t j = 0; j < lastWeightChanges[i].size(); j++) {
lastWeightChanges[i][j].resize(network[i - 1].size());
std::fill(lastWeightChanges[i][j].begin(),lastWeightChanges[i][j].end(),0.1);
}
_changesOfWeightChanges = _lastGradients;
for(std::size_t i = 1; i < _network.size(); i++) {
for(std::size_t j = 0; j < _changesOfWeightChanges[i].size(); j++) {
std::fill(_changesOfWeightChanges[i][j].begin(),_changesOfWeightChanges[i][j].end(),initialWeightChange);
}
}
_lastWeightChanges = _lastGradients;
for(std::size_t i = 1; i < _network.size(); i++) {
for(std::size_t j = 0; j < _lastWeightChanges[i].size(); j++) {
std::fill(_lastWeightChanges[i][j].begin(),_lastWeightChanges[i][j].end(),0.1);
}
}
}
virtual void computeSlopes(const std::vector<float> &expectation);
virtual void computeDeltas(const std::vector<float> &input);
void updateWeightsAndEndBatch() override;
void updateWeights();
virtual void endBatch() {
}
FeedForward::Network &network;
CorrectionFunction::CorrectionFunction *correctionFunction;
std::vector<std::vector<float>> slopes;
std::vector<std::vector<std::vector<float>>> gradients = {};
std::vector<std::vector<std::vector<float>>> lastGradients = {};
std::vector<std::vector<std::vector<float>>> lastWeightChanges = {};
std::vector<std::vector<std::vector<float>>> changesOfWeightChanges = {};
std::size_t batchSize = 1;
std::size_t currentBatchSize = 0;
std::vector<std::vector<std::vector<float>>> _lastGradients = {};
std::vector<std::vector<std::vector<float>>> _lastWeightChanges = {};
std::vector<std::vector<std::vector<float>>> _changesOfWeightChanges = {};
float maxChangeOfWeights = 50;
float minChangeOfWeights = 0.0001;

8
include/NeuralNetwork/Neuron.h
View File

@@ -108,7 +108,7 @@ namespace NeuralNetwork
/**
* @brief getter for activation function of neuron
*/
virtual ActivationFunction::ActivationFunction& getActivationFunction() =0;
virtual const ActivationFunction::ActivationFunction& getActivationFunction() const =0;
virtual void setBasisFunction(const BasisFunction::BasisFunction& basisFunction) =0;
@@ -167,7 +167,7 @@ namespace NeuralNetwork
return *basis;
}
virtual ActivationFunction::ActivationFunction& getActivationFunction() override {
virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override {
return *activation;
}
@@ -216,7 +216,7 @@ namespace NeuralNetwork
throw usageException("basis function");
}
virtual ActivationFunction::ActivationFunction& getActivationFunction() override {
virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override {
throw usageException("biasNeuron - activation function");
}
@@ -267,7 +267,7 @@ namespace NeuralNetwork
throw usageException("basis function");
}
virtual ActivationFunction::ActivationFunction& getActivationFunction() override {
virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override {
throw usageException("input neuron - activation function");
}

80
src/NeuralNetwork/Learning/BackPropagation.cpp
View File

@@ -1,89 +1,23 @@
#include <NeuralNetwork/Learning/BackPropagation.h>
#include <cassert>
#include <immintrin.h>
void NeuralNetwork::Learning::BackPropagation::teach(const std::vector<float> &input, const std::vector<float> &expectation) {
network.computeOutput(input);
resize();
computeSlopes(expectation);
computeDeltas(input);
if(++currentBatchSize >= batchSize) {
updateWeights();
endBatch();
currentBatchSize=0;
}
}
void NeuralNetwork::Learning::BackPropagation::computeSlopes(const std::vector<float> &expectation) {
auto& outputLayer=network[network.size()-1];
for(std::size_t j=1;j<outputLayer.size();j++) {
auto& neuron = outputLayer[j];
slopes[network.size()-1][j]=correctionFunction->operator()( expectation[j-1], neuron.output())*
neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output());
}
for(int layerIndex=static_cast<int>(network.size()-2);layerIndex>0;layerIndex--) {
auto &layer=network[layerIndex];
for(std::size_t j=1;j<layer.size();j++) {
float deltasWeight = 0;
for(std::size_t k=1;k<network[layerIndex+1].size();k++) {
deltasWeight+=slopes[layerIndex+1][k]* network[layerIndex+1][k].weight(j);
}
slopes[layerIndex][j]=deltasWeight*layer[j].getActivationFunction().derivatedOutput(layer[j].value(),layer[j].output());
}
}
}
void NeuralNetwork::Learning::BackPropagation::computeDeltas(const std::vector<float> &input) {
for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
auto &layer=network[layerIndex];
auto &prevLayer=network[layerIndex-1];
std::size_t prevLayerSize=prevLayer.size();
std::size_t layerSize=layer.size();
for(std::size_t j=1;j<layerSize;j++) {
float update = slopes[layerIndex][j];
for(std::size_t k=0;k<prevLayerSize;k++) {
float inputValue = 0.0;
if(layerIndex==1 && k!=0) {
inputValue = input[k-1];
} else {
inputValue= prevLayer[k].output();
}
if(currentBatchSize == 0) {
deltas[layerIndex][j][k] = update * inputValue;
} else {
deltas[layerIndex][j][k] += update * inputValue;
}
}
}
}
}
void NeuralNetwork::Learning::BackPropagation::updateWeights() {
void NeuralNetwork::Learning::BackPropagation::updateWeightsAndEndBatch() {
bool enableMoments = momentumWeight > 0.0;
for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
auto &layer = network[layerIndex];
auto &prevLayer = network[layerIndex - 1];
for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) {
auto &layer = _network[layerIndex];
auto &prevLayer = _network[layerIndex - 1];
std::size_t prevLayerSize = prevLayer.size();
std::size_t layerSize = layer.size();
for(std::size_t j = 1; j < layerSize; j++) {
for(std::size_t k = 0; k < prevLayerSize; k++) {
float delta = deltas[layerIndex][j][k]*learningCoefficient - weightDecay * layer[j].weight(k);
float delta = _gradients[layerIndex][j][k]*learningCoefficient - weightDecay * layer[j].weight(k);
if(enableMoments) {
delta += momentumWeight * lastDeltas[layerIndex][j][k];
lastDeltas[layerIndex][j][k]=delta;
delta += momentumWeight * _lastDeltas[layerIndex][j][k];
_lastDeltas[layerIndex][j][k]=delta;
}
layer[j].weight(k)+= delta;

92
src/NeuralNetwork/Learning/BatchPropagation.cpp Normal file
View File

@@ -0,0 +1,92 @@
#include <NeuralNetwork/Learning/BatchPropagation.h>
void NeuralNetwork::Learning::BatchPropagation::teach(const std::vector<float> &input, const std::vector<float> &expectation) {
_network.computeOutput(input);
if(!init) {
resize();
init = true;
}
computeSlopes(expectation);
computeDeltas(input);
if(++_currentBatchSize >= _batchSize) {
finishTeaching();
}
}
void NeuralNetwork::Learning::BatchPropagation::finishTeaching() {
updateWeightsAndEndBatch();
_currentBatchSize=0;
}
void NeuralNetwork::Learning::BatchPropagation::computeSlopes(const std::vector<float> &expectation) {
const auto& outputLayer=_network[_network.size()-1];
for(std::size_t j=1;j<outputLayer.size();j++) {
const auto& neuron = outputLayer[j];
_slopes[_network.size()-1][j]=_correctionFunction->operator()( expectation[j-1], neuron.output())*
neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output());
}
for(int layerIndex=static_cast<int>(_network.size()-2);layerIndex>0;layerIndex--) {
auto &layer=_network[layerIndex];
for(std::size_t j=1;j<layer.size();j++) {
float deltasWeight = 0;
for(std::size_t k=1;k<_network[layerIndex+1].size();k++) {
deltasWeight+=_slopes[layerIndex+1][k]* _network[layerIndex+1][k].weight(j);
}
_slopes[layerIndex][j]=deltasWeight*layer[j].getActivationFunction().derivatedOutput(layer[j].value(),layer[j].output());
}
}
}
void NeuralNetwork::Learning::BatchPropagation::computeDeltas(const std::vector<float> &input) {
for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) {
auto &layer=_network[layerIndex];
auto &prevLayer=_network[layerIndex-1];
std::size_t prevLayerSize=prevLayer.size();
std::size_t layerSize=layer.size();
for(std::size_t j=1;j<layerSize;j++) {
float update = _slopes[layerIndex][j];
for(std::size_t k=0;k<prevLayerSize;k++) {
float inputValue = 0.0;
if(layerIndex==1 && k!=0) {
inputValue = input[k-1];
} else {
inputValue= prevLayer[k].output();
}
if(_currentBatchSize == 0) {
_gradients[layerIndex][j][k] = update * inputValue;
} else {
_gradients[layerIndex][j][k] += update * inputValue;
}
}
}
}
}
void NeuralNetwork::Learning::BatchPropagation::resize() {
_slopes.resize(_network.size());
for(std::size_t i=0; i < _network.size(); i++) {
_slopes[i].resize(_network[i].size());
}
_gradients.resize(_network.size());
for(std::size_t i = 0; i < _network.size(); i++) {
_gradients[i].resize(_network[i].size());
if(i > 0) {
for(std::size_t j = 0; j < _gradients[i].size(); j++) {
_gradients[i][j].resize(_network[i - 1].size());
std::fill(_gradients[i][j].begin(), _gradients[i][j].end(), 0.0);
}
}
}
}

35
src/NeuralNetwork/Learning/QuickPropagation.cpp
View File

@@ -3,7 +3,7 @@
#include <cassert>
#include <immintrin.h>
void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector<float> &input) {
void NeuralNetwork::Learning::QuickPropagation::computeDeltas(const std::vector<float> &input) {
float shrinkFactor=_maxChange/(_maxChange+1.0);
@@ -15,20 +15,39 @@ void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector<
std::size_t layerSize=layer.size();
for(std::size_t j=1;j<layerSize;j++) {
for(std::size_t k=1;k<layerSize;k++) {
double update = 0.0;
double inputValue = 0.0;
if(layerIndex==1) {
inputValue=input[k-1];
} else {
inputValue=prevLayer[k].output();
}
if(currentBatchSize == 0) {
gradients[layerIndex][j][k] = update * inputValue;
} else {
gradients[layerIndex][j][k] += update * inputValue;
}
}
/*
float newChange=0;
if(fabs (deltas[layerIndex][j])> 0.0001) {
if(std::signbit(deltas[layerIndex][j]) == std::signbit(slopes[layerIndex][j])) {
if(fabs (gradients[layerIndex][j])> 0.0001) {
if(std::signbit(gradients[layerIndex][j]) == std::signbit(slopes[layerIndex][j])) {
newChange+= slopes[layerIndex][j]*_epsilon;
if(fabs(slopes[layerIndex][j]) > fabs(shrinkFactor * previousSlopes[layerIndex][j])) {
newChange += _maxChange * deltas[layerIndex][j];
newChange += _maxChange * gradients[layerIndex][j];
}else {
newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * deltas[layerIndex][j];
newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * gradients[layerIndex][j];
}
} else {
newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * deltas[layerIndex][j];
newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * gradients[layerIndex][j];
}
} else {
newChange+= slopes[layerIndex][j]*_epsilon;
@@ -45,9 +64,7 @@ void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector<
layer[j].weight(k)+=newChange*(prevLayer[k].output());
}
}
*/
}
}
slopes.swap(previousSlopes);
weightChange.swap(deltas);
}

84
src/NeuralNetwork/Learning/RProp.cpp
View File

@@ -1,101 +1,37 @@
#include <NeuralNetwork/Learning/RProp.h>
void NeuralNetwork::Learning::RProp::teach(const std::vector<float> &input, const std::vector<float> &expectation) {
network.computeOutput(input);
resize();
computeSlopes(expectation);
void NeuralNetwork::Learning::RProp::updateWeightsAndEndBatch() {
computeDeltas(input);
if(++currentBatchSize >= batchSize) {
updateWeights();
endBatch();
currentBatchSize=0;
}
}
void NeuralNetwork::Learning::RProp::computeSlopes(const std::vector<float> &expectation) {
auto& outputLayer=network[network.size()-1];
for(std::size_t j=1;j<outputLayer.size();j++) {
auto& neuron = outputLayer[j];
slopes[network.size()-1][j]=correctionFunction->operator()( expectation[j-1], neuron.output())*
neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output());
}
for(int layerIndex=static_cast<int>(network.size()-2);layerIndex>0;layerIndex--) {
auto &layer=network[layerIndex];
for(std::size_t j=1;j<layer.size();j++) {
float deltasWeight = 0;
for(std::size_t k=1;k<network[layerIndex+1].size();k++) {
deltasWeight+=slopes[layerIndex+1][k]* network[layerIndex+1][k].weight(j);
}
slopes[layerIndex][j]=deltasWeight*layer[j].getActivationFunction().derivatedOutput(layer[j].value(),layer[j].output());
}
}
}
void NeuralNetwork::Learning::RProp::computeDeltas(const std::vector<float> &input) {
for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
auto &layer=network[layerIndex];
auto &prevLayer=network[layerIndex-1];
std::size_t prevLayerSize=prevLayer.size();
std::size_t layerSize=layer.size();
for(std::size_t j=1;j<layerSize;j++) {
float update = slopes[layerIndex][j];
for(std::size_t k=0;k<prevLayerSize;k++) {
float inputValue = 0.0;
if(layerIndex==1 && k!=0) {
inputValue = input[k-1];
} else {
inputValue= prevLayer[k].output();
}
if(currentBatchSize == 0) {
gradients[layerIndex][j][k] = update * inputValue;
} else {
gradients[layerIndex][j][k] += update * inputValue;
}
}
}
}
}
void NeuralNetwork::Learning::RProp::updateWeights() {
for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
auto &layer = network[layerIndex];
auto &prevLayer = network[layerIndex - 1];
for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) {
auto &layer = _network[layerIndex];
auto &prevLayer = _network[layerIndex - 1];
std::size_t prevLayerSize = prevLayer.size();
std::size_t layerSize = layer.size();
for(std::size_t j = 1; j < layerSize; j++) {
for(std::size_t k = 0; k < prevLayerSize; k++) {
float gradient = gradients[layerIndex][j][k];
float lastGradient = lastGradients[layerIndex][j][k];
float gradient = _gradients[layerIndex][j][k];
float lastGradient = _lastGradients[layerIndex][j][k];
lastGradients[layerIndex][j][k] = gradient;
_lastGradients[layerIndex][j][k] = gradient;
float weightChangeDelta = lastWeightChanges[layerIndex][j][k];
float weightChangeDelta = _lastWeightChanges[layerIndex][j][k];
if(gradient * lastGradient > 0) {
weightChangeDelta = std::min(weightChangeDelta*weightChangePlus,maxChangeOfWeights);
} else if (gradient * lastGradient < 0) {
weightChangeDelta = std::max(weightChangeDelta*weightChangeMinus,minChangeOfWeights);
} else {
weightChangeDelta = lastWeightChanges[layerIndex][j][k];
weightChangeDelta = _lastWeightChanges[layerIndex][j][k];
}
lastWeightChanges[layerIndex][j][k] = weightChangeDelta;
_lastWeightChanges[layerIndex][j][k] = weightChangeDelta;
if(gradient > 0) {
layer[j].weight(k) += weightChangeDelta;
} else if (gradient < 0){
layer[j].weight(k) -= weightChangeDelta;
} else {
}
}
}

3
tests/CMakeLists.txt
View File

@@ -13,9 +13,6 @@ target_link_libraries(backpropagation NeuralNetwork gtest gtest_main)
add_executable(feedforward feedforward.cpp)
target_link_libraries(feedforward NeuralNetwork gtest gtest_main)
add_executable(optical_backpropagation optical_backpropagation.cpp)
target_link_libraries(optical_backpropagation NeuralNetwork gtest gtest_main)
add_executable(perceptron perceptron.cpp)
target_link_libraries(perceptron NeuralNetwork gtest gtest_main)

14
tests/activation.cpp
View File

@@ -48,20 +48,6 @@ TEST(Sigmoid, ParamMinusFive) {
ASSERT_LT(s(0.7), 0.970788);
}
TEST(SigmoidSSE, ParamMinusZeroPointSeven) {
NeuralNetwork::ActivationFunction::Sigmoid s(-0.7);
SSE comp;
comp.floats[0] = 0.1;
comp.floats[1] = 10;
comp.sse = s(comp.sse);
ASSERT_GT(comp.floats[0], 0.517483);
ASSERT_LT(comp.floats[0], 0.51750);
ASSERT_GT(comp.floats[1], 0.998989);
ASSERT_LT(comp.floats[1], 0.999189);
}
TEST(Linear, ParamOne) {
NeuralNetwork::ActivationFunction::Linear s(1.0);
ASSERT_GT(s(0.5), 0.4999);

3
tests/backpropagation.cpp
View File

@@ -47,6 +47,7 @@ TEST(BackProp,XOR) {
}
TEST(BackProp,XORHyperbolicTangent) {
srand(time(NULL));
NeuralNetwork::FeedForward::Network n(2);
NeuralNetwork::ActivationFunction::HyperbolicTangent a(-1);
n.appendLayer(2,a);
@@ -56,7 +57,7 @@ TEST(BackProp,XORHyperbolicTangent) {
NeuralNetwork::Learning::BackPropagation prop(n);
for(int i=0;i<10000;i++) {
for(int i=0;i<1500;i++) {
prop.teach({1,0},{1});
prop.teach({1,1},{0});
prop.teach({0,0},{0});

18
tests/backpropagation_function_cmp.cpp
View File

@@ -45,41 +45,41 @@ int main() {
std::cout << "\tLinear: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef) << "\n";
std::cout << "\tOptical: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef) << "\n";
std::cout << "\tArcTangent: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::ArcTangent>(arcTangent),arcTangentCoef) << "\n";
}
{
std::cout << "AND:\n";
std::cout << "\tLinear: " <<
LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef) << "\n";
std::cout << "\tOptical: " <<
LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef) << "\n";
std::cout << "\tArcTangent: " <<
LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::ArcTangent>(arcTangent),arcTangentCoef) << "\n";
}
{
std::cout << "AND:\n";
std::cout << "\tLinear: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef) << "\n";
std::cout << "\tOptical: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef) << "\n";
std::cout << "\tArcTangent: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::ArcTangent>(arcTangent),arcTangentCoef) << "\n";
}
}

8
tests/backpropagation_perf.cpp
View File

@@ -4,6 +4,7 @@
#include <iostream>
#include "../include/NeuralNetwork/Learning/BackPropagation.h"
#include <chrono>
int main() {
{ // XOR problem
NeuralNetwork::FeedForward::Network n(2);
@@ -16,11 +17,18 @@ int main() {
n.randomizeWeights();
NeuralNetwork::Learning::BackPropagation prop(n);
// prop.setBatchSize(20);
// prop.setMomentumWeight(0.8);
auto start = std::chrono::system_clock::now();
for(int i=0;i<100;i++) {
prop.teach({1,0},{1});
prop.teach({1,1},{0});
prop.teach({0,0},{0});
prop.teach({0,1},{1});
}
auto end = std::chrono::system_clock::now();
std::chrono::duration<double> elapsed_seconds = end -start;
std::cout << elapsed_seconds.count() << "\n";
}
}

124
tests/optical_backpropagation.cpp
View File

@@ -1,124 +0,0 @@
#include <NeuralNetwork/FeedForward/Network.h>
#include <NeuralNetwork/Learning/OpticalBackPropagation.h>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Weffc++"
#include <gtest/gtest.h>
#pragma GCC diagnostic pop
TEST(OpticalBackPropagation,XOR) {
NeuralNetwork::FeedForward::Network n(2);
NeuralNetwork::ActivationFunction::Sigmoid a(-1);
n.appendLayer(2,a);
n.appendLayer(1,a);
n.randomizeWeights();
NeuralNetwork::Learning::OpticalBackPropagation prop(n);
for(int i=0;i<10000;i++) {
prop.teach({1,0},{1});
prop.teach({1,1},{0});
prop.teach({0,0},{0});
prop.teach({0,1},{1});
}
{
std::vector<float> ret =n.computeOutput({1,1});
ASSERT_LT(ret[0], 0.1);
}
{
std::vector<float> ret =n.computeOutput({0,1});
ASSERT_GT(ret[0], 0.9);
}
{
std::vector<float> ret =n.computeOutput({1,0});
ASSERT_GT(ret[0], 0.9);
}
{
std::vector<float> ret =n.computeOutput({0,0});
ASSERT_LT(ret[0], 0.1);
}
}
TEST(OpticalBackPropagation,AND) {
NeuralNetwork::FeedForward::Network n(2);
NeuralNetwork::ActivationFunction::Sigmoid a(-1);
n.appendLayer(2,a);
n.appendLayer(1,a);
n.randomizeWeights();
NeuralNetwork::Learning::OpticalBackPropagation prop(n);
for(int i=0;i<10000;i++) {
prop.teach({1,1},{1});
prop.teach({0,0},{0});
prop.teach({0,1},{0});
prop.teach({1,0},{0});
}
{
std::vector<float> ret =n.computeOutput({1,1});
ASSERT_GT(ret[0], 0.9);
}
{
std::vector<float> ret =n.computeOutput({0,1});
ASSERT_LT(ret[0], 0.1);
}
{
std::vector<float> ret =n.computeOutput({1,0});
ASSERT_LT(ret[0], 0.1);
}
{
std::vector<float> ret =n.computeOutput({0,0});
ASSERT_LT(ret[0], 0.1);
}
}
TEST(OpticalBackPropagation,NOTAND) {
NeuralNetwork::FeedForward::Network n(2);
NeuralNetwork::ActivationFunction::Sigmoid a(-1);
n.appendLayer(2,a);
n.appendLayer(1,a);
n.randomizeWeights();
NeuralNetwork::Learning::OpticalBackPropagation prop(n);
for(int i=0;i<10000;i++) {
prop.teach({1,1},{0});
prop.teach({0,0},{1});
prop.teach({0,1},{1});
prop.teach({1,0},{1});
}
{
std::vector<float> ret =n.computeOutput({1,1});
ASSERT_LT(ret[0], 0.1);
}
{
std::vector<float> ret =n.computeOutput({0,1});
ASSERT_GT(ret[0], 0.9);
}
{
std::vector<float> ret =n.computeOutput({1,0});
ASSERT_GT(ret[0], 0.9);
}
{
std::vector<float> ret =n.computeOutput({0,0});
ASSERT_GT(ret[0], 0.9);
}
}

31
tests/propagation_cmp.cpp
View File

@@ -4,6 +4,7 @@
#include <iostream>
#include "../include/NeuralNetwork/Learning/BackPropagation.h"
#include "../include/NeuralNetwork/Learning/QuickPropagation.h"
#include "../include/NeuralNetwork/Learning/RProp.h"
#include "../include/NeuralNetwork/Learning/CorrectionFunction/Optical.h"
#include "../include/NeuralNetwork/Learning/CorrectionFunction/ArcTangent.h"
@@ -17,6 +18,7 @@
n.randomizeWeights();\
CLASS prop(n,FUN);\
prop.setLearningCoefficient(COEF);\
prop.setBatchSize(4); \
int error=1; int steps = 0; \
while(error > 0 && steps <99999) {\
steps++;\
@@ -42,36 +44,47 @@ int main() {
const float arcTangent=1.5;
{
std::cout << "XOR:\n";
std::cout << "XOR Linear:\n";
std::cout << "\tBP: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
std::cout << "\tQP: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
std::cout << "\tRProp: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef,NeuralNetwork::Learning::RProp) << "\n";
}
{
std::cout << "AND:\n";
std::cout << "AND Optical:\n";
std::cout << "\tBP: " <<
LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
std::cout << "\tQP: " <<
LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
std::cout << "\tRProp: " <<
LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::RProp) << "\n";
}
{
std::cout << "AND:\n";
std::cout << "XOR Optical:\n";
std::cout << "\tBP: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
std::cout << "\tQP: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
std::cout << "\tRProp: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::RProp) << "\n";
}
}