diff --git a/CMakeLists.txt b/CMakeLists.txt index a3b7b1c..3687582 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,9 +61,12 @@ endif(USE_SSE) set (LIBRARY_SOURCES src/sse_mathfun.cpp + src/NeuralNetwork/Learning/BatchPropagation.cpp src/NeuralNetwork/Learning/BackPropagation.cpp src/NeuralNetwork/Learning/QuickPropagation.cpp src/NeuralNetwork/Learning/PerceptronLearning.cpp + src/NeuralNetwork/Learning/RProp.cpp + src/NeuralNetwork/Learning/iRPropPlus.cpp src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp src/NeuralNetwork/ConstructiveAlgorithms/Cascade2.cpp @@ -118,6 +121,9 @@ IF(ENABLE_TESTS) add_test(quickpropagation tests/quickpropagation) set_property(TEST quickpropagation PROPERTY LABELS unit) + add_test(rprop tests/rprop) + set_property(TEST rprop PROPERTY LABELS unit) + add_test(recurrent tests/recurrent) set_property(TEST recurrent PROPERTY LABELS unit) @@ -136,8 +142,5 @@ IF(ENABLE_TESTS) add_test(recurrent_perf tests/recurrent_perf) set_property(TEST recurrent_perf PROPERTY LABELS perf) - add_test(genetic_programing tests/genetic_programing) - set_property(TEST genetic_programing PROPERTY LABELS unit) - ENDIF(ENABLE_TESTS) diff --git a/include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h b/include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h new file mode 100644 index 0000000..3a16fa4 --- /dev/null +++ b/include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h @@ -0,0 +1,39 @@ +#pragma once + +#include "./ActivationFunction.h" + +#include + +namespace NeuralNetwork { +namespace ActivationFunction { + + class LeakyRectifiedLinear: public ActivationFunction { + public: + LeakyRectifiedLinear(const float &lambdaP=0.04): lambda(lambdaP) {} + + inline virtual float derivatedOutput(const float &inp,const float &) const override { + return inp > 0.0f ? lambda : 0.01f*lambda; + } + + inline virtual float operator()(const float &x) const override { + return x > 0.0? x : 0.001f*x; + }; + + virtual ActivationFunction* clone() const override { + return new LeakyRectifiedLinear(lambda); + } + + virtual SimpleJSON::Type::Object serialize() const override { + return {{"class", "NeuralNetwork::ActivationFunction::LeakyRectifiedLinear"}, {"lambda", lambda}}; + } + + static std::unique_ptr deserialize(const SimpleJSON::Type::Object &obj) { + return std::unique_ptr(new LeakyRectifiedLinear(obj["lambda"].as())); + } + + protected: + float lambda; + NEURAL_NETWORK_REGISTER_ACTIVATION_FUNCTION(NeuralNetwork::ActivationFunction::LeakyRectifiedLinear, LeakyRectifiedLinear::deserialize) + }; +} +} \ No newline at end of file diff --git a/include/NeuralNetwork/ActivationFunction/RectifiedLinear.h b/include/NeuralNetwork/ActivationFunction/RectifiedLinear.h new file mode 100644 index 0000000..93b4d72 --- /dev/null +++ b/include/NeuralNetwork/ActivationFunction/RectifiedLinear.h @@ -0,0 +1,39 @@ +#pragma once + +#include "./ActivationFunction.h" + +#include + +namespace NeuralNetwork { +namespace ActivationFunction { + + class RectifiedLinear: public ActivationFunction { + public: + RectifiedLinear(const float &lambdaP=0.1): lambda(lambdaP) {} + + inline virtual float derivatedOutput(const float &inp,const float &) const override { + return inp > 0.0f ? lambda : 0.0f; + } + + inline virtual float operator()(const float &x) const override { + return std::max(0.0f,x); + }; + + virtual ActivationFunction* clone() const override { + return new RectifiedLinear(lambda); + } + + virtual SimpleJSON::Type::Object serialize() const override { + return {{"class", "NeuralNetwork::ActivationFunction::RectifiedLinear"}, {"lambda", lambda}}; + } + + static std::unique_ptr deserialize(const SimpleJSON::Type::Object &obj) { + return std::unique_ptr(new RectifiedLinear(obj["lambda"].as())); + } + + protected: + float lambda; + NEURAL_NETWORK_REGISTER_ACTIVATION_FUNCTION(NeuralNetwork::ActivationFunction::RectifiedLinear, RectifiedLinear::deserialize) + }; +} +} \ No newline at end of file diff --git a/include/NeuralNetwork/FeedForward/Layer.h b/include/NeuralNetwork/FeedForward/Layer.h index 3c07750..43bf5c3 100644 --- a/include/NeuralNetwork/FeedForward/Layer.h +++ b/include/NeuralNetwork/FeedForward/Layer.h @@ -67,6 +67,15 @@ namespace FeedForward { return *neurons[neuron]; } + /** + * @brief This is a virtual function for selecting neuron + * @param neuron is position in layer + * @returns Specific neuron + */ + const NeuronInterface& operator[](const std::size_t& neuron) const { + return *neurons[neuron]; + } + void solve(const std::vector &input, std::vector &output); /** diff --git a/include/NeuralNetwork/Learning/BackPropagation.h b/include/NeuralNetwork/Learning/BackPropagation.h index c880519..53bf993 100644 --- a/include/NeuralNetwork/Learning/BackPropagation.h +++ b/include/NeuralNetwork/Learning/BackPropagation.h @@ -1,10 +1,6 @@ #pragma once -#include -#include - -#include -#include "CorrectionFunction/Linear.h" +#include "BatchPropagation.h" namespace NeuralNetwork { namespace Learning { @@ -12,24 +8,20 @@ namespace Learning { /** @class BackPropagation * @brief */ - class BackPropagation { + class BackPropagation : public BatchPropagation { public: - inline BackPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()): - network(feedForwardNetwork), correctionFunction(correction),learningCoefficient(0.4), slopes() { + BackPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BatchPropagation(feedForwardNetwork,correction), learningCoefficient(0.4) { resize(); } - virtual ~BackPropagation() { - delete correctionFunction; - } - BackPropagation(const BackPropagation&)=delete; BackPropagation& operator=(const NeuralNetwork::Learning::BackPropagation&) = delete; - void teach(const std::vector &input, const std::vector &output); - - inline virtual void setLearningCoefficient (const float& coefficient) { learningCoefficient=coefficient; } + void setLearningCoefficient (const float& coefficient) { + learningCoefficient=coefficient; + } float getMomentumWeight() const { return momentumWeight; @@ -37,6 +29,7 @@ namespace Learning { void setMomentumWeight(const float& m) { momentumWeight=m; + resize(); } float getWeightDecay() const { @@ -49,47 +42,21 @@ namespace Learning { protected: - virtual inline void resize() { - if(slopes.size()!=network.size()) - slopes.resize(network.size()); - - for(std::size_t i=0; i < network.size(); i++) { - if(slopes[i].size()!=network[i].size()) - slopes[i].resize(network[i].size()); + virtual inline void resize() override { + BatchPropagation::resize(); + if(momentumWeight > 0.0) { + _lastDeltas = _gradients; } - - if(lastDeltas.size()!=network.size()) - lastDeltas.resize(network.size()); - - for(std::size_t i=0; i < network.size(); i++) { - if(lastDeltas[i].size()!=network[i].size()) { - lastDeltas[i].resize(network[i].size()); - - for(std::size_t j = 0; j < lastDeltas[i].size(); j++) { - lastDeltas[i][j] = 0.0; - } - } - } - deltas= lastDeltas; } - virtual void updateWeights(const std::vector &input); - - virtual void computeSlopes(const std::vector &expectation); - - FeedForward::Network &network; - - CorrectionFunction::CorrectionFunction *correctionFunction; + virtual void updateWeightsAndEndBatch() override; float learningCoefficient; - float momentumWeight = 0.0; - float weightDecay = 0.0; - std::vector> slopes; - std::vector> deltas; - std::vector> lastDeltas; + std::vector>> _lastDeltas = {}; + }; } } \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/BatchPropagation.h b/include/NeuralNetwork/Learning/BatchPropagation.h new file mode 100644 index 0000000..b26ddf7 --- /dev/null +++ b/include/NeuralNetwork/Learning/BatchPropagation.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include "CorrectionFunction/Linear.h" + +#include +#include + +namespace NeuralNetwork { +namespace Learning { + class BatchPropagation { + public: + BatchPropagation(FeedForward::Network &ffn, std::shared_ptr correction) : _network(ffn), _correctionFunction(correction) { + + } + + virtual ~BatchPropagation() { + + } + + void teach(const std::vector &input, const std::vector &output); + + void finishTeaching(); + + std::size_t getBatchSize() const { + return _batchSize; + } + + void setBatchSize(std::size_t size) { + _batchSize = size; + } + protected: + virtual void updateWeightsAndEndBatch() = 0; + virtual void resize(); + + FeedForward::Network &_network; + std::shared_ptr _correctionFunction; + + std::size_t _batchSize = 1; + std::size_t _currentBatchSize = 0; + + std::vector> _slopes = {}; + std::vector>> _gradients = {}; + + bool init = false; + private: + void computeSlopes(const std::vector &expectation); + void computeDeltas(const std::vector &input); + }; +} +} \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/OpticalBackPropagation.h b/include/NeuralNetwork/Learning/OpticalBackPropagation.h deleted file mode 100644 index cb70c3f..0000000 --- a/include/NeuralNetwork/Learning/OpticalBackPropagation.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include "./BackPropagation.h" -#include "./CorrectionFunction/Optical.h" - -namespace NeuralNetwork { -namespace Learning { - - /** @class OpticalBackPropagation - * @brief - */ - class OpticalBackPropagation : public BackPropagation { - - public: - OpticalBackPropagation(FeedForward::Network &feedForwardNetwork): BackPropagation(feedForwardNetwork,new CorrectionFunction::Optical()) { - - } - - virtual ~OpticalBackPropagation() { - } - }; -} -} \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/QuickPropagation.h b/include/NeuralNetwork/Learning/QuickPropagation.h index 48f4f98..e790ae7 100644 --- a/include/NeuralNetwork/Learning/QuickPropagation.h +++ b/include/NeuralNetwork/Learning/QuickPropagation.h @@ -4,7 +4,7 @@ #include #include -#include "BackPropagation.h" +#include "BatchPropagation.h" namespace NeuralNetwork { namespace Learning { @@ -12,49 +12,33 @@ namespace NeuralNetwork { /** @class QuickPropagation * @brief */ - class QuickPropagation : public BackPropagation { + class QuickPropagation : public BatchPropagation { public: - inline QuickPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()): - BackPropagation(feedForwardNetwork,correction),previousSlopes() { - resize(); + inline QuickPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BatchPropagation(feedForwardNetwork,correction) { } virtual ~QuickPropagation() { } - protected: - float _maxChange=1.75; - float _epsilon=0.5; - - virtual inline void resize() override { - if(slopes.size()!=network.size()) - slopes.resize(network.size()); - - for(std::size_t i=0; i < network.size(); i++) { - if(slopes[i].size()!=network[i].size()) - slopes[i].resize(network[i].size()); - } - - if(deltas.size()!=network.size()) - deltas.resize(network.size()); - - for(std::size_t i=0; i < network.size(); i++) { - if(deltas[i].size()!=network[i].size()) - deltas[i].resize(network[i].size()); - - for(std::size_t j=0; j < previousSlopes[i].size(); j++) { - deltas[i][j]=1.0; - } - } - weightChange= deltas; + void setLearningCoefficient (const float& coefficient) { } - virtual void updateWeights(const std::vector &input) override; + protected: - std::vector> previousSlopes ={}; - std::vector> deltas ={}; - std::vector> weightChange ={}; + virtual void updateWeightsAndEndBatch() override; + + float _maxChange=1.75; + + virtual inline void resize() override { + BatchPropagation::resize(); + _lastGradients = _gradients; + _lastDeltas = _gradients; + } + + std::vector>> _lastDeltas = {}; + std::vector>> _lastGradients = {}; }; } } \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/RProp.h b/include/NeuralNetwork/Learning/RProp.h new file mode 100644 index 0000000..522fd4a --- /dev/null +++ b/include/NeuralNetwork/Learning/RProp.h @@ -0,0 +1,63 @@ +#pragma once + + +#include "BatchPropagation.h" + +namespace NeuralNetwork { + namespace Learning { + + /** @class Resilient Propagation + * @brief + */ + class RProp : public BatchPropagation { + + public: + RProp(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BatchPropagation(feedForwardNetwork, correction) { + } + + RProp(const RProp&)=delete; + RProp& operator=(const NeuralNetwork::Learning::RProp&) = delete; + + void setInitialWeightChange(float initVal) { + initialWeightChange=initVal; + } + void setLearningCoefficient(float) { + + } + protected: + + virtual inline void resize() override { + BatchPropagation::resize(); + + _lastGradients =_gradients; + + _changesOfWeightChanges = _lastGradients; + for(std::size_t i = 1; i < _network.size(); i++) { + for(std::size_t j = 0; j < _changesOfWeightChanges[i].size(); j++) { + std::fill(_changesOfWeightChanges[i][j].begin(),_changesOfWeightChanges[i][j].end(),initialWeightChange); + } + } + _lastWeightChanges = _lastGradients; + for(std::size_t i = 1; i < _network.size(); i++) { + for(std::size_t j = 0; j < _lastWeightChanges[i].size(); j++) { + std::fill(_lastWeightChanges[i][j].begin(),_lastWeightChanges[i][j].end(),0.1); + } + } + } + + void updateWeightsAndEndBatch() override; + + std::vector>> _lastGradients = {}; + std::vector>> _lastWeightChanges = {}; + std::vector>> _changesOfWeightChanges = {}; + + float maxChangeOfWeights = 50; + float minChangeOfWeights = 0.0001; + + float initialWeightChange=0.02; + float weightChangePlus=1.2; + float weightChangeMinus=0.5; + }; + } +} \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/iRPropPlus.h b/include/NeuralNetwork/Learning/iRPropPlus.h new file mode 100644 index 0000000..2944616 --- /dev/null +++ b/include/NeuralNetwork/Learning/iRPropPlus.h @@ -0,0 +1,64 @@ +#pragma once + +#include "BatchPropagation.h" + +namespace NeuralNetwork { + namespace Learning { + + /** @class Resilient Propagation + * @brief + */ + class iRPropPlus : public BatchPropagation { + + public: + iRPropPlus(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BatchPropagation(feedForwardNetwork, correction) { + } + + iRPropPlus(const iRPropPlus&)=delete; + iRPropPlus& operator=(const NeuralNetwork::Learning::iRPropPlus&) = delete; + + void setInitialWeightChange(float initVal) { + initialWeightChange=initVal; + } + void setLearningCoefficient(float) { + + } + protected: + + virtual inline void resize() override { + BatchPropagation::resize(); + + _lastGradients =_gradients; + + _changesOfWeightChanges = _lastGradients; + for(std::size_t i = 1; i < _network.size(); i++) { + for(std::size_t j = 0; j < _changesOfWeightChanges[i].size(); j++) { + std::fill(_changesOfWeightChanges[i][j].begin(),_changesOfWeightChanges[i][j].end(),initialWeightChange); + } + } + _lastWeightChanges = _lastGradients; + for(std::size_t i = 1; i < _network.size(); i++) { + for(std::size_t j = 0; j < _lastWeightChanges[i].size(); j++) { + std::fill(_lastWeightChanges[i][j].begin(),_lastWeightChanges[i][j].end(),0.1); + } + } + } + + void updateWeightsAndEndBatch() override; + + std::vector>> _lastGradients = {}; + std::vector>> _lastWeightChanges = {}; + std::vector>> _changesOfWeightChanges = {}; + + float _prevError=0; + + float maxChangeOfWeights = 5; + float minChangeOfWeights = 0.0001; + + float initialWeightChange=0.02; + float weightChangePlus=1.2; + float weightChangeMinus=0.5; + }; + } +} \ No newline at end of file diff --git a/include/NeuralNetwork/Neuron.h b/include/NeuralNetwork/Neuron.h index 183df1f..62d8276 100644 --- a/include/NeuralNetwork/Neuron.h +++ b/include/NeuralNetwork/Neuron.h @@ -108,7 +108,7 @@ namespace NeuralNetwork /** * @brief getter for activation function of neuron */ - virtual ActivationFunction::ActivationFunction& getActivationFunction() =0; + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const =0; virtual void setBasisFunction(const BasisFunction::BasisFunction& basisFunction) =0; @@ -167,7 +167,7 @@ namespace NeuralNetwork return *basis; } - virtual ActivationFunction::ActivationFunction& getActivationFunction() override { + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override { return *activation; } @@ -216,7 +216,7 @@ namespace NeuralNetwork throw usageException("basis function"); } - virtual ActivationFunction::ActivationFunction& getActivationFunction() override { + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override { throw usageException("biasNeuron - activation function"); } @@ -267,7 +267,7 @@ namespace NeuralNetwork throw usageException("basis function"); } - virtual ActivationFunction::ActivationFunction& getActivationFunction() override { + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override { throw usageException("input neuron - activation function"); } diff --git a/src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp b/src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp index f05de73..77ff5e3 100644 --- a/src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp +++ b/src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp @@ -1,5 +1,7 @@ #include +#include + using namespace NeuralNetwork::ConstructiveAlgorihtms; float CascadeCorrelation::trainOutputs(Cascade::Network &network, const std::vector &patterns) { diff --git a/src/NeuralNetwork/Learning/BackPropagation.cpp b/src/NeuralNetwork/Learning/BackPropagation.cpp index 9801169..f288989 100644 --- a/src/NeuralNetwork/Learning/BackPropagation.cpp +++ b/src/NeuralNetwork/Learning/BackPropagation.cpp @@ -1,74 +1,27 @@ #include -#include -#include +void NeuralNetwork::Learning::BackPropagation::updateWeightsAndEndBatch() { -void NeuralNetwork::Learning::BackPropagation::teach(const std::vector &input, const std::vector &expectation) { + bool enableMoments = momentumWeight > 0.0; - network.computeOutput(input); + for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) { + auto &layer = _network[layerIndex]; + auto &prevLayer = _network[layerIndex - 1]; - resize(); + std::size_t prevLayerSize = prevLayer.size(); + std::size_t layerSize = layer.size(); - computeSlopes(expectation); + for(std::size_t j = 1; j < layerSize; j++) { + for(std::size_t k = 0; k < prevLayerSize; k++) { + float delta = _gradients[layerIndex][j][k]*learningCoefficient - weightDecay * layer[j].weight(k); - updateWeights(input); - - std::swap(deltas,lastDeltas); -} - - -void NeuralNetwork::Learning::BackPropagation::updateWeights(const std::vector &input) { - - for(std::size_t layerIndex=1;layerIndex &expectation) { - auto& outputLayer=network[network.size()-1]; - for(std::size_t j=1;joperator()( expectation[j-1], neuron.output())* - neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output()); - } - - for(int layerIndex=static_cast(network.size()-2);layerIndex>0;layerIndex--) { - auto &layer=network[layerIndex]; - - for(std::size_t j=1;j + +void NeuralNetwork::Learning::BatchPropagation::teach(const std::vector &input, const std::vector &expectation) { + _network.computeOutput(input); + if(!init) { + resize(); + init = true; + } + + computeSlopes(expectation); + + computeDeltas(input); + if(++_currentBatchSize >= _batchSize) { + finishTeaching(); + } +} + +void NeuralNetwork::Learning::BatchPropagation::finishTeaching() { + updateWeightsAndEndBatch(); + _currentBatchSize=0; +} + +void NeuralNetwork::Learning::BatchPropagation::computeSlopes(const std::vector &expectation) { + const auto& outputLayer=_network[_network.size()-1]; + for(std::size_t j=1;joperator()( expectation[j-1], neuron.output())* + neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output()); + } + + for(int layerIndex=static_cast(_network.size()-2);layerIndex>0;layerIndex--) { + auto &layer=_network[layerIndex]; + + for(std::size_t j=1;j &input) { + for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) { + auto &layer=_network[layerIndex]; + auto &prevLayer=_network[layerIndex-1]; + + std::size_t prevLayerSize=prevLayer.size(); + std::size_t layerSize=layer.size(); + + for(std::size_t j=1;j 0) { + for(std::size_t j = 0; j < _gradients[i].size(); j++) { + _gradients[i][j].resize(_network[i - 1].size()); + std::fill(_gradients[i][j].begin(), _gradients[i][j].end(), 0.0); + } + } + } + +} diff --git a/src/NeuralNetwork/Learning/QuickPropagation.cpp b/src/NeuralNetwork/Learning/QuickPropagation.cpp index c5a0fda..80e5df5 100644 --- a/src/NeuralNetwork/Learning/QuickPropagation.cpp +++ b/src/NeuralNetwork/Learning/QuickPropagation.cpp @@ -1,53 +1,46 @@ #include -#include -#include +void NeuralNetwork::Learning::QuickPropagation::updateWeightsAndEndBatch() { -void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector &input) { + float shrinkFactor=_maxChange/(_maxChange+1.0f); - float shrinkFactor=_maxChange/(_maxChange+1.0); + for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) { + auto &layer = _network[layerIndex]; + auto &prevLayer = _network[layerIndex - 1]; - for(std::size_t layerIndex=1;layerIndex 0.0001) { + if(std::signbit(_gradients[layerIndex][j][k]) == std::signbit(_lastGradients[layerIndex][j][k])) { + newChange+= _gradients[layerIndex][j][k]*_epsilon; - for(std::size_t j=1;j 0.0001) { - if(std::signbit(deltas[layerIndex][j]) == std::signbit(slopes[layerIndex][j])) { - newChange+= slopes[layerIndex][j]*_epsilon; - - if(fabs(slopes[layerIndex][j]) > fabs(shrinkFactor * previousSlopes[layerIndex][j])) { - newChange += _maxChange * deltas[layerIndex][j]; - }else { - newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * deltas[layerIndex][j]; + if(fabs(_gradients[layerIndex][j][k]) > fabs(shrinkFactor * _lastGradients[layerIndex][j][k])) { + newChange += _maxChange * _gradients[layerIndex][j][k]; + }else { + newChange+=_gradients[layerIndex][j][k]/(_lastGradients[layerIndex][j][k]-_gradients[layerIndex][j][k]) * _lastDeltas[layerIndex][j][k]; + } + } else { + newChange+=_gradients[layerIndex][j][k]/(_lastGradients[layerIndex][j][k]-_gradients[layerIndex][j][k]) * _lastDeltas[layerIndex][j][k]; } } else { - newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * deltas[layerIndex][j]; + newChange+= _lastGradients[layerIndex][j][k]*_epsilon; } - } else { - newChange+= slopes[layerIndex][j]*_epsilon; - } + _lastDeltas[layerIndex][j][k]= newChange; + layer[j].weight(k)+= newChange; - weightChange[layerIndex][j]=newChange; - - layer[j].weight(0)+=newChange; - - for(std::size_t k=1;k + +void NeuralNetwork::Learning::RProp::updateWeightsAndEndBatch() { + + for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) { + auto &layer = _network[layerIndex]; + auto &prevLayer = _network[layerIndex - 1]; + + std::size_t prevLayerSize = prevLayer.size(); + std::size_t layerSize = layer.size(); + + for(std::size_t j = 1; j < layerSize; j++) { + for(std::size_t k = 0; k < prevLayerSize; k++) { + float gradient = _gradients[layerIndex][j][k]; + float lastGradient = _lastGradients[layerIndex][j][k]; + + _lastGradients[layerIndex][j][k] = gradient; + + float weightChangeDelta = _lastWeightChanges[layerIndex][j][k]; + + if(gradient * lastGradient > 0) { + weightChangeDelta = std::min(weightChangeDelta*weightChangePlus,maxChangeOfWeights); + } else if (gradient * lastGradient < 0) { + weightChangeDelta = std::max(weightChangeDelta*weightChangeMinus,minChangeOfWeights); + } else { + weightChangeDelta = _lastWeightChanges[layerIndex][j][k]; + } + + _lastWeightChanges[layerIndex][j][k] = weightChangeDelta; + + if(gradient > 0) { + layer[j].weight(k) += weightChangeDelta; + } else if (gradient < 0){ + layer[j].weight(k) -= weightChangeDelta; + } + } + } + } +} \ No newline at end of file diff --git a/src/NeuralNetwork/Learning/iRPropPlus.cpp b/src/NeuralNetwork/Learning/iRPropPlus.cpp new file mode 100644 index 0000000..e3315db --- /dev/null +++ b/src/NeuralNetwork/Learning/iRPropPlus.cpp @@ -0,0 +1,52 @@ +#include + +void NeuralNetwork::Learning::iRPropPlus::updateWeightsAndEndBatch() { + float error = 0.0; + const auto& outputLayer=_network[_network.size()-1]; + for(std::size_t j=1;j 0) { + weightChangeDelta = std::min(weightChangeDelta*weightChangePlus,maxChangeOfWeights); + delta = (std::signbit(gradient)? 1.0f : -1.0f ) * weightChangeDelta; + layer[j].weight(k) -= delta; + } else if (gradient * lastGradient < 0) { + weightChangeDelta = std::max(weightChangeDelta*weightChangeMinus,minChangeOfWeights); + delta = _lastWeightChanges[layerIndex][j][k]; + if(error > _prevError) { + layer[j].weight(k) += delta; + } + _lastGradients[layerIndex][j][k] = 0; + } else { + delta = (std::signbit(gradient)? 1.0f : -1.0f ) * weightChangeDelta; + layer[j].weight(k) -= delta; + } + //std::cout << delta <<"\n"; + + _changesOfWeightChanges[layerIndex][j][k] = weightChangeDelta; + _lastWeightChanges[layerIndex][j][k] = delta; + } + } + } + _prevError = error; +} \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 09ef39a..af6ddd3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -13,9 +13,6 @@ target_link_libraries(backpropagation NeuralNetwork gtest gtest_main) add_executable(feedforward feedforward.cpp) target_link_libraries(feedforward NeuralNetwork gtest gtest_main) -add_executable(optical_backpropagation optical_backpropagation.cpp) -target_link_libraries(optical_backpropagation NeuralNetwork gtest gtest_main) - add_executable(perceptron perceptron.cpp) target_link_libraries(perceptron NeuralNetwork gtest gtest_main) @@ -28,6 +25,9 @@ target_link_libraries(recurrent NeuralNetwork gtest gtest_main) add_executable(quickpropagation quickpropagation.cpp) target_link_libraries(quickpropagation NeuralNetwork gtest gtest_main) +add_executable(rprop rprop.cpp) +target_link_libraries(rprop NeuralNetwork gtest gtest_main) + # PERF add_executable(backpropagation_function_cmp backpropagation_function_cmp.cpp) diff --git a/tests/activation.cpp b/tests/activation.cpp index be77834..04f2d4e 100644 --- a/tests/activation.cpp +++ b/tests/activation.cpp @@ -48,20 +48,6 @@ TEST(Sigmoid, ParamMinusFive) { ASSERT_LT(s(0.7), 0.970788); } -TEST(SigmoidSSE, ParamMinusZeroPointSeven) { - NeuralNetwork::ActivationFunction::Sigmoid s(-0.7); - SSE comp; - comp.floats[0] = 0.1; - comp.floats[1] = 10; - comp.sse = s(comp.sse); - - ASSERT_GT(comp.floats[0], 0.517483); - ASSERT_LT(comp.floats[0], 0.51750); - - ASSERT_GT(comp.floats[1], 0.998989); - ASSERT_LT(comp.floats[1], 0.999189); -} - TEST(Linear, ParamOne) { NeuralNetwork::ActivationFunction::Linear s(1.0); ASSERT_GT(s(0.5), 0.4999); diff --git a/tests/backpropagation.cpp b/tests/backpropagation.cpp index caafcfc..2ae62c3 100644 --- a/tests/backpropagation.cpp +++ b/tests/backpropagation.cpp @@ -47,6 +47,7 @@ TEST(BackProp,XOR) { } TEST(BackProp,XORHyperbolicTangent) { + srand(time(NULL)); NeuralNetwork::FeedForward::Network n(2); NeuralNetwork::ActivationFunction::HyperbolicTangent a(-1); n.appendLayer(2,a); @@ -56,7 +57,7 @@ TEST(BackProp,XORHyperbolicTangent) { NeuralNetwork::Learning::BackPropagation prop(n); - for(int i=0;i<10000;i++) { + for(int i=0;i<1500;i++) { prop.teach({1,0},{1}); prop.teach({1,1},{0}); prop.teach({0,0},{0}); diff --git a/tests/backpropagation_function_cmp.cpp b/tests/backpropagation_function_cmp.cpp index 2f296cb..ae75d64 100644 --- a/tests/backpropagation_function_cmp.cpp +++ b/tests/backpropagation_function_cmp.cpp @@ -45,41 +45,41 @@ int main() { std::cout << "\tLinear: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n"; + std::make_shared(),linearCoef) << "\n"; std::cout << "\tOptical: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n"; + std::make_shared(),opticalCoef) << "\n"; std::cout << "\tArcTangent: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n"; + std::make_shared(arcTangent),arcTangentCoef) << "\n"; } { std::cout << "AND:\n"; std::cout << "\tLinear: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n"; + std::make_shared(),linearCoef) << "\n"; std::cout << "\tOptical: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n"; + std::make_shared(),opticalCoef) << "\n"; std::cout << "\tArcTangent: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n"; + std::make_shared(arcTangent),arcTangentCoef) << "\n"; } { std::cout << "AND:\n"; std::cout << "\tLinear: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n"; + std::make_shared(),linearCoef) << "\n"; std::cout << "\tOptical: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n"; + std::make_shared(),opticalCoef) << "\n"; std::cout << "\tArcTangent: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n"; + std::make_shared(arcTangent),arcTangentCoef) << "\n"; } } diff --git a/tests/backpropagation_perf.cpp b/tests/backpropagation_perf.cpp index 2f31ea1..13732d1 100644 --- a/tests/backpropagation_perf.cpp +++ b/tests/backpropagation_perf.cpp @@ -4,6 +4,7 @@ #include #include "../include/NeuralNetwork/Learning/BackPropagation.h" +#include int main() { { // XOR problem NeuralNetwork::FeedForward::Network n(2); @@ -16,11 +17,18 @@ int main() { n.randomizeWeights(); NeuralNetwork::Learning::BackPropagation prop(n); +// prop.setBatchSize(20); +// prop.setMomentumWeight(0.8); + + auto start = std::chrono::system_clock::now(); for(int i=0;i<100;i++) { prop.teach({1,0},{1}); prop.teach({1,1},{0}); prop.teach({0,0},{0}); prop.teach({0,1},{1}); } + auto end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end -start; + std::cout << elapsed_seconds.count() << "\n"; } } diff --git a/tests/optical_backpropagation.cpp b/tests/optical_backpropagation.cpp deleted file mode 100644 index 037e065..0000000 --- a/tests/optical_backpropagation.cpp +++ /dev/null @@ -1,124 +0,0 @@ -#include - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Weffc++" - -#include - -#pragma GCC diagnostic pop - -TEST(OpticalBackPropagation,XOR) { - NeuralNetwork::FeedForward::Network n(2); - NeuralNetwork::ActivationFunction::Sigmoid a(-1); - n.appendLayer(2,a); - n.appendLayer(1,a); - - n.randomizeWeights(); - - NeuralNetwork::Learning::OpticalBackPropagation prop(n); - - for(int i=0;i<10000;i++) { - prop.teach({1,0},{1}); - prop.teach({1,1},{0}); - prop.teach({0,0},{0}); - prop.teach({0,1},{1}); - } - - { - std::vector ret =n.computeOutput({1,1}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({0,1}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({1,0}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({0,0}); - ASSERT_LT(ret[0], 0.1); - } -} - -TEST(OpticalBackPropagation,AND) { - NeuralNetwork::FeedForward::Network n(2); - NeuralNetwork::ActivationFunction::Sigmoid a(-1); - n.appendLayer(2,a); - n.appendLayer(1,a); - - n.randomizeWeights(); - - NeuralNetwork::Learning::OpticalBackPropagation prop(n); - - for(int i=0;i<10000;i++) { - prop.teach({1,1},{1}); - prop.teach({0,0},{0}); - prop.teach({0,1},{0}); - prop.teach({1,0},{0}); - } - - { - std::vector ret =n.computeOutput({1,1}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({0,1}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({1,0}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({0,0}); - ASSERT_LT(ret[0], 0.1); - } -} - -TEST(OpticalBackPropagation,NOTAND) { - NeuralNetwork::FeedForward::Network n(2); - NeuralNetwork::ActivationFunction::Sigmoid a(-1); - n.appendLayer(2,a); - n.appendLayer(1,a); - - n.randomizeWeights(); - - NeuralNetwork::Learning::OpticalBackPropagation prop(n); - - for(int i=0;i<10000;i++) { - prop.teach({1,1},{0}); - prop.teach({0,0},{1}); - prop.teach({0,1},{1}); - prop.teach({1,0},{1}); - } - - { - std::vector ret =n.computeOutput({1,1}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({0,1}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({1,0}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({0,0}); - ASSERT_GT(ret[0], 0.9); - } -} \ No newline at end of file diff --git a/tests/propagation_cmp.cpp b/tests/propagation_cmp.cpp index 04efd85..50fba6a 100644 --- a/tests/propagation_cmp.cpp +++ b/tests/propagation_cmp.cpp @@ -4,6 +4,7 @@ #include #include "../include/NeuralNetwork/Learning/BackPropagation.h" #include "../include/NeuralNetwork/Learning/QuickPropagation.h" +#include "../include/NeuralNetwork/Learning/RProp.h" #include "../include/NeuralNetwork/Learning/CorrectionFunction/Optical.h" #include "../include/NeuralNetwork/Learning/CorrectionFunction/ArcTangent.h" @@ -16,7 +17,8 @@ n.appendLayer(1,a);\ n.randomizeWeights();\ CLASS prop(n,FUN);\ - prop.setLearningCoefficient(COEF);\ + prop.setLearningCoefficient(COEF);\ + prop.setBatchSize(4); \ int error=1; int steps = 0; \ while(error > 0 && steps <99999) {\ steps++;\ @@ -42,36 +44,47 @@ int main() { const float arcTangent=1.5; { - std::cout << "XOR:\n"; + std::cout << "XOR Linear:\n"; std::cout << "\tBP: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + std::make_shared(),linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; std::cout << "\tQP: " << - LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, + std::make_shared(),linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + std::cout << "\tRProp: " << + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, + std::make_shared(),linearCoef,NeuralNetwork::Learning::RProp) << "\n"; } { - std::cout << "AND:\n"; + std::cout << "AND Optical:\n"; std::cout << "\tBP: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + std::make_shared(),opticalCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; std::cout << "\tQP: " << - LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + + std::cout << "\tRProp: " << + LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::RProp) << "\n"; } { - std::cout << "AND:\n"; + std::cout << "XOR Optical:\n"; std::cout << "\tBP: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + std::make_shared(),opticalCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; std::cout << "\tQP: " << - LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + + std::cout << "\tRProp: " << + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::RProp) << "\n"; } } diff --git a/tests/rprop.cpp b/tests/rprop.cpp new file mode 100644 index 0000000..73e02ef --- /dev/null +++ b/tests/rprop.cpp @@ -0,0 +1,165 @@ +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Weffc++" + +#include + +#pragma GCC diagnostic pop +TEST(RProp,XOR) { + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::Sigmoid a(-1); + n.appendLayer(3,a); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::RProp prop(n); + prop.setBatchSize(4); + + for(int i=0;i<100;i++) { + prop.teach({1,0},{1}); + prop.teach({1,1},{0}); + prop.teach({0,0},{0}); + prop.teach({0,1},{1}); + } + + { + std::vector ret =n.computeOutput({1,1}); + ASSERT_LT(ret[0], 0.1); + } + + { + std::vector ret =n.computeOutput({0,1}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({1,0}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({0,0}); + ASSERT_LT(ret[0], 0.1); + } +} + +TEST(RProp,XORHyperbolicTangent) { + srand(time(NULL)); + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::HyperbolicTangent a(-1); + n.appendLayer(2,a); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::RProp prop(n); + prop.setBatchSize(4); + + for(int i=0;i<15000;i++) { + prop.teach({1,0},{1}); + prop.teach({1,1},{0}); + prop.teach({0,0},{0}); + prop.teach({0,1},{1}); + } + + { + std::vector ret =n.computeOutput({1,1}); + ASSERT_LT(ret[0], 0.1); + } + + { + std::vector ret =n.computeOutput({0,1}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({1,0}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({0,0}); + ASSERT_LT(ret[0], 0.1); + } +} + +TEST(RProp,AND) { + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::Sigmoid a(-1); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::RProp prop(n); + prop.setBatchSize(4); + + for(int i=0;i<100000;i++) { + prop.teach({1,1},{1}); + prop.teach({0,0},{0}); + prop.teach({0,1},{0}); + prop.teach({1,0},{0}); + } + + { + std::vector ret =n.computeOutput({1,1}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({0,1}); + ASSERT_LT(ret[0], 0.1); + } + + { + std::vector ret =n.computeOutput({1,0}); + ASSERT_LT(ret[0], 0.1); + } + + { + std::vector ret =n.computeOutput({0,0}); + ASSERT_LT(ret[0], 0.1); + } +} + +TEST(RProp,NOTAND) { + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::Sigmoid a(-1); + n.appendLayer(2,a); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::RProp prop(n); + prop.setBatchSize(4); + + for(int i=0;i<100000;i++) { + prop.teach({1,1},{0}); + prop.teach({0,0},{1}); + prop.teach({0,1},{1}); + prop.teach({1,0},{1}); + } + + { + std::vector ret =n.computeOutput({1,1}); + ASSERT_LT(ret[0], 0.1); + } + + { + std::vector ret =n.computeOutput({0,1}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({1,0}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({0,0}); + ASSERT_GT(ret[0], 0.9); + } +} \ No newline at end of file