diff --git a/CMakeLists.txt b/CMakeLists.txt index 22ab38f..db2ceab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,8 +61,9 @@ endif(USE_SSE) set (LIBRARY_SOURCES src/sse_mathfun.cpp + src/NeuralNetwork/Learning/BatchPropagation.cpp src/NeuralNetwork/Learning/BackPropagation.cpp - src/NeuralNetwork/Learning/QuickPropagation.cpp + #src/NeuralNetwork/Learning/QuickPropagation.cpp src/NeuralNetwork/Learning/PerceptronLearning.cpp src/NeuralNetwork/Learning/RProp.cpp diff --git a/include/NeuralNetwork/FeedForward/Layer.h b/include/NeuralNetwork/FeedForward/Layer.h index 3c07750..43bf5c3 100644 --- a/include/NeuralNetwork/FeedForward/Layer.h +++ b/include/NeuralNetwork/FeedForward/Layer.h @@ -67,6 +67,15 @@ namespace FeedForward { return *neurons[neuron]; } + /** + * @brief This is a virtual function for selecting neuron + * @param neuron is position in layer + * @returns Specific neuron + */ + const NeuronInterface& operator[](const std::size_t& neuron) const { + return *neurons[neuron]; + } + void solve(const std::vector &input, std::vector &output); /** diff --git a/include/NeuralNetwork/Learning/BackPropagation.h b/include/NeuralNetwork/Learning/BackPropagation.h index 1df9b42..ae7b18a 100644 --- a/include/NeuralNetwork/Learning/BackPropagation.h +++ b/include/NeuralNetwork/Learning/BackPropagation.h @@ -1,10 +1,6 @@ #pragma once -#include -#include - -#include -#include "CorrectionFunction/Linear.h" +#include "BatchPropagation.h" namespace NeuralNetwork { namespace Learning { @@ -12,23 +8,17 @@ namespace Learning { /** @class BackPropagation * @brief */ - class BackPropagation { + class BackPropagation : public BatchPropagation { public: - inline BackPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()): - network(feedForwardNetwork), correctionFunction(correction),learningCoefficient(0.4), slopes() { + BackPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BatchPropagation(feedForwardNetwork,correction), learningCoefficient(0.4) { resize(); } - virtual ~BackPropagation() { - delete correctionFunction; - } - BackPropagation(const BackPropagation&)=delete; BackPropagation& operator=(const NeuralNetwork::Learning::BackPropagation&) = delete; - void teach(const std::vector &input, const std::vector &output); - inline virtual void setLearningCoefficient (const float& coefficient) { learningCoefficient=coefficient; } float getMomentumWeight() const { @@ -48,75 +38,22 @@ namespace Learning { weightDecay=wd; } - std::size_t getBatchSize() const { - return batchSize; - } - - void setBatchSize(std::size_t size) { - batchSize = size; - } - protected: - virtual inline void resize() { - if(slopes.size()!=network.size()) - slopes.resize(network.size()); - - for(std::size_t i=0; i < network.size(); i++) { - if(slopes[i].size()!=network[i].size()) - slopes[i].resize(network[i].size()); - } - - if(deltas.size() != network.size()) - deltas.resize(network.size()); - - bool resized = false; - - for(std::size_t i = 0; i < network.size(); i++) { - if(deltas[i].size() != network[i].size()) { - deltas[i].resize(network[i].size()); - resized = true; - - if(i > 0) { - for(std::size_t j = 0; j < deltas[i].size(); j++) { - deltas[i][j].resize(network[i - 1].size()); - std::fill(deltas[i][j].begin(),deltas[i][j].end(),0.0); - } - } - } - } - - if(momentumWeight > 0.0 && (resized || lastDeltas.size() != deltas.size())) { - lastDeltas = deltas; + virtual inline void resize() override { + BatchPropagation::resize(); + if(momentumWeight > 0.0) { + _lastDeltas = _gradients; } } - virtual void computeDeltas(const std::vector &input); - - void updateWeights(); - - virtual void computeSlopes(const std::vector &expectation); - - virtual void endBatch() { - - } - - FeedForward::Network &network; - - CorrectionFunction::CorrectionFunction *correctionFunction; + virtual void updateWeightsAndEndBatch() override; float learningCoefficient; - float momentumWeight = 0.0; - float weightDecay = 0.0; - std::size_t batchSize = 1; - std::size_t currentBatchSize = 0; - - std::vector> slopes; - std::vector>> deltas = {}; - std::vector>> lastDeltas = {}; + std::vector>> _lastDeltas = {}; }; } diff --git a/include/NeuralNetwork/Learning/BatchPropagation.h b/include/NeuralNetwork/Learning/BatchPropagation.h new file mode 100644 index 0000000..b26ddf7 --- /dev/null +++ b/include/NeuralNetwork/Learning/BatchPropagation.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include "CorrectionFunction/Linear.h" + +#include +#include + +namespace NeuralNetwork { +namespace Learning { + class BatchPropagation { + public: + BatchPropagation(FeedForward::Network &ffn, std::shared_ptr correction) : _network(ffn), _correctionFunction(correction) { + + } + + virtual ~BatchPropagation() { + + } + + void teach(const std::vector &input, const std::vector &output); + + void finishTeaching(); + + std::size_t getBatchSize() const { + return _batchSize; + } + + void setBatchSize(std::size_t size) { + _batchSize = size; + } + protected: + virtual void updateWeightsAndEndBatch() = 0; + virtual void resize(); + + FeedForward::Network &_network; + std::shared_ptr _correctionFunction; + + std::size_t _batchSize = 1; + std::size_t _currentBatchSize = 0; + + std::vector> _slopes = {}; + std::vector>> _gradients = {}; + + bool init = false; + private: + void computeSlopes(const std::vector &expectation); + void computeDeltas(const std::vector &input); + }; +} +} \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/OpticalBackPropagation.h b/include/NeuralNetwork/Learning/OpticalBackPropagation.h deleted file mode 100644 index cb70c3f..0000000 --- a/include/NeuralNetwork/Learning/OpticalBackPropagation.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include "./BackPropagation.h" -#include "./CorrectionFunction/Optical.h" - -namespace NeuralNetwork { -namespace Learning { - - /** @class OpticalBackPropagation - * @brief - */ - class OpticalBackPropagation : public BackPropagation { - - public: - OpticalBackPropagation(FeedForward::Network &feedForwardNetwork): BackPropagation(feedForwardNetwork,new CorrectionFunction::Optical()) { - - } - - virtual ~OpticalBackPropagation() { - } - }; -} -} \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/QuickPropagation.h b/include/NeuralNetwork/Learning/QuickPropagation.h index 48f4f98..358548a 100644 --- a/include/NeuralNetwork/Learning/QuickPropagation.h +++ b/include/NeuralNetwork/Learning/QuickPropagation.h @@ -15,46 +15,24 @@ namespace NeuralNetwork { class QuickPropagation : public BackPropagation { public: - inline QuickPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()): - BackPropagation(feedForwardNetwork,correction),previousSlopes() { - resize(); + inline QuickPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BackPropagation(feedForwardNetwork,correction) { } virtual ~QuickPropagation() { } protected: + float _maxChange=1.75; float _epsilon=0.5; virtual inline void resize() override { - if(slopes.size()!=network.size()) - slopes.resize(network.size()); - - for(std::size_t i=0; i < network.size(); i++) { - if(slopes[i].size()!=network[i].size()) - slopes[i].resize(network[i].size()); - } - - if(deltas.size()!=network.size()) - deltas.resize(network.size()); - - for(std::size_t i=0; i < network.size(); i++) { - if(deltas[i].size()!=network[i].size()) - deltas[i].resize(network[i].size()); - - for(std::size_t j=0; j < previousSlopes[i].size(); j++) { - deltas[i][j]=1.0; - } - } - weightChange= deltas; + BackPropagation::resize(); + _previousSlopes = _slopes; } - virtual void updateWeights(const std::vector &input) override; - - std::vector> previousSlopes ={}; - std::vector> deltas ={}; - std::vector> weightChange ={}; + std::vector> _previousSlopes ={}; }; } } \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/RProp.h b/include/NeuralNetwork/Learning/RProp.h index a838e0e..522fd4a 100644 --- a/include/NeuralNetwork/Learning/RProp.h +++ b/include/NeuralNetwork/Learning/RProp.h @@ -1,10 +1,7 @@ #pragma once -#include -#include -#include -#include "CorrectionFunction/Linear.h" +#include "BatchPropagation.h" namespace NeuralNetwork { namespace Learning { @@ -12,122 +9,48 @@ namespace NeuralNetwork { /** @class Resilient Propagation * @brief */ - class RProp { + class RProp : public BatchPropagation { public: - RProp(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()): - network(feedForwardNetwork), correctionFunction(correction) { - resize(); - } - - virtual ~RProp() { - delete correctionFunction; + RProp(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BatchPropagation(feedForwardNetwork, correction) { } RProp(const RProp&)=delete; RProp& operator=(const NeuralNetwork::Learning::RProp&) = delete; - void teach(const std::vector &input, const std::vector &output); - - std::size_t getBatchSize() const { - return batchSize; + void setInitialWeightChange(float initVal) { + initialWeightChange=initVal; } + void setLearningCoefficient(float) { - void setBatchSize(std::size_t size) { - batchSize = size; - } - - void setInitialWeightChange(float init) { - initialWeightChange=init; } protected: - virtual inline void resize() { - if(slopes.size()!=network.size()) - slopes.resize(network.size()); + virtual inline void resize() override { + BatchPropagation::resize(); - for(std::size_t i=0; i < network.size(); i++) { - if(slopes[i].size()!=network[i].size()) - slopes[i].resize(network[i].size()); - } + _lastGradients =_gradients; - if(gradients.size() != network.size()) - gradients.resize(network.size()); - - bool resized = false; - - for(std::size_t i = 0; i < network.size(); i++) { - if(gradients[i].size() != network[i].size()) { - gradients[i].resize(network[i].size()); - resized = true; - - if(i > 0) { - for(std::size_t j = 0; j < gradients[i].size(); j++) { - gradients[i][j].resize(network[i - 1].size()); - std::fill(gradients[i][j].begin(),gradients[i][j].end(),0.0); - } - } + _changesOfWeightChanges = _lastGradients; + for(std::size_t i = 1; i < _network.size(); i++) { + for(std::size_t j = 0; j < _changesOfWeightChanges[i].size(); j++) { + std::fill(_changesOfWeightChanges[i][j].begin(),_changesOfWeightChanges[i][j].end(),initialWeightChange); } } - - if(resized) { - lastGradients = gradients; - - if(changesOfWeightChanges.size() != network.size()) - changesOfWeightChanges.resize(network.size()); - - for(std::size_t i = 0; i < network.size(); i++) { - if(changesOfWeightChanges[i].size() != network[i].size()) { - changesOfWeightChanges[i].resize(network[i].size()); - if(i > 0) { - for(std::size_t j = 0; j < changesOfWeightChanges[i].size(); j++) { - changesOfWeightChanges[i][j].resize(network[i - 1].size()); - std::fill(changesOfWeightChanges[i][j].begin(),changesOfWeightChanges[i][j].end(),initialWeightChange); - } - } - } - } - } - - if(resized) { - if(lastWeightChanges.size() != network.size()) - lastWeightChanges.resize(network.size()); - - for(std::size_t i = 0; i < network.size(); i++) { - if(lastWeightChanges[i].size() != network[i].size()) { - lastWeightChanges[i].resize(network[i].size()); - if(i > 0) { - for(std::size_t j = 0; j < lastWeightChanges[i].size(); j++) { - lastWeightChanges[i][j].resize(network[i - 1].size()); - std::fill(lastWeightChanges[i][j].begin(),lastWeightChanges[i][j].end(),0.1); - } - } - } + _lastWeightChanges = _lastGradients; + for(std::size_t i = 1; i < _network.size(); i++) { + for(std::size_t j = 0; j < _lastWeightChanges[i].size(); j++) { + std::fill(_lastWeightChanges[i][j].begin(),_lastWeightChanges[i][j].end(),0.1); } } } - virtual void computeSlopes(const std::vector &expectation); - virtual void computeDeltas(const std::vector &input); + void updateWeightsAndEndBatch() override; - void updateWeights(); - - virtual void endBatch() { - - } - - FeedForward::Network &network; - - CorrectionFunction::CorrectionFunction *correctionFunction; - - std::vector> slopes; - std::vector>> gradients = {}; - std::vector>> lastGradients = {}; - std::vector>> lastWeightChanges = {}; - std::vector>> changesOfWeightChanges = {}; - - std::size_t batchSize = 1; - std::size_t currentBatchSize = 0; + std::vector>> _lastGradients = {}; + std::vector>> _lastWeightChanges = {}; + std::vector>> _changesOfWeightChanges = {}; float maxChangeOfWeights = 50; float minChangeOfWeights = 0.0001; diff --git a/include/NeuralNetwork/Neuron.h b/include/NeuralNetwork/Neuron.h index 183df1f..62d8276 100644 --- a/include/NeuralNetwork/Neuron.h +++ b/include/NeuralNetwork/Neuron.h @@ -108,7 +108,7 @@ namespace NeuralNetwork /** * @brief getter for activation function of neuron */ - virtual ActivationFunction::ActivationFunction& getActivationFunction() =0; + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const =0; virtual void setBasisFunction(const BasisFunction::BasisFunction& basisFunction) =0; @@ -167,7 +167,7 @@ namespace NeuralNetwork return *basis; } - virtual ActivationFunction::ActivationFunction& getActivationFunction() override { + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override { return *activation; } @@ -216,7 +216,7 @@ namespace NeuralNetwork throw usageException("basis function"); } - virtual ActivationFunction::ActivationFunction& getActivationFunction() override { + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override { throw usageException("biasNeuron - activation function"); } @@ -267,7 +267,7 @@ namespace NeuralNetwork throw usageException("basis function"); } - virtual ActivationFunction::ActivationFunction& getActivationFunction() override { + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override { throw usageException("input neuron - activation function"); } diff --git a/src/NeuralNetwork/Learning/BackPropagation.cpp b/src/NeuralNetwork/Learning/BackPropagation.cpp index b562393..f288989 100644 --- a/src/NeuralNetwork/Learning/BackPropagation.cpp +++ b/src/NeuralNetwork/Learning/BackPropagation.cpp @@ -1,89 +1,23 @@ #include -#include -#include - -void NeuralNetwork::Learning::BackPropagation::teach(const std::vector &input, const std::vector &expectation) { - network.computeOutput(input); - resize(); - computeSlopes(expectation); - - computeDeltas(input); - if(++currentBatchSize >= batchSize) { - updateWeights(); - endBatch(); - currentBatchSize=0; - } -} - -void NeuralNetwork::Learning::BackPropagation::computeSlopes(const std::vector &expectation) { - auto& outputLayer=network[network.size()-1]; - for(std::size_t j=1;joperator()( expectation[j-1], neuron.output())* - neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output()); - } - - for(int layerIndex=static_cast(network.size()-2);layerIndex>0;layerIndex--) { - auto &layer=network[layerIndex]; - - for(std::size_t j=1;j &input) { - for(std::size_t layerIndex=1;layerIndex 0.0; - for(std::size_t layerIndex=1;layerIndex + +void NeuralNetwork::Learning::BatchPropagation::teach(const std::vector &input, const std::vector &expectation) { + _network.computeOutput(input); + if(!init) { + resize(); + init = true; + } + + computeSlopes(expectation); + + computeDeltas(input); + if(++_currentBatchSize >= _batchSize) { + finishTeaching(); + } +} + +void NeuralNetwork::Learning::BatchPropagation::finishTeaching() { + updateWeightsAndEndBatch(); + _currentBatchSize=0; +} + +void NeuralNetwork::Learning::BatchPropagation::computeSlopes(const std::vector &expectation) { + const auto& outputLayer=_network[_network.size()-1]; + for(std::size_t j=1;joperator()( expectation[j-1], neuron.output())* + neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output()); + } + + for(int layerIndex=static_cast(_network.size()-2);layerIndex>0;layerIndex--) { + auto &layer=_network[layerIndex]; + + for(std::size_t j=1;j &input) { + for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) { + auto &layer=_network[layerIndex]; + auto &prevLayer=_network[layerIndex-1]; + + std::size_t prevLayerSize=prevLayer.size(); + std::size_t layerSize=layer.size(); + + for(std::size_t j=1;j 0) { + for(std::size_t j = 0; j < _gradients[i].size(); j++) { + _gradients[i][j].resize(_network[i - 1].size()); + std::fill(_gradients[i][j].begin(), _gradients[i][j].end(), 0.0); + } + } + } + +} diff --git a/src/NeuralNetwork/Learning/QuickPropagation.cpp b/src/NeuralNetwork/Learning/QuickPropagation.cpp index c5a0fda..3939c50 100644 --- a/src/NeuralNetwork/Learning/QuickPropagation.cpp +++ b/src/NeuralNetwork/Learning/QuickPropagation.cpp @@ -3,7 +3,7 @@ #include #include -void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector &input) { +void NeuralNetwork::Learning::QuickPropagation::computeDeltas(const std::vector &input) { float shrinkFactor=_maxChange/(_maxChange+1.0); @@ -15,20 +15,39 @@ void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector< std::size_t layerSize=layer.size(); for(std::size_t j=1;j 0.0001) { - if(std::signbit(deltas[layerIndex][j]) == std::signbit(slopes[layerIndex][j])) { + if(fabs (gradients[layerIndex][j])> 0.0001) { + if(std::signbit(gradients[layerIndex][j]) == std::signbit(slopes[layerIndex][j])) { newChange+= slopes[layerIndex][j]*_epsilon; if(fabs(slopes[layerIndex][j]) > fabs(shrinkFactor * previousSlopes[layerIndex][j])) { - newChange += _maxChange * deltas[layerIndex][j]; + newChange += _maxChange * gradients[layerIndex][j]; }else { - newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * deltas[layerIndex][j]; + newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * gradients[layerIndex][j]; } } else { - newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * deltas[layerIndex][j]; + newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * gradients[layerIndex][j]; } } else { newChange+= slopes[layerIndex][j]*_epsilon; @@ -45,9 +64,7 @@ void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector< layer[j].weight(k)+=newChange*(prevLayer[k].output()); } } + */ } } - - slopes.swap(previousSlopes); - weightChange.swap(deltas); } \ No newline at end of file diff --git a/src/NeuralNetwork/Learning/RProp.cpp b/src/NeuralNetwork/Learning/RProp.cpp index ab17e8e..fb95703 100644 --- a/src/NeuralNetwork/Learning/RProp.cpp +++ b/src/NeuralNetwork/Learning/RProp.cpp @@ -1,101 +1,37 @@ #include -void NeuralNetwork::Learning::RProp::teach(const std::vector &input, const std::vector &expectation) { - network.computeOutput(input); - resize(); - computeSlopes(expectation); +void NeuralNetwork::Learning::RProp::updateWeightsAndEndBatch() { - computeDeltas(input); - if(++currentBatchSize >= batchSize) { - updateWeights(); - endBatch(); - currentBatchSize=0; - } -} - -void NeuralNetwork::Learning::RProp::computeSlopes(const std::vector &expectation) { - auto& outputLayer=network[network.size()-1]; - for(std::size_t j=1;joperator()( expectation[j-1], neuron.output())* - neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output()); - } - - for(int layerIndex=static_cast(network.size()-2);layerIndex>0;layerIndex--) { - auto &layer=network[layerIndex]; - - for(std::size_t j=1;j &input) { - for(std::size_t layerIndex=1;layerIndex 0) { weightChangeDelta = std::min(weightChangeDelta*weightChangePlus,maxChangeOfWeights); } else if (gradient * lastGradient < 0) { weightChangeDelta = std::max(weightChangeDelta*weightChangeMinus,minChangeOfWeights); } else { - weightChangeDelta = lastWeightChanges[layerIndex][j][k]; + weightChangeDelta = _lastWeightChanges[layerIndex][j][k]; } - lastWeightChanges[layerIndex][j][k] = weightChangeDelta; + _lastWeightChanges[layerIndex][j][k] = weightChangeDelta; if(gradient > 0) { layer[j].weight(k) += weightChangeDelta; } else if (gradient < 0){ layer[j].weight(k) -= weightChangeDelta; - } else { } } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3f5f17a..af6ddd3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -13,9 +13,6 @@ target_link_libraries(backpropagation NeuralNetwork gtest gtest_main) add_executable(feedforward feedforward.cpp) target_link_libraries(feedforward NeuralNetwork gtest gtest_main) -add_executable(optical_backpropagation optical_backpropagation.cpp) -target_link_libraries(optical_backpropagation NeuralNetwork gtest gtest_main) - add_executable(perceptron perceptron.cpp) target_link_libraries(perceptron NeuralNetwork gtest gtest_main) diff --git a/tests/activation.cpp b/tests/activation.cpp index be77834..04f2d4e 100644 --- a/tests/activation.cpp +++ b/tests/activation.cpp @@ -48,20 +48,6 @@ TEST(Sigmoid, ParamMinusFive) { ASSERT_LT(s(0.7), 0.970788); } -TEST(SigmoidSSE, ParamMinusZeroPointSeven) { - NeuralNetwork::ActivationFunction::Sigmoid s(-0.7); - SSE comp; - comp.floats[0] = 0.1; - comp.floats[1] = 10; - comp.sse = s(comp.sse); - - ASSERT_GT(comp.floats[0], 0.517483); - ASSERT_LT(comp.floats[0], 0.51750); - - ASSERT_GT(comp.floats[1], 0.998989); - ASSERT_LT(comp.floats[1], 0.999189); -} - TEST(Linear, ParamOne) { NeuralNetwork::ActivationFunction::Linear s(1.0); ASSERT_GT(s(0.5), 0.4999); diff --git a/tests/backpropagation.cpp b/tests/backpropagation.cpp index caafcfc..2ae62c3 100644 --- a/tests/backpropagation.cpp +++ b/tests/backpropagation.cpp @@ -47,6 +47,7 @@ TEST(BackProp,XOR) { } TEST(BackProp,XORHyperbolicTangent) { + srand(time(NULL)); NeuralNetwork::FeedForward::Network n(2); NeuralNetwork::ActivationFunction::HyperbolicTangent a(-1); n.appendLayer(2,a); @@ -56,7 +57,7 @@ TEST(BackProp,XORHyperbolicTangent) { NeuralNetwork::Learning::BackPropagation prop(n); - for(int i=0;i<10000;i++) { + for(int i=0;i<1500;i++) { prop.teach({1,0},{1}); prop.teach({1,1},{0}); prop.teach({0,0},{0}); diff --git a/tests/backpropagation_function_cmp.cpp b/tests/backpropagation_function_cmp.cpp index 2f296cb..ae75d64 100644 --- a/tests/backpropagation_function_cmp.cpp +++ b/tests/backpropagation_function_cmp.cpp @@ -45,41 +45,41 @@ int main() { std::cout << "\tLinear: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n"; + std::make_shared(),linearCoef) << "\n"; std::cout << "\tOptical: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n"; + std::make_shared(),opticalCoef) << "\n"; std::cout << "\tArcTangent: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n"; + std::make_shared(arcTangent),arcTangentCoef) << "\n"; } { std::cout << "AND:\n"; std::cout << "\tLinear: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n"; + std::make_shared(),linearCoef) << "\n"; std::cout << "\tOptical: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n"; + std::make_shared(),opticalCoef) << "\n"; std::cout << "\tArcTangent: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n"; + std::make_shared(arcTangent),arcTangentCoef) << "\n"; } { std::cout << "AND:\n"; std::cout << "\tLinear: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n"; + std::make_shared(),linearCoef) << "\n"; std::cout << "\tOptical: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n"; + std::make_shared(),opticalCoef) << "\n"; std::cout << "\tArcTangent: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n"; + std::make_shared(arcTangent),arcTangentCoef) << "\n"; } } diff --git a/tests/backpropagation_perf.cpp b/tests/backpropagation_perf.cpp index 2f31ea1..13732d1 100644 --- a/tests/backpropagation_perf.cpp +++ b/tests/backpropagation_perf.cpp @@ -4,6 +4,7 @@ #include #include "../include/NeuralNetwork/Learning/BackPropagation.h" +#include int main() { { // XOR problem NeuralNetwork::FeedForward::Network n(2); @@ -16,11 +17,18 @@ int main() { n.randomizeWeights(); NeuralNetwork::Learning::BackPropagation prop(n); +// prop.setBatchSize(20); +// prop.setMomentumWeight(0.8); + + auto start = std::chrono::system_clock::now(); for(int i=0;i<100;i++) { prop.teach({1,0},{1}); prop.teach({1,1},{0}); prop.teach({0,0},{0}); prop.teach({0,1},{1}); } + auto end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end -start; + std::cout << elapsed_seconds.count() << "\n"; } } diff --git a/tests/optical_backpropagation.cpp b/tests/optical_backpropagation.cpp deleted file mode 100644 index 037e065..0000000 --- a/tests/optical_backpropagation.cpp +++ /dev/null @@ -1,124 +0,0 @@ -#include - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Weffc++" - -#include - -#pragma GCC diagnostic pop - -TEST(OpticalBackPropagation,XOR) { - NeuralNetwork::FeedForward::Network n(2); - NeuralNetwork::ActivationFunction::Sigmoid a(-1); - n.appendLayer(2,a); - n.appendLayer(1,a); - - n.randomizeWeights(); - - NeuralNetwork::Learning::OpticalBackPropagation prop(n); - - for(int i=0;i<10000;i++) { - prop.teach({1,0},{1}); - prop.teach({1,1},{0}); - prop.teach({0,0},{0}); - prop.teach({0,1},{1}); - } - - { - std::vector ret =n.computeOutput({1,1}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({0,1}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({1,0}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({0,0}); - ASSERT_LT(ret[0], 0.1); - } -} - -TEST(OpticalBackPropagation,AND) { - NeuralNetwork::FeedForward::Network n(2); - NeuralNetwork::ActivationFunction::Sigmoid a(-1); - n.appendLayer(2,a); - n.appendLayer(1,a); - - n.randomizeWeights(); - - NeuralNetwork::Learning::OpticalBackPropagation prop(n); - - for(int i=0;i<10000;i++) { - prop.teach({1,1},{1}); - prop.teach({0,0},{0}); - prop.teach({0,1},{0}); - prop.teach({1,0},{0}); - } - - { - std::vector ret =n.computeOutput({1,1}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({0,1}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({1,0}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({0,0}); - ASSERT_LT(ret[0], 0.1); - } -} - -TEST(OpticalBackPropagation,NOTAND) { - NeuralNetwork::FeedForward::Network n(2); - NeuralNetwork::ActivationFunction::Sigmoid a(-1); - n.appendLayer(2,a); - n.appendLayer(1,a); - - n.randomizeWeights(); - - NeuralNetwork::Learning::OpticalBackPropagation prop(n); - - for(int i=0;i<10000;i++) { - prop.teach({1,1},{0}); - prop.teach({0,0},{1}); - prop.teach({0,1},{1}); - prop.teach({1,0},{1}); - } - - { - std::vector ret =n.computeOutput({1,1}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({0,1}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({1,0}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({0,0}); - ASSERT_GT(ret[0], 0.9); - } -} \ No newline at end of file diff --git a/tests/propagation_cmp.cpp b/tests/propagation_cmp.cpp index 04efd85..50fba6a 100644 --- a/tests/propagation_cmp.cpp +++ b/tests/propagation_cmp.cpp @@ -4,6 +4,7 @@ #include #include "../include/NeuralNetwork/Learning/BackPropagation.h" #include "../include/NeuralNetwork/Learning/QuickPropagation.h" +#include "../include/NeuralNetwork/Learning/RProp.h" #include "../include/NeuralNetwork/Learning/CorrectionFunction/Optical.h" #include "../include/NeuralNetwork/Learning/CorrectionFunction/ArcTangent.h" @@ -16,7 +17,8 @@ n.appendLayer(1,a);\ n.randomizeWeights();\ CLASS prop(n,FUN);\ - prop.setLearningCoefficient(COEF);\ + prop.setLearningCoefficient(COEF);\ + prop.setBatchSize(4); \ int error=1; int steps = 0; \ while(error > 0 && steps <99999) {\ steps++;\ @@ -42,36 +44,47 @@ int main() { const float arcTangent=1.5; { - std::cout << "XOR:\n"; + std::cout << "XOR Linear:\n"; std::cout << "\tBP: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + std::make_shared(),linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; std::cout << "\tQP: " << - LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, + std::make_shared(),linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + std::cout << "\tRProp: " << + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, + std::make_shared(),linearCoef,NeuralNetwork::Learning::RProp) << "\n"; } { - std::cout << "AND:\n"; + std::cout << "AND Optical:\n"; std::cout << "\tBP: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + std::make_shared(),opticalCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; std::cout << "\tQP: " << - LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + + std::cout << "\tRProp: " << + LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::RProp) << "\n"; } { - std::cout << "AND:\n"; + std::cout << "XOR Optical:\n"; std::cout << "\tBP: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + std::make_shared(),opticalCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; std::cout << "\tQP: " << - LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + + std::cout << "\tRProp: " << + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::RProp) << "\n"; } }