Merge branch 'bp'

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "lib/SimpleJSON"]
 	path = lib/SimpleJSON
 	url = http://gitlab.ishin.cz/shin/SimpleJSON.git
-[submodule "lib/EvolutionaryAlgorithm"]
-	path = lib/EvolutionaryAlgorithm
-	url = http://gitlab.ishin.cz/shin/EvolutionaryAlgorithm.git

CMakeLists.txt

@@ -61,9 +61,12 @@ endif(USE_SSE)
 set (LIBRARY_SOURCES
 	src/sse_mathfun.cpp
 
+	src/NeuralNetwork/Learning/BatchPropagation.cpp
 	src/NeuralNetwork/Learning/BackPropagation.cpp
 	src/NeuralNetwork/Learning/QuickPropagation.cpp
 	src/NeuralNetwork/Learning/PerceptronLearning.cpp
+	src/NeuralNetwork/Learning/RProp.cpp
+	src/NeuralNetwork/Learning/iRPropPlus.cpp
 
 	src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp
 	src/NeuralNetwork/ConstructiveAlgorithms/Cascade2.cpp
@@ -118,6 +121,9 @@ IF(ENABLE_TESTS)
 	add_test(quickpropagation tests/quickpropagation)
 	set_property(TEST quickpropagation PROPERTY LABELS unit)
 
+	add_test(rprop tests/rprop)
+	set_property(TEST rprop PROPERTY LABELS unit)
+
 	add_test(recurrent tests/recurrent)
 	set_property(TEST recurrent PROPERTY LABELS unit)
 
@@ -136,8 +142,5 @@ IF(ENABLE_TESTS)
 	add_test(recurrent_perf tests/recurrent_perf)
 	set_property(TEST recurrent_perf PROPERTY LABELS perf)
 
-	add_test(genetic_programing tests/genetic_programing)
-	set_property(TEST genetic_programing PROPERTY LABELS unit)
-
 ENDIF(ENABLE_TESTS)
 

include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "./ActivationFunction.h"
+
+#include <cassert>
+
+namespace NeuralNetwork {
+namespace ActivationFunction {
+
+	class LeakyRectifiedLinear: public ActivationFunction {
+		public:
+			LeakyRectifiedLinear(const float &lambdaP=0.04): lambda(lambdaP) {}
+
+			inline virtual float derivatedOutput(const float &inp,const float &) const override {
+				return inp > 0.0f ? lambda : 0.01f*lambda;
+			}
+
+			inline virtual float operator()(const float &x) const override {
+				return x > 0.0? x : 0.001f*x;
+			};
+
+			virtual ActivationFunction* clone() const override {
+				return new LeakyRectifiedLinear(lambda);
+			}
+
+			virtual SimpleJSON::Type::Object serialize() const override {
+				return {{"class", "NeuralNetwork::ActivationFunction::LeakyRectifiedLinear"}, {"lambda", lambda}};
+			}
+
+			static std::unique_ptr<LeakyRectifiedLinear> deserialize(const SimpleJSON::Type::Object &obj) {
+				return std::unique_ptr<LeakyRectifiedLinear>(new LeakyRectifiedLinear(obj["lambda"].as<double>()));
+			}
+
+		protected:
+			float lambda;
+			NEURAL_NETWORK_REGISTER_ACTIVATION_FUNCTION(NeuralNetwork::ActivationFunction::LeakyRectifiedLinear, LeakyRectifiedLinear::deserialize)
+	};
+}
+}

include/NeuralNetwork/ActivationFunction/RectifiedLinear.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "./ActivationFunction.h"
+
+#include <cassert>
+
+namespace NeuralNetwork {
+namespace ActivationFunction {
+
+	class RectifiedLinear: public ActivationFunction {
+		public:
+			RectifiedLinear(const float &lambdaP=0.1): lambda(lambdaP) {}
+
+			inline virtual float derivatedOutput(const float &inp,const float &) const override {
+				return inp > 0.0f ? lambda : 0.0f;
+			}
+
+			inline virtual float operator()(const float &x) const override {
+				return std::max(0.0f,x);
+			};
+
+			virtual ActivationFunction* clone() const override {
+				return new RectifiedLinear(lambda);
+			}
+
+			virtual SimpleJSON::Type::Object serialize() const override {
+				return {{"class", "NeuralNetwork::ActivationFunction::RectifiedLinear"}, {"lambda", lambda}};
+			}
+
+			static std::unique_ptr<RectifiedLinear> deserialize(const SimpleJSON::Type::Object &obj) {
+				return std::unique_ptr<RectifiedLinear>(new RectifiedLinear(obj["lambda"].as<double>()));
+			}
+
+		protected:
+			float lambda;
+			NEURAL_NETWORK_REGISTER_ACTIVATION_FUNCTION(NeuralNetwork::ActivationFunction::RectifiedLinear, RectifiedLinear::deserialize)
+	};
+}
+}

include/NeuralNetwork/FeedForward/Layer.h

@@ -67,6 +67,15 @@ namespace FeedForward {
 				return *neurons[neuron];
 			}
 
+			/**
+			 * @brief This is a virtual function for selecting neuron
+			 * @param neuron is position in layer
+			 * @returns Specific neuron
+			 */
+			const NeuronInterface& operator[](const std::size_t& neuron) const {
+				return *neurons[neuron];
+			}
+
 			void solve(const std::vector<float> &input, std::vector<float> &output);
 
 			/**

include/NeuralNetwork/Learning/BackPropagation.h

@@ -1,10 +1,6 @@
 #pragma once
 
-#include <vector>
+#include "BatchPropagation.h"
-#include <cmath>
-
-#include <NeuralNetwork/FeedForward/Network.h>
-#include "CorrectionFunction/Linear.h"
 
 namespace NeuralNetwork {
 namespace Learning {
@@ -12,24 +8,20 @@ namespace Learning {
 	/** @class BackPropagation
 	 * @brief 
 	 */
-	class BackPropagation {
+	class BackPropagation : public BatchPropagation {
 
 		public:
-			inline BackPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()):
+			BackPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr<CorrectionFunction::CorrectionFunction> correction = std::make_shared<CorrectionFunction::Linear>()):
-									network(feedForwardNetwork), correctionFunction(correction),learningCoefficient(0.4), slopes() {
+				BatchPropagation(feedForwardNetwork,correction), learningCoefficient(0.4) {
 				resize();
 			}
 
-			virtual ~BackPropagation() {
-				delete correctionFunction;
-			}
-
 			BackPropagation(const BackPropagation&)=delete;
 			BackPropagation& operator=(const NeuralNetwork::Learning::BackPropagation&) = delete;
 
-			void teach(const std::vector<float> &input, const std::vector<float> &output);
+			void setLearningCoefficient (const float& coefficient) {
-
+				learningCoefficient=coefficient;
-			inline virtual void setLearningCoefficient (const float& coefficient) { learningCoefficient=coefficient; }
+			}
 
 			float getMomentumWeight() const {
 				return momentumWeight;
@@ -48,75 +40,22 @@ namespace Learning {
 				weightDecay=wd;
 			}
 
-			std::size_t getBatchSize() const {
-				return batchSize;
-			}
-
-			void setBatchSize(std::size_t size) {
-				batchSize = size;
-			}
-
 		protected:
 
-			virtual inline void resize() {
+			virtual inline void resize() override {
-				if(slopes.size()!=network.size())
+				BatchPropagation::resize();
-					slopes.resize(network.size());
+				if(momentumWeight > 0.0) {
-
+					_lastDeltas = _gradients;
-				for(std::size_t i=0; i < network.size(); i++) {
-					if(slopes[i].size()!=network[i].size())
-						slopes[i].resize(network[i].size());
-				}
-
-				if(deltas.size() != network.size())
-					deltas.resize(network.size());
-
-				bool resized = false;
-
-				for(std::size_t i = 0; i < network.size(); i++) {
-					if(deltas[i].size() != network[i].size()) {
-						deltas[i].resize(network[i].size());
-						resized = true;
-
-						if(i > 0) {
-							for(std::size_t j = 0; j < deltas[i].size(); j++) {
-								deltas[i][j].resize(network[i - 1].size());
-								std::fill(deltas[i][j].begin(),deltas[i][j].end(),0.0);
-							}
-						}
-					}
-				}
-
-				if(momentumWeight > 0.0 && (resized || lastDeltas.size() != deltas.size())) {
-					lastDeltas = deltas;
 				}
 			}
 
-			virtual void computeDeltas(const std::vector<float> &input);
+			virtual void updateWeightsAndEndBatch() override;
-
-			void updateWeights();
-
-			virtual void computeSlopes(const std::vector<float> &expectation);
-
-			virtual void endBatch() {
-
-			}
-
-			FeedForward::Network &network;
-
-			CorrectionFunction::CorrectionFunction *correctionFunction;
 
 			float learningCoefficient;
-
 			float momentumWeight = 0.0;
-
 			float weightDecay = 0.0;
 
-			std::size_t batchSize = 1;
+			std::vector<std::vector<std::vector<float>>> _lastDeltas = {};
-			std::size_t currentBatchSize = 0;
-
-			std::vector<std::vector<float>> slopes;
-			std::vector<std::vector<std::vector<float>>> deltas = {};
-			std::vector<std::vector<std::vector<float>>> lastDeltas = {};
 
 	};
 }

include/NeuralNetwork/Learning/BatchPropagation.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <NeuralNetwork/FeedForward/Network.h>
+
+#include "CorrectionFunction/Linear.h"
+
+#include <vector>
+#include <memory>
+
+namespace NeuralNetwork {
+namespace Learning {
+	class BatchPropagation {
+		public:
+			BatchPropagation(FeedForward::Network &ffn, std::shared_ptr<CorrectionFunction::CorrectionFunction> correction) : _network(ffn), _correctionFunction(correction) {
+
+			}
+
+			virtual ~BatchPropagation() {
+
+			}
+
+			void teach(const std::vector<float> &input, const std::vector<float> &output);
+
+			void finishTeaching();
+
+			std::size_t getBatchSize() const {
+				return _batchSize;
+			}
+
+			void setBatchSize(std::size_t size) {
+				_batchSize = size;
+			}
+		protected:
+			virtual void updateWeightsAndEndBatch() = 0;
+			virtual void resize();
+
+			FeedForward::Network &_network;
+			std::shared_ptr<CorrectionFunction::CorrectionFunction> _correctionFunction;
+
+			std::size_t _batchSize = 1;
+			std::size_t _currentBatchSize = 0;
+
+			std::vector<std::vector<float>> _slopes = {};
+			std::vector<std::vector<std::vector<float>>> _gradients = {};
+
+			bool init = false;
+		private:
+			void computeSlopes(const std::vector<float> &expectation);
+			void computeDeltas(const std::vector<float> &input);
+	};
+}
+}

include/NeuralNetwork/Learning/OpticalBackPropagation.h

@@ -1,23 +0,0 @@
-#pragma once
-
-#include "./BackPropagation.h"
-#include "./CorrectionFunction/Optical.h"
-
-namespace NeuralNetwork {
-namespace Learning {
-
-	/** @class OpticalBackPropagation
-	 * @brief 
-	 */
-	class OpticalBackPropagation : public BackPropagation {
-
-		public:
-			OpticalBackPropagation(FeedForward::Network &feedForwardNetwork): BackPropagation(feedForwardNetwork,new CorrectionFunction::Optical()) {
-
-			}
-
-			virtual ~OpticalBackPropagation() {
-			}
-	};
-}
-}

include/NeuralNetwork/Learning/QuickPropagation.h

@@ -4,7 +4,7 @@
 #include <cmath>
 
 #include <NeuralNetwork/FeedForward/Network.h>
-#include "BackPropagation.h"
+#include "BatchPropagation.h"
 
 namespace NeuralNetwork {
 	namespace Learning {
@@ -12,49 +12,33 @@ namespace NeuralNetwork {
 		/** @class QuickPropagation
 		 * @brief
 		 */
-		class QuickPropagation : public BackPropagation {
+		class QuickPropagation : public BatchPropagation {
 
 		public:
-			inline QuickPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()):
+			inline QuickPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr<CorrectionFunction::CorrectionFunction> correction = std::make_shared<CorrectionFunction::Linear>()):
-				BackPropagation(feedForwardNetwork,correction),previousSlopes() {
+				BatchPropagation(feedForwardNetwork,correction) {
-				resize();
 			}
 
 			virtual ~QuickPropagation() {
 			}
 
-		protected:
+			void setLearningCoefficient (const float& coefficient) {
-			float _maxChange=1.75;
-			float _epsilon=0.5;
-
-			virtual inline void resize() override {
-				if(slopes.size()!=network.size())
-					slopes.resize(network.size());
-
-				for(std::size_t i=0; i < network.size(); i++) {
-					if(slopes[i].size()!=network[i].size())
-						slopes[i].resize(network[i].size());
-				}
-
-				if(deltas.size()!=network.size())
-					deltas.resize(network.size());
-
-				for(std::size_t i=0; i < network.size(); i++) {
-					if(deltas[i].size()!=network[i].size())
-						deltas[i].resize(network[i].size());
-
-					for(std::size_t j=0; j < previousSlopes[i].size(); j++) {
-						deltas[i][j]=1.0;
-					}
-				}
-				weightChange= deltas;
 			}
 
-			virtual void updateWeights(const std::vector<float> &input) override;
+			protected:
 
-			std::vector<std::vector<float>> previousSlopes ={};
+			virtual void updateWeightsAndEndBatch() override;
-			std::vector<std::vector<float>> deltas ={};
+
-			std::vector<std::vector<float>> weightChange ={};
+			float _maxChange=1.75;
+
+			virtual inline void resize() override {
+				BatchPropagation::resize();
+				_lastGradients = _gradients;
+				_lastDeltas = _gradients;
+			}
+
+			std::vector<std::vector<std::vector<float>>> _lastDeltas = {};
+			std::vector<std::vector<std::vector<float>>> _lastGradients = {};
 		};
 	}
 }

include/NeuralNetwork/Learning/RProp.h

@@ -0,0 +1,63 @@
+#pragma once
+
+
+#include "BatchPropagation.h"
+
+namespace NeuralNetwork {
+	namespace Learning {
+
+		/** @class Resilient Propagation
+		 * @brief
+		 */
+		class RProp : public BatchPropagation {
+
+			public:
+				RProp(FeedForward::Network &feedForwardNetwork, std::shared_ptr<CorrectionFunction::CorrectionFunction> correction = std::make_shared<CorrectionFunction::Linear>()):
+					BatchPropagation(feedForwardNetwork, correction) {
+				}
+
+				RProp(const RProp&)=delete;
+				RProp& operator=(const NeuralNetwork::Learning::RProp&) = delete;
+
+				void setInitialWeightChange(float initVal) {
+					initialWeightChange=initVal;
+				}
+				void setLearningCoefficient(float) {
+
+				}
+			protected:
+
+				virtual inline void resize() override {
+					BatchPropagation::resize();
+
+					_lastGradients =_gradients;
+
+					_changesOfWeightChanges = _lastGradients;
+					for(std::size_t i = 1; i < _network.size(); i++) {
+						for(std::size_t j = 0; j < _changesOfWeightChanges[i].size(); j++) {
+							std::fill(_changesOfWeightChanges[i][j].begin(),_changesOfWeightChanges[i][j].end(),initialWeightChange);
+						}
+					}
+					_lastWeightChanges = _lastGradients;
+					for(std::size_t i = 1; i < _network.size(); i++) {
+						for(std::size_t j = 0; j < _lastWeightChanges[i].size(); j++) {
+							std::fill(_lastWeightChanges[i][j].begin(),_lastWeightChanges[i][j].end(),0.1);
+						}
+					}
+				}
+
+				void updateWeightsAndEndBatch() override;
+
+				std::vector<std::vector<std::vector<float>>> _lastGradients = {};
+				std::vector<std::vector<std::vector<float>>> _lastWeightChanges = {};
+				std::vector<std::vector<std::vector<float>>> _changesOfWeightChanges = {};
+
+				float maxChangeOfWeights = 50;
+				float minChangeOfWeights = 0.0001;
+
+				float initialWeightChange=0.02;
+				float weightChangePlus=1.2;
+				float weightChangeMinus=0.5;
+		};
+	}
+}

include/NeuralNetwork/Learning/iRPropPlus.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include "BatchPropagation.h"
+
+namespace NeuralNetwork {
+	namespace Learning {
+
+		/** @class Resilient Propagation
+		 * @brief
+		 */
+		class iRPropPlus : public BatchPropagation {
+
+			public:
+				iRPropPlus(FeedForward::Network &feedForwardNetwork, std::shared_ptr<CorrectionFunction::CorrectionFunction> correction = std::make_shared<CorrectionFunction::Linear>()):
+					BatchPropagation(feedForwardNetwork, correction) {
+				}
+
+				iRPropPlus(const iRPropPlus&)=delete;
+				iRPropPlus& operator=(const NeuralNetwork::Learning::iRPropPlus&) = delete;
+
+				void setInitialWeightChange(float initVal) {
+					initialWeightChange=initVal;
+				}
+				void setLearningCoefficient(float) {
+
+				}
+			protected:
+
+				virtual inline void resize() override {
+					BatchPropagation::resize();
+
+					_lastGradients =_gradients;
+
+					_changesOfWeightChanges = _lastGradients;
+					for(std::size_t i = 1; i < _network.size(); i++) {
+						for(std::size_t j = 0; j < _changesOfWeightChanges[i].size(); j++) {
+							std::fill(_changesOfWeightChanges[i][j].begin(),_changesOfWeightChanges[i][j].end(),initialWeightChange);
+						}
+					}
+					_lastWeightChanges = _lastGradients;
+					for(std::size_t i = 1; i < _network.size(); i++) {
+						for(std::size_t j = 0; j < _lastWeightChanges[i].size(); j++) {
+							std::fill(_lastWeightChanges[i][j].begin(),_lastWeightChanges[i][j].end(),0.1);
+						}
+					}
+				}
+
+				void updateWeightsAndEndBatch() override;
+
+				std::vector<std::vector<std::vector<float>>> _lastGradients = {};
+				std::vector<std::vector<std::vector<float>>> _lastWeightChanges = {};
+				std::vector<std::vector<std::vector<float>>> _changesOfWeightChanges = {};
+
+				float _prevError=0;
+
+				float maxChangeOfWeights = 5;
+				float minChangeOfWeights = 0.0001;
+
+				float initialWeightChange=0.02;
+				float weightChangePlus=1.2;
+				float weightChangeMinus=0.5;
+		};
+	}
+}

include/NeuralNetwork/Neuron.h

@@ -108,7 +108,7 @@ namespace NeuralNetwork
 			/**
 			 * @brief getter for activation function of neuron
 			 */
-			virtual ActivationFunction::ActivationFunction& getActivationFunction() =0;
+			virtual const ActivationFunction::ActivationFunction& getActivationFunction() const =0;
 
 			virtual void setBasisFunction(const BasisFunction::BasisFunction& basisFunction) =0;
 
@@ -167,7 +167,7 @@ namespace NeuralNetwork
 				return *basis;
 			}
 
-			virtual ActivationFunction::ActivationFunction& getActivationFunction() override {
+			virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override {
 				return *activation;
 			}
 
@@ -216,7 +216,7 @@ namespace NeuralNetwork
 				throw usageException("basis function");
 			}
 
-			virtual ActivationFunction::ActivationFunction& getActivationFunction() override {
+			virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override {
 				throw usageException("biasNeuron - activation function");
 			}
 
@@ -267,7 +267,7 @@ namespace NeuralNetwork
 				throw usageException("basis function");
 			}
 
-			virtual ActivationFunction::ActivationFunction& getActivationFunction() override {
+			virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override {
 				throw usageException("input neuron - activation function");
 			}
 

src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp

@@ -1,5 +1,7 @@
 #include <NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.h>
 
+#include <NeuralNetwork/Learning/BackPropagation.h>
+
 using namespace NeuralNetwork::ConstructiveAlgorihtms;
 
 float CascadeCorrelation::trainOutputs(Cascade::Network &network, const std::vector <CascadeCorrelation::TrainingPattern> &patterns) {

src/NeuralNetwork/Learning/BackPropagation.cpp

@@ -1,89 +1,23 @@
 #include <NeuralNetwork/Learning/BackPropagation.h>
 
-#include <cassert>
+void NeuralNetwork::Learning::BackPropagation::updateWeightsAndEndBatch() {
-#include <immintrin.h>
-
-void NeuralNetwork::Learning::BackPropagation::teach(const std::vector<float> &input, const std::vector<float> &expectation) {
-	network.computeOutput(input);
-	resize();
-	computeSlopes(expectation);
-
-	computeDeltas(input);
-	if(++currentBatchSize >= batchSize) {
-		updateWeights();
-		endBatch();
-		currentBatchSize=0;
-	}
-}
-
-void NeuralNetwork::Learning::BackPropagation::computeSlopes(const std::vector<float> &expectation) {
-	auto& outputLayer=network[network.size()-1];
-	for(std::size_t j=1;j<outputLayer.size();j++) {
-		auto& neuron = outputLayer[j];
-		slopes[network.size()-1][j]=correctionFunction->operator()( expectation[j-1], neuron.output())*
-									neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output());
-	}
-
-	for(int layerIndex=static_cast<int>(network.size()-2);layerIndex>0;layerIndex--) {
-		auto &layer=network[layerIndex];
-
-		for(std::size_t j=1;j<layer.size();j++) {
-			float deltasWeight = 0;
-
-			for(std::size_t k=1;k<network[layerIndex+1].size();k++) {
-				deltasWeight+=slopes[layerIndex+1][k]* network[layerIndex+1][k].weight(j);
-			}
-
-			slopes[layerIndex][j]=deltasWeight*layer[j].getActivationFunction().derivatedOutput(layer[j].value(),layer[j].output());
-		}
-	}
-}
-
-void NeuralNetwork::Learning::BackPropagation::computeDeltas(const std::vector<float> &input) {
-	for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
-		auto &layer=network[layerIndex];
-		auto &prevLayer=network[layerIndex-1];
-
-		std::size_t prevLayerSize=prevLayer.size();
-		std::size_t layerSize=layer.size();
-
-		for(std::size_t j=1;j<layerSize;j++)  {
-			float update = slopes[layerIndex][j];
-			for(std::size_t k=0;k<prevLayerSize;k++) {
-				float inputValue = 0.0;
-				if(layerIndex==1 && k!=0) {
-					inputValue = input[k-1];
-				} else {
-					inputValue= prevLayer[k].output();
-				}
-				if(currentBatchSize == 0) {
-					deltas[layerIndex][j][k] = update * inputValue;
-				} else {
-					deltas[layerIndex][j][k] += update * inputValue;
-				}
-			}
-		}
-	}
-}
-
-void NeuralNetwork::Learning::BackPropagation::updateWeights() {
 
 	bool enableMoments = momentumWeight > 0.0;
 
-	for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
+	for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) {
-		auto &layer = network[layerIndex];
+		auto &layer = _network[layerIndex];
-		auto &prevLayer = network[layerIndex - 1];
+		auto &prevLayer = _network[layerIndex - 1];
 
 		std::size_t prevLayerSize = prevLayer.size();
 		std::size_t layerSize = layer.size();
 
 		for(std::size_t j = 1; j < layerSize; j++) {
 			for(std::size_t k = 0; k < prevLayerSize; k++) {
-				float delta = deltas[layerIndex][j][k]*learningCoefficient - weightDecay * layer[j].weight(k);
+				float delta = _gradients[layerIndex][j][k]*learningCoefficient - weightDecay * layer[j].weight(k);
 
 				if(enableMoments) {
-					delta += momentumWeight * lastDeltas[layerIndex][j][k];
+					delta += momentumWeight * _lastDeltas[layerIndex][j][k];
-					lastDeltas[layerIndex][j][k]=delta;
+					_lastDeltas[layerIndex][j][k]=delta;
 				}
 
 				layer[j].weight(k)+= delta;

src/NeuralNetwork/Learning/BatchPropagation.cpp

@@ -0,0 +1,92 @@
+#include <NeuralNetwork/Learning/BatchPropagation.h>
+
+void NeuralNetwork::Learning::BatchPropagation::teach(const std::vector<float> &input, const std::vector<float> &expectation) {
+	_network.computeOutput(input);
+	if(!init) {
+		resize();
+		init = true;
+	}
+
+	computeSlopes(expectation);
+
+	computeDeltas(input);
+	if(++_currentBatchSize >= _batchSize) {
+		finishTeaching();
+	}
+}
+
+void NeuralNetwork::Learning::BatchPropagation::finishTeaching() {
+	updateWeightsAndEndBatch();
+	_currentBatchSize=0;
+}
+
+void NeuralNetwork::Learning::BatchPropagation::computeSlopes(const std::vector<float> &expectation) {
+	const auto& outputLayer=_network[_network.size()-1];
+	for(std::size_t j=1;j<outputLayer.size();j++) {
+		const auto& neuron = outputLayer[j];
+		_slopes[_network.size()-1][j]=_correctionFunction->operator()( expectation[j-1], neuron.output())*
+									neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output());
+	}
+
+	for(int layerIndex=static_cast<int>(_network.size()-2);layerIndex>0;layerIndex--) {
+		auto &layer=_network[layerIndex];
+
+		for(std::size_t j=1;j<layer.size();j++) {
+			float deltasWeight = 0;
+
+			for(std::size_t k=1;k<_network[layerIndex+1].size();k++) {
+				deltasWeight+=_slopes[layerIndex+1][k]* _network[layerIndex+1][k].weight(j);
+			}
+
+			_slopes[layerIndex][j]=deltasWeight*layer[j].getActivationFunction().derivatedOutput(layer[j].value(),layer[j].output());
+		}
+	}
+}
+
+void NeuralNetwork::Learning::BatchPropagation::computeDeltas(const std::vector<float> &input) {
+	for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) {
+		auto &layer=_network[layerIndex];
+		auto &prevLayer=_network[layerIndex-1];
+
+		std::size_t prevLayerSize=prevLayer.size();
+		std::size_t layerSize=layer.size();
+
+		for(std::size_t j=1;j<layerSize;j++)  {
+			float update = _slopes[layerIndex][j];
+			for(std::size_t k=0;k<prevLayerSize;k++) {
+				float inputValue = 0.0;
+				if(layerIndex==1 && k!=0) {
+					inputValue = input[k-1];
+				} else {
+					inputValue= prevLayer[k].output();
+				}
+				if(_currentBatchSize == 0) {
+					_gradients[layerIndex][j][k] = update * inputValue;
+				} else {
+					_gradients[layerIndex][j][k] += update * inputValue;
+				}
+			}
+		}
+	}
+}
+
+void NeuralNetwork::Learning::BatchPropagation::resize() {
+	_slopes.resize(_network.size());
+
+	for(std::size_t i=0; i < _network.size(); i++) {
+		_slopes[i].resize(_network[i].size());
+	}
+
+	_gradients.resize(_network.size());
+
+	for(std::size_t i = 0; i < _network.size(); i++) {
+		_gradients[i].resize(_network[i].size());
+		if(i > 0) {
+			for(std::size_t j = 0; j < _gradients[i].size(); j++) {
+				_gradients[i][j].resize(_network[i - 1].size());
+				std::fill(_gradients[i][j].begin(), _gradients[i][j].end(), 0.0);
+			}
+		}
+	}
+
+}

src/NeuralNetwork/Learning/QuickPropagation.cpp

@@ -1,53 +1,46 @@
 #include <NeuralNetwork/Learning/QuickPropagation.h>
 
-#include <cassert>
+void NeuralNetwork::Learning::QuickPropagation::updateWeightsAndEndBatch() {
-#include <immintrin.h>
 
-void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector<float> &input) {
+	float shrinkFactor=_maxChange/(_maxChange+1.0f);
 
-	float shrinkFactor=_maxChange/(_maxChange+1.0);
+	for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) {
+		auto &layer = _network[layerIndex];
+		auto &prevLayer = _network[layerIndex - 1];
 
-	for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
+		std::size_t prevLayerSize = prevLayer.size();
-		auto &layer=network[layerIndex];
+		std::size_t layerSize = layer.size();
-		auto &prevLayer=network[layerIndex-1];
 
-		std::size_t prevLayerSize=prevLayer.size();
+		for(std::size_t j = 1; j < layerSize; j++) {
-		std::size_t layerSize=layer.size();
+			for(std::size_t k = 0; k < prevLayerSize; k++) {
+				float newChange = 0.0f;
+				float _epsilon = 0.9;
+				if(fabs (_gradients[layerIndex][j][k])> 0.0001) {
+					if(std::signbit(_gradients[layerIndex][j][k]) == std::signbit(_lastGradients[layerIndex][j][k])) {
+						newChange+= _gradients[layerIndex][j][k]*_epsilon;
 
-		for(std::size_t j=1;j<layerSize;j++)  {
+						if(fabs(_gradients[layerIndex][j][k]) > fabs(shrinkFactor * _lastGradients[layerIndex][j][k])) {
-
+							newChange += _maxChange * _gradients[layerIndex][j][k];
-			float newChange=0;
+						}else {
-
+							newChange+=_gradients[layerIndex][j][k]/(_lastGradients[layerIndex][j][k]-_gradients[layerIndex][j][k]) * _lastDeltas[layerIndex][j][k];
-			if(fabs (deltas[layerIndex][j])> 0.0001) {
+						}
-				if(std::signbit(deltas[layerIndex][j]) == std::signbit(slopes[layerIndex][j])) {
+					} else {
-					newChange+= slopes[layerIndex][j]*_epsilon;
+						newChange+=_gradients[layerIndex][j][k]/(_lastGradients[layerIndex][j][k]-_gradients[layerIndex][j][k]) * _lastDeltas[layerIndex][j][k];
-
-					if(fabs(slopes[layerIndex][j]) > fabs(shrinkFactor * previousSlopes[layerIndex][j])) {
-						newChange += _maxChange * deltas[layerIndex][j];
-					}else {
-						newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * deltas[layerIndex][j];
 					}
 				} else {
-					newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * deltas[layerIndex][j];
+					newChange+= _lastGradients[layerIndex][j][k]*_epsilon;
 				}
-			} else {
+				_lastDeltas[layerIndex][j][k]= newChange;
-				newChange+= slopes[layerIndex][j]*_epsilon;
+				layer[j].weight(k)+= newChange;
-			}
 
-			weightChange[layerIndex][j]=newChange;
+	/*          This is according to paper?
-
+	//			delta = _gradients[layerIndex][j][k] / (_lastGradients[layerIndex][j][k]-_gradients[layerIndex][j][k]) * _lastDeltas[layerIndex][j][k];
-			layer[j].weight(0)+=newChange;
+	//			delta = std::min(_maxChange,delta);
-
+				_lastDeltas[layerIndex][j][k] = delta;
-			for(std::size_t k=1;k<prevLayerSize;k++) {
+				layer[j].weight(k)+= delta;
-				if(layerIndex==1) {
+	 */
-					layer[j].weight(k)+=newChange*(input[k-1]);
-				} else {
-					layer[j].weight(k)+=newChange*(prevLayer[k].output());
-				}
 			}
 		}
 	}
-
+	_lastGradients.swap(_gradients);
-	slopes.swap(previousSlopes);
-	weightChange.swap(deltas);
 }

src/NeuralNetwork/Learning/RProp.cpp

@@ -0,0 +1,39 @@
+#include <NeuralNetwork/Learning/RProp.h>
+
+void NeuralNetwork::Learning::RProp::updateWeightsAndEndBatch() {
+
+	for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) {
+		auto &layer = _network[layerIndex];
+		auto &prevLayer = _network[layerIndex - 1];
+
+		std::size_t prevLayerSize = prevLayer.size();
+		std::size_t layerSize = layer.size();
+
+		for(std::size_t j = 1; j < layerSize; j++) {
+			for(std::size_t k = 0; k < prevLayerSize; k++) {
+				float gradient = _gradients[layerIndex][j][k];
+				float lastGradient = _lastGradients[layerIndex][j][k];
+
+				_lastGradients[layerIndex][j][k] = gradient;
+
+				float weightChangeDelta = _lastWeightChanges[layerIndex][j][k];
+
+				if(gradient * lastGradient > 0) {
+					weightChangeDelta = std::min(weightChangeDelta*weightChangePlus,maxChangeOfWeights);
+				} else if (gradient * lastGradient < 0) {
+					weightChangeDelta = std::max(weightChangeDelta*weightChangeMinus,minChangeOfWeights);
+				} else {
+					weightChangeDelta = _lastWeightChanges[layerIndex][j][k];
+				}
+
+				_lastWeightChanges[layerIndex][j][k] = weightChangeDelta;
+
+				if(gradient > 0) {
+					layer[j].weight(k) += weightChangeDelta;
+				} else if (gradient < 0){
+					layer[j].weight(k) -= weightChangeDelta;
+				}
+			}
+		}
+	}
+}

src/NeuralNetwork/Learning/iRPropPlus.cpp

@@ -0,0 +1,52 @@
+#include <NeuralNetwork/Learning/iRPropPlus.h>
+
+void NeuralNetwork::Learning::iRPropPlus::updateWeightsAndEndBatch() {
+	float error = 0.0;
+	const auto& outputLayer=_network[_network.size()-1];
+	for(std::size_t j=1;j<outputLayer.size();j++) {
+		error+=_slopes[_network.size()-1][j];
+	}
+
+	error /= outputLayer.size();
+
+	for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) {
+		auto &layer = _network[layerIndex];
+		auto &prevLayer = _network[layerIndex - 1];
+
+		std::size_t prevLayerSize = prevLayer.size();
+		std::size_t layerSize = layer.size();
+
+		for(std::size_t j = 1; j < layerSize; j++) {
+			for(std::size_t k = 0; k < prevLayerSize; k++) {
+				float gradient = _gradients[layerIndex][j][k];
+				float lastGradient = _lastGradients[layerIndex][j][k];
+
+				_lastGradients[layerIndex][j][k] = gradient;
+
+				float weightChangeDelta = _changesOfWeightChanges[layerIndex][j][k];
+				float delta;
+
+				if(gradient * lastGradient > 0) {
+					weightChangeDelta = std::min(weightChangeDelta*weightChangePlus,maxChangeOfWeights);
+					delta = (std::signbit(gradient)? 1.0f : -1.0f ) * weightChangeDelta;
+					layer[j].weight(k) -= delta;
+				} else if (gradient * lastGradient < 0) {
+					weightChangeDelta = std::max(weightChangeDelta*weightChangeMinus,minChangeOfWeights);
+					delta = _lastWeightChanges[layerIndex][j][k];
+					if(error > _prevError) {
+						layer[j].weight(k) += delta;
+					}
+					_lastGradients[layerIndex][j][k] = 0;
+				} else {
+					delta = (std::signbit(gradient)? 1.0f : -1.0f ) * weightChangeDelta;
+					layer[j].weight(k) -= delta;
+				}
+				//std::cout << delta <<"\n";
+
+				_changesOfWeightChanges[layerIndex][j][k] = weightChangeDelta;
+				_lastWeightChanges[layerIndex][j][k] = delta;
+			}
+		}
+	}
+	_prevError = error;
+}

tests/CMakeLists.txt

@@ -13,9 +13,6 @@ target_link_libraries(backpropagation NeuralNetwork gtest gtest_main)
 add_executable(feedforward feedforward.cpp)
 target_link_libraries(feedforward NeuralNetwork gtest gtest_main)
 
-add_executable(optical_backpropagation optical_backpropagation.cpp)
-target_link_libraries(optical_backpropagation NeuralNetwork gtest gtest_main)
-
 add_executable(perceptron perceptron.cpp)
 target_link_libraries(perceptron NeuralNetwork gtest gtest_main)
 
@@ -28,6 +25,9 @@ target_link_libraries(recurrent NeuralNetwork gtest gtest_main)
 add_executable(quickpropagation quickpropagation.cpp)
 target_link_libraries(quickpropagation NeuralNetwork gtest gtest_main)
 
+add_executable(rprop rprop.cpp)
+target_link_libraries(rprop NeuralNetwork gtest gtest_main)
+
 # PERF
 
 add_executable(backpropagation_function_cmp backpropagation_function_cmp.cpp)

tests/activation.cpp

@@ -48,20 +48,6 @@ TEST(Sigmoid, ParamMinusFive) {
 	ASSERT_LT(s(0.7), 0.970788);
 }
 
-TEST(SigmoidSSE, ParamMinusZeroPointSeven) {
-	NeuralNetwork::ActivationFunction::Sigmoid s(-0.7);
-	SSE comp;
-	comp.floats[0] = 0.1;
-	comp.floats[1] = 10;
-	comp.sse = s(comp.sse);
-
-	ASSERT_GT(comp.floats[0], 0.517483);
-	ASSERT_LT(comp.floats[0], 0.51750);
-
-	ASSERT_GT(comp.floats[1], 0.998989);
-	ASSERT_LT(comp.floats[1], 0.999189);
-}
-
 TEST(Linear, ParamOne) {
 	NeuralNetwork::ActivationFunction::Linear s(1.0);
 	ASSERT_GT(s(0.5), 0.4999);

tests/backpropagation.cpp

@@ -47,6 +47,7 @@ TEST(BackProp,XOR) {
 }
 
 TEST(BackProp,XORHyperbolicTangent) {
+	srand(time(NULL));
 	NeuralNetwork::FeedForward::Network n(2);
 	NeuralNetwork::ActivationFunction::HyperbolicTangent a(-1);
 	n.appendLayer(2,a);
@@ -56,7 +57,7 @@ TEST(BackProp,XORHyperbolicTangent) {
 
 	NeuralNetwork::Learning::BackPropagation prop(n);
 
-	for(int i=0;i<10000;i++) {
+	for(int i=0;i<1500;i++) {
 		prop.teach({1,0},{1});
 		prop.teach({1,1},{0});
 		prop.teach({0,0},{0});

tests/backpropagation_function_cmp.cpp

@@ -45,41 +45,41 @@ int main() {
 
 		std::cout << "\tLinear: " <<
 		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
-			  new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef) << "\n";
 
 		std::cout << "\tOptical: " <<
 		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
-			  new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef) << "\n";
 		std::cout << "\tArcTangent: " <<
 		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
-			  new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::ArcTangent>(arcTangent),arcTangentCoef) << "\n";
 	}
 	{
 		std::cout << "AND:\n";
 		std::cout << "\tLinear: " <<
 		LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
-			  new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef) << "\n";
 
 		std::cout << "\tOptical: " <<
 		LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
-			  new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef) << "\n";
 
 		std::cout << "\tArcTangent: " <<
 		LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
-			  new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::ArcTangent>(arcTangent),arcTangentCoef) << "\n";
 	}
 	{
 		std::cout << "AND:\n";
 		std::cout << "\tLinear: " <<
 		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
-			  new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef) << "\n";
 
 		std::cout << "\tOptical: " <<
 		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
-			  new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef) << "\n";
 
 		std::cout << "\tArcTangent: " <<
 		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
-			  new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::ArcTangent>(arcTangent),arcTangentCoef) << "\n";
 	}
 }

tests/backpropagation_perf.cpp

@@ -4,6 +4,7 @@
 #include <iostream>
 #include "../include/NeuralNetwork/Learning/BackPropagation.h"
 
+#include <chrono>
 int main() {
 	{	// XOR problem
 		NeuralNetwork::FeedForward::Network n(2);
@@ -16,11 +17,18 @@ int main() {
 		n.randomizeWeights();
 
 		NeuralNetwork::Learning::BackPropagation prop(n);
+//		prop.setBatchSize(20);
+// 		prop.setMomentumWeight(0.8);
+
+		auto start = std::chrono::system_clock::now();
 		for(int i=0;i<100;i++) {
 			prop.teach({1,0},{1});
 			prop.teach({1,1},{0});
 			prop.teach({0,0},{0});
 			prop.teach({0,1},{1});
 		}
+		auto end = std::chrono::system_clock::now();
+		std::chrono::duration<double> elapsed_seconds = end -start;
+		std::cout << elapsed_seconds.count() << "\n";
 	}
 }

tests/optical_backpropagation.cpp

@@ -1,124 +0,0 @@
-#include <NeuralNetwork/FeedForward/Network.h>
-
-#include <NeuralNetwork/Learning/OpticalBackPropagation.h>
-
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Weffc++"
-
-#include <gtest/gtest.h>
-
-#pragma GCC diagnostic pop
-
-TEST(OpticalBackPropagation,XOR) {
-	NeuralNetwork::FeedForward::Network n(2);
-	NeuralNetwork::ActivationFunction::Sigmoid a(-1);
-	n.appendLayer(2,a);
-	n.appendLayer(1,a);
-
-	n.randomizeWeights();
-
-	NeuralNetwork::Learning::OpticalBackPropagation prop(n);
-
-	for(int i=0;i<10000;i++) {
-		prop.teach({1,0},{1});
-		prop.teach({1,1},{0});
-		prop.teach({0,0},{0});
-		prop.teach({0,1},{1});
-	}
-
-	{
-		std::vector<float> ret =n.computeOutput({1,1});
-		ASSERT_LT(ret[0], 0.1);
-	}
-
-	{
-		std::vector<float> ret =n.computeOutput({0,1});
-		ASSERT_GT(ret[0], 0.9);
-	}
-
-	{
-		std::vector<float> ret =n.computeOutput({1,0});
-		ASSERT_GT(ret[0], 0.9);
-	}
-
-	{
-		std::vector<float> ret =n.computeOutput({0,0});
-		ASSERT_LT(ret[0], 0.1);
-	}
-}
-
-TEST(OpticalBackPropagation,AND) {
-	NeuralNetwork::FeedForward::Network n(2);
-	NeuralNetwork::ActivationFunction::Sigmoid a(-1);
-	n.appendLayer(2,a);
-	n.appendLayer(1,a);
-
-	n.randomizeWeights();
-
-	NeuralNetwork::Learning::OpticalBackPropagation prop(n);
-
-	for(int i=0;i<10000;i++) {
-		prop.teach({1,1},{1});
-		prop.teach({0,0},{0});
-		prop.teach({0,1},{0});
-		prop.teach({1,0},{0});
-	}
-
-	{
-	std::vector<float> ret =n.computeOutput({1,1});
-	ASSERT_GT(ret[0], 0.9);
-	}
-
-	{
-	std::vector<float> ret =n.computeOutput({0,1});
-	ASSERT_LT(ret[0], 0.1);
-	}
-
-	{
-	std::vector<float> ret =n.computeOutput({1,0});
-	ASSERT_LT(ret[0], 0.1);
-	}
-
-	{
-	std::vector<float> ret =n.computeOutput({0,0});
-	ASSERT_LT(ret[0], 0.1);
-	}
-}
-
-TEST(OpticalBackPropagation,NOTAND) {
-	NeuralNetwork::FeedForward::Network n(2);
-	NeuralNetwork::ActivationFunction::Sigmoid a(-1);
-	n.appendLayer(2,a);
-	n.appendLayer(1,a);
-
-	n.randomizeWeights();
-
-	NeuralNetwork::Learning::OpticalBackPropagation prop(n);
-
-	for(int i=0;i<10000;i++) {
-		prop.teach({1,1},{0});
-		prop.teach({0,0},{1});
-		prop.teach({0,1},{1});
-		prop.teach({1,0},{1});
-	}
-
-	{
-	std::vector<float> ret =n.computeOutput({1,1});
-	ASSERT_LT(ret[0], 0.1);
-	}
-
-	{
-	std::vector<float> ret =n.computeOutput({0,1});
-	ASSERT_GT(ret[0], 0.9);
-	}
-
-	{
-	std::vector<float> ret =n.computeOutput({1,0});
-	ASSERT_GT(ret[0], 0.9);
-	}
-
-	{
-	std::vector<float> ret =n.computeOutput({0,0});
-	ASSERT_GT(ret[0], 0.9);
-	}
-}

tests/propagation_cmp.cpp

@@ -4,6 +4,7 @@
 #include <iostream>
 #include "../include/NeuralNetwork/Learning/BackPropagation.h"
 #include "../include/NeuralNetwork/Learning/QuickPropagation.h"
+#include "../include/NeuralNetwork/Learning/RProp.h"
 #include "../include/NeuralNetwork/Learning/CorrectionFunction/Optical.h"
 #include "../include/NeuralNetwork/Learning/CorrectionFunction/ArcTangent.h"
 
@@ -16,7 +17,8 @@
 	n.appendLayer(1,a);\
 	n.randomizeWeights();\
 	CLASS prop(n,FUN);\
-	prop.setLearningCoefficient(COEF);\
+ 	prop.setLearningCoefficient(COEF);\
+	prop.setBatchSize(4); \
 	int error=1; int steps = 0; \
 	while(error > 0 && steps <99999) {\
 		steps++;\
@@ -42,36 +44,47 @@ int main() {
 	const float arcTangent=1.5;
 
 	{
-		std::cout << "XOR:\n";
+		std::cout << "XOR Linear:\n";
 
 		std::cout << "\tBP: " <<
 					 LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
-						   new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
+						   std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
 		std::cout << "\tQP: " <<
-		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
+				  LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
-			  new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
+						std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
+		std::cout << "\tRProp: " <<
+				  LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
+						std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Linear>(),linearCoef,NeuralNetwork::Learning::RProp) << "\n";
 
 	}
 	{
-		std::cout << "AND:\n";
+		std::cout << "AND Optical:\n";
 
 		std::cout << "\tBP: " <<
 		LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
-			  new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
 
 		std::cout << "\tQP: " <<
-		LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
+				  LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
-			  new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
+						std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
+
+		std::cout << "\tRProp: " <<
+				  LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
+						std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::RProp) << "\n";
 	}
 	{
-		std::cout << "AND:\n";
+		std::cout << "XOR Optical:\n";
 
 		std::cout << "\tBP: " <<
 		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
-			  new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
+			  std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
 
 		std::cout << "\tQP: " <<
-		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
+				  LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
-			  new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
+						std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
+
+		std::cout << "\tRProp: " <<
+				  LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
+						std::make_shared<NeuralNetwork::Learning::CorrectionFunction::Optical>(),opticalCoef,NeuralNetwork::Learning::RProp) << "\n";
 	}
 }

tests/rprop.cpp

@@ -0,0 +1,165 @@
+#include <NeuralNetwork/FeedForward/Network.h>
+#include <NeuralNetwork/Learning/RProp.h>
+#include <NeuralNetwork/ActivationFunction/HyperbolicTangent.h>
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Weffc++"
+
+#include <gtest/gtest.h>
+
+#pragma GCC diagnostic pop
+TEST(RProp,XOR) {
+	NeuralNetwork::FeedForward::Network n(2);
+	NeuralNetwork::ActivationFunction::Sigmoid a(-1);
+	n.appendLayer(3,a);
+	n.appendLayer(1,a);
+
+	n.randomizeWeights();
+
+	NeuralNetwork::Learning::RProp prop(n);
+	prop.setBatchSize(4);
+
+	for(int i=0;i<100;i++) {
+		prop.teach({1,0},{1});
+		prop.teach({1,1},{0});
+		prop.teach({0,0},{0});
+		prop.teach({0,1},{1});
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({1,1});
+		ASSERT_LT(ret[0], 0.1);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({0,1});
+		ASSERT_GT(ret[0], 0.9);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({1,0});
+		ASSERT_GT(ret[0], 0.9);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({0,0});
+		ASSERT_LT(ret[0], 0.1);
+	}
+}
+
+TEST(RProp,XORHyperbolicTangent) {
+	srand(time(NULL));
+	NeuralNetwork::FeedForward::Network n(2);
+	NeuralNetwork::ActivationFunction::HyperbolicTangent a(-1);
+	n.appendLayer(2,a);
+	n.appendLayer(1,a);
+
+	n.randomizeWeights();
+
+	NeuralNetwork::Learning::RProp prop(n);
+	prop.setBatchSize(4);
+
+	for(int i=0;i<15000;i++) {
+		prop.teach({1,0},{1});
+		prop.teach({1,1},{0});
+		prop.teach({0,0},{0});
+		prop.teach({0,1},{1});
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({1,1});
+		ASSERT_LT(ret[0], 0.1);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({0,1});
+		ASSERT_GT(ret[0], 0.9);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({1,0});
+		ASSERT_GT(ret[0], 0.9);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({0,0});
+		ASSERT_LT(ret[0], 0.1);
+	}
+}
+
+TEST(RProp,AND) {
+	NeuralNetwork::FeedForward::Network n(2);
+	NeuralNetwork::ActivationFunction::Sigmoid a(-1);
+	n.appendLayer(1,a);
+
+	n.randomizeWeights();
+
+	NeuralNetwork::Learning::RProp prop(n);
+	prop.setBatchSize(4);
+
+	for(int i=0;i<100000;i++) {
+		prop.teach({1,1},{1});
+		prop.teach({0,0},{0});
+		prop.teach({0,1},{0});
+		prop.teach({1,0},{0});
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({1,1});
+		ASSERT_GT(ret[0], 0.9);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({0,1});
+		ASSERT_LT(ret[0], 0.1);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({1,0});
+		ASSERT_LT(ret[0], 0.1);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({0,0});
+		ASSERT_LT(ret[0], 0.1);
+	}
+}
+
+TEST(RProp,NOTAND) {
+	NeuralNetwork::FeedForward::Network n(2);
+	NeuralNetwork::ActivationFunction::Sigmoid a(-1);
+	n.appendLayer(2,a);
+	n.appendLayer(1,a);
+
+	n.randomizeWeights();
+
+	NeuralNetwork::Learning::RProp prop(n);
+	prop.setBatchSize(4);
+
+	for(int i=0;i<100000;i++) {
+		prop.teach({1,1},{0});
+		prop.teach({0,0},{1});
+		prop.teach({0,1},{1});
+		prop.teach({1,0},{1});
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({1,1});
+		ASSERT_LT(ret[0], 0.1);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({0,1});
+		ASSERT_GT(ret[0], 0.9);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({1,0});
+		ASSERT_GT(ret[0], 0.9);
+	}
+
+	{
+		std::vector<float> ret =n.computeOutput({0,0});
+		ASSERT_GT(ret[0], 0.9);
+	}
+}