quickProapagtion and tests

CMakeLists.txt

@@ -6,15 +6,15 @@ include(cmake/CPUFeatures)
 OPTION(BUILD_SHARED_LIBS "Build also shared library." ON)
 
 if(CPU_AVX_AVAILABLE)
-    SET(USE_AVX ON CACHE BOOL "")
+	SET(USE_AVX ON CACHE BOOL "")
 endif(CPU_AVX_AVAILABLE)
 
 if(CPU_SSE3_AVAILABLE)
-    SET(USE_SSE ON CACHE BOOL "")
+	SET(USE_SSE ON CACHE BOOL "")
 
-    if(CPU_SSE4.2_AVAILABLE)
+	if(CPU_SSE4.2_AVAILABLE)
-        SET(USE_SSE2 OFF CACHE BOOL "")
+		SET(USE_SSE2 OFF CACHE BOOL "")
-    endif(CPU_SSE4.2_AVAILABLE)
+	endif(CPU_SSE4.2_AVAILABLE)
 
 endif(CPU_SSE3_AVAILABLE)
 
@@ -33,45 +33,29 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++14")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -mtune=native -O3")
 
 if(USE_AVX)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx -DUSE_AVX")
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx -DUSE_AVX")
 endif(USE_AVX)
 
 if(USE_SSE)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -DUSE_SSE")
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -DUSE_SSE")
-    if(USE_SSE2)
+	if(USE_SSE2)
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_SSE2")
+		set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_SSE2")
-    endif(USE_SSE2)
+	endif(USE_SSE2)
 endif(USE_SSE)
 
 include_directories(./include/)
 
 set (LIBRARY_SOURCES
-        include/NeuralNetwork/ActivationFunction/ActivationFunction.h
+	src/sse_mathfun.cpp
-        include/NeuralNetwork/ActivationFunction/Heaviside.h
+
-        include/NeuralNetwork/ActivationFunction/HyperbolicTangent.h
+	src/NeuralNetwork/Learning/BackPropagation.cpp
-        include/NeuralNetwork/ActivationFunction/Linear.h
+	src/NeuralNetwork/Learning/QuickPropagation.cpp
-        include/NeuralNetwork/ActivationFunction/Sigmoid.h
+
-        include/NeuralNetwork/ActivationFunction/StreamingActivationFunction.h
+	src/NeuralNetwork/BasisFunction/Linear.cpp
-        include/NeuralNetwork/BasisFunction/BasisFunction.h
+	src/NeuralNetwork/FeedForward/Layer.cpp
-        include/NeuralNetwork/BasisFunction/Linear.h
+	src/NeuralNetwork/FeedForward/Network.cpp
-        include/NeuralNetwork/BasisFunction/Product.h
+	src/NeuralNetwork/Recurrent/Network.cpp
-        include/NeuralNetwork/BasisFunction/Radial.h
+	src/NeuralNetwork/Neuron.cpp
-        include/NeuralNetwork/BasisFunction/StreamingBasisFunction.h
-        include/NeuralNetwork/FeedForward/Layer.h
-        include/NeuralNetwork/FeedForward/Network.h
-        include/NeuralNetwork/Recurrent/Network.h
-        include/NeuralNetwork/Network.h
-        include/NeuralNetwork/Neuron.h
-        include/NeuralNetwork/Stringifiable.h
-        include/NeuralNetwork/Stringifiable.h
-        src/NeuralNetwork/Learning/BackPropagation.cpp include/NeuralNetwork/Learning/BackPropagation.h
-        include/sse_mathfun.h
-        src/NeuralNetwork/BasisFunction/Linear.cpp
-        src/NeuralNetwork/FeedForward/Layer.cpp
-        src/NeuralNetwork/FeedForward/Network.cpp
-        src/NeuralNetwork/Recurrent/Network.cpp
-        src/NeuralNetwork/Neuron.cpp
-        src/sse_mathfun.cpp
 )
 
 add_library(NeuralNetwork STATIC ${LIBRARY_SOURCES})
@@ -79,8 +63,8 @@ add_library(NeuralNetwork STATIC ${LIBRARY_SOURCES})
 link_libraries(NeuralNetwork pthread)
 
 IF(BUILD_SHARED_LIBS)
-    add_library(NeuralNetworkShared SHARED ${LIBRARY_SOURCES})
+	add_library(NeuralNetworkShared SHARED ${LIBRARY_SOURCES})
-    set_target_properties(NeuralNetworkShared PROPERTIES OUTPUT_NAME NeuralNetwork)
+	set_target_properties(NeuralNetworkShared PROPERTIES OUTPUT_NAME NeuralNetwork)
 ENDIF(BUILD_SHARED_LIBS)
 
 IF(ENABLE_TESTS)
@@ -106,15 +90,20 @@ set_property(TEST feedforward PROPERTY LABELS unit)
 add_test(recurrent tests/recurrent)
 set_property(TEST recurrent PROPERTY LABELS unit)
 
-add_test(feedforward_perf tests/feedforward_perf)
-set_property(TEST feedforward_perf PROPERTY LABELS perf)
-
 add_test(optical_backpropagation tests/optical_backpropagation)
 set_property(TEST optical_backpropagation PROPERTY LABELS unit)
 
+add_test(quickpropagation tests/quickpropagation)
+set_property(TEST quickpropagation PROPERTY LABELS unit)
 
 
 
+add_test(feedforward_perf tests/feedforward_perf)
+set_property(TEST feedforward_perf PROPERTY LABELS perf)
+
+add_test(quickpropagation_perf tests/quickpropagation_perf)
+set_property(TEST quickpropagation_perf PROPERTY LABELS perf)
+
 add_test(backpropagation_perf tests/backpropagation_perf)
 set_property(TEST backpropagation_perf PROPERTY LABELS perf)
 

include/NeuralNetwork/Learning/BackPropagation.h

@@ -33,7 +33,7 @@ namespace Learning {
 
 		protected:
 
-			inline void resize() {
+			virtual inline void resize() {
 				if(deltas.size()!=network.size())
 					deltas.resize(network.size());
 
@@ -43,6 +43,8 @@ namespace Learning {
 				}
 			}
 
+			virtual void updateWeights(const std::vector<float> &input);
+
 			FeedForward::Network &network;
 
 			CorrectionFunction::CorrectionFunction *correctionFunction;

include/NeuralNetwork/Learning/QuickPropagation.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <vector>
+#include <cmath>
+
+#include <NeuralNetwork/FeedForward/Network.h>
+#include "BackPropagation.h"
+
+namespace NeuralNetwork {
+	namespace Learning {
+
+		/** @class QuickPropagation
+		 * @brief
+		 */
+		class QuickPropagation : public BackPropagation {
+
+		public:
+			inline QuickPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()):
+				BackPropagation(feedForwardNetwork,correction),deltasPrev() {
+				resize();
+			}
+
+			virtual ~QuickPropagation() {
+			}
+
+		protected:
+
+			virtual inline void resize() override {
+				if(deltas.size()!=network.size())
+					deltas.resize(network.size());
+
+				for(std::size_t i=0; i < network.size(); i++) {
+					if(deltas[i].size()!=network[i].size())
+						deltas[i].resize(network[i].size());
+				}
+
+				if(deltasPrev.size()!=network.size())
+					deltasPrev.resize(network.size());
+
+				for(std::size_t i=0; i < network.size(); i++) {
+					if(deltasPrev[i].size()!=network[i].size())
+						deltasPrev[i].resize(network[i].size());
+
+					for(std::size_t j=0; j < deltasPrev[i].size(); j++) {
+						deltasPrev[i][j]=1.0;
+					}
+				}
+			}
+
+			virtual void updateWeights(const std::vector<float> &input) override;
+
+			std::vector<std::vector<float>> deltasPrev;
+		};
+	}
+}

src/NeuralNetwork/Learning/BackPropagation.cpp

@@ -30,6 +30,11 @@ void NeuralNetwork::Learning::BackPropagation::teach(const std::vector<float> &i
 		}
 	}
 
+	updateWeights(input);
+}
+
+void NeuralNetwork::Learning::BackPropagation::updateWeights(const std::vector<float> &input) {
+
 	for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
 		auto &layer=network[layerIndex];
 		auto &prevLayer=network[layerIndex-1];
@@ -52,4 +57,5 @@ void NeuralNetwork::Learning::BackPropagation::teach(const std::vector<float> &i
 			}
 		}
 	}
-}
+
+}

src/NeuralNetwork/Learning/QuickPropagation.cpp

@@ -0,0 +1,35 @@
+#include <NeuralNetwork/Learning/QuickPropagation.h>
+
+#include <cassert>
+#include <immintrin.h>
+
+void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector<float> &input) {
+
+	for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
+		auto &layer=network[layerIndex];
+		auto &prevLayer=network[layerIndex-1];
+
+		std::size_t prevLayerSize=prevLayer.size();
+		std::size_t layerSize=layer.size();
+
+		for(std::size_t j=1;j<layerSize;j++)  {
+
+			//TODO: is this correct??
+			float delta=deltas[layerIndex][j]/(deltasPrev[layerIndex][j]-deltas[layerIndex][j]);
+
+			deltas[layerIndex][j]=delta;
+
+			layer[j].weight(0)+=delta;
+
+			for(std::size_t k=1;k<prevLayerSize;k++) {
+				if(layerIndex==1) {
+					layer[j].weight(k)+=delta*input[k-1];
+				} else {
+					layer[j].weight(k)+=delta*prevLayer[k].output();
+				}
+			}
+		}
+	}
+
+	deltas.swap(deltasPrev);
+}

tests/CMakeLists.txt

@@ -29,4 +29,13 @@ add_executable(recurrent recurrent.cpp)
 target_link_libraries(recurrent NeuralNetwork)
 
 add_executable(recurrent_perf recurrent_perf.cpp)
-target_link_libraries(recurrent_perf NeuralNetwork)
+target_link_libraries(recurrent_perf NeuralNetwork)
+
+add_executable(quickpropagation quickpropagation.cpp)
+target_link_libraries(quickpropagation NeuralNetwork)
+
+add_executable(quickpropagation_perf quickpropagation_perf.cpp)
+target_link_libraries(quickpropagation_perf NeuralNetwork)
+
+add_executable(propagation_cmp propagation_cmp.cpp)
+target_link_libraries(propagation_cmp NeuralNetwork)

tests/propagation_cmp.cpp

@@ -0,0 +1,77 @@
+#include <NeuralNetwork/FeedForward/Network.h>
+
+#include <cassert>
+#include <iostream>
+#include "../include/NeuralNetwork/Learning/BackPropagation.h"
+#include "../include/NeuralNetwork/Learning/QuickPropagation.h"
+#include "../include/NeuralNetwork/Learning/CorrectionFunction/Optical.h"
+#include "../include/NeuralNetwork/Learning/CorrectionFunction/ArcTangent.h"
+
+#define LEARN(A,AR,B,BR,C,CR,D,DR,FUN,COEF,CLASS) \
+({\
+	srand(rand);\
+	NeuralNetwork::FeedForward::Network n(2);\
+	NeuralNetwork::ActivationFunction::Sigmoid a(-1);\
+	n.appendLayer(2,a);\
+	n.appendLayer(1,a);\
+	n.randomizeWeights();\
+	CLASS prop(n,FUN);\
+	prop.setLearningCoefficient(COEF);\
+	int error=1; int steps = 0; \
+	while(error > 0 && steps <99999) {\
+		steps++;\
+		error=0;\
+		prop.teach(A,{AR});\
+		prop.teach(B,{BR});\
+		prop.teach(C,{CR});\
+		prop.teach(D,{DR});\
+		error+=fabs(n.computeOutput(A)[0]-AR) > 0.1 ? 1:0;\
+		error+=fabs(n.computeOutput(B)[0]-BR) > 0.1 ? 1:0;\
+		error+=fabs(n.computeOutput(C)[0]-CR) > 0.1 ? 1:0;\
+		error+=fabs(n.computeOutput(D)[0]-DR) > 0.1 ? 1:0;\
+	}\
+steps;\
+})
+
+int main() {
+	long rand=(time(NULL));
+
+	const float linearCoef=0.7;
+	const float opticalCoef=0.11;
+	const float arcTangentCoef=0.6;
+	const float arcTangent=1.5;
+
+	{
+		std::cout << "XOR:\n";
+
+		std::cout << "\tBP: " <<
+					 LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
+						   new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
+		std::cout << "\tQP: " <<
+		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
+			  new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
+
+	}
+	{
+		std::cout << "AND:\n";
+
+		std::cout << "\tBP: " <<
+		LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
+			  new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
+
+		std::cout << "\tQP: " <<
+		LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
+			  new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
+	}
+	{
+		std::cout << "AND:\n";
+
+		std::cout << "\tBP: " <<
+		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
+			  new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
+
+		std::cout << "\tQP: " <<
+		LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
+			  new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
+	}
+}

tests/quickpropagation.cpp

@@ -0,0 +1,116 @@
+#include <NeuralNetwork/FeedForward/Network.h>
+
+#include <cassert>
+#include <iostream>
+#include "../include/NeuralNetwork/Learning/QuickPropagation.h"
+
+int main() {
+	{	// XOR problem
+		NeuralNetwork::FeedForward::Network n(2);
+		NeuralNetwork::ActivationFunction::Sigmoid a(-1);
+		n.appendLayer(2,a);
+		n.appendLayer(1,a);
+
+		n.randomizeWeights();
+
+		NeuralNetwork::Learning::QuickPropagation prop(n);
+		for(int i=0;i<10000;i++) {
+			prop.teach({1,0},{1});
+			prop.teach({1,1},{0});
+			prop.teach({0,0},{0});
+			prop.teach({0,1},{1});
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({1,1});
+			assert(ret[0] < 0.1);
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({0,1});
+			assert(ret[0] > 0.9);
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({1,0});
+			assert(ret[0] > 0.9);
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({0,0});
+			assert(ret[0] < 0.1);
+		}
+	}
+	{	// AND problem
+		NeuralNetwork::FeedForward::Network n(2);
+		NeuralNetwork::ActivationFunction::Sigmoid a(-1);
+		n.appendLayer(2,a);
+		n.appendLayer(1,a);
+
+		n.randomizeWeights();
+
+		NeuralNetwork::Learning::QuickPropagation prop(n);
+		for(int i=0;i<10000;i++) {
+			prop.teach({1,1},{1});
+			prop.teach({0,0},{0});
+			prop.teach({0,1},{0});
+			prop.teach({1,0},{0});
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({1,1});
+			assert(ret[0] > 0.9);
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({0,1});
+			assert(ret[0] < 0.1);
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({1,0});
+			assert(ret[0] < 0.1);
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({0,0});
+			assert(ret[0] < 0.1);
+		}
+	}
+	{	// NOT AND problem
+		NeuralNetwork::FeedForward::Network n(2);
+		NeuralNetwork::ActivationFunction::Sigmoid a(-1);
+		n.appendLayer(2,a);
+		n.appendLayer(1,a);
+
+		n.randomizeWeights();
+
+		NeuralNetwork::Learning::QuickPropagation prop(n);
+		for(int i=0;i<10000;i++) {
+			prop.teach({1,1},{0});
+			prop.teach({0,0},{1});
+			prop.teach({0,1},{1});
+			prop.teach({1,0},{1});
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({1,1});
+			assert(ret[0] < 0.1);
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({0,1});
+			assert(ret[0] > 0.9);
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({1,0});
+			assert(ret[0] > 0.9);
+		}
+
+		{
+			std::vector<float> ret =n.computeOutput({0,0});
+			assert(ret[0] > 0.9);
+		}
+	}
+}

tests/quickpropagation_perf.cpp

@@ -0,0 +1,26 @@
+#include <NeuralNetwork/FeedForward/Network.h>
+
+#include <cassert>
+#include <iostream>
+#include "../include/NeuralNetwork/Learning/QuickPropagation.h"
+
+int main() {
+	{	// XOR problem
+		NeuralNetwork::FeedForward::Network n(2);
+		NeuralNetwork::ActivationFunction::Sigmoid a(-1);
+		n.appendLayer(200,a);
+		n.appendLayer(500,a);
+		n.appendLayer(900,a);
+		n.appendLayer(1,a);
+
+		n.randomizeWeights();
+
+		NeuralNetwork::Learning::QuickPropagation prop(n);
+		for(int i=0;i<100;i++) {
+			prop.teach({1,0},{1});
+			prop.teach({1,1},{0});
+			prop.teach({0,0},{0});
+			prop.teach({0,1},{1});
+		}
+	}
+}