Added transfer function support, yet not colplete

Makefile.const

@@ -5,7 +5,7 @@ CXXFLAGS+= -std=c++14
 CXXFLAGS+= -pg -fPIC
 CXXFLAGS+= -g
 CXXFLAGS+= -fPIC -pthread
-CXXFLAGS+= -DUSE_SSE2
+#CXXFLAGS+= -DUSE_SSE2
 OPTIMALIZATION = -O3 -march=native -mtune=native 
 
 %.o : %.cpp %.h

src/NeuralNetwork/FeedForward.cpp

@@ -35,6 +35,7 @@ FFNeuron& FFLayer::operator[](const size_t& neuron)
 
 FeedForward::FeedForward(std::initializer_list<size_t> s, double lam): ACyclicNetwork(lam),layers(s.size())
 {
+	transfer = new TransferFunction::TransferFunction*[s.size()];
 	weights= new float**[s.size()];
 	potentials= new float*[s.size()];
 	layerSizes= new size_t[s.size()];
@@ -44,6 +45,7 @@ FeedForward::FeedForward(std::initializer_list<size_t> s, double lam): ACyclicNe
 	int prev_size=1;
 	for(int layeSize:s) // TODO rename
 	{
+		transfer[i]= new TransferFunction::Sigmoid(lam);
 		layeSize+=1;
 		if(i==0)
 		{
@@ -103,13 +105,12 @@ FeedForward::~FeedForward()
 
 void FeedForward::solvePart(float *newSolution, register size_t begin, size_t end,size_t prevSize, float *sol,size_t layer)
 {
-	if(prevSize >4)
+	TransferFunction::StreamingTransferFunction *function=dynamic_cast<TransferFunction::StreamingTransferFunction*>(transfer[layer]);
+	if(prevSize >=4 && function !=nullptr)
 	{
 		__m128 partialSolution;
 		__m128 w;
 		__m128 sols;
-		__m128 temporaryConst1=_mm_set1_ps(1.0);
-		__m128 temporaryConstLambda=_mm_set1_ps(-lambda);
 		register size_t alignedPrev=prevSize>16?(prevSize-(prevSize%16)):0;
 		for( size_t j=begin;j<end;j++)
 		{
@@ -122,9 +123,10 @@ void FeedForward::solvePart(float *newSolution, register size_t begin, size_t en
 				w=_mm_mul_ps(w,sols);
 				partialSolution=_mm_add_ps(partialSolution,w);
 			}
+			register float* memory=this->weights[layer][j];
 			for(register size_t k=0;k<alignedPrev;k+=4)
 			{
-				w = _mm_load_ps(this->weights[layer][j]+k);
+				w = _mm_load_ps(memory+k);
 				sols  = _mm_load_ps(sol+k);
 				w=_mm_mul_ps(w,sols);
 				partialSolution=_mm_add_ps(partialSolution,w);
@@ -142,22 +144,19 @@ void FeedForward::solvePart(float *newSolution, register size_t begin, size_t en
 			partialSolution = _mm_hadd_ps(partialSolution, partialSolution);
 #endif
 			_mm_store_ss(inputs[layer]+j,partialSolution);
-			partialSolution=_mm_mul_ps(temporaryConstLambda,partialSolution); //-lambda*sol[k]
+			partialSolution=function->operator()(partialSolution);
-			partialSolution=exp_ps(partialSolution); //exp(sols)
-			partialSolution= _mm_add_ps(partialSolution,temporaryConst1); //1+exp()
-			partialSolution= _mm_div_ps(temporaryConst1,partialSolution);//1/....*/
 			_mm_store_ss(newSolution+j,partialSolution);
 		}
 	}else
 	{
 		for( size_t j=begin;j<end;j++)
 		{
-			register float tmp=0;
+			float tmp=0;
 			for(register size_t k=0;k<prevSize;k++)
 			{
 				tmp+=sol[k]*weights[layer][j][k];
 			}
-			newSolution[j]=(1.0/(1.0+exp(-lambda*tmp)));
+			newSolution[j]=transfer[layer]->operator()(tmp);
 			inputs[layer][j]=tmp;
 		}
 	}

src/NeuralNetwork/FeedForward.h

@@ -4,6 +4,8 @@
 #include "../Problem"
 #include "../Solution"
 #include "Network"
+#include "TransferFunction/Sigmoid.h"
+#include "TransferFunction/TransferFunction.h"
 
 #include <vector>
 #include <initializer_list>
@@ -93,6 +95,7 @@ namespace NeuralNetwork
 			float **potentials=nullptr;
 			float **sums=nullptr;
 			float **inputs=nullptr;
+			TransferFunction::TransferFunction **transfer=nullptr;
 			size_t *layerSizes=nullptr;
 			size_t layers;
 	};

src/NeuralNetwork/Makefile

@@ -15,7 +15,7 @@ lib: $(LIBNAME).so $(LIBNAME).a
 $(LIBNAME).so:  $(OBJFILES)
 	$(CXX) -shared $(CXXFLAGS) $(OBJFILES) $(LINKFILES) -o $(LIBNAME).so
 
-$(LIBNAME).a: $(OBJFILES) ./Neuron.h ./Network.h ../Solution.h ../Problem.h
+$(LIBNAME).a: $(OBJFILES) ./Neuron.h ./Network.h ../Solution.h ../Problem.h ./TransferFunction/TransferFunction.h ./TransferFunction/Sigmoid.h
 	rm -f $(LIBNAME).a  # create new library
 	ar rcv $(LIBNAME).a $(OBJFILES) $(LINKFILES)
 	ranlib $(LIBNAME).a

src/NeuralNetwork/TransferFunction/Heaviside.h

@@ -0,0 +1,25 @@
+#ifndef __TRAN_HEAVISIDE_H_
+#define  __TRAN_HEAVISIDE_H_
+
+#include "./TransferFunction.h"
+
+namespace Shin
+{
+namespace NeuralNetwork
+{
+namespace TransferFunction
+{
+	class Heaviside: public TransferFunction
+	{
+		public:
+			Sigmoid(const float &lambdaP): lambda(lambdaP) {}
+			inline virtual float derivatedOutput(const float &input,const float &output) override { return 1.0;  }
+			inline virtual float operator()(const float &x) override { return x>k ? 1.0f : 0.0f; };
+		protected:
+			float lambda;
+			
+	};
+}
+}
+}
+#endif

src/NeuralNetwork/TransferFunction/HyperbolicTangent.h

@@ -0,0 +1,23 @@
+#ifndef __TRAN_SIGMOID_H_
+#define  __TRAN_SIGMOID_H_
+
+#include "./TransferFunction.h"
+
+namespace Shin
+{
+namespace NeuralNetwork
+{
+namespace TransferFunction
+{
+	class HyperbolicTangent: public TransferFunction
+	{
+		public:
+			HyperbolicTangent() {}
+			inline virtual float derivatedOutput(const float&,const float &output) override { return 1-pow(output); }
+			inline virtual float operator()(const float &x) override { return tanh(x); };
+		protected:
+	};
+}
+}
+}
+#endif

src/NeuralNetwork/TransferFunction/Sigmoid.h

@@ -0,0 +1,33 @@
+#ifndef __TRAN_SIGMOID_H_
+#define  __TRAN_SIGMOID_H_
+
+#include "./StreamingTransferFunction.h"
+
+namespace Shin
+{
+namespace NeuralNetwork
+{
+namespace TransferFunction
+{
+	class Sigmoid: public StreamingTransferFunction
+	{
+		public:
+			Sigmoid(const float lambdaP = 0.8): lambda(lambdaP) {}
+			inline virtual float derivatedOutput(const float&,const float &output) override { return lambda*output*(1.0f-output); }
+			inline virtual float operator()(const float &x) override { return 1.0f / (1.0f +exp(-lambda*x) ); };
+			inline virtual __m128 operator()(__m128 x) override {
+				x=_mm_mul_ps(temporaryConstLambda,x); //-lambda*sol[k]
+				x=exp_ps(x); //exp(x)
+				x= _mm_add_ps(x,temporaryConst1); //1+exp()
+				x= _mm_div_ps(temporaryConst1,x);//1/....
+				return x;
+			}
+		protected:
+			float lambda;
+			__m128 temporaryConst1=_mm_set1_ps(1.0);
+			__m128 temporaryConstLambda=_mm_set1_ps(-lambda);
+	};
+}
+}
+}
+#endif

src/NeuralNetwork/TransferFunction/StreamingTransferFunction.h

@@ -0,0 +1,26 @@
+#ifndef __STREAMINGTRAN_FUN_H_
+#define  __STREAMINGTRAN_FUN_H_
+
+#include <xmmintrin.h>
+
+#include "../../sse_mathfun.h"
+
+#include "./TransferFunction.h"
+
+namespace Shin
+{
+namespace NeuralNetwork
+{
+namespace TransferFunction
+{
+	class StreamingTransferFunction : public TransferFunction
+	{
+		public:
+			virtual float derivatedOutput(const float &input,const float &output)=0;
+			virtual float operator()(const float &x)=0;
+			virtual __m128 operator()(__m128)=0; // it must be overriden to be used!
+	};
+}
+}
+}
+#endif

src/NeuralNetwork/TransferFunction/TransferFunction.h

@@ -0,0 +1,22 @@
+#ifndef __TRAN_FUN_H_
+#define  __TRAN_FUN_H_
+
+#include <math.h>
+
+namespace Shin
+{
+namespace NeuralNetwork
+{
+namespace TransferFunction
+{
+	class TransferFunction
+	{
+		public:
+			virtual ~TransferFunction() {}
+			virtual float derivatedOutput(const float &input,const float &output)=0;
+			virtual float operator()(const float &x)=0;
+	};
+}
+}
+}
+#endif