reinforcement with randomising

src/NeuronNetwork/FeedForwardQuick.cpp

@@ -21,7 +21,7 @@ FFNeuron* FFLayer::operator[](int neuron)
 		neurons=new FFNeuron*[layerSize];
 		for(size_t i=0;i<layerSize;i++)
 		{
-			neurons[i]=new FFNeuron(&potentials[i],weights[i],&sums[i]);
+			neurons[i]=new FFNeuron(&potentials[i],weights[i],&sums[i],lambda);
 		}
 	}
 	return neurons[neuron];
@@ -81,7 +81,7 @@ Solution FeedForwardNetworkQuick::solve(const Problem& p)
 					q+=sol[k]*weights[i][j][k];
 				}else
 				{
-					q+=(1.0/(1.0+exp(-0.5*sol[k])))*weights[i][j][k];
+					q+=(1.0/(1.0+exp(-lambda*sol[k])))*weights[i][j][k];
 				}
 			}
 			newSolution[j]=q;
@@ -92,7 +92,7 @@ Solution FeedForwardNetworkQuick::solve(const Problem& p)
 	std::vector<double> ret;
 	for(size_t i=1;i<prevSize;i++)
 	{
-		ret.push_back((1.0/(1.0+exp(-0.5*sol[i]))));
+		ret.push_back((1.0/(1.0+exp(-lambda*sol[i]))));
 	}
 	return ret;
 }
@@ -104,7 +104,7 @@ FFLayer* FeedForwardNetworkQuick::operator[](int l)
 		ffLayers=new FFLayer*[layers];
 		for(size_t i=0;i<layers;i++)
 		{
-			ffLayers[i]=new FFLayer(layerSizes[i],potentials[i],weights[i],sums[i+1]);
+			ffLayers[i]=new FFLayer(layerSizes[i],potentials[i],weights[i],sums[i+1],lambda);
 		}
 	}
 	return ffLayers[l];

src/NeuronNetwork/FeedForwardQuick.h

@@ -12,6 +12,8 @@
 #include <iostream>
 #include <math.h>
 
+#define LAMBDA 0.8
+
 namespace Shin
 {
 namespace NeuronNetwork
@@ -22,7 +24,7 @@ namespace NeuronNetwork
 			FFNeuron() = delete;
 			FFNeuron(const FFNeuron&) = delete;
 			FFNeuron& operator=(const FFNeuron&) = delete;
-			FFNeuron(double *pot, double *w, double*s):potential(pot),weights(w),sum(s) { }
+			FFNeuron(double *pot, double *w, double*s,double lam):potential(pot),weights(w),sum(s),lambda(lam) { }
 
 			double getPotential() {return *potential;}
 			void setPotential(double p) { *potential=p;}
@@ -30,17 +32,18 @@ namespace NeuronNetwork
 			void setWeight(unsigned int i,double p) { weights[i]=p; }
 			inline double output()
 			{
-				return 1.0/(1.0+(exp(-0.5*input())));
+				return 1.0/(1.0+(exp(-lambda*input())));
 				return input();
 //				register double tmp=;
 //				return NAN==tmp?0:tmp;
 			/* > *potential? 1 :0;*/	}
 			inline double input() { return *sum; }
-			inline double derivatedOutput() { return output()*(1.0-output()); };
+			inline double derivatedOutput() { return lambda*output()*(1.0-output()); };
 		protected:
 			double *potential;
 			double *weights;
 			double *sum;
+			double lambda;
 		private:	
 	};
 
@@ -49,7 +52,7 @@ namespace NeuronNetwork
 		public:
 			FFLayer(const FFLayer &) =delete;
 			FFLayer operator=(const FFLayer &) = delete;
-			FFLayer(size_t s, double *p,double **w,double *su): neurons(nullptr),layerSize(s),potentials(p),weights(w),sums(su) {}
+			FFLayer(size_t s, double *p,double **w,double *su,double lam): neurons(nullptr),layerSize(s),potentials(p),weights(w),sums(su),lambda(lam) {}
 			~FFLayer();
 			FFNeuron* operator[](int neuron);
 			size_t size() const {return layerSize;};
@@ -59,6 +62,7 @@ namespace NeuronNetwork
 			double *potentials;
 			double **weights;
 			double *sums;
+			double lambda;
 	};
 
 	class FeedForwardNetworkQuick:public ACyclicNetwork
@@ -66,7 +70,7 @@ namespace NeuronNetwork
 		public:
 			FeedForwardNetworkQuick(const FeedForwardNetworkQuick &f) = delete; //TODO
 			FeedForwardNetworkQuick operator=(const FeedForwardNetworkQuick &f)=delete;
-			template<typename... Args>inline FeedForwardNetworkQuick(std::initializer_list<int> s):ffLayers(nullptr),weights(nullptr),potentials(nullptr),sums(nullptr),layerSizes(nullptr),layers(s.size())
+			template<typename... Args>inline FeedForwardNetworkQuick(std::initializer_list<int> s, double lam=LAMBDA):ffLayers(nullptr),weights(nullptr),potentials(nullptr),sums(nullptr),layerSizes(nullptr),layers(s.size()),lambda(lam)
 			{
 				weights= new double**[s.size()];
 				potentials= new double*[s.size()];
@@ -114,8 +118,10 @@ namespace NeuronNetwork
 			double **potentials;
 		public:
 			double **sums;
+		private:
 			size_t *layerSizes;
 			size_t layers;
+			double lambda;
 	};
 
 }

src/NeuronNetwork/Learning/BackPropagation.cpp

@@ -15,15 +15,12 @@ double Shin::NeuronNetwork::Learning::BackPropagation::calculateError(const Shin
 	return a;
 }
 
-const double    LAMBDA           = 0.5;
-
 void Shin::NeuronNetwork::Learning::BackPropagation::propagate(const Shin::NeuronNetwork::Solution& expectation)
 {
 	double **deltas;
 	deltas=new double*[network.size()];
 	for(int i=(int)network.size()-1;i>=0;i--)
 	{
-		std::cerr << i << "XXXXXXXXXXXXXX\n";
 		deltas[i]=new double[network[i]->size()];
 		deltas[i][0]=0.0;
 		if(i==(int)network.size()-1)
@@ -57,7 +54,7 @@ void Shin::NeuronNetwork::Learning::BackPropagation::propagate(const Shin::Neuro
 
 		for(size_t j=1;j<network[i]->size();j++)
 		{
-			network[i]->operator[](j)->setWeight(0,network[i]->operator[](j)->getWeight(0)+0.5*deltas[i][j]);
+			network[i]->operator[](j)->setWeight(0,network[i]->operator[](j)->getWeight(0)+deltas[i][j]*learningCoeficient);
 			for(size_t k=1;k<max;k++)
 			{
 				network[i]->operator[](j)->setWeight(k,
@@ -85,3 +82,10 @@ double Shin::NeuronNetwork::Learning::BackPropagation::teach(const Shin::NeuronN
 //	std::cerr << "error: " << error  << "\n";
 	return error;
 }
+
+
+void Shin::NeuronNetwork::Learning::BackPropagation::setLearningCoeficient(double c)
+{
+	learningCoeficient=c;
+}
+

src/NeuronNetwork/Learning/BackPropagation.h

@@ -27,8 +27,9 @@ namespace Learning
 			double calculateError(const Solution &expectation,const Solution &solution);
 			void propagate(const Shin::NeuronNetwork::Solution& expectation);
 			double teach(const Shin::NeuronNetwork::Problem &p,const Solution &solution);
+			void setLearningCoeficient (double);
 		protected:
-			double learningCoeficient=0.8;
+			double learningCoeficient=0.4;
 	};
 }
 }

src/NeuronNetwork/Learning/Reinforcement.cpp

@@ -1 +1,91 @@
 #include "./Reinforcement"
+
+Shin::NeuronNetwork::Learning::Reinforcement::Reinforcement(Shin::NeuronNetwork::FeedForwardNetworkQuick& n): Unsupervised(n), p(n)
+{
+	p.setLearningCoeficient(4.5);
+}
+
+void Shin::NeuronNetwork::Learning::Reinforcement::setQualityFunction(std::function< double(const Solution &s) > f)
+{
+	qualityFunction=f;
+}
+
+double Shin::NeuronNetwork::Learning::Reinforcement::learn(const Shin::NeuronNetwork::Problem& problem)
+{
+	Solution s=network.solve(problem);
+	double quality=qualityFunction(s);
+	std::vector<double> q;
+	for(register size_t j=0;j<s.size();j++)
+	{
+		q.push_back(s[j]*((double)(990+(rand()%21))/1000.0));
+	}
+	if(quality <= 0)
+	{
+		for(register size_t j=0;j<s.size();j++)
+		{
+			do{
+				q[j]=((double)(10+rand()%80))/100.0;
+			}while(fabs(q[j]-s[j]) < 0.1);
+		}
+	}
+	for(register int i=abs((int)quality);i>=0;i--)
+	{
+		p.propagate(q);
+	}
+	return quality;
+}
+
+void Shin::NeuronNetwork::Learning::Reinforcement::propagate(const Shin::NeuronNetwork::Solution& expectation,bool random)
+{
+	double **deltas;
+	deltas=new double*[network.size()];
+	for(int i=(int)network.size()-1;i>=0;i--)
+	{
+		deltas[i]=new double[network[i]->size()];
+		deltas[i][0]=0.0;
+		if(i==(int)network.size()-1)
+		{
+			for(size_t j=1;j<network[i]->size();j++)
+			{
+				deltas[i][j]= (expectation[j-1]-network[i]->operator[](j)->output())*network[i]->operator[](j)->derivatedOutput();
+//				std::cerr << "X "<< deltas[i][j] <" Z ";
+			}
+		}else
+		{
+			for(size_t j=1;j<network[i]->size();j++)
+			{
+				register double deltasWeight = 0;
+				for(size_t k=1;k<network[i+1]->size();k++)
+				{
+					deltasWeight+=deltas[i+1][k]*network[i+1]->operator[](k)->getWeight(j);
+				}
+				deltas[i][j]=deltasWeight*network[i]->operator[](j)->derivatedOutput();
+			}
+		}
+	}
+	
+	for(size_t i=0;i<network.size();i++)
+	{
+		size_t max;
+		if(i==0)
+			max=network[i]->size();
+		else
+			max=network[i-1]->size();
+
+		for(size_t j=1;j<network[i]->size();j++)
+		{
+			network[i]->operator[](j)->setWeight(0,network[i]->operator[](j)->getWeight(0)+deltas[i][j]*learningCoeficient);
+			for(size_t k=1;k<max;k++)
+			{
+				network[i]->operator[](j)->setWeight(k,
+					network[i]->operator[](j)->getWeight(k)+learningCoeficient*	deltas[i][j]*
+					(i==0? network.sums[0][k]:(double)network[i-1]->operator[](k)->output()));
+			}
+		}
+	}
+	for(size_t i=0;i<network.size();i++)
+	{
+		delete[] deltas[i];
+	}
+	delete[] deltas;
+}

src/NeuronNetwork/Learning/Reinforcement.h

@@ -4,9 +4,11 @@
 #include <math.h>
 #include <cstddef>
 
-#include "../Solution.h"
+#include "../Problem.h"
 #include "../FeedForwardQuick.h"
+#include "BackPropagation"
 #include  "Unsupervised"
+#include "functional"
 
 /*
  * 
@@ -24,11 +26,15 @@ namespace Learning
 	{
 		public:
 			Reinforcement(FeedForwardNetworkQuick &n);
-			double calculateError(const Solution &expectation,const Solution &solution);
+
-			void propagate(const Shin::NeuronNetwork::Solution& expectation);
+			void setQualityFunction(std::function<double(const Solution &s)>);
-			double teach(const Shin::NeuronNetwork::Problem &p,const Solution &solution);
+			double learn(const Shin::NeuronNetwork::Problem &p);
+			void propagate(const Shin::NeuronNetwork::Solution& expectation,bool random=0);
+			void setCoef(double q) {p.setLearningCoeficient(q);}
 		protected:
-			double learningCoeficient=0.8;
+			double learningCoeficient=3;
+			std::function<double(const Solution &s)> qualityFunction=nullptr;
+			BackPropagation p;
 	};
 }
 }

src/NeuronNetwork/Learning/Unsupervised.h

@@ -19,8 +19,6 @@ namespace Learning
 			Unsupervised() =delete;
 			Unsupervised(FeedForwardNetworkQuick &n);
 			virtual ~Unsupervised() {};
-			virtual double calculateError(const Solution &expectation,const Solution &solution)=0;
-			virtual double teach(const Shin::NeuronNetwork::Problem &p,const Solution &solution)=0;
 			void debugOn();
 			void debugOff();
 		protected:

src/NeuronNetwork/Makefile

@@ -1,6 +1,6 @@
-OBJFILES= Neuron.o ./Network.o FeedForward.o FeedForwardQuick.o \
+OBJFILES= Neuron.o Network.o FeedForward.o FeedForwardQuick.o \
 	Learning/Supervised.o Learning/Unsupervised.o Learning/Reinforcement.o Learning/BackPropagation.o \
-	./Solution.o ./Problem.o
+	Solution.o Problem.o
 
 LIBNAME=NeuronNetwork
 

tests/Makefile

@@ -2,7 +2,7 @@ include ../Makefile.const
 
 LIB_DIR = ../lib
 GEN_TESTS=g-01 g-02
-NN_TESTS=nn-01 nn-02 nn-03 nn-04
+NN_TESTS= nn-reinforcement nn-01 nn-02 nn-03 nn-04
 ALL_TESTS=$(NN_TESTS) $(GEN_TESTS)
 
 LIBS=$(LIB_DIR)/Genetics.a $(LIB_DIR)/NeuronNetwork.a

tests/nn-reinforcement.cpp

@@ -0,0 +1,94 @@
+#include "../src/NeuronNetwork/FeedForwardQuick"
+#include "../src/NeuronNetwork/Learning/Reinforcement.h"
+#include "../src/NeuronNetwork/Solution.h"
+
+#include <iostream>
+#include <vector>
+
+class X: public Shin::NeuronNetwork::Problem
+{
+	public:
+		X(const X& a) :q(a.q) {}
+		X(const std::vector<bool> &a):q(a) {}
+		std::vector<bool> representation() const
+		{
+			return q;
+		}
+	protected:
+		std::vector<bool> q;
+};
+
+int main()
+{
+	srand(time(NULL));
+
+	std::vector<X> p;
+
+	p.push_back(X(std::vector<bool>({0,0})));
+
+	p.push_back(X(std::vector<bool>({1,1})));
+
+	Shin::NeuronNetwork::FeedForwardNetworkQuick q({2,6,2});
+	Shin::NeuronNetwork::Learning::Reinforcement b(q);
+	int i=0;
+	b.setQualityFunction(
+		[&i](const Shin::NeuronNetwork::Solution &s)->double
+		{
+			if(i%2==0)
+			{
+				//ocekavame 1
+				int e=(s[0]-0.80)*15.0;//+(abs(s[1])-0.5)*100.0;
+				return e;
+			}else
+			{
+				//ocekavame 0
+				int e=(0.20-s[0])*15.0;//+(0.4-abs(s[1]))*100.0;
+				return e;
+			}
+			return 1.0;
+		});
+	for(i=0;i < 500000000;i++)
+	{
+		if(i==75000)
+		{
+			std::cerr << "SSSSSS1XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n";
+			b.setCoef(1);
+		}
+		if(i==150000)
+		{
+			std::cerr << "SSSSSS1XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n";
+			b.setCoef(0.51);
+		}
+		if(i==300000)
+		{
+			std::cerr << "SSSSSS2XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n";
+			b.setCoef(0.15);
+		}
+		b.learn(p[i%2]);
+	
+		if(i%100000==0)
+			srand(time(NULL));
+		if(i%10000==0)
+			for(int j=0;j<2;j++)
+			{
+				std::cerr << j%4 <<". FOR: [" << p[j%4].representation()[0] << "," <<p[j%4].representation()[0] << "] res: " << q.solve(p[j%4])[0] << "\n";
+			}
+	}
+
+/*	int i=0;
+	std::cerr << i%4 <<". FOR: [" << p[i%2].representation()[0] << "] res: " << q.solve(p[i%2])[0] << " should be " << s[i%2][0]<<"\n";
+
+	for(int i=0;i<2000;i++)sa	
+	{
+		b.teach(p[i%2],s[i%2]);
+		std::cerr << i%2 <<". FOR: [" << p[i%2].representation()[0] << "] res: " << q.solve(p[i%2])[0] << " should be " << s[i%2][0]<<"\n";
+	}
+	b.debugOn();
+	for(int i=0;i<2;i++)
+	{
+		b.teach(p[i%2],s[i%2]);
+		std::cerr << i%4 <<". FOR: [" << p[i%4].representation()[0] << "," <<p[i%4].representation()[0] << "] res: " << q.solve(p[i%4])[0] << " should be " <<
+		s[i%4][0]<<"\n";
+	}
+	b.debugOff();*/
+}