diff --git a/CMakeLists.txt b/CMakeLists.txt index 2228453..1de2cab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,15 +6,15 @@ include(cmake/CPUFeatures) OPTION(BUILD_SHARED_LIBS "Build also shared library." ON) if(CPU_AVX_AVAILABLE) - SET(USE_AVX ON CACHE BOOL "") + SET(USE_AVX ON CACHE BOOL "") endif(CPU_AVX_AVAILABLE) if(CPU_SSE3_AVAILABLE) - SET(USE_SSE ON CACHE BOOL "") + SET(USE_SSE ON CACHE BOOL "") - if(CPU_SSE4.2_AVAILABLE) - SET(USE_SSE2 OFF CACHE BOOL "") - endif(CPU_SSE4.2_AVAILABLE) + if(CPU_SSE4.2_AVAILABLE) + SET(USE_SSE2 OFF CACHE BOOL "") + endif(CPU_SSE4.2_AVAILABLE) endif(CPU_SSE3_AVAILABLE) @@ -33,45 +33,29 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++14") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -mtune=native -O3") if(USE_AVX) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx -DUSE_AVX") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx -DUSE_AVX") endif(USE_AVX) if(USE_SSE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -DUSE_SSE") - if(USE_SSE2) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_SSE2") - endif(USE_SSE2) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -DUSE_SSE") + if(USE_SSE2) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_SSE2") + endif(USE_SSE2) endif(USE_SSE) include_directories(./include/) set (LIBRARY_SOURCES - include/NeuralNetwork/ActivationFunction/ActivationFunction.h - include/NeuralNetwork/ActivationFunction/Heaviside.h - include/NeuralNetwork/ActivationFunction/HyperbolicTangent.h - include/NeuralNetwork/ActivationFunction/Linear.h - include/NeuralNetwork/ActivationFunction/Sigmoid.h - include/NeuralNetwork/ActivationFunction/StreamingActivationFunction.h - include/NeuralNetwork/BasisFunction/BasisFunction.h - include/NeuralNetwork/BasisFunction/Linear.h - include/NeuralNetwork/BasisFunction/Product.h - include/NeuralNetwork/BasisFunction/Radial.h - include/NeuralNetwork/BasisFunction/StreamingBasisFunction.h - include/NeuralNetwork/FeedForward/Layer.h - include/NeuralNetwork/FeedForward/Network.h - include/NeuralNetwork/Recurrent/Network.h - include/NeuralNetwork/Network.h - include/NeuralNetwork/Neuron.h - include/NeuralNetwork/Stringifiable.h - include/NeuralNetwork/Stringifiable.h - src/NeuralNetwork/Learning/BackPropagation.cpp include/NeuralNetwork/Learning/BackPropagation.h - include/sse_mathfun.h - src/NeuralNetwork/BasisFunction/Linear.cpp - src/NeuralNetwork/FeedForward/Layer.cpp - src/NeuralNetwork/FeedForward/Network.cpp - src/NeuralNetwork/Recurrent/Network.cpp - src/NeuralNetwork/Neuron.cpp - src/sse_mathfun.cpp + src/sse_mathfun.cpp + + src/NeuralNetwork/Learning/BackPropagation.cpp + src/NeuralNetwork/Learning/QuickPropagation.cpp + + src/NeuralNetwork/BasisFunction/Linear.cpp + src/NeuralNetwork/FeedForward/Layer.cpp + src/NeuralNetwork/FeedForward/Network.cpp + src/NeuralNetwork/Recurrent/Network.cpp + src/NeuralNetwork/Neuron.cpp ) add_library(NeuralNetwork STATIC ${LIBRARY_SOURCES}) @@ -79,8 +63,8 @@ add_library(NeuralNetwork STATIC ${LIBRARY_SOURCES}) link_libraries(NeuralNetwork pthread) IF(BUILD_SHARED_LIBS) - add_library(NeuralNetworkShared SHARED ${LIBRARY_SOURCES}) - set_target_properties(NeuralNetworkShared PROPERTIES OUTPUT_NAME NeuralNetwork) + add_library(NeuralNetworkShared SHARED ${LIBRARY_SOURCES}) + set_target_properties(NeuralNetworkShared PROPERTIES OUTPUT_NAME NeuralNetwork) ENDIF(BUILD_SHARED_LIBS) IF(ENABLE_TESTS) @@ -106,15 +90,20 @@ set_property(TEST feedforward PROPERTY LABELS unit) add_test(recurrent tests/recurrent) set_property(TEST recurrent PROPERTY LABELS unit) -add_test(feedforward_perf tests/feedforward_perf) -set_property(TEST feedforward_perf PROPERTY LABELS perf) - add_test(optical_backpropagation tests/optical_backpropagation) set_property(TEST optical_backpropagation PROPERTY LABELS unit) +add_test(quickpropagation tests/quickpropagation) +set_property(TEST quickpropagation PROPERTY LABELS unit) +add_test(feedforward_perf tests/feedforward_perf) +set_property(TEST feedforward_perf PROPERTY LABELS perf) + +add_test(quickpropagation_perf tests/quickpropagation_perf) +set_property(TEST quickpropagation_perf PROPERTY LABELS perf) + add_test(backpropagation_perf tests/backpropagation_perf) set_property(TEST backpropagation_perf PROPERTY LABELS perf) diff --git a/include/NeuralNetwork/Learning/BackPropagation.h b/include/NeuralNetwork/Learning/BackPropagation.h index ef306f1..e8c4304 100644 --- a/include/NeuralNetwork/Learning/BackPropagation.h +++ b/include/NeuralNetwork/Learning/BackPropagation.h @@ -33,7 +33,7 @@ namespace Learning { protected: - inline void resize() { + virtual inline void resize() { if(deltas.size()!=network.size()) deltas.resize(network.size()); @@ -43,6 +43,8 @@ namespace Learning { } } + virtual void updateWeights(const std::vector &input); + FeedForward::Network &network; CorrectionFunction::CorrectionFunction *correctionFunction; diff --git a/include/NeuralNetwork/Learning/QuickPropagation.h b/include/NeuralNetwork/Learning/QuickPropagation.h new file mode 100644 index 0000000..ba612b0 --- /dev/null +++ b/include/NeuralNetwork/Learning/QuickPropagation.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include + +#include +#include "BackPropagation.h" + +namespace NeuralNetwork { + namespace Learning { + + /** @class QuickPropagation + * @brief + */ + class QuickPropagation : public BackPropagation { + + public: + inline QuickPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()): + BackPropagation(feedForwardNetwork,correction),deltasPrev() { + resize(); + } + + virtual ~QuickPropagation() { + } + + protected: + + virtual inline void resize() override { + if(deltas.size()!=network.size()) + deltas.resize(network.size()); + + for(std::size_t i=0; i < network.size(); i++) { + if(deltas[i].size()!=network[i].size()) + deltas[i].resize(network[i].size()); + } + + if(deltasPrev.size()!=network.size()) + deltasPrev.resize(network.size()); + + for(std::size_t i=0; i < network.size(); i++) { + if(deltasPrev[i].size()!=network[i].size()) + deltasPrev[i].resize(network[i].size()); + + for(std::size_t j=0; j < deltasPrev[i].size(); j++) { + deltasPrev[i][j]=1.0; + } + } + } + + virtual void updateWeights(const std::vector &input) override; + + std::vector> deltasPrev; + }; + } +} \ No newline at end of file diff --git a/src/NeuralNetwork/Learning/BackPropagation.cpp b/src/NeuralNetwork/Learning/BackPropagation.cpp index 11453d8..fba88ce 100644 --- a/src/NeuralNetwork/Learning/BackPropagation.cpp +++ b/src/NeuralNetwork/Learning/BackPropagation.cpp @@ -30,6 +30,11 @@ void NeuralNetwork::Learning::BackPropagation::teach(const std::vector &i } } + updateWeights(input); +} + +void NeuralNetwork::Learning::BackPropagation::updateWeights(const std::vector &input) { + for(std::size_t layerIndex=1;layerIndex &i } } } -} \ No newline at end of file + +} diff --git a/src/NeuralNetwork/Learning/QuickPropagation.cpp b/src/NeuralNetwork/Learning/QuickPropagation.cpp new file mode 100644 index 0000000..68638dd --- /dev/null +++ b/src/NeuralNetwork/Learning/QuickPropagation.cpp @@ -0,0 +1,35 @@ +#include + +#include +#include + +void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector &input) { + + for(std::size_t layerIndex=1;layerIndex + +#include +#include +#include "../include/NeuralNetwork/Learning/BackPropagation.h" +#include "../include/NeuralNetwork/Learning/QuickPropagation.h" +#include "../include/NeuralNetwork/Learning/CorrectionFunction/Optical.h" +#include "../include/NeuralNetwork/Learning/CorrectionFunction/ArcTangent.h" + +#define LEARN(A,AR,B,BR,C,CR,D,DR,FUN,COEF,CLASS) \ +({\ + srand(rand);\ + NeuralNetwork::FeedForward::Network n(2);\ + NeuralNetwork::ActivationFunction::Sigmoid a(-1);\ + n.appendLayer(2,a);\ + n.appendLayer(1,a);\ + n.randomizeWeights();\ + CLASS prop(n,FUN);\ + prop.setLearningCoefficient(COEF);\ + int error=1; int steps = 0; \ + while(error > 0 && steps <99999) {\ + steps++;\ + error=0;\ + prop.teach(A,{AR});\ + prop.teach(B,{BR});\ + prop.teach(C,{CR});\ + prop.teach(D,{DR});\ + error+=fabs(n.computeOutput(A)[0]-AR) > 0.1 ? 1:0;\ + error+=fabs(n.computeOutput(B)[0]-BR) > 0.1 ? 1:0;\ + error+=fabs(n.computeOutput(C)[0]-CR) > 0.1 ? 1:0;\ + error+=fabs(n.computeOutput(D)[0]-DR) > 0.1 ? 1:0;\ + }\ +steps;\ +}) + +int main() { + long rand=(time(NULL)); + + const float linearCoef=0.7; + const float opticalCoef=0.11; + const float arcTangentCoef=0.6; + const float arcTangent=1.5; + + { + std::cout << "XOR:\n"; + + std::cout << "\tBP: " << + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, + new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + std::cout << "\tQP: " << + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, + new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + + } + { + std::cout << "AND:\n"; + + std::cout << "\tBP: " << + LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, + new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + + std::cout << "\tQP: " << + LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, + new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + } + { + std::cout << "AND:\n"; + + std::cout << "\tBP: " << + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, + new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + + std::cout << "\tQP: " << + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, + new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + } +} diff --git a/tests/quickpropagation.cpp b/tests/quickpropagation.cpp new file mode 100644 index 0000000..a7feec1 --- /dev/null +++ b/tests/quickpropagation.cpp @@ -0,0 +1,116 @@ +#include + +#include +#include +#include "../include/NeuralNetwork/Learning/QuickPropagation.h" + +int main() { + { // XOR problem + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::Sigmoid a(-1); + n.appendLayer(2,a); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::QuickPropagation prop(n); + for(int i=0;i<10000;i++) { + prop.teach({1,0},{1}); + prop.teach({1,1},{0}); + prop.teach({0,0},{0}); + prop.teach({0,1},{1}); + } + + { + std::vector ret =n.computeOutput({1,1}); + assert(ret[0] < 0.1); + } + + { + std::vector ret =n.computeOutput({0,1}); + assert(ret[0] > 0.9); + } + + { + std::vector ret =n.computeOutput({1,0}); + assert(ret[0] > 0.9); + } + + { + std::vector ret =n.computeOutput({0,0}); + assert(ret[0] < 0.1); + } + } + { // AND problem + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::Sigmoid a(-1); + n.appendLayer(2,a); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::QuickPropagation prop(n); + for(int i=0;i<10000;i++) { + prop.teach({1,1},{1}); + prop.teach({0,0},{0}); + prop.teach({0,1},{0}); + prop.teach({1,0},{0}); + } + + { + std::vector ret =n.computeOutput({1,1}); + assert(ret[0] > 0.9); + } + + { + std::vector ret =n.computeOutput({0,1}); + assert(ret[0] < 0.1); + } + + { + std::vector ret =n.computeOutput({1,0}); + assert(ret[0] < 0.1); + } + + { + std::vector ret =n.computeOutput({0,0}); + assert(ret[0] < 0.1); + } + } + { // NOT AND problem + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::Sigmoid a(-1); + n.appendLayer(2,a); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::QuickPropagation prop(n); + for(int i=0;i<10000;i++) { + prop.teach({1,1},{0}); + prop.teach({0,0},{1}); + prop.teach({0,1},{1}); + prop.teach({1,0},{1}); + } + + { + std::vector ret =n.computeOutput({1,1}); + assert(ret[0] < 0.1); + } + + { + std::vector ret =n.computeOutput({0,1}); + assert(ret[0] > 0.9); + } + + { + std::vector ret =n.computeOutput({1,0}); + assert(ret[0] > 0.9); + } + + { + std::vector ret =n.computeOutput({0,0}); + assert(ret[0] > 0.9); + } + } +} diff --git a/tests/quickpropagation_perf.cpp b/tests/quickpropagation_perf.cpp new file mode 100644 index 0000000..40ed2c3 --- /dev/null +++ b/tests/quickpropagation_perf.cpp @@ -0,0 +1,26 @@ +#include + +#include +#include +#include "../include/NeuralNetwork/Learning/QuickPropagation.h" + +int main() { + { // XOR problem + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::Sigmoid a(-1); + n.appendLayer(200,a); + n.appendLayer(500,a); + n.appendLayer(900,a); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::QuickPropagation prop(n); + for(int i=0;i<100;i++) { + prop.teach({1,0},{1}); + prop.teach({1,1},{0}); + prop.teach({0,0},{0}); + prop.teach({0,1},{1}); + } + } +}