shin/NeuralNetworkLib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

quickProapagtion and tests

Browse Source
This commit is contained in:
Shin
2016-02-24 20:23:16 +01:00
parent c45f12f53c
commit 3c924d01f3
9 changed files with 359 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
CMakeLists.txt
View File

@@ -6,15 +6,15 @@ include(cmake/CPUFeatures)
OPTION(BUILD_SHARED_LIBS "Build also shared library." ON)
if(CPU_AVX_AVAILABLE)
SET(USE_AVX ON CACHE BOOL "")
SET(USE_AVX ON CACHE BOOL "")
endif(CPU_AVX_AVAILABLE)
if(CPU_SSE3_AVAILABLE)
SET(USE_SSE ON CACHE BOOL "")
SET(USE_SSE ON CACHE BOOL "")
if(CPU_SSE4.2_AVAILABLE)
SET(USE_SSE2 OFF CACHE BOOL "")
endif(CPU_SSE4.2_AVAILABLE)
if(CPU_SSE4.2_AVAILABLE)
SET(USE_SSE2 OFF CACHE BOOL "")
endif(CPU_SSE4.2_AVAILABLE)
endif(CPU_SSE3_AVAILABLE)
@@ -33,45 +33,29 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++14")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -mtune=native -O3")
if(USE_AVX)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx -DUSE_AVX")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx -DUSE_AVX")
endif(USE_AVX)
if(USE_SSE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -DUSE_SSE")
if(USE_SSE2)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_SSE2")
endif(USE_SSE2)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -DUSE_SSE")
if(USE_SSE2)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_SSE2")
endif(USE_SSE2)
endif(USE_SSE)
include_directories(./include/)
set (LIBRARY_SOURCES
include/NeuralNetwork/ActivationFunction/ActivationFunction.h
include/NeuralNetwork/ActivationFunction/Heaviside.h
include/NeuralNetwork/ActivationFunction/HyperbolicTangent.h
include/NeuralNetwork/ActivationFunction/Linear.h
include/NeuralNetwork/ActivationFunction/Sigmoid.h
include/NeuralNetwork/ActivationFunction/StreamingActivationFunction.h
include/NeuralNetwork/BasisFunction/BasisFunction.h
include/NeuralNetwork/BasisFunction/Linear.h
include/NeuralNetwork/BasisFunction/Product.h
include/NeuralNetwork/BasisFunction/Radial.h
include/NeuralNetwork/BasisFunction/StreamingBasisFunction.h
include/NeuralNetwork/FeedForward/Layer.h
include/NeuralNetwork/FeedForward/Network.h
include/NeuralNetwork/Recurrent/Network.h
include/NeuralNetwork/Network.h
include/NeuralNetwork/Neuron.h
include/NeuralNetwork/Stringifiable.h
include/NeuralNetwork/Stringifiable.h
src/NeuralNetwork/Learning/BackPropagation.cpp include/NeuralNetwork/Learning/BackPropagation.h
include/sse_mathfun.h
src/NeuralNetwork/BasisFunction/Linear.cpp
src/NeuralNetwork/FeedForward/Layer.cpp
src/NeuralNetwork/FeedForward/Network.cpp
src/NeuralNetwork/Recurrent/Network.cpp
src/NeuralNetwork/Neuron.cpp
src/sse_mathfun.cpp
src/sse_mathfun.cpp
src/NeuralNetwork/Learning/BackPropagation.cpp
src/NeuralNetwork/Learning/QuickPropagation.cpp
src/NeuralNetwork/BasisFunction/Linear.cpp
src/NeuralNetwork/FeedForward/Layer.cpp
src/NeuralNetwork/FeedForward/Network.cpp
src/NeuralNetwork/Recurrent/Network.cpp
src/NeuralNetwork/Neuron.cpp
)
add_library(NeuralNetwork STATIC ${LIBRARY_SOURCES})
@@ -79,8 +63,8 @@ add_library(NeuralNetwork STATIC ${LIBRARY_SOURCES})
link_libraries(NeuralNetwork pthread)
IF(BUILD_SHARED_LIBS)
add_library(NeuralNetworkShared SHARED ${LIBRARY_SOURCES})
set_target_properties(NeuralNetworkShared PROPERTIES OUTPUT_NAME NeuralNetwork)
add_library(NeuralNetworkShared SHARED ${LIBRARY_SOURCES})
set_target_properties(NeuralNetworkShared PROPERTIES OUTPUT_NAME NeuralNetwork)
ENDIF(BUILD_SHARED_LIBS)
IF(ENABLE_TESTS)
@@ -106,15 +90,20 @@ set_property(TEST feedforward PROPERTY LABELS unit)
add_test(recurrent tests/recurrent)
set_property(TEST recurrent PROPERTY LABELS unit)
add_test(feedforward_perf tests/feedforward_perf)
set_property(TEST feedforward_perf PROPERTY LABELS perf)
add_test(optical_backpropagation tests/optical_backpropagation)
set_property(TEST optical_backpropagation PROPERTY LABELS unit)
add_test(quickpropagation tests/quickpropagation)
set_property(TEST quickpropagation PROPERTY LABELS unit)
add_test(feedforward_perf tests/feedforward_perf)
set_property(TEST feedforward_perf PROPERTY LABELS perf)
add_test(quickpropagation_perf tests/quickpropagation_perf)
set_property(TEST quickpropagation_perf PROPERTY LABELS perf)
add_test(backpropagation_perf tests/backpropagation_perf)
set_property(TEST backpropagation_perf PROPERTY LABELS perf)

4
include/NeuralNetwork/Learning/BackPropagation.h
View File

@@ -33,7 +33,7 @@ namespace Learning {
protected:
inline void resize() {
virtual inline void resize() {
if(deltas.size()!=network.size())
deltas.resize(network.size());
@@ -43,6 +43,8 @@ namespace Learning {
}
}
virtual void updateWeights(const std::vector<float> &input);
FeedForward::Network &network;
CorrectionFunction::CorrectionFunction *correctionFunction;

55
include/NeuralNetwork/Learning/QuickPropagation.h Normal file
View File

@@ -0,0 +1,55 @@
#pragma once
#include <vector>
#include <cmath>
#include <NeuralNetwork/FeedForward/Network.h>
#include "BackPropagation.h"
namespace NeuralNetwork {
namespace Learning {
/** @class QuickPropagation
* @brief
*/
class QuickPropagation : public BackPropagation {
public:
inline QuickPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()):
BackPropagation(feedForwardNetwork,correction),deltasPrev() {
resize();
}
virtual ~QuickPropagation() {
}
protected:
virtual inline void resize() override {
if(deltas.size()!=network.size())
deltas.resize(network.size());
for(std::size_t i=0; i < network.size(); i++) {
if(deltas[i].size()!=network[i].size())
deltas[i].resize(network[i].size());
}
if(deltasPrev.size()!=network.size())
deltasPrev.resize(network.size());
for(std::size_t i=0; i < network.size(); i++) {
if(deltasPrev[i].size()!=network[i].size())
deltasPrev[i].resize(network[i].size());
for(std::size_t j=0; j < deltasPrev[i].size(); j++) {
deltasPrev[i][j]=1.0;
}
}
}
virtual void updateWeights(const std::vector<float> &input) override;
std::vector<std::vector<float>> deltasPrev;
};
}
}

8
src/NeuralNetwork/Learning/BackPropagation.cpp
View File

@@ -30,6 +30,11 @@ void NeuralNetwork::Learning::BackPropagation::teach(const std::vector<float> &i
}
}
updateWeights(input);
}
void NeuralNetwork::Learning::BackPropagation::updateWeights(const std::vector<float> &input) {
for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
auto &layer=network[layerIndex];
auto &prevLayer=network[layerIndex-1];
@@ -52,4 +57,5 @@ void NeuralNetwork::Learning::BackPropagation::teach(const std::vector<float> &i
}
}
}
}
}

35
src/NeuralNetwork/Learning/QuickPropagation.cpp Normal file
View File

@@ -0,0 +1,35 @@
#include <NeuralNetwork/Learning/QuickPropagation.h>
#include <cassert>
#include <immintrin.h>
void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector<float> &input) {
for(std::size_t layerIndex=1;layerIndex<network.size();layerIndex++) {
auto &layer=network[layerIndex];
auto &prevLayer=network[layerIndex-1];
std::size_t prevLayerSize=prevLayer.size();
std::size_t layerSize=layer.size();
for(std::size_t j=1;j<layerSize;j++) {
//TODO: is this correct??
float delta=deltas[layerIndex][j]/(deltasPrev[layerIndex][j]-deltas[layerIndex][j]);
deltas[layerIndex][j]=delta;
layer[j].weight(0)+=delta;
for(std::size_t k=1;k<prevLayerSize;k++) {
if(layerIndex==1) {
layer[j].weight(k)+=delta*input[k-1];
} else {
layer[j].weight(k)+=delta*prevLayer[k].output();
}
}
}
}
deltas.swap(deltasPrev);
}

11
tests/CMakeLists.txt
View File

@@ -29,4 +29,13 @@ add_executable(recurrent recurrent.cpp)
target_link_libraries(recurrent NeuralNetwork)
add_executable(recurrent_perf recurrent_perf.cpp)
target_link_libraries(recurrent_perf NeuralNetwork)
target_link_libraries(recurrent_perf NeuralNetwork)
add_executable(quickpropagation quickpropagation.cpp)
target_link_libraries(quickpropagation NeuralNetwork)
add_executable(quickpropagation_perf quickpropagation_perf.cpp)
target_link_libraries(quickpropagation_perf NeuralNetwork)
add_executable(propagation_cmp propagation_cmp.cpp)
target_link_libraries(propagation_cmp NeuralNetwork)

77
tests/propagation_cmp.cpp Normal file
View File

@@ -0,0 +1,77 @@
#include <NeuralNetwork/FeedForward/Network.h>
#include <cassert>
#include <iostream>
#include "../include/NeuralNetwork/Learning/BackPropagation.h"
#include "../include/NeuralNetwork/Learning/QuickPropagation.h"
#include "../include/NeuralNetwork/Learning/CorrectionFunction/Optical.h"
#include "../include/NeuralNetwork/Learning/CorrectionFunction/ArcTangent.h"
#define LEARN(A,AR,B,BR,C,CR,D,DR,FUN,COEF,CLASS) \
({\
srand(rand);\
NeuralNetwork::FeedForward::Network n(2);\
NeuralNetwork::ActivationFunction::Sigmoid a(-1);\
n.appendLayer(2,a);\
n.appendLayer(1,a);\
n.randomizeWeights();\
CLASS prop(n,FUN);\
prop.setLearningCoefficient(COEF);\
int error=1; int steps = 0; \
while(error > 0 && steps <99999) {\
steps++;\
error=0;\
prop.teach(A,{AR});\
prop.teach(B,{BR});\
prop.teach(C,{CR});\
prop.teach(D,{DR});\
error+=fabs(n.computeOutput(A)[0]-AR) > 0.1 ? 1:0;\
error+=fabs(n.computeOutput(B)[0]-BR) > 0.1 ? 1:0;\
error+=fabs(n.computeOutput(C)[0]-CR) > 0.1 ? 1:0;\
error+=fabs(n.computeOutput(D)[0]-DR) > 0.1 ? 1:0;\
}\
steps;\
})
int main() {
long rand=(time(NULL));
const float linearCoef=0.7;
const float opticalCoef=0.11;
const float arcTangentCoef=0.6;
const float arcTangent=1.5;
{
std::cout << "XOR:\n";
std::cout << "\tBP: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
std::cout << "\tQP: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),0,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
}
{
std::cout << "AND:\n";
std::cout << "\tBP: " <<
LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
std::cout << "\tQP: " <<
LEARN(std::vector<float>({1,0}),0,std::vector<float>({1,1}),1,std::vector<float>({0,0}),0,std::vector<float>({0,1}),0,
new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
}
{
std::cout << "AND:\n";
std::cout << "\tBP: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n";
std::cout << "\tQP: " <<
LEARN(std::vector<float>({1,0}),1,std::vector<float>({1,1}),0,std::vector<float>({0,0}),1,std::vector<float>({0,1}),1,
new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n";
}
}

116
tests/quickpropagation.cpp Normal file
View File

@@ -0,0 +1,116 @@
#include <NeuralNetwork/FeedForward/Network.h>
#include <cassert>
#include <iostream>
#include "../include/NeuralNetwork/Learning/QuickPropagation.h"
int main() {
{ // XOR problem
NeuralNetwork::FeedForward::Network n(2);
NeuralNetwork::ActivationFunction::Sigmoid a(-1);
n.appendLayer(2,a);
n.appendLayer(1,a);
n.randomizeWeights();
NeuralNetwork::Learning::QuickPropagation prop(n);
for(int i=0;i<10000;i++) {
prop.teach({1,0},{1});
prop.teach({1,1},{0});
prop.teach({0,0},{0});
prop.teach({0,1},{1});
}
{
std::vector<float> ret =n.computeOutput({1,1});
assert(ret[0] < 0.1);
}
{
std::vector<float> ret =n.computeOutput({0,1});
assert(ret[0] > 0.9);
}
{
std::vector<float> ret =n.computeOutput({1,0});
assert(ret[0] > 0.9);
}
{
std::vector<float> ret =n.computeOutput({0,0});
assert(ret[0] < 0.1);
}
}
{ // AND problem
NeuralNetwork::FeedForward::Network n(2);
NeuralNetwork::ActivationFunction::Sigmoid a(-1);
n.appendLayer(2,a);
n.appendLayer(1,a);
n.randomizeWeights();
NeuralNetwork::Learning::QuickPropagation prop(n);
for(int i=0;i<10000;i++) {
prop.teach({1,1},{1});
prop.teach({0,0},{0});
prop.teach({0,1},{0});
prop.teach({1,0},{0});
}
{
std::vector<float> ret =n.computeOutput({1,1});
assert(ret[0] > 0.9);
}
{
std::vector<float> ret =n.computeOutput({0,1});
assert(ret[0] < 0.1);
}
{
std::vector<float> ret =n.computeOutput({1,0});
assert(ret[0] < 0.1);
}
{
std::vector<float> ret =n.computeOutput({0,0});
assert(ret[0] < 0.1);
}
}
{ // NOT AND problem
NeuralNetwork::FeedForward::Network n(2);
NeuralNetwork::ActivationFunction::Sigmoid a(-1);
n.appendLayer(2,a);
n.appendLayer(1,a);
n.randomizeWeights();
NeuralNetwork::Learning::QuickPropagation prop(n);
for(int i=0;i<10000;i++) {
prop.teach({1,1},{0});
prop.teach({0,0},{1});
prop.teach({0,1},{1});
prop.teach({1,0},{1});
}
{
std::vector<float> ret =n.computeOutput({1,1});
assert(ret[0] < 0.1);
}
{
std::vector<float> ret =n.computeOutput({0,1});
assert(ret[0] > 0.9);
}
{
std::vector<float> ret =n.computeOutput({1,0});
assert(ret[0] > 0.9);
}
{
std::vector<float> ret =n.computeOutput({0,0});
assert(ret[0] > 0.9);
}
}
}

26
tests/quickpropagation_perf.cpp Normal file
View File

@@ -0,0 +1,26 @@
#include <NeuralNetwork/FeedForward/Network.h>
#include <cassert>
#include <iostream>
#include "../include/NeuralNetwork/Learning/QuickPropagation.h"
int main() {
{ // XOR problem
NeuralNetwork::FeedForward::Network n(2);
NeuralNetwork::ActivationFunction::Sigmoid a(-1);
n.appendLayer(200,a);
n.appendLayer(500,a);
n.appendLayer(900,a);
n.appendLayer(1,a);
n.randomizeWeights();
NeuralNetwork::Learning::QuickPropagation prop(n);
for(int i=0;i<100;i++) {
prop.teach({1,0},{1});
prop.teach({1,1},{0});
prop.teach({0,0},{0});
prop.teach({0,1},{1});
}
}
}