From 006810a49c592edfb0225ee2e4f10668f3e3689a Mon Sep 17 00:00:00 2001 From: Shin Date: Sun, 30 Oct 2016 20:40:07 +0100 Subject: [PATCH 01/11] bp: refactored --- .../NeuralNetwork/Learning/BackPropagation.h | 50 +++++++--- .../Learning/BackPropagation.cpp | 93 +++++++++++-------- 2 files changed, 95 insertions(+), 48 deletions(-) diff --git a/include/NeuralNetwork/Learning/BackPropagation.h b/include/NeuralNetwork/Learning/BackPropagation.h index c880519..1df9b42 100644 --- a/include/NeuralNetwork/Learning/BackPropagation.h +++ b/include/NeuralNetwork/Learning/BackPropagation.h @@ -37,6 +37,7 @@ namespace Learning { void setMomentumWeight(const float& m) { momentumWeight=m; + resize(); } float getWeightDecay() const { @@ -47,6 +48,14 @@ namespace Learning { weightDecay=wd; } + std::size_t getBatchSize() const { + return batchSize; + } + + void setBatchSize(std::size_t size) { + batchSize = size; + } + protected: virtual inline void resize() { @@ -58,25 +67,40 @@ namespace Learning { slopes[i].resize(network[i].size()); } - if(lastDeltas.size()!=network.size()) - lastDeltas.resize(network.size()); + if(deltas.size() != network.size()) + deltas.resize(network.size()); - for(std::size_t i=0; i < network.size(); i++) { - if(lastDeltas[i].size()!=network[i].size()) { - lastDeltas[i].resize(network[i].size()); + bool resized = false; - for(std::size_t j = 0; j < lastDeltas[i].size(); j++) { - lastDeltas[i][j] = 0.0; + for(std::size_t i = 0; i < network.size(); i++) { + if(deltas[i].size() != network[i].size()) { + deltas[i].resize(network[i].size()); + resized = true; + + if(i > 0) { + for(std::size_t j = 0; j < deltas[i].size(); j++) { + deltas[i][j].resize(network[i - 1].size()); + std::fill(deltas[i][j].begin(),deltas[i][j].end(),0.0); + } } } } - deltas= lastDeltas; + + if(momentumWeight > 0.0 && (resized || lastDeltas.size() != deltas.size())) { + lastDeltas = deltas; + } } - virtual void updateWeights(const std::vector &input); + virtual void computeDeltas(const std::vector &input); + + void updateWeights(); virtual void computeSlopes(const std::vector &expectation); + virtual void endBatch() { + + } + FeedForward::Network &network; CorrectionFunction::CorrectionFunction *correctionFunction; @@ -87,9 +111,13 @@ namespace Learning { float weightDecay = 0.0; + std::size_t batchSize = 1; + std::size_t currentBatchSize = 0; + std::vector> slopes; - std::vector> deltas; - std::vector> lastDeltas; + std::vector>> deltas = {}; + std::vector>> lastDeltas = {}; + }; } } \ No newline at end of file diff --git a/src/NeuralNetwork/Learning/BackPropagation.cpp b/src/NeuralNetwork/Learning/BackPropagation.cpp index 9801169..b562393 100644 --- a/src/NeuralNetwork/Learning/BackPropagation.cpp +++ b/src/NeuralNetwork/Learning/BackPropagation.cpp @@ -4,47 +4,15 @@ #include void NeuralNetwork::Learning::BackPropagation::teach(const std::vector &input, const std::vector &expectation) { - network.computeOutput(input); - resize(); - computeSlopes(expectation); - updateWeights(input); - - std::swap(deltas,lastDeltas); -} - - -void NeuralNetwork::Learning::BackPropagation::updateWeights(const std::vector &input) { - - for(std::size_t layerIndex=1;layerIndex= batchSize) { + updateWeights(); + endBatch(); + currentBatchSize=0; } } @@ -71,4 +39,55 @@ void NeuralNetwork::Learning::BackPropagation::computeSlopes(const std::vector &input) { + for(std::size_t layerIndex=1;layerIndex 0.0; + + for(std::size_t layerIndex=1;layerIndex Date: Sun, 30 Oct 2016 20:45:06 +0100 Subject: [PATCH 02/11] removed Cellular encoding --- CMakeLists.txt | 3 - .../CelularEncoding/Cell.h | 182 ------------- .../CelularEncoding/CelularEncoding.h | 182 ------------- .../CelularEncoding/Exception.h | 23 -- .../CelularEncoding/Instruction/Bias.h | 143 ---------- .../CelularEncoding/Instruction/Division.h | 101 -------- .../CelularEncoding/Instruction/Finish.h | 58 ----- .../CelularEncoding/Instruction/Instruction.h | 32 --- .../CelularEncoding/Instruction/Link.h | 245 ------------------ .../CelularEncoding/Instruction/Other.h | 34 --- .../CellularEncoding/Cell.cpp | 12 - .../CellularEncoding/CellularEncoding.cpp | 20 -- tests/CMakeLists.txt | 6 - 13 files changed, 1041 deletions(-) delete mode 100644 include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Cell.h delete mode 100644 include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/CelularEncoding.h delete mode 100644 include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Exception.h delete mode 100644 include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Bias.h delete mode 100644 include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Division.h delete mode 100644 include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Finish.h delete mode 100644 include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Instruction.h delete mode 100644 include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Link.h delete mode 100644 include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Other.h delete mode 100644 src/NeuralNetwork/ConstructiveAlgorithms/CellularEncoding/Cell.cpp delete mode 100644 src/NeuralNetwork/ConstructiveAlgorithms/CellularEncoding/CellularEncoding.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cc802dc..a3b7b1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,9 +76,6 @@ set (LIBRARY_SOURCES src/NeuralNetwork/Neuron.cpp src/NeuralNetwork/IMPL.cpp - - src/NeuralNetwork/ConstructiveAlgorithms/CellularEncoding/CellularEncoding.cpp - src/NeuralNetwork/ConstructiveAlgorithms/CellularEncoding/Cell.cpp ) add_library(NeuralNetwork STATIC ${LIBRARY_SOURCES}) diff --git a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Cell.h b/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Cell.h deleted file mode 100644 index 03fa359..0000000 --- a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Cell.h +++ /dev/null @@ -1,182 +0,0 @@ -#pragma once - -#include -#include "./Instruction/Instruction.h" - -#include -#include - -namespace NeuralNetworks { -namespace ConstructiveAlgorithms { -namespace CelularEncoding { - class CelularEncoding; - - class Link { - public: - Link(bool status_, float value_, std::size_t neuron_) : status(status_), value(value_), neuron(neuron_) { - - } - - bool status=false; - float value=0.0; - std::size_t neuron; - }; - - class Cell { - friend Instruction::Instruction; - - public: - Cell(const Cell&)=delete; - Cell&operator=(const Cell&)=delete; - - Cell(Cell &&r): id(r.id), codePtr(r.codePtr), _isOutput(r._isOutput), _isInput(r._isInput), alive(r.alive), life(r.life), bias(r.bias), links(r.links) { - - } - - Cell(const std::size_t& id_, const EvolutionaryAlgorithm::GeneticPrograming::CodeTree* codePtr_): id(id_), codePtr(codePtr_) { - - } - - void step(CelularEncoding&); - - bool isAlive() const { - return alive; - }; - - void die() { - alive=false; - }; - - std::size_t getLife() const { - return life; - } - - void setLife(const std::size_t &newLife) { - life=newLife; - } - - float getBias() const { - return bias; - } - void setBias(const float &newBias) { - bias=newBias; - } - - std::size_t getLinkRegister() const { - return linkRegister; - } - - void setLinkRegister(const std::size_t &newLinkRegister) { - linkRegister=newLinkRegister; - } - - const std::vector& getLinks() const { - return links; - } - - void setLinks(const std::vector &links_) { - links=links_; - } - - void addLink(const Link& l) { - for(auto &link:links) { - if(link.neuron ==l.neuron) { - link.value=l.value; - link.status=l.status; - return; - } - } - links.push_back(l); - } - - bool getLinkStatus(const std::size_t index) const { - if(index < links.size()) { - return links[index].status; - } - return false; - } - - void setLinkStatus(const std::size_t index, const bool &status) { - if(index < links.size()) { - links[index].status=status; - } - } - - float getLinkValue(const std::size_t index) const { - if(index < links.size()) { - return links[index].value; - } - return 0.0; - } - - void setLinkValue(const std::size_t index, const float &value) { - if(index < links.size()) { - links[index].value = value; - } - } - - void setCodePointer(const EvolutionaryAlgorithm::GeneticPrograming::CodeTree *c) { - codePtr=c; - } - - const EvolutionaryAlgorithm::GeneticPrograming::CodeTree* getCodePointer() const { - return codePtr; - } - - std::size_t getID() const { - return id; - } - - bool isOutput() const { - return _isOutput; - } - - void setOutput() { - _isOutput=true; - } - void disableOutput() { - _isOutput=false; - } - - bool isInput() const { - return _isInput; - } - - void setInput() { - _isInput=true; - } - void disableInput() { - _isInput=false; - } - - protected: - std::size_t id; - - const EvolutionaryAlgorithm::GeneticPrograming::CodeTree *codePtr; - //const Code &code; - - bool _isOutput=false; - bool _isInput=false; - - bool alive=true; - std::size_t life=1; - float bias = 0.0; - - std::size_t linkRegister=0; - - std::vector links = std::vector(); -/* - std::size_t codePosition; - std::size_t linkRegister; - - std::size_t sigme; - std::size_t dyn; - std::size_t link; - std::size_t simplif; -*/ - // 150 of paper - private: - }; -} -} -} \ No newline at end of file diff --git a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/CelularEncoding.h b/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/CelularEncoding.h deleted file mode 100644 index 7f38fab..0000000 --- a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/CelularEncoding.h +++ /dev/null @@ -1,182 +0,0 @@ -#pragma once - -#include "Exception.h" -#include "Cell.h" - -#include - -#include -#include - -namespace NeuralNetworks { - namespace ConstructiveAlgorithms { - namespace CelularEncoding { - class CelularEncoding { - public: - CelularEncoding(const CelularEncoding &) = delete; - - CelularEncoding &operator=(const CelularEncoding &) = delete; - - CelularEncoding() { - - } - - void setActivationFunction(const std::shared_ptr &fun) { - _activationFunction=fun; - } - - void setMaxSteps(std::size_t steps) { - _maxSteps=steps; - } - - NeuralNetwork::Recurrent::Network run() { - std::size_t cellsStep = 0; - std::size_t steps=0; - do { - cellsStep = step(); - steps++; - } - while(cellsStep > 0 && steps < _maxSteps); - - if(steps >= _maxSteps) { - throw Exception("Went over max steps"); - } - - if(cells.size() > _maxCells) { - throw Exception("Went over max cells"); - } - - std::size_t outputs = 0; - std::size_t inputs = 0; - - std::vector cells2Neurons; - cells2Neurons.resize(cells.size()); - - std::size_t indexOfNeuronTmp=1; - - for(std::size_t i = 0; i < cells.size(); i++) { - if(cells[i]->isInput()) { - cells2Neurons[i] = indexOfNeuronTmp++; - inputs++; - } - } - - for(std::size_t i = 0; i < cells.size(); i++) { - if(cells[i]->isOutput()) { - if(!cells[i]->isInput()) { - cells2Neurons[i] = indexOfNeuronTmp++; - } - outputs++; - } - } - - for(std::size_t i = 0; i < cells.size(); i++) { - if(!cells[i]->isOutput() && !cells[i]->isInput()) { - cells2Neurons[i] = indexOfNeuronTmp++; - } - } - - std::size_t hiddenNeurons = static_cast(cells.size()) - static_cast(inputs) - static_cast(outputs) < 0 ? 0 : cells.size() - inputs - outputs; - - NeuralNetwork::Recurrent::Network netw(inputs, outputs, hiddenNeurons); - for(std::size_t i = 0; i < cells.size(); i++) { - - const auto &cell = cells[i]; - - std::size_t indexOfNeuron = cells2Neurons[i]; - auto& neuron = netw[indexOfNeuron]; - if(cells2Neurons[i] > inputs) { - neuron.setActivationFunction(*_activationFunction); - } - - neuron.weight(0)=cell->getBias(); - - for(auto &link: cells[i]->getLinks()) { - if(link.status == true) { - neuron.weight(cells2Neurons[link.neuron]) = link.value; - } else { - neuron.weight(cells2Neurons[link.neuron]) = 0.0; - } - } - } - - return netw; - } - - Cell &addCell(const EvolutionaryAlgorithm::GeneticPrograming::CodeTree *c) { - cells.push_back(std::make_shared(cells.size(), c)); - - return (*cells.back()); - } - - void setAcyclicTopology() { - cells.clear(); -/* - for(std::size_t i = 0; i < inputSize; i++) { - addCell(code).die(); - } -*/ - Cell &cell = addCell(code); - cell.setLife(_initialLife); - _processingOrder.push_back(cell.getID()); - - cell.setOutput(); - cell.setInput(); -/* - for(std::size_t i = 0; i < inputSize; i++) { - Link l(true, 1.0, i); - cell.addLink(l); - } -*/ - } - - void setCyclicTopology() { - setAcyclicTopology(); - // Acyclic + reccurent link - Link l(true, 1.0, cells.back()->getID()); - cells.back()->addLink(l); - } - - void setCode(const EvolutionaryAlgorithm::GeneticPrograming::CodeTree *code_) { - code = code_; - } - - const EvolutionaryAlgorithm::GeneticPrograming::CodeTree *getCodeStart() const { - return code; - } - - std::vector> &getCells() { - return cells; - } - - void addCellToProcessingOrder(std::size_t id) { - auto position = std::find(_processingOrder.begin(),_processingOrder.end(),currentID); - if(position == _processingOrder.end()) { - _processingOrder.push_back(id); - } else { - _processingOrder.insert(position+1,id); - } - } - - void setInitiaLife(std::size_t life) { - _initialLife=life; - } - - protected: - std::size_t step(); - - private: - std::size_t _maxCells= 15; - std::size_t _maxSteps = std::numeric_limits::max(); - std::size_t _initialLife = 2.0; - - std::shared_ptr _activationFunction = std::make_shared(); - std::vector _processingOrder = {}; - std::size_t currentID = 0; - const EvolutionaryAlgorithm::GeneticPrograming::CodeTree *code = nullptr; - std::vector> cells = {}; - }; - - } - } -} \ No newline at end of file diff --git a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Exception.h b/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Exception.h deleted file mode 100644 index 9830a23..0000000 --- a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Exception.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include -#include - -namespace NeuralNetworks { - namespace ConstructiveAlgorithms { - namespace CelularEncoding { - class Exception : public std::exception { - public: - Exception (const std::string &e) : _what(e) { - - } - - virtual const char* what() const noexcept override { - return _what.c_str(); - } - protected: - std::string _what; - }; - } - } -} \ No newline at end of file diff --git a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Bias.h b/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Bias.h deleted file mode 100644 index c040140..0000000 --- a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Bias.h +++ /dev/null @@ -1,143 +0,0 @@ -#pragma once - -#include "./Instruction.h" -#include "../Cell.h" -#include "../CelularEncoding.h" - -namespace NeuralNetworks { -namespace ConstructiveAlgorithms { -namespace CelularEncoding { -namespace Instruction{ - class IncBias : public Instruction{ - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setBias(cell.getBias()+1.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "IncBias"; - } - }; - - class DecBias : public Instruction{ - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setBias(cell.getBias()-1.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "DecBias"; - } - - }; - class MulBias : public Instruction{ - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setBias(cell.getBias()*2.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "MulBias"; - } - }; - - class DivBias : public Instruction{ - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setBias(cell.getBias()*0.5); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "DivBias"; - } - }; - - class SetBiasZero : public Instruction{ - public: - SetBiasZero(bool terminal = true) : _nodes(terminal ? 0 : 1) { - - } - - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setBias(0.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return _nodes; - } - - virtual std::string toString() const override { - return "SetBiasZero"; - } - protected: - std::size_t _nodes; - }; - - class SetBiasOne : public Instruction{ - public: - SetBiasOne(bool terminal = true) : _nodes(terminal ? 0 : 1) { - - } - - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setBias(1.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return _nodes; - } - - virtual std::string toString() const override { - return "SetBiasZero"; - } - protected: - std::size_t _nodes; - }; - - class SetBiasMinusOne : public Instruction{ - public: - SetBiasMinusOne(bool terminal = true) : _nodes(terminal ? 0 : 1) { - - } - - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setBias(-1.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return _nodes; - } - - virtual std::string toString() const override { - return "SetBiasZero"; - } - protected: - std::size_t _nodes; - }; -} -} -} -} \ No newline at end of file diff --git a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Division.h b/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Division.h deleted file mode 100644 index 5e30fff..0000000 --- a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Division.h +++ /dev/null @@ -1,101 +0,0 @@ -#pragma once - -#include "./Instruction.h" -#include "../Cell.h" -#include "../CelularEncoding.h" - -namespace NeuralNetworks { -namespace ConstructiveAlgorithms { -namespace CelularEncoding { -namespace Instruction{ - class Par : public Instruction{ - public: - virtual void run(Cell &cell, CelularEncoding &encoding, const std::vector &) override { - Cell &newCell = encoding.addCell(cell.getCodePointer()->c2); - - if(cell.isOutput()) { - newCell.setOutput(); - } - - if(cell.isInput()) { - newCell.setInput(); - } - - newCell.setLife(cell.getLife()); - newCell.setBias(cell.getBias()); - - newCell.setLinks(cell.getLinks()); - - for(auto &cellI:encoding.getCells()) { - for( const Link &link: cellI->getLinks()) { - if(link.neuron == cell.getID()) { - Link newLink (link); - newLink.neuron=newCell.getID(); - cellI->addLink(newLink); - break; - } - } - } - encoding.addCellToProcessingOrder(newCell.getID()); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 2; - } - - virtual std::string toString() const override { - return "Par"; - } - - protected: - private: - }; - - class Seq : public Instruction{ - public: - virtual void run(Cell &cell, CelularEncoding &encoding, const std::vector &) override { - Cell &newCell = encoding.addCell(cell.getCodePointer()->c2); - - Link l(true, 1.0, cell.getID()); - - if(cell.isOutput()) { - newCell.setOutput(); - cell.disableOutput(); - } - - newCell.addLink(l); - - for(auto& cellIter:encoding.getCells()) { - if(cellIter->getID() != newCell.getID()) { - auto links = cellIter->getLinks(); - for(auto& link: links) { - if(link.neuron==cell.getID()) { - link.neuron=newCell.getID(); - } - } - cellIter->setLinks(links); - } - } - - newCell.setLife(cell.getLife()); - encoding.addCellToProcessingOrder(newCell.getID()); - cell.setCodePointer(cell.getCodePointer()->c1); - //TODO: copy registers - } - - virtual std::size_t numberOfNodes() const override { - return 2; - } - - virtual std::string toString() const override { - return "Seq"; - } - - protected: - private: - }; -} -} -} -} \ No newline at end of file diff --git a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Finish.h b/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Finish.h deleted file mode 100644 index 89d4bbd..0000000 --- a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Finish.h +++ /dev/null @@ -1,58 +0,0 @@ -#pragma once - -#include "./Instruction.h" -#include "../Cell.h" -#include "../CelularEncoding.h" - -namespace NeuralNetworks { -namespace ConstructiveAlgorithms { -namespace CelularEncoding { -namespace Instruction{ - class End : public Instruction{ - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.die(); - } - - virtual std::size_t numberOfNodes() const override { - return 0; - } - - virtual std::string toString() const override { - return "End"; - } - protected: - private: - }; - - class Rec : public Instruction{ - public: - Rec(bool terminal = true) : _nodes(terminal ? 0 : 1) { - - } - - virtual void run(Cell &cell, CelularEncoding &encoding, const std::vector &) override { - if(cell.getLife()<=1) { - cell.setCodePointer(cell.getCodePointer()->c1); - cell.die(); - }else { - cell.setLife(cell.getLife()-1); - cell.setCodePointer(encoding.getCodeStart()); - } - } - - virtual std::size_t numberOfNodes() const override { - return _nodes; - } - - virtual std::string toString() const override { - return "Rec"; - } - protected: - std::size_t _nodes; - private: - }; -} -} -} -} \ No newline at end of file diff --git a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Instruction.h b/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Instruction.h deleted file mode 100644 index 8d785cb..0000000 --- a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Instruction.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include - -#include - -namespace NeuralNetworks { -namespace ConstructiveAlgorithms { -namespace CelularEncoding { - - class Cell; - class CelularEncoding; - - namespace Instruction { - class Instruction : public EvolutionaryAlgorithm::GeneticPrograming::Instruction { - public: - virtual ~Instruction() { - - } - - // todo what implement?? - void operator()() {} - - virtual void run(Cell &, CelularEncoding &, const std::vector &) = 0; - - protected: - private: - }; - } -} -} -} \ No newline at end of file diff --git a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Link.h b/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Link.h deleted file mode 100644 index b52860b..0000000 --- a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Link.h +++ /dev/null @@ -1,245 +0,0 @@ -#pragma once - -#include "./Instruction.h" -#include "../Cell.h" -#include "../CelularEncoding.h" - -namespace NeuralNetworks { - namespace ConstructiveAlgorithms { - namespace CelularEncoding { - namespace Instruction { - class Inclr : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - if(cell.getLinkRegister() == cell.getLinks().size() - 1) { - cell.setLinkRegister(0); - } else { - cell.setLinkRegister(cell.getLinkRegister() + 1); - } - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "IncLr"; - } - - protected: - private: - }; - - class Declr : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - if(cell.getLinkRegister() == 0) { - if(cell.getLinks().size() > 1) { - cell.setLinkRegister(cell.getLinks().size() - 1); - } - } else { - cell.setLinkRegister(cell.getLinkRegister() - 1); - } - - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "Declr"; - } - - protected: - private: - }; - - class ValPlus : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setLinkValue(cell.getLinkRegister(), 1.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "Val+"; - } - - protected: - private: - }; - - class ValMinus : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setLinkValue(cell.getLinkRegister(), -1.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "Val-"; - } - - protected: - private: - }; - - class Inc : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setLinkValue(cell.getLinkRegister(), cell.getLinkValue(cell.getLinkRegister()) + 1.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "Inc"; - } - - protected: - private: - }; - - class Dec : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setLinkValue(cell.getLinkRegister(), cell.getLinkValue(cell.getLinkRegister()) - 1.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "Dec"; - } - - protected: - private: - }; - - class Mult : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setLinkValue(cell.getLinkRegister(), cell.getLinkValue(cell.getLinkRegister()) * 2.0); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "Mult"; - } - - protected: - private: - }; - - class Div : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setLinkValue(cell.getLinkRegister(), cell.getLinkValue(cell.getLinkRegister()) * 0.5); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "Div"; - } - - protected: - private: - }; - - class On : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setLinkStatus(cell.getLinkRegister(), true); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "On"; - } - - protected: - private: - }; - - class Cyc : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - bool exists = false; - for(std::size_t i = 0; i < cell.getLinks().size(); i++) { - auto &link = cell.getLinks()[i]; - if(link.neuron == cell.getID()) { - exists = true; - cell.setLinkStatus(i,true); - } - } - - if(!exists) { - cell.addLink(Link(true, 1.0, cell.getID())); - } - - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "Cyc"; - } - - protected: - private: - }; - - class Off : public Instruction { - public: - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setLinkStatus(cell.getLinkRegister(), false); - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return 1; - } - - virtual std::string toString() const override { - return "Off"; - } - - protected: - private: - }; - } - } - } -} \ No newline at end of file diff --git a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Other.h b/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Other.h deleted file mode 100644 index e4c0a1f..0000000 --- a/include/NeuralNetwork/ConstructiveAlgorithms/CelularEncoding/Instruction/Other.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include "./Instruction.h" -#include "../Cell.h" -#include "../CelularEncoding.h" - -namespace NeuralNetworks { - namespace ConstructiveAlgorithms { - namespace CelularEncoding { - namespace Instruction{ - class Wait : public Instruction{ - public: - Wait(bool terminal = false) : _numberOfNodes(terminal? 0 : 1) { - - } - - virtual void run(Cell &cell, CelularEncoding &, const std::vector &) override { - cell.setCodePointer(cell.getCodePointer()->c1); - } - - virtual std::size_t numberOfNodes() const override { - return _numberOfNodes; - } - - virtual std::string toString() const override { - return "Wait"; - } - protected: - std::size_t _numberOfNodes; - }; - } - } - } -} \ No newline at end of file diff --git a/src/NeuralNetwork/ConstructiveAlgorithms/CellularEncoding/Cell.cpp b/src/NeuralNetwork/ConstructiveAlgorithms/CellularEncoding/Cell.cpp deleted file mode 100644 index f5e1aed..0000000 --- a/src/NeuralNetwork/ConstructiveAlgorithms/CellularEncoding/Cell.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include -#include - -void NeuralNetworks::ConstructiveAlgorithms::CelularEncoding::Cell::step(CelularEncoding &c) { - if(isAlive()) { - reinterpret_cast(codePtr->instruction.lock().get())->run(*this, c, codePtr->parameters); - } - - if(codePtr == nullptr) { - die(); - } -} \ No newline at end of file diff --git a/src/NeuralNetwork/ConstructiveAlgorithms/CellularEncoding/CellularEncoding.cpp b/src/NeuralNetwork/ConstructiveAlgorithms/CellularEncoding/CellularEncoding.cpp deleted file mode 100644 index ae62066..0000000 --- a/src/NeuralNetwork/ConstructiveAlgorithms/CellularEncoding/CellularEncoding.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include - -std::size_t NeuralNetworks::ConstructiveAlgorithms::CelularEncoding::CelularEncoding::step() { - std::size_t activeCellCount = 0; - std::vector processingOrder(_processingOrder); - for(std::size_t i = 0; i < processingOrder.size(); i++) { - std::size_t cell = processingOrder[i]; - if(cells[cell]->isAlive()) { - currentID = cells[cell]->getID(); - cells[cell]->step(*this); - activeCellCount++; - } else { - auto iter = std::find(_processingOrder.begin(), _processingOrder.end(), cell); - if(iter != _processingOrder.end()) { - _processingOrder.erase(iter); - } - } - } - return activeCellCount; -} \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c3aa6d5..09ef39a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -28,12 +28,6 @@ target_link_libraries(recurrent NeuralNetwork gtest gtest_main) add_executable(quickpropagation quickpropagation.cpp) target_link_libraries(quickpropagation NeuralNetwork gtest gtest_main) -add_executable(celular_code celular_code.cpp) -target_link_libraries(celular_code NeuralNetwork gtest gtest_main) - -add_executable(genetic_programing genetic_programing.cpp) -target_link_libraries(genetic_programing NeuralNetwork gtest gtest_main) - # PERF add_executable(backpropagation_function_cmp backpropagation_function_cmp.cpp) From cb7e3a9f3501607f6a8f1f964a8f2962717f1d30 Mon Sep 17 00:00:00 2001 From: Shin Date: Sun, 30 Oct 2016 20:47:21 +0100 Subject: [PATCH 03/11] removed Evo library --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 480d6f7..e3cbfa2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "lib/SimpleJSON"] path = lib/SimpleJSON url = http://gitlab.ishin.cz/shin/SimpleJSON.git -[submodule "lib/EvolutionaryAlgorithm"] - path = lib/EvolutionaryAlgorithm - url = http://gitlab.ishin.cz/shin/EvolutionaryAlgorithm.git From 554ef1b46bda72c47adfa865cc8a0f5ea59b412f Mon Sep 17 00:00:00 2001 From: Shin Date: Sun, 30 Oct 2016 20:47:33 +0100 Subject: [PATCH 04/11] removed evolAlgorithm --- lib/EvolutionaryAlgorithm | 1 - 1 file changed, 1 deletion(-) delete mode 160000 lib/EvolutionaryAlgorithm diff --git a/lib/EvolutionaryAlgorithm b/lib/EvolutionaryAlgorithm deleted file mode 160000 index a0bb388..0000000 --- a/lib/EvolutionaryAlgorithm +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a0bb388dbabc661d36b1af871fda89d1af6e3adf From 8749b3eb03db9af787c53880f23d5ba7f148383a Mon Sep 17 00:00:00 2001 From: Shin Date: Sun, 30 Oct 2016 23:00:50 +0100 Subject: [PATCH 05/11] Rprop implementation --- CMakeLists.txt | 7 +- include/NeuralNetwork/Learning/RProp.h | 140 +++++++++++++++++++++ src/NeuralNetwork/Learning/RProp.cpp | 103 +++++++++++++++ tests/CMakeLists.txt | 3 + tests/rprop.cpp | 165 +++++++++++++++++++++++++ 5 files changed, 415 insertions(+), 3 deletions(-) create mode 100644 include/NeuralNetwork/Learning/RProp.h create mode 100644 src/NeuralNetwork/Learning/RProp.cpp create mode 100644 tests/rprop.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a3b7b1c..22ab38f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,7 @@ set (LIBRARY_SOURCES src/NeuralNetwork/Learning/BackPropagation.cpp src/NeuralNetwork/Learning/QuickPropagation.cpp src/NeuralNetwork/Learning/PerceptronLearning.cpp + src/NeuralNetwork/Learning/RProp.cpp src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp src/NeuralNetwork/ConstructiveAlgorithms/Cascade2.cpp @@ -118,6 +119,9 @@ IF(ENABLE_TESTS) add_test(quickpropagation tests/quickpropagation) set_property(TEST quickpropagation PROPERTY LABELS unit) + add_test(rprop tests/rprop) + set_property(TEST rprop PROPERTY LABELS unit) + add_test(recurrent tests/recurrent) set_property(TEST recurrent PROPERTY LABELS unit) @@ -136,8 +140,5 @@ IF(ENABLE_TESTS) add_test(recurrent_perf tests/recurrent_perf) set_property(TEST recurrent_perf PROPERTY LABELS perf) - add_test(genetic_programing tests/genetic_programing) - set_property(TEST genetic_programing PROPERTY LABELS unit) - ENDIF(ENABLE_TESTS) diff --git a/include/NeuralNetwork/Learning/RProp.h b/include/NeuralNetwork/Learning/RProp.h new file mode 100644 index 0000000..a838e0e --- /dev/null +++ b/include/NeuralNetwork/Learning/RProp.h @@ -0,0 +1,140 @@ +#pragma once + +#include +#include + +#include +#include "CorrectionFunction/Linear.h" + +namespace NeuralNetwork { + namespace Learning { + + /** @class Resilient Propagation + * @brief + */ + class RProp { + + public: + RProp(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()): + network(feedForwardNetwork), correctionFunction(correction) { + resize(); + } + + virtual ~RProp() { + delete correctionFunction; + } + + RProp(const RProp&)=delete; + RProp& operator=(const NeuralNetwork::Learning::RProp&) = delete; + + void teach(const std::vector &input, const std::vector &output); + + std::size_t getBatchSize() const { + return batchSize; + } + + void setBatchSize(std::size_t size) { + batchSize = size; + } + + void setInitialWeightChange(float init) { + initialWeightChange=init; + } + protected: + + virtual inline void resize() { + if(slopes.size()!=network.size()) + slopes.resize(network.size()); + + for(std::size_t i=0; i < network.size(); i++) { + if(slopes[i].size()!=network[i].size()) + slopes[i].resize(network[i].size()); + } + + if(gradients.size() != network.size()) + gradients.resize(network.size()); + + bool resized = false; + + for(std::size_t i = 0; i < network.size(); i++) { + if(gradients[i].size() != network[i].size()) { + gradients[i].resize(network[i].size()); + resized = true; + + if(i > 0) { + for(std::size_t j = 0; j < gradients[i].size(); j++) { + gradients[i][j].resize(network[i - 1].size()); + std::fill(gradients[i][j].begin(),gradients[i][j].end(),0.0); + } + } + } + } + + if(resized) { + lastGradients = gradients; + + if(changesOfWeightChanges.size() != network.size()) + changesOfWeightChanges.resize(network.size()); + + for(std::size_t i = 0; i < network.size(); i++) { + if(changesOfWeightChanges[i].size() != network[i].size()) { + changesOfWeightChanges[i].resize(network[i].size()); + if(i > 0) { + for(std::size_t j = 0; j < changesOfWeightChanges[i].size(); j++) { + changesOfWeightChanges[i][j].resize(network[i - 1].size()); + std::fill(changesOfWeightChanges[i][j].begin(),changesOfWeightChanges[i][j].end(),initialWeightChange); + } + } + } + } + } + + if(resized) { + if(lastWeightChanges.size() != network.size()) + lastWeightChanges.resize(network.size()); + + for(std::size_t i = 0; i < network.size(); i++) { + if(lastWeightChanges[i].size() != network[i].size()) { + lastWeightChanges[i].resize(network[i].size()); + if(i > 0) { + for(std::size_t j = 0; j < lastWeightChanges[i].size(); j++) { + lastWeightChanges[i][j].resize(network[i - 1].size()); + std::fill(lastWeightChanges[i][j].begin(),lastWeightChanges[i][j].end(),0.1); + } + } + } + } + } + } + + virtual void computeSlopes(const std::vector &expectation); + virtual void computeDeltas(const std::vector &input); + + void updateWeights(); + + virtual void endBatch() { + + } + + FeedForward::Network &network; + + CorrectionFunction::CorrectionFunction *correctionFunction; + + std::vector> slopes; + std::vector>> gradients = {}; + std::vector>> lastGradients = {}; + std::vector>> lastWeightChanges = {}; + std::vector>> changesOfWeightChanges = {}; + + std::size_t batchSize = 1; + std::size_t currentBatchSize = 0; + + float maxChangeOfWeights = 50; + float minChangeOfWeights = 0.0001; + + float initialWeightChange=0.02; + float weightChangePlus=1.2; + float weightChangeMinus=0.5; + }; + } +} \ No newline at end of file diff --git a/src/NeuralNetwork/Learning/RProp.cpp b/src/NeuralNetwork/Learning/RProp.cpp new file mode 100644 index 0000000..ab17e8e --- /dev/null +++ b/src/NeuralNetwork/Learning/RProp.cpp @@ -0,0 +1,103 @@ +#include + +void NeuralNetwork::Learning::RProp::teach(const std::vector &input, const std::vector &expectation) { + network.computeOutput(input); + resize(); + computeSlopes(expectation); + + computeDeltas(input); + if(++currentBatchSize >= batchSize) { + updateWeights(); + endBatch(); + currentBatchSize=0; + } +} + +void NeuralNetwork::Learning::RProp::computeSlopes(const std::vector &expectation) { + auto& outputLayer=network[network.size()-1]; + for(std::size_t j=1;joperator()( expectation[j-1], neuron.output())* + neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output()); + } + + for(int layerIndex=static_cast(network.size()-2);layerIndex>0;layerIndex--) { + auto &layer=network[layerIndex]; + + for(std::size_t j=1;j &input) { + for(std::size_t layerIndex=1;layerIndex 0) { + weightChangeDelta = std::min(weightChangeDelta*weightChangePlus,maxChangeOfWeights); + } else if (gradient * lastGradient < 0) { + weightChangeDelta = std::max(weightChangeDelta*weightChangeMinus,minChangeOfWeights); + } else { + weightChangeDelta = lastWeightChanges[layerIndex][j][k]; + } + + lastWeightChanges[layerIndex][j][k] = weightChangeDelta; + + if(gradient > 0) { + layer[j].weight(k) += weightChangeDelta; + } else if (gradient < 0){ + layer[j].weight(k) -= weightChangeDelta; + } else { + } + } + } + } +} \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 09ef39a..3f5f17a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -28,6 +28,9 @@ target_link_libraries(recurrent NeuralNetwork gtest gtest_main) add_executable(quickpropagation quickpropagation.cpp) target_link_libraries(quickpropagation NeuralNetwork gtest gtest_main) +add_executable(rprop rprop.cpp) +target_link_libraries(rprop NeuralNetwork gtest gtest_main) + # PERF add_executable(backpropagation_function_cmp backpropagation_function_cmp.cpp) diff --git a/tests/rprop.cpp b/tests/rprop.cpp new file mode 100644 index 0000000..73e02ef --- /dev/null +++ b/tests/rprop.cpp @@ -0,0 +1,165 @@ +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Weffc++" + +#include + +#pragma GCC diagnostic pop +TEST(RProp,XOR) { + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::Sigmoid a(-1); + n.appendLayer(3,a); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::RProp prop(n); + prop.setBatchSize(4); + + for(int i=0;i<100;i++) { + prop.teach({1,0},{1}); + prop.teach({1,1},{0}); + prop.teach({0,0},{0}); + prop.teach({0,1},{1}); + } + + { + std::vector ret =n.computeOutput({1,1}); + ASSERT_LT(ret[0], 0.1); + } + + { + std::vector ret =n.computeOutput({0,1}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({1,0}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({0,0}); + ASSERT_LT(ret[0], 0.1); + } +} + +TEST(RProp,XORHyperbolicTangent) { + srand(time(NULL)); + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::HyperbolicTangent a(-1); + n.appendLayer(2,a); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::RProp prop(n); + prop.setBatchSize(4); + + for(int i=0;i<15000;i++) { + prop.teach({1,0},{1}); + prop.teach({1,1},{0}); + prop.teach({0,0},{0}); + prop.teach({0,1},{1}); + } + + { + std::vector ret =n.computeOutput({1,1}); + ASSERT_LT(ret[0], 0.1); + } + + { + std::vector ret =n.computeOutput({0,1}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({1,0}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({0,0}); + ASSERT_LT(ret[0], 0.1); + } +} + +TEST(RProp,AND) { + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::Sigmoid a(-1); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::RProp prop(n); + prop.setBatchSize(4); + + for(int i=0;i<100000;i++) { + prop.teach({1,1},{1}); + prop.teach({0,0},{0}); + prop.teach({0,1},{0}); + prop.teach({1,0},{0}); + } + + { + std::vector ret =n.computeOutput({1,1}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({0,1}); + ASSERT_LT(ret[0], 0.1); + } + + { + std::vector ret =n.computeOutput({1,0}); + ASSERT_LT(ret[0], 0.1); + } + + { + std::vector ret =n.computeOutput({0,0}); + ASSERT_LT(ret[0], 0.1); + } +} + +TEST(RProp,NOTAND) { + NeuralNetwork::FeedForward::Network n(2); + NeuralNetwork::ActivationFunction::Sigmoid a(-1); + n.appendLayer(2,a); + n.appendLayer(1,a); + + n.randomizeWeights(); + + NeuralNetwork::Learning::RProp prop(n); + prop.setBatchSize(4); + + for(int i=0;i<100000;i++) { + prop.teach({1,1},{0}); + prop.teach({0,0},{1}); + prop.teach({0,1},{1}); + prop.teach({1,0},{1}); + } + + { + std::vector ret =n.computeOutput({1,1}); + ASSERT_LT(ret[0], 0.1); + } + + { + std::vector ret =n.computeOutput({0,1}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({1,0}); + ASSERT_GT(ret[0], 0.9); + } + + { + std::vector ret =n.computeOutput({0,0}); + ASSERT_GT(ret[0], 0.9); + } +} \ No newline at end of file From 77b38dec650b76b5bbeffbdd34addd01dde275db Mon Sep 17 00:00:00 2001 From: Shin Date: Mon, 31 Oct 2016 15:03:27 +0100 Subject: [PATCH 06/11] refactored propagation --- CMakeLists.txt | 3 +- include/NeuralNetwork/FeedForward/Layer.h | 9 ++ .../NeuralNetwork/Learning/BackPropagation.h | 83 ++---------- .../NeuralNetwork/Learning/BatchPropagation.h | 52 ++++++++ .../Learning/OpticalBackPropagation.h | 23 ---- .../NeuralNetwork/Learning/QuickPropagation.h | 34 +---- include/NeuralNetwork/Learning/RProp.h | 121 ++++------------- include/NeuralNetwork/Neuron.h | 8 +- .../Learning/BackPropagation.cpp | 80 +---------- .../Learning/BatchPropagation.cpp | 92 +++++++++++++ .../Learning/QuickPropagation.cpp | 35 +++-- src/NeuralNetwork/Learning/RProp.cpp | 84 ++---------- tests/CMakeLists.txt | 3 - tests/activation.cpp | 14 -- tests/backpropagation.cpp | 3 +- tests/backpropagation_function_cmp.cpp | 18 +-- tests/backpropagation_perf.cpp | 8 ++ tests/optical_backpropagation.cpp | 124 ------------------ tests/propagation_cmp.cpp | 39 ++++-- 19 files changed, 285 insertions(+), 548 deletions(-) create mode 100644 include/NeuralNetwork/Learning/BatchPropagation.h delete mode 100644 include/NeuralNetwork/Learning/OpticalBackPropagation.h create mode 100644 src/NeuralNetwork/Learning/BatchPropagation.cpp delete mode 100644 tests/optical_backpropagation.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 22ab38f..db2ceab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,8 +61,9 @@ endif(USE_SSE) set (LIBRARY_SOURCES src/sse_mathfun.cpp + src/NeuralNetwork/Learning/BatchPropagation.cpp src/NeuralNetwork/Learning/BackPropagation.cpp - src/NeuralNetwork/Learning/QuickPropagation.cpp + #src/NeuralNetwork/Learning/QuickPropagation.cpp src/NeuralNetwork/Learning/PerceptronLearning.cpp src/NeuralNetwork/Learning/RProp.cpp diff --git a/include/NeuralNetwork/FeedForward/Layer.h b/include/NeuralNetwork/FeedForward/Layer.h index 3c07750..43bf5c3 100644 --- a/include/NeuralNetwork/FeedForward/Layer.h +++ b/include/NeuralNetwork/FeedForward/Layer.h @@ -67,6 +67,15 @@ namespace FeedForward { return *neurons[neuron]; } + /** + * @brief This is a virtual function for selecting neuron + * @param neuron is position in layer + * @returns Specific neuron + */ + const NeuronInterface& operator[](const std::size_t& neuron) const { + return *neurons[neuron]; + } + void solve(const std::vector &input, std::vector &output); /** diff --git a/include/NeuralNetwork/Learning/BackPropagation.h b/include/NeuralNetwork/Learning/BackPropagation.h index 1df9b42..ae7b18a 100644 --- a/include/NeuralNetwork/Learning/BackPropagation.h +++ b/include/NeuralNetwork/Learning/BackPropagation.h @@ -1,10 +1,6 @@ #pragma once -#include -#include - -#include -#include "CorrectionFunction/Linear.h" +#include "BatchPropagation.h" namespace NeuralNetwork { namespace Learning { @@ -12,23 +8,17 @@ namespace Learning { /** @class BackPropagation * @brief */ - class BackPropagation { + class BackPropagation : public BatchPropagation { public: - inline BackPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()): - network(feedForwardNetwork), correctionFunction(correction),learningCoefficient(0.4), slopes() { + BackPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BatchPropagation(feedForwardNetwork,correction), learningCoefficient(0.4) { resize(); } - virtual ~BackPropagation() { - delete correctionFunction; - } - BackPropagation(const BackPropagation&)=delete; BackPropagation& operator=(const NeuralNetwork::Learning::BackPropagation&) = delete; - void teach(const std::vector &input, const std::vector &output); - inline virtual void setLearningCoefficient (const float& coefficient) { learningCoefficient=coefficient; } float getMomentumWeight() const { @@ -48,75 +38,22 @@ namespace Learning { weightDecay=wd; } - std::size_t getBatchSize() const { - return batchSize; - } - - void setBatchSize(std::size_t size) { - batchSize = size; - } - protected: - virtual inline void resize() { - if(slopes.size()!=network.size()) - slopes.resize(network.size()); - - for(std::size_t i=0; i < network.size(); i++) { - if(slopes[i].size()!=network[i].size()) - slopes[i].resize(network[i].size()); - } - - if(deltas.size() != network.size()) - deltas.resize(network.size()); - - bool resized = false; - - for(std::size_t i = 0; i < network.size(); i++) { - if(deltas[i].size() != network[i].size()) { - deltas[i].resize(network[i].size()); - resized = true; - - if(i > 0) { - for(std::size_t j = 0; j < deltas[i].size(); j++) { - deltas[i][j].resize(network[i - 1].size()); - std::fill(deltas[i][j].begin(),deltas[i][j].end(),0.0); - } - } - } - } - - if(momentumWeight > 0.0 && (resized || lastDeltas.size() != deltas.size())) { - lastDeltas = deltas; + virtual inline void resize() override { + BatchPropagation::resize(); + if(momentumWeight > 0.0) { + _lastDeltas = _gradients; } } - virtual void computeDeltas(const std::vector &input); - - void updateWeights(); - - virtual void computeSlopes(const std::vector &expectation); - - virtual void endBatch() { - - } - - FeedForward::Network &network; - - CorrectionFunction::CorrectionFunction *correctionFunction; + virtual void updateWeightsAndEndBatch() override; float learningCoefficient; - float momentumWeight = 0.0; - float weightDecay = 0.0; - std::size_t batchSize = 1; - std::size_t currentBatchSize = 0; - - std::vector> slopes; - std::vector>> deltas = {}; - std::vector>> lastDeltas = {}; + std::vector>> _lastDeltas = {}; }; } diff --git a/include/NeuralNetwork/Learning/BatchPropagation.h b/include/NeuralNetwork/Learning/BatchPropagation.h new file mode 100644 index 0000000..b26ddf7 --- /dev/null +++ b/include/NeuralNetwork/Learning/BatchPropagation.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include "CorrectionFunction/Linear.h" + +#include +#include + +namespace NeuralNetwork { +namespace Learning { + class BatchPropagation { + public: + BatchPropagation(FeedForward::Network &ffn, std::shared_ptr correction) : _network(ffn), _correctionFunction(correction) { + + } + + virtual ~BatchPropagation() { + + } + + void teach(const std::vector &input, const std::vector &output); + + void finishTeaching(); + + std::size_t getBatchSize() const { + return _batchSize; + } + + void setBatchSize(std::size_t size) { + _batchSize = size; + } + protected: + virtual void updateWeightsAndEndBatch() = 0; + virtual void resize(); + + FeedForward::Network &_network; + std::shared_ptr _correctionFunction; + + std::size_t _batchSize = 1; + std::size_t _currentBatchSize = 0; + + std::vector> _slopes = {}; + std::vector>> _gradients = {}; + + bool init = false; + private: + void computeSlopes(const std::vector &expectation); + void computeDeltas(const std::vector &input); + }; +} +} \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/OpticalBackPropagation.h b/include/NeuralNetwork/Learning/OpticalBackPropagation.h deleted file mode 100644 index cb70c3f..0000000 --- a/include/NeuralNetwork/Learning/OpticalBackPropagation.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include "./BackPropagation.h" -#include "./CorrectionFunction/Optical.h" - -namespace NeuralNetwork { -namespace Learning { - - /** @class OpticalBackPropagation - * @brief - */ - class OpticalBackPropagation : public BackPropagation { - - public: - OpticalBackPropagation(FeedForward::Network &feedForwardNetwork): BackPropagation(feedForwardNetwork,new CorrectionFunction::Optical()) { - - } - - virtual ~OpticalBackPropagation() { - } - }; -} -} \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/QuickPropagation.h b/include/NeuralNetwork/Learning/QuickPropagation.h index 48f4f98..358548a 100644 --- a/include/NeuralNetwork/Learning/QuickPropagation.h +++ b/include/NeuralNetwork/Learning/QuickPropagation.h @@ -15,46 +15,24 @@ namespace NeuralNetwork { class QuickPropagation : public BackPropagation { public: - inline QuickPropagation(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()): - BackPropagation(feedForwardNetwork,correction),previousSlopes() { - resize(); + inline QuickPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BackPropagation(feedForwardNetwork,correction) { } virtual ~QuickPropagation() { } protected: + float _maxChange=1.75; float _epsilon=0.5; virtual inline void resize() override { - if(slopes.size()!=network.size()) - slopes.resize(network.size()); - - for(std::size_t i=0; i < network.size(); i++) { - if(slopes[i].size()!=network[i].size()) - slopes[i].resize(network[i].size()); - } - - if(deltas.size()!=network.size()) - deltas.resize(network.size()); - - for(std::size_t i=0; i < network.size(); i++) { - if(deltas[i].size()!=network[i].size()) - deltas[i].resize(network[i].size()); - - for(std::size_t j=0; j < previousSlopes[i].size(); j++) { - deltas[i][j]=1.0; - } - } - weightChange= deltas; + BackPropagation::resize(); + _previousSlopes = _slopes; } - virtual void updateWeights(const std::vector &input) override; - - std::vector> previousSlopes ={}; - std::vector> deltas ={}; - std::vector> weightChange ={}; + std::vector> _previousSlopes ={}; }; } } \ No newline at end of file diff --git a/include/NeuralNetwork/Learning/RProp.h b/include/NeuralNetwork/Learning/RProp.h index a838e0e..522fd4a 100644 --- a/include/NeuralNetwork/Learning/RProp.h +++ b/include/NeuralNetwork/Learning/RProp.h @@ -1,10 +1,7 @@ #pragma once -#include -#include -#include -#include "CorrectionFunction/Linear.h" +#include "BatchPropagation.h" namespace NeuralNetwork { namespace Learning { @@ -12,122 +9,48 @@ namespace NeuralNetwork { /** @class Resilient Propagation * @brief */ - class RProp { + class RProp : public BatchPropagation { public: - RProp(FeedForward::Network &feedForwardNetwork, CorrectionFunction::CorrectionFunction *correction = new CorrectionFunction::Linear()): - network(feedForwardNetwork), correctionFunction(correction) { - resize(); - } - - virtual ~RProp() { - delete correctionFunction; + RProp(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BatchPropagation(feedForwardNetwork, correction) { } RProp(const RProp&)=delete; RProp& operator=(const NeuralNetwork::Learning::RProp&) = delete; - void teach(const std::vector &input, const std::vector &output); - - std::size_t getBatchSize() const { - return batchSize; + void setInitialWeightChange(float initVal) { + initialWeightChange=initVal; } + void setLearningCoefficient(float) { - void setBatchSize(std::size_t size) { - batchSize = size; - } - - void setInitialWeightChange(float init) { - initialWeightChange=init; } protected: - virtual inline void resize() { - if(slopes.size()!=network.size()) - slopes.resize(network.size()); + virtual inline void resize() override { + BatchPropagation::resize(); - for(std::size_t i=0; i < network.size(); i++) { - if(slopes[i].size()!=network[i].size()) - slopes[i].resize(network[i].size()); - } + _lastGradients =_gradients; - if(gradients.size() != network.size()) - gradients.resize(network.size()); - - bool resized = false; - - for(std::size_t i = 0; i < network.size(); i++) { - if(gradients[i].size() != network[i].size()) { - gradients[i].resize(network[i].size()); - resized = true; - - if(i > 0) { - for(std::size_t j = 0; j < gradients[i].size(); j++) { - gradients[i][j].resize(network[i - 1].size()); - std::fill(gradients[i][j].begin(),gradients[i][j].end(),0.0); - } - } + _changesOfWeightChanges = _lastGradients; + for(std::size_t i = 1; i < _network.size(); i++) { + for(std::size_t j = 0; j < _changesOfWeightChanges[i].size(); j++) { + std::fill(_changesOfWeightChanges[i][j].begin(),_changesOfWeightChanges[i][j].end(),initialWeightChange); } } - - if(resized) { - lastGradients = gradients; - - if(changesOfWeightChanges.size() != network.size()) - changesOfWeightChanges.resize(network.size()); - - for(std::size_t i = 0; i < network.size(); i++) { - if(changesOfWeightChanges[i].size() != network[i].size()) { - changesOfWeightChanges[i].resize(network[i].size()); - if(i > 0) { - for(std::size_t j = 0; j < changesOfWeightChanges[i].size(); j++) { - changesOfWeightChanges[i][j].resize(network[i - 1].size()); - std::fill(changesOfWeightChanges[i][j].begin(),changesOfWeightChanges[i][j].end(),initialWeightChange); - } - } - } - } - } - - if(resized) { - if(lastWeightChanges.size() != network.size()) - lastWeightChanges.resize(network.size()); - - for(std::size_t i = 0; i < network.size(); i++) { - if(lastWeightChanges[i].size() != network[i].size()) { - lastWeightChanges[i].resize(network[i].size()); - if(i > 0) { - for(std::size_t j = 0; j < lastWeightChanges[i].size(); j++) { - lastWeightChanges[i][j].resize(network[i - 1].size()); - std::fill(lastWeightChanges[i][j].begin(),lastWeightChanges[i][j].end(),0.1); - } - } - } + _lastWeightChanges = _lastGradients; + for(std::size_t i = 1; i < _network.size(); i++) { + for(std::size_t j = 0; j < _lastWeightChanges[i].size(); j++) { + std::fill(_lastWeightChanges[i][j].begin(),_lastWeightChanges[i][j].end(),0.1); } } } - virtual void computeSlopes(const std::vector &expectation); - virtual void computeDeltas(const std::vector &input); + void updateWeightsAndEndBatch() override; - void updateWeights(); - - virtual void endBatch() { - - } - - FeedForward::Network &network; - - CorrectionFunction::CorrectionFunction *correctionFunction; - - std::vector> slopes; - std::vector>> gradients = {}; - std::vector>> lastGradients = {}; - std::vector>> lastWeightChanges = {}; - std::vector>> changesOfWeightChanges = {}; - - std::size_t batchSize = 1; - std::size_t currentBatchSize = 0; + std::vector>> _lastGradients = {}; + std::vector>> _lastWeightChanges = {}; + std::vector>> _changesOfWeightChanges = {}; float maxChangeOfWeights = 50; float minChangeOfWeights = 0.0001; diff --git a/include/NeuralNetwork/Neuron.h b/include/NeuralNetwork/Neuron.h index 183df1f..62d8276 100644 --- a/include/NeuralNetwork/Neuron.h +++ b/include/NeuralNetwork/Neuron.h @@ -108,7 +108,7 @@ namespace NeuralNetwork /** * @brief getter for activation function of neuron */ - virtual ActivationFunction::ActivationFunction& getActivationFunction() =0; + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const =0; virtual void setBasisFunction(const BasisFunction::BasisFunction& basisFunction) =0; @@ -167,7 +167,7 @@ namespace NeuralNetwork return *basis; } - virtual ActivationFunction::ActivationFunction& getActivationFunction() override { + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override { return *activation; } @@ -216,7 +216,7 @@ namespace NeuralNetwork throw usageException("basis function"); } - virtual ActivationFunction::ActivationFunction& getActivationFunction() override { + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override { throw usageException("biasNeuron - activation function"); } @@ -267,7 +267,7 @@ namespace NeuralNetwork throw usageException("basis function"); } - virtual ActivationFunction::ActivationFunction& getActivationFunction() override { + virtual const ActivationFunction::ActivationFunction& getActivationFunction() const override { throw usageException("input neuron - activation function"); } diff --git a/src/NeuralNetwork/Learning/BackPropagation.cpp b/src/NeuralNetwork/Learning/BackPropagation.cpp index b562393..f288989 100644 --- a/src/NeuralNetwork/Learning/BackPropagation.cpp +++ b/src/NeuralNetwork/Learning/BackPropagation.cpp @@ -1,89 +1,23 @@ #include -#include -#include - -void NeuralNetwork::Learning::BackPropagation::teach(const std::vector &input, const std::vector &expectation) { - network.computeOutput(input); - resize(); - computeSlopes(expectation); - - computeDeltas(input); - if(++currentBatchSize >= batchSize) { - updateWeights(); - endBatch(); - currentBatchSize=0; - } -} - -void NeuralNetwork::Learning::BackPropagation::computeSlopes(const std::vector &expectation) { - auto& outputLayer=network[network.size()-1]; - for(std::size_t j=1;joperator()( expectation[j-1], neuron.output())* - neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output()); - } - - for(int layerIndex=static_cast(network.size()-2);layerIndex>0;layerIndex--) { - auto &layer=network[layerIndex]; - - for(std::size_t j=1;j &input) { - for(std::size_t layerIndex=1;layerIndex 0.0; - for(std::size_t layerIndex=1;layerIndex + +void NeuralNetwork::Learning::BatchPropagation::teach(const std::vector &input, const std::vector &expectation) { + _network.computeOutput(input); + if(!init) { + resize(); + init = true; + } + + computeSlopes(expectation); + + computeDeltas(input); + if(++_currentBatchSize >= _batchSize) { + finishTeaching(); + } +} + +void NeuralNetwork::Learning::BatchPropagation::finishTeaching() { + updateWeightsAndEndBatch(); + _currentBatchSize=0; +} + +void NeuralNetwork::Learning::BatchPropagation::computeSlopes(const std::vector &expectation) { + const auto& outputLayer=_network[_network.size()-1]; + for(std::size_t j=1;joperator()( expectation[j-1], neuron.output())* + neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output()); + } + + for(int layerIndex=static_cast(_network.size()-2);layerIndex>0;layerIndex--) { + auto &layer=_network[layerIndex]; + + for(std::size_t j=1;j &input) { + for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) { + auto &layer=_network[layerIndex]; + auto &prevLayer=_network[layerIndex-1]; + + std::size_t prevLayerSize=prevLayer.size(); + std::size_t layerSize=layer.size(); + + for(std::size_t j=1;j 0) { + for(std::size_t j = 0; j < _gradients[i].size(); j++) { + _gradients[i][j].resize(_network[i - 1].size()); + std::fill(_gradients[i][j].begin(), _gradients[i][j].end(), 0.0); + } + } + } + +} diff --git a/src/NeuralNetwork/Learning/QuickPropagation.cpp b/src/NeuralNetwork/Learning/QuickPropagation.cpp index c5a0fda..3939c50 100644 --- a/src/NeuralNetwork/Learning/QuickPropagation.cpp +++ b/src/NeuralNetwork/Learning/QuickPropagation.cpp @@ -3,7 +3,7 @@ #include #include -void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector &input) { +void NeuralNetwork::Learning::QuickPropagation::computeDeltas(const std::vector &input) { float shrinkFactor=_maxChange/(_maxChange+1.0); @@ -15,20 +15,39 @@ void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector< std::size_t layerSize=layer.size(); for(std::size_t j=1;j 0.0001) { - if(std::signbit(deltas[layerIndex][j]) == std::signbit(slopes[layerIndex][j])) { + if(fabs (gradients[layerIndex][j])> 0.0001) { + if(std::signbit(gradients[layerIndex][j]) == std::signbit(slopes[layerIndex][j])) { newChange+= slopes[layerIndex][j]*_epsilon; if(fabs(slopes[layerIndex][j]) > fabs(shrinkFactor * previousSlopes[layerIndex][j])) { - newChange += _maxChange * deltas[layerIndex][j]; + newChange += _maxChange * gradients[layerIndex][j]; }else { - newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * deltas[layerIndex][j]; + newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * gradients[layerIndex][j]; } } else { - newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * deltas[layerIndex][j]; + newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * gradients[layerIndex][j]; } } else { newChange+= slopes[layerIndex][j]*_epsilon; @@ -45,9 +64,7 @@ void NeuralNetwork::Learning::QuickPropagation::updateWeights(const std::vector< layer[j].weight(k)+=newChange*(prevLayer[k].output()); } } + */ } } - - slopes.swap(previousSlopes); - weightChange.swap(deltas); } \ No newline at end of file diff --git a/src/NeuralNetwork/Learning/RProp.cpp b/src/NeuralNetwork/Learning/RProp.cpp index ab17e8e..fb95703 100644 --- a/src/NeuralNetwork/Learning/RProp.cpp +++ b/src/NeuralNetwork/Learning/RProp.cpp @@ -1,101 +1,37 @@ #include -void NeuralNetwork::Learning::RProp::teach(const std::vector &input, const std::vector &expectation) { - network.computeOutput(input); - resize(); - computeSlopes(expectation); +void NeuralNetwork::Learning::RProp::updateWeightsAndEndBatch() { - computeDeltas(input); - if(++currentBatchSize >= batchSize) { - updateWeights(); - endBatch(); - currentBatchSize=0; - } -} - -void NeuralNetwork::Learning::RProp::computeSlopes(const std::vector &expectation) { - auto& outputLayer=network[network.size()-1]; - for(std::size_t j=1;joperator()( expectation[j-1], neuron.output())* - neuron.getActivationFunction().derivatedOutput(neuron.value(),neuron.output()); - } - - for(int layerIndex=static_cast(network.size()-2);layerIndex>0;layerIndex--) { - auto &layer=network[layerIndex]; - - for(std::size_t j=1;j &input) { - for(std::size_t layerIndex=1;layerIndex 0) { weightChangeDelta = std::min(weightChangeDelta*weightChangePlus,maxChangeOfWeights); } else if (gradient * lastGradient < 0) { weightChangeDelta = std::max(weightChangeDelta*weightChangeMinus,minChangeOfWeights); } else { - weightChangeDelta = lastWeightChanges[layerIndex][j][k]; + weightChangeDelta = _lastWeightChanges[layerIndex][j][k]; } - lastWeightChanges[layerIndex][j][k] = weightChangeDelta; + _lastWeightChanges[layerIndex][j][k] = weightChangeDelta; if(gradient > 0) { layer[j].weight(k) += weightChangeDelta; } else if (gradient < 0){ layer[j].weight(k) -= weightChangeDelta; - } else { } } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3f5f17a..af6ddd3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -13,9 +13,6 @@ target_link_libraries(backpropagation NeuralNetwork gtest gtest_main) add_executable(feedforward feedforward.cpp) target_link_libraries(feedforward NeuralNetwork gtest gtest_main) -add_executable(optical_backpropagation optical_backpropagation.cpp) -target_link_libraries(optical_backpropagation NeuralNetwork gtest gtest_main) - add_executable(perceptron perceptron.cpp) target_link_libraries(perceptron NeuralNetwork gtest gtest_main) diff --git a/tests/activation.cpp b/tests/activation.cpp index be77834..04f2d4e 100644 --- a/tests/activation.cpp +++ b/tests/activation.cpp @@ -48,20 +48,6 @@ TEST(Sigmoid, ParamMinusFive) { ASSERT_LT(s(0.7), 0.970788); } -TEST(SigmoidSSE, ParamMinusZeroPointSeven) { - NeuralNetwork::ActivationFunction::Sigmoid s(-0.7); - SSE comp; - comp.floats[0] = 0.1; - comp.floats[1] = 10; - comp.sse = s(comp.sse); - - ASSERT_GT(comp.floats[0], 0.517483); - ASSERT_LT(comp.floats[0], 0.51750); - - ASSERT_GT(comp.floats[1], 0.998989); - ASSERT_LT(comp.floats[1], 0.999189); -} - TEST(Linear, ParamOne) { NeuralNetwork::ActivationFunction::Linear s(1.0); ASSERT_GT(s(0.5), 0.4999); diff --git a/tests/backpropagation.cpp b/tests/backpropagation.cpp index caafcfc..2ae62c3 100644 --- a/tests/backpropagation.cpp +++ b/tests/backpropagation.cpp @@ -47,6 +47,7 @@ TEST(BackProp,XOR) { } TEST(BackProp,XORHyperbolicTangent) { + srand(time(NULL)); NeuralNetwork::FeedForward::Network n(2); NeuralNetwork::ActivationFunction::HyperbolicTangent a(-1); n.appendLayer(2,a); @@ -56,7 +57,7 @@ TEST(BackProp,XORHyperbolicTangent) { NeuralNetwork::Learning::BackPropagation prop(n); - for(int i=0;i<10000;i++) { + for(int i=0;i<1500;i++) { prop.teach({1,0},{1}); prop.teach({1,1},{0}); prop.teach({0,0},{0}); diff --git a/tests/backpropagation_function_cmp.cpp b/tests/backpropagation_function_cmp.cpp index 2f296cb..ae75d64 100644 --- a/tests/backpropagation_function_cmp.cpp +++ b/tests/backpropagation_function_cmp.cpp @@ -45,41 +45,41 @@ int main() { std::cout << "\tLinear: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n"; + std::make_shared(),linearCoef) << "\n"; std::cout << "\tOptical: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n"; + std::make_shared(),opticalCoef) << "\n"; std::cout << "\tArcTangent: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n"; + std::make_shared(arcTangent),arcTangentCoef) << "\n"; } { std::cout << "AND:\n"; std::cout << "\tLinear: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n"; + std::make_shared(),linearCoef) << "\n"; std::cout << "\tOptical: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n"; + std::make_shared(),opticalCoef) << "\n"; std::cout << "\tArcTangent: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n"; + std::make_shared(arcTangent),arcTangentCoef) << "\n"; } { std::cout << "AND:\n"; std::cout << "\tLinear: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef) << "\n"; + std::make_shared(),linearCoef) << "\n"; std::cout << "\tOptical: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef) << "\n"; + std::make_shared(),opticalCoef) << "\n"; std::cout << "\tArcTangent: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::ArcTangent(arcTangent),arcTangentCoef) << "\n"; + std::make_shared(arcTangent),arcTangentCoef) << "\n"; } } diff --git a/tests/backpropagation_perf.cpp b/tests/backpropagation_perf.cpp index 2f31ea1..13732d1 100644 --- a/tests/backpropagation_perf.cpp +++ b/tests/backpropagation_perf.cpp @@ -4,6 +4,7 @@ #include #include "../include/NeuralNetwork/Learning/BackPropagation.h" +#include int main() { { // XOR problem NeuralNetwork::FeedForward::Network n(2); @@ -16,11 +17,18 @@ int main() { n.randomizeWeights(); NeuralNetwork::Learning::BackPropagation prop(n); +// prop.setBatchSize(20); +// prop.setMomentumWeight(0.8); + + auto start = std::chrono::system_clock::now(); for(int i=0;i<100;i++) { prop.teach({1,0},{1}); prop.teach({1,1},{0}); prop.teach({0,0},{0}); prop.teach({0,1},{1}); } + auto end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end -start; + std::cout << elapsed_seconds.count() << "\n"; } } diff --git a/tests/optical_backpropagation.cpp b/tests/optical_backpropagation.cpp deleted file mode 100644 index 037e065..0000000 --- a/tests/optical_backpropagation.cpp +++ /dev/null @@ -1,124 +0,0 @@ -#include - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Weffc++" - -#include - -#pragma GCC diagnostic pop - -TEST(OpticalBackPropagation,XOR) { - NeuralNetwork::FeedForward::Network n(2); - NeuralNetwork::ActivationFunction::Sigmoid a(-1); - n.appendLayer(2,a); - n.appendLayer(1,a); - - n.randomizeWeights(); - - NeuralNetwork::Learning::OpticalBackPropagation prop(n); - - for(int i=0;i<10000;i++) { - prop.teach({1,0},{1}); - prop.teach({1,1},{0}); - prop.teach({0,0},{0}); - prop.teach({0,1},{1}); - } - - { - std::vector ret =n.computeOutput({1,1}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({0,1}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({1,0}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({0,0}); - ASSERT_LT(ret[0], 0.1); - } -} - -TEST(OpticalBackPropagation,AND) { - NeuralNetwork::FeedForward::Network n(2); - NeuralNetwork::ActivationFunction::Sigmoid a(-1); - n.appendLayer(2,a); - n.appendLayer(1,a); - - n.randomizeWeights(); - - NeuralNetwork::Learning::OpticalBackPropagation prop(n); - - for(int i=0;i<10000;i++) { - prop.teach({1,1},{1}); - prop.teach({0,0},{0}); - prop.teach({0,1},{0}); - prop.teach({1,0},{0}); - } - - { - std::vector ret =n.computeOutput({1,1}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({0,1}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({1,0}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({0,0}); - ASSERT_LT(ret[0], 0.1); - } -} - -TEST(OpticalBackPropagation,NOTAND) { - NeuralNetwork::FeedForward::Network n(2); - NeuralNetwork::ActivationFunction::Sigmoid a(-1); - n.appendLayer(2,a); - n.appendLayer(1,a); - - n.randomizeWeights(); - - NeuralNetwork::Learning::OpticalBackPropagation prop(n); - - for(int i=0;i<10000;i++) { - prop.teach({1,1},{0}); - prop.teach({0,0},{1}); - prop.teach({0,1},{1}); - prop.teach({1,0},{1}); - } - - { - std::vector ret =n.computeOutput({1,1}); - ASSERT_LT(ret[0], 0.1); - } - - { - std::vector ret =n.computeOutput({0,1}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({1,0}); - ASSERT_GT(ret[0], 0.9); - } - - { - std::vector ret =n.computeOutput({0,0}); - ASSERT_GT(ret[0], 0.9); - } -} \ No newline at end of file diff --git a/tests/propagation_cmp.cpp b/tests/propagation_cmp.cpp index 04efd85..50fba6a 100644 --- a/tests/propagation_cmp.cpp +++ b/tests/propagation_cmp.cpp @@ -4,6 +4,7 @@ #include #include "../include/NeuralNetwork/Learning/BackPropagation.h" #include "../include/NeuralNetwork/Learning/QuickPropagation.h" +#include "../include/NeuralNetwork/Learning/RProp.h" #include "../include/NeuralNetwork/Learning/CorrectionFunction/Optical.h" #include "../include/NeuralNetwork/Learning/CorrectionFunction/ArcTangent.h" @@ -16,7 +17,8 @@ n.appendLayer(1,a);\ n.randomizeWeights();\ CLASS prop(n,FUN);\ - prop.setLearningCoefficient(COEF);\ + prop.setLearningCoefficient(COEF);\ + prop.setBatchSize(4); \ int error=1; int steps = 0; \ while(error > 0 && steps <99999) {\ steps++;\ @@ -42,36 +44,47 @@ int main() { const float arcTangent=1.5; { - std::cout << "XOR:\n"; + std::cout << "XOR Linear:\n"; std::cout << "\tBP: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + std::make_shared(),linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; std::cout << "\tQP: " << - LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, + std::make_shared(),linearCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + std::cout << "\tRProp: " << + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),0,std::vector({0,1}),1, + std::make_shared(),linearCoef,NeuralNetwork::Learning::RProp) << "\n"; } { - std::cout << "AND:\n"; + std::cout << "AND Optical:\n"; std::cout << "\tBP: " << LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + std::make_shared(),opticalCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; std::cout << "\tQP: " << - LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + + std::cout << "\tRProp: " << + LEARN(std::vector({1,0}),0,std::vector({1,1}),1,std::vector({0,0}),0,std::vector({0,1}),0, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::RProp) << "\n"; } { - std::cout << "AND:\n"; + std::cout << "XOR Optical:\n"; std::cout << "\tBP: " << LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Linear,linearCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; + std::make_shared(),opticalCoef,NeuralNetwork::Learning::BackPropagation) << "\n"; std::cout << "\tQP: " << - LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, - new NeuralNetwork::Learning::CorrectionFunction::Optical,opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::QuickPropagation) << "\n"; + + std::cout << "\tRProp: " << + LEARN(std::vector({1,0}),1,std::vector({1,1}),0,std::vector({0,0}),1,std::vector({0,1}),1, + std::make_shared(),opticalCoef,NeuralNetwork::Learning::RProp) << "\n"; } } From 913a5cc41f1548f1c0ebf37edbf4c19b2a657daf Mon Sep 17 00:00:00 2001 From: Shin Date: Tue, 1 Nov 2016 21:38:15 +0100 Subject: [PATCH 07/11] bp: cleaning code --- include/NeuralNetwork/Learning/BackPropagation.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/NeuralNetwork/Learning/BackPropagation.h b/include/NeuralNetwork/Learning/BackPropagation.h index ae7b18a..53bf993 100644 --- a/include/NeuralNetwork/Learning/BackPropagation.h +++ b/include/NeuralNetwork/Learning/BackPropagation.h @@ -19,7 +19,9 @@ namespace Learning { BackPropagation(const BackPropagation&)=delete; BackPropagation& operator=(const NeuralNetwork::Learning::BackPropagation&) = delete; - inline virtual void setLearningCoefficient (const float& coefficient) { learningCoefficient=coefficient; } + void setLearningCoefficient (const float& coefficient) { + learningCoefficient=coefficient; + } float getMomentumWeight() const { return momentumWeight; From 173cfc9789a3787fc81e617dd51650a4bb95110e Mon Sep 17 00:00:00 2001 From: Shin Date: Tue, 1 Nov 2016 21:38:56 +0100 Subject: [PATCH 08/11] iRprop+ implementation --- include/NeuralNetwork/Learning/iRPropPlus.h | 64 +++++++++++++++++++++ src/NeuralNetwork/Learning/iRPropPlus.cpp | 52 +++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 include/NeuralNetwork/Learning/iRPropPlus.h create mode 100644 src/NeuralNetwork/Learning/iRPropPlus.cpp diff --git a/include/NeuralNetwork/Learning/iRPropPlus.h b/include/NeuralNetwork/Learning/iRPropPlus.h new file mode 100644 index 0000000..2944616 --- /dev/null +++ b/include/NeuralNetwork/Learning/iRPropPlus.h @@ -0,0 +1,64 @@ +#pragma once + +#include "BatchPropagation.h" + +namespace NeuralNetwork { + namespace Learning { + + /** @class Resilient Propagation + * @brief + */ + class iRPropPlus : public BatchPropagation { + + public: + iRPropPlus(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): + BatchPropagation(feedForwardNetwork, correction) { + } + + iRPropPlus(const iRPropPlus&)=delete; + iRPropPlus& operator=(const NeuralNetwork::Learning::iRPropPlus&) = delete; + + void setInitialWeightChange(float initVal) { + initialWeightChange=initVal; + } + void setLearningCoefficient(float) { + + } + protected: + + virtual inline void resize() override { + BatchPropagation::resize(); + + _lastGradients =_gradients; + + _changesOfWeightChanges = _lastGradients; + for(std::size_t i = 1; i < _network.size(); i++) { + for(std::size_t j = 0; j < _changesOfWeightChanges[i].size(); j++) { + std::fill(_changesOfWeightChanges[i][j].begin(),_changesOfWeightChanges[i][j].end(),initialWeightChange); + } + } + _lastWeightChanges = _lastGradients; + for(std::size_t i = 1; i < _network.size(); i++) { + for(std::size_t j = 0; j < _lastWeightChanges[i].size(); j++) { + std::fill(_lastWeightChanges[i][j].begin(),_lastWeightChanges[i][j].end(),0.1); + } + } + } + + void updateWeightsAndEndBatch() override; + + std::vector>> _lastGradients = {}; + std::vector>> _lastWeightChanges = {}; + std::vector>> _changesOfWeightChanges = {}; + + float _prevError=0; + + float maxChangeOfWeights = 5; + float minChangeOfWeights = 0.0001; + + float initialWeightChange=0.02; + float weightChangePlus=1.2; + float weightChangeMinus=0.5; + }; + } +} \ No newline at end of file diff --git a/src/NeuralNetwork/Learning/iRPropPlus.cpp b/src/NeuralNetwork/Learning/iRPropPlus.cpp new file mode 100644 index 0000000..e3315db --- /dev/null +++ b/src/NeuralNetwork/Learning/iRPropPlus.cpp @@ -0,0 +1,52 @@ +#include + +void NeuralNetwork::Learning::iRPropPlus::updateWeightsAndEndBatch() { + float error = 0.0; + const auto& outputLayer=_network[_network.size()-1]; + for(std::size_t j=1;j 0) { + weightChangeDelta = std::min(weightChangeDelta*weightChangePlus,maxChangeOfWeights); + delta = (std::signbit(gradient)? 1.0f : -1.0f ) * weightChangeDelta; + layer[j].weight(k) -= delta; + } else if (gradient * lastGradient < 0) { + weightChangeDelta = std::max(weightChangeDelta*weightChangeMinus,minChangeOfWeights); + delta = _lastWeightChanges[layerIndex][j][k]; + if(error > _prevError) { + layer[j].weight(k) += delta; + } + _lastGradients[layerIndex][j][k] = 0; + } else { + delta = (std::signbit(gradient)? 1.0f : -1.0f ) * weightChangeDelta; + layer[j].weight(k) -= delta; + } + //std::cout << delta <<"\n"; + + _changesOfWeightChanges[layerIndex][j][k] = weightChangeDelta; + _lastWeightChanges[layerIndex][j][k] = delta; + } + } + } + _prevError = error; +} \ No newline at end of file From 8b2a4e89b349badf694ae60bb57f1ea529526bd1 Mon Sep 17 00:00:00 2001 From: Shin Date: Tue, 1 Nov 2016 21:40:00 +0100 Subject: [PATCH 09/11] quickProp implementation after refactoring --- .../NeuralNetwork/Learning/QuickPropagation.h | 30 ++++--- .../CascadeCorrelation.cpp | 2 + .../Learning/QuickPropagation.cpp | 88 +++++++------------ 3 files changed, 52 insertions(+), 68 deletions(-) diff --git a/include/NeuralNetwork/Learning/QuickPropagation.h b/include/NeuralNetwork/Learning/QuickPropagation.h index 358548a..e790ae7 100644 --- a/include/NeuralNetwork/Learning/QuickPropagation.h +++ b/include/NeuralNetwork/Learning/QuickPropagation.h @@ -4,7 +4,7 @@ #include #include -#include "BackPropagation.h" +#include "BatchPropagation.h" namespace NeuralNetwork { namespace Learning { @@ -12,27 +12,33 @@ namespace NeuralNetwork { /** @class QuickPropagation * @brief */ - class QuickPropagation : public BackPropagation { + class QuickPropagation : public BatchPropagation { public: inline QuickPropagation(FeedForward::Network &feedForwardNetwork, std::shared_ptr correction = std::make_shared()): - BackPropagation(feedForwardNetwork,correction) { + BatchPropagation(feedForwardNetwork,correction) { } virtual ~QuickPropagation() { } - protected: - - float _maxChange=1.75; - float _epsilon=0.5; - - virtual inline void resize() override { - BackPropagation::resize(); - _previousSlopes = _slopes; + void setLearningCoefficient (const float& coefficient) { } - std::vector> _previousSlopes ={}; + protected: + + virtual void updateWeightsAndEndBatch() override; + + float _maxChange=1.75; + + virtual inline void resize() override { + BatchPropagation::resize(); + _lastGradients = _gradients; + _lastDeltas = _gradients; + } + + std::vector>> _lastDeltas = {}; + std::vector>> _lastGradients = {}; }; } } \ No newline at end of file diff --git a/src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp b/src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp index f05de73..77ff5e3 100644 --- a/src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp +++ b/src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp @@ -1,5 +1,7 @@ #include +#include + using namespace NeuralNetwork::ConstructiveAlgorihtms; float CascadeCorrelation::trainOutputs(Cascade::Network &network, const std::vector &patterns) { diff --git a/src/NeuralNetwork/Learning/QuickPropagation.cpp b/src/NeuralNetwork/Learning/QuickPropagation.cpp index 3939c50..80e5df5 100644 --- a/src/NeuralNetwork/Learning/QuickPropagation.cpp +++ b/src/NeuralNetwork/Learning/QuickPropagation.cpp @@ -1,70 +1,46 @@ #include -#include -#include +void NeuralNetwork::Learning::QuickPropagation::updateWeightsAndEndBatch() { -void NeuralNetwork::Learning::QuickPropagation::computeDeltas(const std::vector &input) { + float shrinkFactor=_maxChange/(_maxChange+1.0f); - float shrinkFactor=_maxChange/(_maxChange+1.0); + for(std::size_t layerIndex=1;layerIndex<_network.size();layerIndex++) { + auto &layer = _network[layerIndex]; + auto &prevLayer = _network[layerIndex - 1]; - for(std::size_t layerIndex=1;layerIndex 0.0001) { + if(std::signbit(_gradients[layerIndex][j][k]) == std::signbit(_lastGradients[layerIndex][j][k])) { + newChange+= _gradients[layerIndex][j][k]*_epsilon; - for(std::size_t j=1;j 0.0001) { - if(std::signbit(gradients[layerIndex][j]) == std::signbit(slopes[layerIndex][j])) { - newChange+= slopes[layerIndex][j]*_epsilon; - - if(fabs(slopes[layerIndex][j]) > fabs(shrinkFactor * previousSlopes[layerIndex][j])) { - newChange += _maxChange * gradients[layerIndex][j]; - }else { - newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * gradients[layerIndex][j]; + if(fabs(_gradients[layerIndex][j][k]) > fabs(shrinkFactor * _lastGradients[layerIndex][j][k])) { + newChange += _maxChange * _gradients[layerIndex][j][k]; + }else { + newChange+=_gradients[layerIndex][j][k]/(_lastGradients[layerIndex][j][k]-_gradients[layerIndex][j][k]) * _lastDeltas[layerIndex][j][k]; + } + } else { + newChange+=_gradients[layerIndex][j][k]/(_lastGradients[layerIndex][j][k]-_gradients[layerIndex][j][k]) * _lastDeltas[layerIndex][j][k]; } } else { - newChange+=slopes[layerIndex][j]/(previousSlopes[layerIndex][j]-slopes[layerIndex][j]) * gradients[layerIndex][j]; + newChange+= _lastGradients[layerIndex][j][k]*_epsilon; } - } else { - newChange+= slopes[layerIndex][j]*_epsilon; + _lastDeltas[layerIndex][j][k]= newChange; + layer[j].weight(k)+= newChange; + + /* This is according to paper? + // delta = _gradients[layerIndex][j][k] / (_lastGradients[layerIndex][j][k]-_gradients[layerIndex][j][k]) * _lastDeltas[layerIndex][j][k]; + // delta = std::min(_maxChange,delta); + _lastDeltas[layerIndex][j][k] = delta; + layer[j].weight(k)+= delta; + */ } - - weightChange[layerIndex][j]=newChange; - - layer[j].weight(0)+=newChange; - - for(std::size_t k=1;k Date: Tue, 1 Nov 2016 21:40:23 +0100 Subject: [PATCH 10/11] cmake: iRProp+ --- CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index db2ceab..3687582 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,11 +61,12 @@ endif(USE_SSE) set (LIBRARY_SOURCES src/sse_mathfun.cpp - src/NeuralNetwork/Learning/BatchPropagation.cpp + src/NeuralNetwork/Learning/BatchPropagation.cpp src/NeuralNetwork/Learning/BackPropagation.cpp - #src/NeuralNetwork/Learning/QuickPropagation.cpp + src/NeuralNetwork/Learning/QuickPropagation.cpp src/NeuralNetwork/Learning/PerceptronLearning.cpp src/NeuralNetwork/Learning/RProp.cpp + src/NeuralNetwork/Learning/iRPropPlus.cpp src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp src/NeuralNetwork/ConstructiveAlgorithms/Cascade2.cpp From 9f476b42336b289fce6df1a65f5aa0daac5f9a7b Mon Sep 17 00:00:00 2001 From: Shin Date: Tue, 1 Nov 2016 21:59:22 +0100 Subject: [PATCH 11/11] add RectifiedUnit and LeakyRectifiedUnit --- .../ActivationFunction/LeakyRectifiedLinear.h | 39 +++++++++++++++++++ .../ActivationFunction/RectifiedLinear.h | 39 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h create mode 100644 include/NeuralNetwork/ActivationFunction/RectifiedLinear.h diff --git a/include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h b/include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h new file mode 100644 index 0000000..3a16fa4 --- /dev/null +++ b/include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h @@ -0,0 +1,39 @@ +#pragma once + +#include "./ActivationFunction.h" + +#include + +namespace NeuralNetwork { +namespace ActivationFunction { + + class LeakyRectifiedLinear: public ActivationFunction { + public: + LeakyRectifiedLinear(const float &lambdaP=0.04): lambda(lambdaP) {} + + inline virtual float derivatedOutput(const float &inp,const float &) const override { + return inp > 0.0f ? lambda : 0.01f*lambda; + } + + inline virtual float operator()(const float &x) const override { + return x > 0.0? x : 0.001f*x; + }; + + virtual ActivationFunction* clone() const override { + return new LeakyRectifiedLinear(lambda); + } + + virtual SimpleJSON::Type::Object serialize() const override { + return {{"class", "NeuralNetwork::ActivationFunction::LeakyRectifiedLinear"}, {"lambda", lambda}}; + } + + static std::unique_ptr deserialize(const SimpleJSON::Type::Object &obj) { + return std::unique_ptr(new LeakyRectifiedLinear(obj["lambda"].as())); + } + + protected: + float lambda; + NEURAL_NETWORK_REGISTER_ACTIVATION_FUNCTION(NeuralNetwork::ActivationFunction::LeakyRectifiedLinear, LeakyRectifiedLinear::deserialize) + }; +} +} \ No newline at end of file diff --git a/include/NeuralNetwork/ActivationFunction/RectifiedLinear.h b/include/NeuralNetwork/ActivationFunction/RectifiedLinear.h new file mode 100644 index 0000000..93b4d72 --- /dev/null +++ b/include/NeuralNetwork/ActivationFunction/RectifiedLinear.h @@ -0,0 +1,39 @@ +#pragma once + +#include "./ActivationFunction.h" + +#include + +namespace NeuralNetwork { +namespace ActivationFunction { + + class RectifiedLinear: public ActivationFunction { + public: + RectifiedLinear(const float &lambdaP=0.1): lambda(lambdaP) {} + + inline virtual float derivatedOutput(const float &inp,const float &) const override { + return inp > 0.0f ? lambda : 0.0f; + } + + inline virtual float operator()(const float &x) const override { + return std::max(0.0f,x); + }; + + virtual ActivationFunction* clone() const override { + return new RectifiedLinear(lambda); + } + + virtual SimpleJSON::Type::Object serialize() const override { + return {{"class", "NeuralNetwork::ActivationFunction::RectifiedLinear"}, {"lambda", lambda}}; + } + + static std::unique_ptr deserialize(const SimpleJSON::Type::Object &obj) { + return std::unique_ptr(new RectifiedLinear(obj["lambda"].as())); + } + + protected: + float lambda; + NEURAL_NETWORK_REGISTER_ACTIVATION_FUNCTION(NeuralNetwork::ActivationFunction::RectifiedLinear, RectifiedLinear::deserialize) + }; +} +} \ No newline at end of file