shin/NeuralNetworkLib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cascade correlation: sticking to implementation by Fahlman and not paper

Browse Source
This commit is contained in:
Shin
2016-05-07 23:21:40 +02:00
parent eaafc27211
commit 0221158a56
2 changed files with 73 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
include/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.h
View File

@@ -3,7 +3,6 @@
#include "../Cascade/Network.h"
#include "../FeedForward/Network.h"
#include "../Learning/QuickPropagation.h"
#include "../ActivationFunction/Tangents.h"
#include <random>
#include <algorithm>
@@ -22,16 +21,16 @@ namespace NeuralNetwork {
std::size_t inputs = patterns[0].first.size();
std::size_t outputs = patterns[0].second.size();
Cascade::Network network(inputs, outputs, NeuralNetwork::ActivationFunction::Tangents());
Cascade::Network network(inputs, outputs, *_activFunction.get());
network.randomizeWeights();
int step = 0;
std::size_t step = 0;
float error = trainOutputs(network, patterns);
while(step++ < 15 && error > _maxError) {
while(step++ < _maxHiddenUnits && error > _maxError) {
std::vector<std::shared_ptr<Neuron>> candidates = createCandidates(network.getNeuronSize() - outputs);
std::shared_ptr<Neuron> candidate=trainCandidates(network, candidates, patterns);
std::pair<std::shared_ptr<Neuron>, std::vector<float>> candidate = trainCandidates(network, candidates, patterns);
addBestCandidate(network, candidate);
error = trainOutputs(network, patterns);
@@ -59,10 +58,20 @@ namespace NeuralNetwork {
_distribution = std::uniform_real_distribution<>(-weightRange, weightRange);
}
void setMaximumHiddenNeurons(std::size_t neurons) {
_maxHiddenUnits = neurons;
}
void setActivationFunction(const ActivationFunction::ActivationFunction &function) {
_activFunction = std::shared_ptr<ActivationFunction::ActivationFunction>(function.clone());
}
protected:
std::shared_ptr<ActivationFunction::ActivationFunction> _activFunction = std::make_shared<ActivationFunction::Sigmoid>(-4.9);
float _minimalErrorStep = 0.00005;
float _maxError;
float _weightRange;
std::size_t _maxHiddenUnits = 20;
std::size_t _numberOfCandidates;
std::mt19937 _generator;
std::uniform_real_distribution<> _distribution;
@@ -84,19 +93,23 @@ namespace NeuralNetwork {
float trainOutputs(Cascade::Network &network, const std::vector<TrainingPattern> &patterns);
std::shared_ptr<Neuron> trainCandidates(Cascade::Network &network, std::vector<std::shared_ptr<Neuron>> &candidates, const std::vector<TrainingPattern> &patterns);
std::pair<std::shared_ptr<Neuron>, std::vector<float>> trainCandidates(Cascade::Network &network, std::vector<std::shared_ptr<Neuron>> &candidates,
const std::vector<TrainingPattern> &patterns);
void addBestCandidate(Cascade::Network &network, const std::shared_ptr<Neuron> &candidate) {
void addBestCandidate(Cascade::Network &network, const std::pair<std::shared_ptr<Neuron>, std::vector<float>> &candidate) {
auto neuron = network.addNeuron();
neuron->setWeights(candidate->getWeights());
neuron->setActivationFunction(candidate->getActivationFunction());
float weightPortion = network.getNeuronSize() - network.outputs();
neuron->setWeights(candidate.first->getWeights());
neuron->setActivationFunction(candidate.first->getActivationFunction());
std::size_t outIndex = 0;
for(auto &n :network.getOutputNeurons()) {
auto weights = n->getWeights();
for(auto& weight: weights) {
weight *=0.7;
for(auto &weight: weights) {
weight *= 0.7;
}
weights[weights.size()-1] = _distribution(_generator);
weights[weights.size() - 1] = -candidate.second[outIndex] * weightPortion;//_distribution(_generator);
outIndex++;
n->setWeights(weights);
}
}
@@ -107,7 +120,7 @@ namespace NeuralNetwork {
for(std::size_t i = 0; i < _numberOfCandidates; i++) {
candidates.push_back(std::make_shared<Neuron>(id));
candidates.back()->setInputSize(id);
candidates.back()->setActivationFunction(NeuralNetwork::ActivationFunction::Tangents());
candidates.back()->setActivationFunction(*_activFunction.get());
for(std::size_t weightIndex = 0; weightIndex < id; weightIndex++) {
candidates.back()->weight(weightIndex) = _distribution(_generator);

84
src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp
View File

@@ -5,55 +5,58 @@ using namespace NeuralNetwork::ConstructiveAlgorihtms;
float CascadeCorrelation::trainOutputs(Cascade::Network &network, const std::vector<CascadeCorrelation::TrainingPattern> &patterns) {
std::size_t outputs = patterns[0].second.size();
FeedForward::Network p(network.getNeuronSize() - outputs-1);
FeedForward::Network p(network.getNeuronSize() - outputs - 1);
p.appendLayer(outputs);
Learning::QuickPropagation learner(p);
for(std::size_t neuron = 0; neuron < outputs; neuron++) {
p[1][neuron+1].setWeights(network.getNeuron(network.getNeuronSize() - outputs + neuron)->getWeights());
p[1][neuron+1].setActivationFunction(network.getNeuron(network.getNeuronSize() - outputs + neuron)->getActivationFunction());
p[1][neuron + 1].setWeights(network.getNeuron(network.getNeuronSize() - outputs + neuron)->getWeights());
p[1][neuron + 1].setActivationFunction(network.getNeuron(network.getNeuronSize() - outputs + neuron)->getActivationFunction());
}
//std::cout << p.stringify() << "\n";
std::vector<TrainingPattern> patternsForOutput;
for(auto &pattern:patterns) {
patternsForOutput.emplace_back(getInnerNeuronsOutput(network,pattern.first), pattern.second);
patternsForOutput.emplace_back(getInnerNeuronsOutput(network, pattern.first), pattern.second);
}
float lastError = std::numeric_limits<float>::max();
float lastError;
float error = std::numeric_limits<float>::max();
std::size_t iteration = 0;
std::size_t iterWithoutImporvement=0;
std::size_t iterWithoutImporvement = 0;
do {
lastError=error;
lastError = error;
for(auto &pattern:patternsForOutput) {
learner.teach({pattern.first.begin()+1,pattern.first.end()}, pattern.second);
learner.teach({pattern.first.begin() + 1, pattern.first.end()}, pattern.second);
}
error = 0;
for(auto &pattern:patternsForOutput) {
std::vector<float> output = p.computeOutput(pattern.first);
std::vector<float> output = p.computeOutput({pattern.first.begin() + 1, pattern.first.end()});
for(std::size_t outputIndex = 0; outputIndex < output.size(); outputIndex++) {
error += pow(output[outputIndex] - pattern.second[outputIndex],2);
error += pow(output[outputIndex] - pattern.second[outputIndex], 2);
}
}
if(fabs(lastError - error) < _minimalErrorStep) {
iterWithoutImporvement++;
}else {
iterWithoutImporvement=0;
} else {
iterWithoutImporvement = 0;
}
} while (iteration++ < 1000 && iterWithoutImporvement < 3);
std::cout << "iter: " << iteration << ", error: " << error << ", " << (lastError-error) << "\n";
}
while(iteration++ < 1000 && iterWithoutImporvement < 400);
std::cout << "iter: " << iteration << ", error: " << error << ", " << (lastError - error) << "\n";
for(std::size_t neuron = 0; neuron < outputs; neuron++) {
network.getNeuron(network.getNeuronSize() - outputs + neuron)->setWeights(p[1][neuron+1].getWeights());
network.getNeuron(network.getNeuronSize() - outputs + neuron)->setWeights(p[1][neuron + 1].getWeights());
}
return error;
}
std::shared_ptr<NeuralNetwork::Neuron> CascadeCorrelation::trainCandidates(Cascade::Network &network, std::vector<std::shared_ptr<Neuron>> &candidates, const std::vector<TrainingPattern> &patterns) {
std::pair<std::shared_ptr<NeuralNetwork::Neuron>, std::vector<float>> CascadeCorrelation::trainCandidates(Cascade::Network &network,
std::vector<std::shared_ptr<Neuron>> &candidates,
const std::vector<TrainingPattern> &patterns) {
std::size_t outputs = patterns[0].second.size();
std::vector<TrainingPattern> patternsForOutput;
@@ -62,31 +65,34 @@ std::shared_ptr<NeuralNetwork::Neuron> CascadeCorrelation::trainCandidates(Casca
patternsForOutput.emplace_back(getInnerNeuronsOutput(network, pattern.first), pattern.second);
}
std::vector<std::vector<float>> errors (patterns.size());
std::vector<float> meanErrors (outputs);
std::vector<std::vector<float>> errors(patterns.size());
std::vector<float> meanErrors(outputs);
for(std::size_t patternNumber = 0; patternNumber < patterns.size(); patternNumber++) {
auto &pattern=patterns[patternNumber];
auto &pattern = patterns[patternNumber];
errors[patternNumber].resize(network.outputs());
std::vector<float> output = network.computeOutput(patterns[patternNumber].first);
for(std::size_t outputIndex = 0; outputIndex < network.outputs(); outputIndex++) {
float error = pow(pattern.second[outputIndex] - output[outputIndex],2);
errors[patternNumber][outputIndex]=error;
float error = pow(pattern.second[outputIndex] - output[outputIndex], 2);
errors[patternNumber][outputIndex] = error;
meanErrors[outputIndex] += error;
}
}
std::for_each(meanErrors.begin(), meanErrors.end(), [&patterns](float &n){ n/=patterns.size(); });
std::size_t iterations=0;
std::size_t iterationsWithoutIprovement=0;
float bestCorrelation=0;
float lastCorrelation=0;
std::shared_ptr<Neuron> bestCandidate=nullptr;
std::for_each(meanErrors.begin(), meanErrors.end(), [&patterns](float &n) { n /= patterns.size(); });
std::size_t iterations = 0;
std::size_t iterationsWithoutIprovement = 0;
float bestCorrelation = 0;
float lastCorrelation = 0;
std::shared_ptr<Neuron> bestCandidate = nullptr;
std::vector<float> bestCorrelations(errors[0].size());
do {
lastCorrelation = bestCorrelation;
bool firstStep=true;
for(auto&candidate : candidates) {
bool firstStep = true;
for(auto &candidate : candidates) {
float correlation;
std::vector<float> activations;
@@ -104,7 +110,8 @@ std::shared_ptr<NeuralNetwork::Neuron> CascadeCorrelation::trainCandidates(Casca
correlationSigns[err] = correlations[err] > 0 ? 1.0 : -1.0;
}
correlation = std::accumulate(correlations.begin(), correlations.end(), 0.0);
correlation = std::accumulate(correlations.begin(), correlations.end(), 0.0, [](const float &a, float b) { return a + fabs(b); });
std::vector<float> derivatives(candidate->getWeights().size());
for(std::size_t input = 0; input < candidate->getWeights().size(); input++) {
float dcdw = 0.0;
@@ -122,13 +129,14 @@ std::shared_ptr<NeuralNetwork::Neuron> CascadeCorrelation::trainCandidates(Casca
}
for(std::size_t weightIndex = 0; weightIndex < derivatives.size(); weightIndex++) {
candidate->weight(weightIndex) += derivatives[weightIndex] * 0.1;
candidate->weight(weightIndex) += derivatives[weightIndex] * 0.7;
}
if(firstStep || correlation > bestCorrelation) {
bestCorrelation = correlation;
bestCandidate=candidate;
firstStep=false;
bestCandidate = candidate;
std::swap(bestCorrelations, correlations);
firstStep = false;
}
}
@@ -136,8 +144,10 @@ std::shared_ptr<NeuralNetwork::Neuron> CascadeCorrelation::trainCandidates(Casca
iterationsWithoutIprovement++;
}
} while (iterations ++ < 200 && iterationsWithoutIprovement <3);
std::cout << "iter: " << iterations << ", correlation: " << bestCorrelation << ", " << lastCorrelation << "\n";
}
while(iterations++ < 200 && iterationsWithoutIprovement < 300);
std::cout << "iter: " << iterations << ", correlation: " << bestCorrelation << ", " << lastCorrelation << "\n";
return {bestCandidate, bestCorrelations};
}
return bestCandidate;
}