shin/NeuralNetworkLib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cascade correlation: sticking to implementation by Fahlman and not paper

Browse Source
This commit is contained in:
Shin
2016-05-07 23:21:40 +02:00
parent eaafc27211
commit 0221158a56
2 changed files with 73 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
include/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.h
View File

@@ -3,7 +3,6 @@
#include "../Cascade/Network.h"
#include "../FeedForward/Network.h"
#include "../Learning/QuickPropagation.h"
#include "../ActivationFunction/Tangents.h"
#include <random>
#include <algorithm>
@@ -22,16 +21,16 @@ namespace NeuralNetwork {
std::size_t inputs = patterns[0].first.size();
std::size_t outputs = patterns[0].second.size();
Cascade::Network network(inputs, outputs, NeuralNetwork::ActivationFunction::Tangents());
Cascade::Network network(inputs, outputs, *_activFunction.get());
network.randomizeWeights();
int step = 0;
std::size_t step = 0;
float error = trainOutputs(network, patterns);
while(step++ < 15 && error > _maxError) {
while(step++ < _maxHiddenUnits && error > _maxError) {
std::vector<std::shared_ptr<Neuron>> candidates = createCandidates(network.getNeuronSize() - outputs);
std::shared_ptr<Neuron> candidate=trainCandidates(network, candidates, patterns);
std::pair<std::shared_ptr<Neuron>, std::vector<float>> candidate = trainCandidates(network, candidates, patterns);
addBestCandidate(network, candidate);
error = trainOutputs(network, patterns);
@@ -59,10 +58,20 @@ namespace NeuralNetwork {
_distribution = std::uniform_real_distribution<>(-weightRange, weightRange);
}
void setMaximumHiddenNeurons(std::size_t neurons) {
_maxHiddenUnits = neurons;
}
void setActivationFunction(const ActivationFunction::ActivationFunction &function) {
_activFunction = std::shared_ptr<ActivationFunction::ActivationFunction>(function.clone());
}
protected:
std::shared_ptr<ActivationFunction::ActivationFunction> _activFunction = std::make_shared<ActivationFunction::Sigmoid>(-4.9);
float _minimalErrorStep = 0.00005;
float _maxError;
float _weightRange;
std::size_t _maxHiddenUnits = 20;
std::size_t _numberOfCandidates;
std::mt19937 _generator;
std::uniform_real_distribution<> _distribution;
@@ -84,19 +93,23 @@ namespace NeuralNetwork {
float trainOutputs(Cascade::Network &network, const std::vector<TrainingPattern> &patterns);
std::shared_ptr<Neuron> trainCandidates(Cascade::Network &network, std::vector<std::shared_ptr<Neuron>> &candidates, const std::vector<TrainingPattern> &patterns);
std::pair<std::shared_ptr<Neuron>, std::vector<float>> trainCandidates(Cascade::Network &network, std::vector<std::shared_ptr<Neuron>> &candidates,
const std::vector<TrainingPattern> &patterns);
void addBestCandidate(Cascade::Network &network, const std::shared_ptr<Neuron> &candidate) {
void addBestCandidate(Cascade::Network &network, const std::pair<std::shared_ptr<Neuron>, std::vector<float>> &candidate) {
auto neuron = network.addNeuron();
neuron->setWeights(candidate->getWeights());
neuron->setActivationFunction(candidate->getActivationFunction());
float weightPortion = network.getNeuronSize() - network.outputs();
neuron->setWeights(candidate.first->getWeights());
neuron->setActivationFunction(candidate.first->getActivationFunction());
std::size_t outIndex = 0;
for(auto &n :network.getOutputNeurons()) {
auto weights = n->getWeights();
for(auto &weight: weights) {
weight *= 0.7;
}
weights[weights.size()-1] = _distribution(_generator);
weights[weights.size() - 1] = -candidate.second[outIndex] * weightPortion;//_distribution(_generator);
outIndex++;
n->setWeights(weights);
}
}
@@ -107,7 +120,7 @@ namespace NeuralNetwork {
for(std::size_t i = 0; i < _numberOfCandidates; i++) {
candidates.push_back(std::make_shared<Neuron>(id));
candidates.back()->setInputSize(id);
candidates.back()->setActivationFunction(NeuralNetwork::ActivationFunction::Tangents());
candidates.back()->setActivationFunction(*_activFunction.get());
for(std::size_t weightIndex = 0; weightIndex < id; weightIndex++) {
candidates.back()->weight(weightIndex) = _distribution(_generator);

26
src/NeuralNetwork/ConstructiveAlgorithms/CascadeCorrelation.cpp
View File

@@ -21,7 +21,7 @@ float CascadeCorrelation::trainOutputs(Cascade::Network &network, const std::vec
patternsForOutput.emplace_back(getInnerNeuronsOutput(network, pattern.first), pattern.second);
}
float lastError = std::numeric_limits<float>::max();
float lastError;
float error = std::numeric_limits<float>::max();
std::size_t iteration = 0;
std::size_t iterWithoutImporvement = 0;
@@ -33,7 +33,7 @@ float CascadeCorrelation::trainOutputs(Cascade::Network &network, const std::vec
error = 0;
for(auto &pattern:patternsForOutput) {
std::vector<float> output = p.computeOutput(pattern.first);
std::vector<float> output = p.computeOutput({pattern.first.begin() + 1, pattern.first.end()});
for(std::size_t outputIndex = 0; outputIndex < output.size(); outputIndex++) {
error += pow(output[outputIndex] - pattern.second[outputIndex], 2);
}
@@ -44,7 +44,8 @@ float CascadeCorrelation::trainOutputs(Cascade::Network &network, const std::vec
} else {
iterWithoutImporvement = 0;
}
} while (iteration++ < 1000 && iterWithoutImporvement < 3);
}
while(iteration++ < 1000 && iterWithoutImporvement < 400);
std::cout << "iter: " << iteration << ", error: " << error << ", " << (lastError - error) << "\n";
for(std::size_t neuron = 0; neuron < outputs; neuron++) {
@@ -53,7 +54,9 @@ float CascadeCorrelation::trainOutputs(Cascade::Network &network, const std::vec
return error;
}
std::shared_ptr<NeuralNetwork::Neuron> CascadeCorrelation::trainCandidates(Cascade::Network &network, std::vector<std::shared_ptr<Neuron>> &candidates, const std::vector<TrainingPattern> &patterns) {
std::pair<std::shared_ptr<NeuralNetwork::Neuron>, std::vector<float>> CascadeCorrelation::trainCandidates(Cascade::Network &network,
std::vector<std::shared_ptr<Neuron>> &candidates,
const std::vector<TrainingPattern> &patterns) {
std::size_t outputs = patterns[0].second.size();
std::vector<TrainingPattern> patternsForOutput;
@@ -77,12 +80,15 @@ std::shared_ptr<NeuralNetwork::Neuron> CascadeCorrelation::trainCandidates(Casca
}
std::for_each(meanErrors.begin(), meanErrors.end(), [&patterns](float &n) { n /= patterns.size(); });
std::size_t iterations = 0;
std::size_t iterationsWithoutIprovement = 0;
float bestCorrelation = 0;
float lastCorrelation = 0;
std::shared_ptr<Neuron> bestCandidate = nullptr;
std::vector<float> bestCorrelations(errors[0].size());
do {
lastCorrelation = bestCorrelation;
bool firstStep = true;
@@ -104,7 +110,8 @@ std::shared_ptr<NeuralNetwork::Neuron> CascadeCorrelation::trainCandidates(Casca
correlationSigns[err] = correlations[err] > 0 ? 1.0 : -1.0;
}
correlation = std::accumulate(correlations.begin(), correlations.end(), 0.0);
correlation = std::accumulate(correlations.begin(), correlations.end(), 0.0, [](const float &a, float b) { return a + fabs(b); });
std::vector<float> derivatives(candidate->getWeights().size());
for(std::size_t input = 0; input < candidate->getWeights().size(); input++) {
float dcdw = 0.0;
@@ -122,12 +129,13 @@ std::shared_ptr<NeuralNetwork::Neuron> CascadeCorrelation::trainCandidates(Casca
}
for(std::size_t weightIndex = 0; weightIndex < derivatives.size(); weightIndex++) {
candidate->weight(weightIndex) += derivatives[weightIndex] * 0.1;
candidate->weight(weightIndex) += derivatives[weightIndex] * 0.7;
}
if(firstStep || correlation > bestCorrelation) {
bestCorrelation = correlation;
bestCandidate = candidate;
std::swap(bestCorrelations, correlations);
firstStep = false;
}
}
@@ -136,8 +144,10 @@ std::shared_ptr<NeuralNetwork::Neuron> CascadeCorrelation::trainCandidates(Casca
iterationsWithoutIprovement++;
}
} while (iterations ++ < 200 && iterationsWithoutIprovement <3);
}
while(iterations++ < 200 && iterationsWithoutIprovement < 300);
std::cout << "iter: " << iterations << ", correlation: " << bestCorrelation << ", " << lastCorrelation << "\n";
return bestCandidate;
return {bestCandidate, bestCorrelations};
}