diff --git a/include/NeuralNetwork/ConstructiveAlgorithms/Cascade2.h b/include/NeuralNetwork/ConstructiveAlgorithms/Cascade2.h new file mode 100644 index 0000000..5e35ce9 --- /dev/null +++ b/include/NeuralNetwork/ConstructiveAlgorithms/Cascade2.h @@ -0,0 +1,31 @@ +#pragma once + +#include "../Cascade/Network.h" +#include "../FeedForward/Network.h" +#include "../Learning/QuickPropagation.h" + +#include "CascadeCorrelation.h" + +#include +#include + +// http://fann.cvs.sourceforge.net/viewvc/fann/fann/src/fann_cascade.c?view=markup +// https://github.com/gtomar/cascade + +namespace NeuralNetwork { + namespace ConstructiveAlgorihtms { + class Cascade2 : public CascadeCorrelation { + public: + typedef std::pair, std::vector> TrainingPattern; + + Cascade2(std::size_t numberOfCandidate = 18, float maxError = 0.7) : CascadeCorrelation(numberOfCandidate, maxError) { + } + + protected: + + virtual std::pair, std::vector> trainCandidates(Cascade::Network &network, std::vector> &candidates, + const std::vector &patterns) override; + }; + + } +} \ No newline at end of file diff --git a/src/NeuralNetwork/ConstructiveAlgorithms/Cascade2.cpp b/src/NeuralNetwork/ConstructiveAlgorithms/Cascade2.cpp new file mode 100644 index 0000000..df62f07 --- /dev/null +++ b/src/NeuralNetwork/ConstructiveAlgorithms/Cascade2.cpp @@ -0,0 +1,327 @@ +#include + +using namespace NeuralNetwork::ConstructiveAlgorihtms; + +std::pair, std::vector> Cascade2::trainCandidates(Cascade::Network &network, + std::vector> &candidates, + const std::vector &patterns) { + std::size_t outputs = patterns[0].second.size(); + + std::vector patternsForOutput; + + float sumSqDiffs=0.0; + + for(auto &pattern:patterns) { + patternsForOutput.emplace_back(getInnerNeuronsOutput(network, pattern.first), pattern.second); + } + + std::vector > errors(patterns.size()); + + for(std::size_t patternNumber = 0; patternNumber < patterns.size(); patternNumber++) { + auto &pattern = patterns[patternNumber]; + errors[patternNumber].resize(network.outputs()); + + std::vector output = network.computeOutput(patterns[patternNumber].first); + for(std::size_t outputIndex = 0; outputIndex < outputs; outputIndex++) { + float diff = output[outputIndex]-pattern.second[outputIndex]; + errors[patternNumber][outputIndex] = diff; + sumSqDiffs+=diff*diff; + } + } + + std::size_t iterations = 0; + std::size_t iterationsWithoutIprovement = 0; + float bestCorrelation = 0; + float lastCorrelation = 0; + std::size_t bestCandidateIndex=0; + std::shared_ptr bestCandidate = nullptr; + + std::vector> candidateWeights(candidates.size()); + for(auto &w: candidateWeights) { + w.resize(outputs); + for(auto &output: w) { + output = fabs(_distribution(_generator))*0.5; + } + } + + //compute Correlation Epoch + do { + lastCorrelation = bestCorrelation; + bool firstStep = true; + std::size_t candidateIndex=0; + + for(auto &candidate : candidates) { + + float score=sumSqDiffs; + std::vector slopes(candidate->getWeights().size()); + std::vector outSlopes(outputs); + + std::size_t patternIndex=0; + for(auto &pattern:patternsForOutput) { + float errSum = 0.0; + float activationValue =(*candidate)(pattern.first); + float derivatived = candidate->getActivationFunction().derivatedOutput(candidate->value(), candidate->output()); + + for(std::size_t output = 0; output < outputs; output++) { + float weight = candidateWeights[candidateIndex][output]; + float diff = activationValue * weight - errors[patternIndex][output]; + + float goalDir= pattern.second[output] <0.0? -1.0 :1.0; + float diffDir= diff >0.0? -1.0 :1.0; + score -= (diff * diff); + outSlopes[output] -= 2.0 * diff * activationValue; + errSum += diff * weight; + patternIndex++; + } + errSum*= derivatived; + + for(std::size_t input = 0; input < pattern.first.size(); input++) { + slopes[input] -= errSum*pattern.first[input]; + } + } + + for(std::size_t weightIndex = 0; weightIndex < slopes.size(); weightIndex++) { + candidate->weight(weightIndex) += slopes[weightIndex] * 0.7 / (patterns.size());/// (patterns.size() * patterns[0].first.size()); + } + + for(std::size_t weightIndex = 0; weightIndex < outSlopes.size(); weightIndex++) { + candidateWeights[candidateIndex][weightIndex] += outSlopes[weightIndex] * 0.7 / (patterns.size());/// (patterns.size() * patterns[0].first.size()); + } + + if(firstStep || score > bestCorrelation) { + bestCorrelation = score; + bestCandidate = candidate; + firstStep = false; + bestCandidateIndex=candidateIndex; + } + candidateIndex++; + } + + if(bestCorrelation <= lastCorrelation) { + iterationsWithoutIprovement++; + } + + } + while(iterations++ < _maxCandidateIterations && iterationsWithoutIprovement < _maxCandidateIterationsWithoutChange); + std::cout << "iter: " << iterations << ", correlation: " << bestCorrelation << ", " << lastCorrelation << "\n"; + + for(auto &a : candidateWeights[bestCandidateIndex]) { + a*=-1.0; + } + + return {bestCandidate, candidateWeights[bestCandidateIndex]}; +} + +/* + * +std::pair, std::vector> Cascade2::trainCandidates(Cascade::Network &network, + std::vector> &candidates, + const std::vector &patterns) { + std::size_t outputs = patterns[0].second.size(); + + std::vector patternsForOutput; + + float sumSqDiffs=0.0; + + for(auto &pattern:patterns) { + patternsForOutput.emplace_back(getInnerNeuronsOutput(network, pattern.first), pattern.second); + } + + std::vector errors(patterns.size()); + + for(std::size_t patternNumber = 0; patternNumber < patterns.size(); patternNumber++) { + auto &pattern = patterns[patternNumber]; + + std::vector output = network.computeOutput(patterns[patternNumber].first); + for(std::size_t outputIndex = 0; outputIndex < outputs; outputIndex++) { + float diff = output[outputIndex]-pattern.second[outputIndex]; + errors[outputIndex] += diff; + sumSqDiffs+=diff*diff; + } + } + + std::size_t iterations = 0; + std::size_t iterationsWithoutIprovement = 0; + float bestCorrelation = 0; + float lastCorrelation = 0; + std::size_t bestCandidateIndex=0; + std::shared_ptr bestCandidate = nullptr; + + std::vector> candidateWeights(candidates.size()); + for(auto &w: candidateWeights) { + w.resize(outputs); + for(auto &output: w) { + output = fabs(_distribution(_generator)); + } + } + + //compute Correlation Epoch + do { + lastCorrelation = bestCorrelation; + bool firstStep = true; + std::size_t candidateIndex=0; + + for(auto &candidate : candidates) { + + float score=sumSqDiffs; + std::vector slopes(candidate->getWeights().size()); + std::vector outSlopes(outputs); + + for(auto &pattern:patternsForOutput) { + float errSum = 0.0; + float activationValue =(*candidate)(pattern.first); + float derivatived = candidate->getActivationFunction().derivatedOutput(candidate->value(), candidate->output()); + + for(std::size_t output = 0; output < outputs; output++) { + float weight = candidateWeights[candidateIndex][output]; + float diff = activationValue * weight - errors[output]; + + float goalDir= pattern.second[output] <0.0? -1.0 :1.0; + float diffDir= diff >0.0? -1.0 :1.0; + score -= (diff * diff); + outSlopes[output] += diff * activationValue; + errSum += diff * weight; + } + errSum*= derivatived; + + for(std::size_t input = 0; input < pattern.first.size(); input++) { + slopes[input] += errSum*pattern.first[input]; + } + } + + for(std::size_t weightIndex = 0; weightIndex < slopes.size(); weightIndex++) { + candidate->weight(weightIndex) += slopes[weightIndex] * 0.7/ (patterns.size() * patterns[0].first.size()); + } + + for(std::size_t weightIndex = 0; weightIndex < outSlopes.size(); weightIndex++) { + candidateWeights[candidateIndex][weightIndex] += outSlopes[weightIndex] * 0.7/ (patterns.size() * patterns[0].first.size()); + } + + if(firstStep || score > bestCorrelation) { + bestCorrelation = score; + bestCandidate = candidate; + firstStep = false; + bestCandidateIndex=candidateIndex; + } + candidateIndex++; + } + + if(bestCorrelation <= lastCorrelation) { + iterationsWithoutIprovement++; + } + + } + while(iterations++ < _maxCandidateIterations && iterationsWithoutIprovement < _maxCandidateIterationsWithoutChange); + std::cout << "iter: " << iterations << ", correlation: " << bestCorrelation << ", " << lastCorrelation << "\n"; + + for(auto &a : candidateWeights[bestCandidateIndex]) { + a*=-1.0; + } + + return {bestCandidate, candidateWeights[bestCandidateIndex]}; +} +*/ + + +/* +std::pair, std::vector> Cascade2::trainCandidates(Cascade::Network &network, + std::vector> &candidates, + const std::vector &patterns) { + std::size_t outputs = patterns[0].second.size(); + + std::vector patternsForOutput; + std::vector patternNets; + + for(auto &pattern:patterns) { + patternsForOutput.emplace_back(getInnerNeuronsOutput(network, pattern.first), pattern.second); + } + + std::vector errors(patterns.size()); + + for(std::size_t patternNumber = 0; patternNumber < patterns.size(); patternNumber++) { + auto &pattern = patterns[patternNumber]; + + std::vector output = network.computeOutput(patterns[patternNumber].first); + patternNets.push_back(new FeedForward::Network(patternsForOutput[patternNumber].first.size()-1)); + auto patternNetwork = patternNets.back(); + + auto &hidden = patternNetwork->appendLayer(2); + auto &outputLayer = patternNetwork->appendLayer(outputs); + + for(std::size_t outputIndex = 0; outputIndex < outputs; outputIndex++) { + outputLayer[outputIndex+1].weight(0) = network.getOutputNeurons()[outputIndex]->value(); + float diff = pattern.second[outputIndex] - output[outputIndex]; + errors[outputIndex] += diff; + } + } + + std::size_t iterations = 0; + std::size_t iterationsWithoutIprovement = 0; + float bestCorrelation = 0; + float lastCorrelation = 0; + std::size_t bestCandidateIndex=0; + + std::vector> candidateWeights(candidates.size()); + + for(auto &w: candidateWeights) { + w.resize(outputs); + for(auto &output: w) { + output = fabs(_distribution(_generator)); + } + } + std::vectorcandidateScores(candidates.size()); + //compute Correlation Epoch + do { + std::fill(candidateScores.begin(),candidateScores.end(),0.0); + lastCorrelation = bestCorrelation; + + for(std::size_t patternIndex=0;patternIndexgetWeights()); + + for(std::size_t outputNeuron=0;outputNeuroncomputeOutput(pattern.first); + + for(std::size_t outputNeuron=0;outputNeuronsetWeights((*net)[1][1].getWeights()); + candidateIndex++; + } + + } + + bestCorrelation=candidateScores[0]; + bestCandidateIndex=0; + + for(std::size_t index=1;index < candidateScores.size();index++) { + if(bestCorrelation > candidateScores[index]) { + bestCandidateIndex = index; + } + } + + if(bestCorrelation <= lastCorrelation) { + iterationsWithoutIprovement++; + } + + } + while(iterations++ < _maxCandidateIterations && iterationsWithoutIprovement < _maxCandidateIterationsWithoutChange); + std::cout << "iter: " << iterations << ", correlation: " << bestCorrelation << ", " << lastCorrelation << "\n"; + + for(auto &net:patternNets) { + delete net; + } + return {candidates[bestCandidateIndex], candidateWeights[bestCandidateIndex]}; +} +*/ \ No newline at end of file