From 9f476b42336b289fce6df1a65f5aa0daac5f9a7b Mon Sep 17 00:00:00 2001
From: Shin <Tom.Cernik@gmail.com>
Date: Tue, 1 Nov 2016 21:59:22 +0100
Subject: [PATCH] add RectifiedUnit and LeakyRectifiedUnit

---
 .../ActivationFunction/LeakyRectifiedLinear.h | 39 +++++++++++++++++++
 .../ActivationFunction/RectifiedLinear.h      | 39 +++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h
 create mode 100644 include/NeuralNetwork/ActivationFunction/RectifiedLinear.h

diff --git a/include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h b/include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h
new file mode 100644
index 0000000..3a16fa4
--- /dev/null
+++ b/include/NeuralNetwork/ActivationFunction/LeakyRectifiedLinear.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#include "./ActivationFunction.h"
+
+#include <cassert>
+
+namespace NeuralNetwork {
+namespace ActivationFunction {
+
+	class LeakyRectifiedLinear: public ActivationFunction {
+		public:
+			LeakyRectifiedLinear(const float &lambdaP=0.04): lambda(lambdaP) {}
+
+			inline virtual float derivatedOutput(const float &inp,const float &) const override {
+				return inp > 0.0f ? lambda : 0.01f*lambda;
+			}
+
+			inline virtual float operator()(const float &x) const override {
+				return x > 0.0? x : 0.001f*x;
+			};
+
+			virtual ActivationFunction* clone() const override {
+				return new LeakyRectifiedLinear(lambda);
+			}
+
+			virtual SimpleJSON::Type::Object serialize() const override {
+				return {{"class", "NeuralNetwork::ActivationFunction::LeakyRectifiedLinear"}, {"lambda", lambda}};
+			}
+
+			static std::unique_ptr<LeakyRectifiedLinear> deserialize(const SimpleJSON::Type::Object &obj) {
+				return std::unique_ptr<LeakyRectifiedLinear>(new LeakyRectifiedLinear(obj["lambda"].as<double>()));
+			}
+
+		protected:
+			float lambda;
+			NEURAL_NETWORK_REGISTER_ACTIVATION_FUNCTION(NeuralNetwork::ActivationFunction::LeakyRectifiedLinear, LeakyRectifiedLinear::deserialize)
+	};
+}
+}
\ No newline at end of file
diff --git a/include/NeuralNetwork/ActivationFunction/RectifiedLinear.h b/include/NeuralNetwork/ActivationFunction/RectifiedLinear.h
new file mode 100644
index 0000000..93b4d72
--- /dev/null
+++ b/include/NeuralNetwork/ActivationFunction/RectifiedLinear.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#include "./ActivationFunction.h"
+
+#include <cassert>
+
+namespace NeuralNetwork {
+namespace ActivationFunction {
+
+	class RectifiedLinear: public ActivationFunction {
+		public:
+			RectifiedLinear(const float &lambdaP=0.1): lambda(lambdaP) {}
+
+			inline virtual float derivatedOutput(const float &inp,const float &) const override {
+				return inp > 0.0f ? lambda : 0.0f;
+			}
+
+			inline virtual float operator()(const float &x) const override {
+				return std::max(0.0f,x);
+			};
+
+			virtual ActivationFunction* clone() const override {
+				return new RectifiedLinear(lambda);
+			}
+
+			virtual SimpleJSON::Type::Object serialize() const override {
+				return {{"class", "NeuralNetwork::ActivationFunction::RectifiedLinear"}, {"lambda", lambda}};
+			}
+
+			static std::unique_ptr<RectifiedLinear> deserialize(const SimpleJSON::Type::Object &obj) {
+				return std::unique_ptr<RectifiedLinear>(new RectifiedLinear(obj["lambda"].as<double>()));
+			}
+
+		protected:
+			float lambda;
+			NEURAL_NETWORK_REGISTER_ACTIVATION_FUNCTION(NeuralNetwork::ActivationFunction::RectifiedLinear, RectifiedLinear::deserialize)
+	};
+}
+}
\ No newline at end of file