Line data Source code
1 : /**
2 : * @file ActivationFunctions.h
3 : * @author Damien Balima (www.dams-labs.net)
4 : * @brief Activation functions
5 : * @date 2024-03-07
6 : *
7 : * @copyright Damien Balima (c) CC-BY-NC-SA-4.0 2024
8 : *
9 : */
10 :
11 : #pragma once
12 : #include "Common.h"
13 : #include <algorithm>
14 : #include <map>
15 : #include <math.h>
16 : #include <opencv2/opencv.hpp>
17 : #include <string>
18 :
19 : namespace sipai {
20 : /**
21 : * @brief Activation Function enum.
22 : * Beware the int values are used in the Vulkan GLSL shader
23 : */
24 : enum class EActivationFunction {
25 : ELU = 0,
26 : LReLU = 1,
27 : PReLU = 2,
28 : ReLU = 3,
29 : Sigmoid = 4,
30 : Tanh = 5
31 : };
32 :
33 : const std::map<std::string, EActivationFunction, std::less<>> activation_map{
34 : {"ELU", EActivationFunction::ELU},
35 : {"LReLU", EActivationFunction::LReLU},
36 : {"PReLU", EActivationFunction::PReLU},
37 : {"ReLU", EActivationFunction::ReLU},
38 : {"Sigmoid", EActivationFunction::Sigmoid},
39 : {"Tanh", EActivationFunction::Tanh}};
40 :
41 0 : inline std::string getActivationStr(EActivationFunction activation) {
42 0 : for (const auto &[key, value] : activation_map) {
43 0 : if (value == activation) {
44 0 : return key;
45 : }
46 : }
47 0 : return "";
48 : }
49 :
50 : /**
51 : * @brief the sigmoid function is commonly used as the
52 : * activation function during the forward propagation step. The reason for this
53 : * is that the sigmoid function maps any input value into a range between 0 and
54 : * 1, which can be useful for outputting probabilities, among other things.
55 : * The sigmoid derivative can be expressed in terms of the output of
56 : * the sigmoid function itself: if σ(x) is the sigmoid function, then its
57 : * derivative σ'(x) can be computed as σ(x) * (1 - σ(x)).
58 : */
59 4 : inline auto sigmoid = [](const cv::Vec4f &rgba) {
60 4 : cv::Vec4f result;
61 4 : cv::exp(-rgba, result);
62 4 : cv::divide(cv::Vec4f::all(1.0f), (cv::Vec4f::all(1.0f) + result), result);
63 4 : return Common::clamp4f(result);
64 : };
65 :
66 2 : inline auto sigmoidDerivative = [](const cv::Vec4f &rgba) {
67 2 : cv::Vec4f sigmoidValue = sigmoid(rgba);
68 4 : return sigmoidValue.mul(cv::Vec4f::all(1.0f) - sigmoidValue);
69 : };
70 :
71 : /**
72 : * @brief Tanh Function (Hyperbolic Tangent): This function is similar to the
73 : * sigmoid function but maps the input to a range between -1 and 1. It is often
74 : * used in the hidden layers of a neural network.
75 : */
76 6 : inline auto tanhFunc = [](const cv::Vec4f &rgba) {
77 6 : cv::Vec4f result;
78 6 : std::transform(rgba.val, rgba.val + 4, result.val,
79 24 : [](float v) { return (std::tanh(v) / 2.0f) + 0.5f; });
80 6 : return result;
81 : };
82 :
83 3 : inline auto tanhDerivative = [](const cv::Vec4f &rgba) {
84 3 : cv::Vec4f tanhValue = tanhFunc(rgba);
85 6 : return cv::Vec4f::all(1.0f) - tanhValue.mul(tanhValue);
86 : };
87 :
88 : /**
89 : * @brief ReLU Function (Rectified Linear Unit): This function outputs the input
90 : * directly if it’s positive; otherwise, it outputs zero. It has become very
91 : * popular in recent years because it helps to alleviate the vanishing gradient
92 : * problem.
93 : * Combine ReLU with clamping to [0, 1] range
94 : * @param rgba
95 : * @return ReLU
96 : */
97 :
98 2 : inline auto relu = [](const cv::Vec4f &rgba) { return Common::clamp4f(rgba); };
99 2 : inline auto reluDerivative = [](const cv::Vec4f &rgba) {
100 2 : cv::Vec4f result;
101 2 : std::transform(rgba.val, rgba.val + 4, result.val,
102 8 : [](float v) { return v > 0.0f ? 1.0f : 0.0f; });
103 2 : return result;
104 : };
105 :
106 : /**
107 : * @brief Leaky ReLU: This is a variant of ReLU that allows small negative
108 : * values when the input is less than zero. It can help to alleviate the dying
109 : * ReLU problem where neurons become inactive and only output zero.
110 : * Combine LReLU with clamping to [0, 1] range
111 : */
112 923 : inline auto leakyRelu = [](const cv::Vec4f &rgba) {
113 1846 : return Common::clamp4f(rgba * 0.01f);
114 : };
115 :
116 254 : inline auto leakyReluDerivative = [](const cv::Vec4f &rgba) {
117 508 : return Common::clamp4f(rgba, cv::Vec4f::all(0.01f), cv::Vec4f::all(1.0f));
118 : };
119 :
120 : /**
121 : * @brief Parametric ReLU (PReLU) is a type of leaky ReLU that, instead of
122 : * having a predetermined slope like 0.01, learns the slope during training.
123 : * This can give it a bit more flexibility and help it to learn more complex
124 : * patterns
125 : */
126 2 : inline auto parametricRelu = [](const cv::Vec4f &rgba, float alpha) {
127 2 : cv::Vec4f result;
128 2 : std::transform(rgba.val, rgba.val + 4, result.val, [alpha](float v) {
129 8 : float value = std::max(alpha * v, v);
130 8 : return std::clamp(value, 0.f, 1.f);
131 : });
132 2 : return result;
133 : };
134 :
135 2 : inline auto parametricReluDerivative = [](const cv::Vec4f &rgba, float alpha) {
136 2 : cv::Vec4f result;
137 2 : std::transform(rgba.val, rgba.val + 4, result.val,
138 8 : [alpha](float v) { return v > 0.0f ? 1.0f : alpha; });
139 2 : return result;
140 : };
141 :
142 : /**
143 : * @brief the Exponential Linear Units (ELUs) are a great choice as they
144 : * take on negative values when the input is less than zero, which allows them
145 : * to push mean unit activations closer to zero like batch normalization. Unlike
146 : * ReLUs, ELUs have a nonzero gradient for negative input, which avoids the
147 : * “dead neuron” problem.
148 : *
149 : */
150 3 : inline auto elu = [](const cv::Vec4f &rgba, float alpha) {
151 3 : cv::Vec4f result;
152 3 : std::transform(rgba.val, rgba.val + 4, result.val, [alpha](float v) {
153 12 : float value = v >= 0 ? v : alpha * (exp(v) - 1);
154 12 : return std::clamp(value, 0.f, 1.f);
155 : });
156 3 : return result;
157 : };
158 :
159 2 : inline auto eluDerivative = [](const cv::Vec4f &rgba, float alpha) {
160 2 : cv::Vec4f result;
161 2 : std::transform(rgba.val, rgba.val + 4, result.val,
162 8 : [alpha](float v) { return v > 0.0f ? 1.0f : alpha * exp(v); });
163 2 : return result;
164 : };
165 : } // namespace sipai
|
