1 /**
2  * @file methods/ann/activation_functions/rectifier_function.hpp
3  * @author Marcus Edel
4  *
5  * Definition and implementation of the rectifier function as described by
6  * V. Nair and G. E. Hinton.
7  *
8  * For more information, see the following paper.
9  *
10  * @code
11  * @misc{NairHinton2010,
12  *   author = {Vinod Nair, Geoffrey E. Hinton},
13  *   title = {Rectified Linear Units Improve Restricted Boltzmann Machines},
14  *   year = {2010}
15  * }
16  * @endcode
17  *
18  * mlpack is free software; you may redistribute it and/or modify it under the
19  * terms of the 3-clause BSD license.  You should have received a copy of the
20  * 3-clause BSD license along with mlpack.  If not, see
21  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
22  */
23 #ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_RECTIFIER_FUNCTION_HPP
24 #define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_RECTIFIER_FUNCTION_HPP
25 
26 #include <mlpack/prereqs.hpp>
27 #include <algorithm>
28 
29 namespace mlpack {
30 namespace ann /** Artificial Neural Network. */ {
31 
32 /**
33  * The rectifier function, defined by
34  *
35  * @f{eqnarray*}{
36  * f(x) &=& \max(0, x) \\
37  * f'(x) &=& \left\{
38  *   \begin{array}{lr}
39  *     1 & : x > 0 \\
40  *     0 & : x \le 0
41  *   \end{array}
42  * \right.
43  * @f}
44  */
45 class RectifierFunction
46 {
47  public:
48   /**
49    * Computes the rectifier function.
50    *
51    * @param x Input data.
52    * @return f(x).
53    */
Fn(const double x)54   static double Fn(const double x)
55   {
56     return std::max(0.0, x);
57   }
58 
59   /**
60    * Computes the rectifier function using a dense matrix as input.
61    *
62    * @param x Input data.
63    * @param y The resulting output activation.
64    */
65   template<typename eT>
Fn(const arma::Mat<eT> & x,arma::Mat<eT> & y)66   static void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
67   {
68     y.zeros(x.n_rows, x.n_cols);
69     y = arma::max(y, x);
70   }
71 
72   /**
73    * Computes the rectifier function using a 3rd-order tensor as input.
74    *
75    * @param x Input data.
76    * @param y The resulting output activation.
77    */
78   template<typename eT>
Fn(const arma::Cube<eT> & x,arma::Cube<eT> & y)79   static void Fn(const arma::Cube<eT>& x, arma::Cube<eT>& y)
80   {
81     y.zeros(x.n_rows, x.n_cols, x.n_slices);
82     y = arma::max(y, x);
83   }
84 
85   /**
86    * Computes the first derivative of the rectifier function.
87    *
88    * @param x Input data.
89    * @return f'(x)
90    */
Deriv(const double x)91   static double Deriv(const double x)
92   {
93     return (double)(x > 0);
94   }
95 
96   /**
97    * Computes the first derivatives of the rectifier function.
98    *
99    * @param y Input data.
100    * @param x The resulting derivatives.
101    */
102   template<typename InputType, typename OutputType>
Deriv(const InputType & y,OutputType & x)103   static void Deriv(const InputType& y, OutputType& x)
104   {
105     x.set_size(arma::size(y));
106 
107     for (size_t i = 0; i < y.n_elem; ++i)
108       x(i) = Deriv(y(i));
109   }
110 }; // class RectifierFunction
111 
112 } // namespace ann
113 } // namespace mlpack
114 
115 #endif
116