1 /** 2 * @file methods/ann/activation_functions/rectifier_function.hpp 3 * @author Marcus Edel 4 * 5 * Definition and implementation of the rectifier function as described by 6 * V. Nair and G. E. Hinton. 7 * 8 * For more information, see the following paper. 9 * 10 * @code 11 * @misc{NairHinton2010, 12 * author = {Vinod Nair, Geoffrey E. Hinton}, 13 * title = {Rectified Linear Units Improve Restricted Boltzmann Machines}, 14 * year = {2010} 15 * } 16 * @endcode 17 * 18 * mlpack is free software; you may redistribute it and/or modify it under the 19 * terms of the 3-clause BSD license. You should have received a copy of the 20 * 3-clause BSD license along with mlpack. If not, see 21 * http://www.opensource.org/licenses/BSD-3-Clause for more information. 22 */ 23 #ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_RECTIFIER_FUNCTION_HPP 24 #define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_RECTIFIER_FUNCTION_HPP 25 26 #include <mlpack/prereqs.hpp> 27 #include <algorithm> 28 29 namespace mlpack { 30 namespace ann /** Artificial Neural Network. */ { 31 32 /** 33 * The rectifier function, defined by 34 * 35 * @f{eqnarray*}{ 36 * f(x) &=& \max(0, x) \\ 37 * f'(x) &=& \left\{ 38 * \begin{array}{lr} 39 * 1 & : x > 0 \\ 40 * 0 & : x \le 0 41 * \end{array} 42 * \right. 43 * @f} 44 */ 45 class RectifierFunction 46 { 47 public: 48 /** 49 * Computes the rectifier function. 50 * 51 * @param x Input data. 52 * @return f(x). 53 */ Fn(const double x)54 static double Fn(const double x) 55 { 56 return std::max(0.0, x); 57 } 58 59 /** 60 * Computes the rectifier function using a dense matrix as input. 61 * 62 * @param x Input data. 63 * @param y The resulting output activation. 64 */ 65 template<typename eT> Fn(const arma::Mat<eT> & x,arma::Mat<eT> & y)66 static void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y) 67 { 68 y.zeros(x.n_rows, x.n_cols); 69 y = arma::max(y, x); 70 } 71 72 /** 73 * Computes the rectifier function using a 3rd-order tensor as input. 74 * 75 * @param x Input data. 76 * @param y The resulting output activation. 77 */ 78 template<typename eT> Fn(const arma::Cube<eT> & x,arma::Cube<eT> & y)79 static void Fn(const arma::Cube<eT>& x, arma::Cube<eT>& y) 80 { 81 y.zeros(x.n_rows, x.n_cols, x.n_slices); 82 y = arma::max(y, x); 83 } 84 85 /** 86 * Computes the first derivative of the rectifier function. 87 * 88 * @param x Input data. 89 * @return f'(x) 90 */ Deriv(const double x)91 static double Deriv(const double x) 92 { 93 return (double)(x > 0); 94 } 95 96 /** 97 * Computes the first derivatives of the rectifier function. 98 * 99 * @param y Input data. 100 * @param x The resulting derivatives. 101 */ 102 template<typename InputType, typename OutputType> Deriv(const InputType & y,OutputType & x)103 static void Deriv(const InputType& y, OutputType& x) 104 { 105 x.set_size(arma::size(y)); 106 107 for (size_t i = 0; i < y.n_elem; ++i) 108 x(i) = Deriv(y(i)); 109 } 110 }; // class RectifierFunction 111 112 } // namespace ann 113 } // namespace mlpack 114 115 #endif 116