1 // OpenNN: Open Neural Networks Library 2 // www.opennn.net 3 // 4 // O P T I M I Z A T I O N A L G O R I T H M C L A S S H E A D E R 5 // 6 // Artificial Intelligence Techniques SL 7 // artelnics@artelnics.com 8 9 #ifndef OPTIMIZATIONALGORITHM_H 10 #define OPTIMIZATIONALGORITHM_H 11 12 // System includes 13 14 #include <iostream> 15 #include <fstream> 16 #include <algorithm> 17 #include <functional> 18 #include <limits> 19 #include <cmath> 20 #include <ctime> 21 #include <iomanip> 22 23 // OpenNN includes 24 25 #include "config.h" 26 #include "loss_index.h" 27 28 using namespace std; 29 using namespace Eigen; 30 31 32 namespace OpenNN 33 { 34 35 /// This abstract class represents the concept of optimization algorithm for a neural network in OpenNN library. 36 /// Any derived class must implement the perform_training() method. 37 38 class OptimizationAlgorithm 39 { 40 41 public: 42 43 explicit OptimizationAlgorithm(); 44 45 explicit OptimizationAlgorithm(LossIndex*); 46 47 virtual ~OptimizationAlgorithm(); 48 49 /// Enumeration of all possibles condition of stop for the algorithms. 50 51 enum StoppingCondition{MinimumParametersIncrementNorm, MinimumLossDecrease, LossGoal, GradientNormGoal, 52 MaximumSelectionErrorIncreases, MaximumEpochsNumber, MaximumTime}; 53 54 struct OptimizationData 55 { OptimizationDataOptimizationData56 explicit OptimizationData() 57 { 58 } 59 ~OptimizationDataOptimizationData60 virtual ~OptimizationData() 61 { 62 } 63 printOptimizationData64 void print() const 65 { 66 cout << "Parameters:" << endl; 67 cout << parameters << endl; 68 69 cout << "Potential parameters:" << endl; 70 cout << potential_parameters << endl; 71 72 cout << "Training direction:" << endl; 73 cout << training_direction << endl; 74 75 cout << "Initial learning rate:" << endl; 76 cout << initial_learning_rate << endl; 77 } 78 79 Tensor<type, 1> parameters; 80 Tensor<type, 1> potential_parameters; 81 Tensor<type, 1> training_direction; 82 type initial_learning_rate = 0; 83 84 85 86 }; 87 88 /// This structure contains the optimization algorithm results. 89 90 struct Results 91 { ResultsResults92 explicit Results() 93 { 94 } 95 ~ResultsResults96 virtual ~Results() 97 { 98 } 99 100 string write_stopping_condition() const; 101 102 /// Stopping condition of the algorithm. 103 104 StoppingCondition stopping_condition; 105 106 /// Returns a string representation of the results structure. 107 108 109 110 void save(const string&) const; 111 112 /// Returns a default(empty) string matrix with the final results from training. 113 114 Tensor<string, 2> write_final_results(const Index& = 3) const; 115 116 /// Resizes training history variables. 117 118 void resize_training_history(const Index&); 119 120 /// Resizes selection history variables. 121 122 void resize_selection_history(const Index&); 123 124 /// Resizes the training error history keeping the values. 125 126 void resize_training_error_history(const Index&); 127 128 /// Resizes the selection error history keeping the values. 129 130 void resize_selection_error_history(const Index&); 131 132 /// Writes final results of the training. 133 /* 134 Tensor<string, 2> write_final_results(const Index& precision = 3) const; 135 */ 136 137 // Training history 138 139 /// History of the loss function loss over the training iterations. 140 141 Tensor<type, 1> training_error_history; 142 143 /// History of the selection error over the training iterations. 144 145 Tensor<type, 1> selection_error_history; 146 147 // Final values 148 149 /// Final neural network parameters vector. 150 151 Tensor<type, 1> final_parameters; 152 153 /// Final neural network parameters norm. 154 155 type final_parameters_norm; 156 157 /// Final loss function evaluation. 158 159 type final_training_error; 160 161 /// Final selection error. 162 163 type final_selection_error; 164 165 /// Final gradient norm. 166 167 type final_gradient_norm; 168 169 /// Elapsed time of the training process. 170 171 string elapsed_time; 172 173 /// Maximum number of training iterations. 174 175 Index epochs_number; 176 177 /// Stopping criterion. 178 179 string stopping_criterion; 180 }; 181 182 183 // Get methods 184 185 LossIndex* get_loss_index_pointer() const; 186 187 /// Hardware use. 188 string get_hardware_use() const; 189 void set_hardware_use(const string&); 190 191 bool has_loss_index() const; 192 193 // Utilities 194 195 const bool& get_display() const; 196 197 const Index& get_display_period() const; 198 199 const Index& get_save_period() const; 200 201 const string& get_neural_network_file_name() const; 202 203 /// Writes the time from seconds in format HH:mm:ss. 204 205 const string write_elapsed_time(const type&) const; 206 207 // Set methods 208 209 void set(); 210 void set(LossIndex*); 211 212 virtual void set_default(); 213 214 virtual void set_threads_number(const int&); 215 216 virtual void set_loss_index_pointer(LossIndex*); 217 218 virtual void set_display(const bool&); 219 220 void set_display_period(const Index&); 221 222 void set_save_period(const Index&); 223 void set_neural_network_file_name(const string&); 224 225 virtual void set_reserve_selection_error_history(const bool&) = 0; 226 227 // Training methods 228 229 virtual void check() const; 230 231 /// Trains a neural network which has a loss index associated. 232 233 virtual Results perform_training() = 0; 234 write_optimization_algorithm_type()235 virtual string write_optimization_algorithm_type() const {return string();} 236 237 // Serialization methods 238 239 virtual 240 void print() const; 241 242 virtual Tensor<string, 2> to_string_matrix() const; 243 244 virtual void from_XML(const tinyxml2::XMLDocument&); 245 246 virtual void write_XML(tinyxml2::XMLPrinter&) const; 247 248 void save(const string&) const; 249 void load(const string&); 250 251 protected: 252 253 NonBlockingThreadPool* non_blocking_thread_pool = nullptr; 254 ThreadPoolDevice* thread_pool_device; 255 256 /// Pointer to a loss index for a neural network object. 257 258 259 LossIndex* loss_index_pointer = nullptr; 260 261 /// Number of training epochs in the neural network. 262 263 Index epochs_number = 10000; 264 265 // UTILITIES 266 267 ///Hardware use 268 string hardware_use = "Multi-core"; 269 270 271 /// Number of iterations between the training showing progress. 272 273 Index display_period = 5; 274 275 /// Number of iterations between the training saving progress. 276 277 Index save_period = 100; 278 279 /// Path where the neural network is saved. 280 281 string neural_network_file_name = "neural_network.xml"; 282 283 /// Display messages to screen. 284 285 bool display = true; 286 287 const Eigen::array<IndexPair<Index>, 1> AT_B = {IndexPair<Index>(0, 0)}; 288 const Eigen::array<IndexPair<Index>, 1> product_vector_matrix = {IndexPair<Index>(0, 1)}; // Normal product vector times matrix 289 const Eigen::array<IndexPair<Index>, 1> A_B = {IndexPair<Index>(1, 0)}; 290 normalized(Tensor<type,1> & tensor)291 void normalized(Tensor<type, 1>& tensor) const 292 { 293 const type norm = l2_norm(tensor); 294 295 tensor.device(*thread_pool_device) = tensor/norm; 296 } 297 298 l2_norm(const Tensor<type,1> & tensor)299 type l2_norm(const Tensor<type, 1>& tensor) const 300 { 301 Tensor<type, 0> norm; 302 303 norm.device(*thread_pool_device) = tensor.square().sum().sqrt(); 304 305 return norm(0); 306 } 307 is_zero(const Tensor<type,1> & tensor)308 bool is_zero(const Tensor<type, 1>& tensor) const 309 { 310 const Index size = tensor.size(); 311 312 for(Index i = 0; i < size; i++) 313 { 314 if(abs(tensor[i]) > numeric_limits<type>::min()) return false; 315 } 316 317 return true; 318 } 319 320 #ifdef OPENNN_CUDA 321 #include "../../opennn-cuda/opennn_cuda/optimization_algorithm_cuda.h" 322 #endif 323 324 }; 325 326 } 327 328 #endif 329 330 331 // OpenNN: Open Neural Networks Library. 332 // Copyright(C) 2005-2020 Artificial Intelligence Techniques, SL. 333 // 334 // This library is free software; you can redistribute it and/or 335 // modify it under the terms of the GNU Lesser General Public 336 // License as published by the Free Software Foundation; either 337 // version 2.1 of the License, or any later version. 338 // 339 // This library is distributed in the hope that it will be useful, 340 // but WITHOUT ANY WARRANTY; without even the implied warranty of 341 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 342 // Lesser General Public License for more details. 343 344 // You should have received a copy of the GNU Lesser General Public 345 // License along with this library; if not, write to the Free Software 346 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 347