1 // OpenNN: Open Neural Networks Library
2 // www.opennn.net
3 //
4 // O P T I M I Z A T I O N A L G O R I T H M C L A S S
5 //
6 // Artificial Intelligence Techniques SL
7 // artelnics@artelnics.com
8
9 #include "optimization_algorithm.h"
10
11 namespace OpenNN
12 {
13
14 /// Default constructor.
15 /// It creates a optimization algorithm object not associated to any loss index object.
16
OptimizationAlgorithm()17 OptimizationAlgorithm::OptimizationAlgorithm()
18 : loss_index_pointer(nullptr)
19 {
20 const int n = omp_get_max_threads();
21 non_blocking_thread_pool = new NonBlockingThreadPool(n);
22 thread_pool_device = new ThreadPoolDevice(non_blocking_thread_pool, n);
23
24 set_default();
25 }
26
27
28 /// It creates a optimization algorithm object associated to a loss index object.
29 /// @param new_loss_index_pointer Pointer to a loss index object.
30
OptimizationAlgorithm(LossIndex * new_loss_index_pointer)31 OptimizationAlgorithm::OptimizationAlgorithm(LossIndex* new_loss_index_pointer)
32 : loss_index_pointer(new_loss_index_pointer)
33 {
34 const int n = omp_get_max_threads();
35 non_blocking_thread_pool = new NonBlockingThreadPool(n);
36 thread_pool_device = new ThreadPoolDevice(non_blocking_thread_pool, n);
37
38 set_default();
39 }
40
41
42 /// Destructor.
43
~OptimizationAlgorithm()44 OptimizationAlgorithm::~OptimizationAlgorithm()
45 {
46 delete non_blocking_thread_pool;
47 delete thread_pool_device;
48 }
49
50
51 /// Returns a pointer to the loss index object to which the optimization algorithm is
52 /// associated.
53
get_loss_index_pointer() const54 LossIndex* OptimizationAlgorithm::get_loss_index_pointer() const
55 {
56 #ifdef __OPENNN_DEBUG__
57
58 if(!loss_index_pointer)
59 {
60 ostringstream buffer;
61
62 buffer << "OpenNN Exception: OptimizationAlgorithm class.\n"
63 << "LossIndex* get_loss_index_pointer() const method.\n"
64 << "Loss index pointer is nullptr.\n";
65
66 throw logic_error(buffer.str());
67 }
68
69 #endif
70
71 return loss_index_pointer;
72 }
73
74
75 /// Returns the hardware used. Default: Multi-core
76
get_hardware_use() const77 string OptimizationAlgorithm::get_hardware_use() const
78 {
79 return hardware_use;
80 }
81
82
83 /// Set hardware to use. Default: Multi-core.
84
set_hardware_use(const string & new_hardware_use)85 void OptimizationAlgorithm::set_hardware_use(const string& new_hardware_use)
86 {
87 hardware_use = new_hardware_use;
88 }
89
90 /// Returns true if this optimization algorithm object has an associated loss index object,
91 /// and false otherwise.
92
has_loss_index() const93 bool OptimizationAlgorithm::has_loss_index() const
94 {
95 if(loss_index_pointer)
96 {
97 return true;
98 }
99 else
100 {
101 return false;
102 }
103 }
104
105
106 /// Returns true if messages from this class can be displayed on the screen, or false if messages from
107 /// this class can't be displayed on the screen.
108
get_display() const109 const bool& OptimizationAlgorithm::get_display() const
110 {
111 return display;
112 }
113
114
115 /// Returns the number of iterations between the training showing progress.
116
get_display_period() const117 const Index& OptimizationAlgorithm::get_display_period() const
118 {
119 return display_period;
120 }
121
122
123 /// Returns the number of iterations between the training saving progress.
124
get_save_period() const125 const Index& OptimizationAlgorithm::get_save_period() const
126 {
127 return save_period;
128 }
129
130
131 /// Returns the file name where the neural network will be saved.
132
get_neural_network_file_name() const133 const string& OptimizationAlgorithm::get_neural_network_file_name() const
134 {
135 return neural_network_file_name;
136 }
137
138
139 /// Sets the loss index pointer to nullptr.
140 /// It also sets the rest of members to their default values.
141
set()142 void OptimizationAlgorithm::set()
143 {
144 loss_index_pointer = nullptr;
145
146 set_default();
147 }
148
149
150 /// Sets a new loss index pointer.
151 /// It also sets the rest of members to their default values.
152 /// @param new_loss_index_pointer Pointer to a loss index object.
153
set(LossIndex * new_loss_index_pointer)154 void OptimizationAlgorithm::set(LossIndex* new_loss_index_pointer)
155 {
156 loss_index_pointer = new_loss_index_pointer;
157
158 set_default();
159 }
160
161
set_threads_number(const int & new_threads_number)162 void OptimizationAlgorithm::set_threads_number(const int& new_threads_number)
163 {
164 if(non_blocking_thread_pool != nullptr) delete this->non_blocking_thread_pool;
165 if(thread_pool_device != nullptr) delete this->thread_pool_device;
166
167 non_blocking_thread_pool = new NonBlockingThreadPool(new_threads_number);
168 thread_pool_device = new ThreadPoolDevice(non_blocking_thread_pool, new_threads_number);
169 }
170
171
172 /// Sets a pointer to a loss index object to be associated to the optimization algorithm.
173 /// @param new_loss_index_pointer Pointer to a loss index object.
174
set_loss_index_pointer(LossIndex * new_loss_index_pointer)175 void OptimizationAlgorithm::set_loss_index_pointer(LossIndex* new_loss_index_pointer)
176 {
177 loss_index_pointer = new_loss_index_pointer;
178 }
179
180
181 /// Sets a new display value.
182 /// If it is set to true messages from this class are to be displayed on the screen;
183 /// if it is set to false messages from this class are not to be displayed on the screen.
184 /// @param new_display Display value.
185
set_display(const bool & new_display)186 void OptimizationAlgorithm::set_display(const bool& new_display)
187 {
188 display = new_display;
189 }
190
191
192 /// Sets a new number of iterations between the training showing progress.
193 /// @param new_display_period
194 /// Number of iterations between the training showing progress.
195
set_display_period(const Index & new_display_period)196 void OptimizationAlgorithm::set_display_period(const Index& new_display_period)
197 {
198
199
200 #ifdef __OPENNN_DEBUG__
201
202 if(new_display_period <= 0)
203 {
204 ostringstream buffer;
205
206 buffer << "OpenNN Exception: ConjugateGradient class.\n"
207 << "void set_display_period(const Index&) method.\n"
208 << "Display period must be greater than 0.\n";
209
210 throw logic_error(buffer.str());
211 }
212
213 #endif
214
215 display_period = new_display_period;
216 }
217
218
219 /// Sets a new number of iterations between the training saving progress.
220 /// @param new_save_period
221 /// Number of iterations between the training saving progress.
222
set_save_period(const Index & new_save_period)223 void OptimizationAlgorithm::set_save_period(const Index& new_save_period)
224 {
225
226
227 #ifdef __OPENNN_DEBUG__
228
229 if(new_save_period <= 0)
230 {
231 ostringstream buffer;
232
233 buffer << "OpenNN Exception: ConjugateGradient class.\n"
234 << "void set_save_period(const Index&) method.\n"
235 << "Save period must be greater than 0.\n";
236
237 throw logic_error(buffer.str());
238 }
239
240 #endif
241
242 save_period = new_save_period;
243 }
244
245
246 /// Sets a new file name where the neural network will be saved.
247 /// @param new_neural_network_file_name
248 /// File name for the neural network object.
249
set_neural_network_file_name(const string & new_neural_network_file_name)250 void OptimizationAlgorithm::set_neural_network_file_name(const string& new_neural_network_file_name)
251 {
252 neural_network_file_name = new_neural_network_file_name;
253 }
254
255
256 /// Sets the members of the optimization algorithm object to their default values.
257
set_default()258 void OptimizationAlgorithm::set_default()
259 {
260 display = true;
261
262 display_period = 5;
263
264 save_period = UINT_MAX;
265
266 neural_network_file_name = "neural_network.xml";
267 }
268
269
270 /// Performs a default checking for optimization algorithms.
271 /// In particular, it checks that the loss index pointer associated to the optimization algorithm is not nullptr,
272 /// and that the neural network associated to that loss index is neither nullptr.
273 /// If that checkings are not hold, an exception is thrown.
274
check() const275 void OptimizationAlgorithm::check() const
276 {
277 #ifdef __OPENNN_DEBUG__
278
279 ostringstream buffer;
280
281 if(!loss_index_pointer)
282 {
283 buffer << "OpenNN Exception: OptimizationAlgorithm class.\n"
284 << "void check() const method.\n"
285 << "Pointer to loss index is nullptr.\n";
286
287 throw logic_error(buffer.str());
288 }
289
290 const NeuralNetwork* neural_network_pointer = loss_index_pointer->get_neural_network_pointer();
291
292 if(neural_network_pointer == nullptr)
293 {
294 buffer << "OpenNN Exception: OptimizationAlgorithm class.\n"
295 << "void check() const method.\n"
296 << "Pointer to neural network is nullptr.\n";
297
298 throw logic_error(buffer.str());
299 }
300
301 #endif
302 }
303
304
305 /// Serializes the optimization algorithm object into a XML document of the TinyXML library without keep the DOM tree in memory.
306 /// See the OpenNN manual for more information about the format of this document.
307
write_XML(tinyxml2::XMLPrinter & file_stream) const308 void OptimizationAlgorithm::write_XML(tinyxml2::XMLPrinter& file_stream) const
309 {
310 ostringstream buffer;
311
312 file_stream.OpenElement("OptimizationAlgorithm");
313
314 // Display
315
316 file_stream.OpenElement("Display");
317
318 buffer.str("");
319 buffer << display;
320
321 file_stream.PushText(buffer.str().c_str());
322
323 file_stream.CloseElement();
324
325
326 file_stream.CloseElement();
327 }
328
329
330 /// Loads a default optimization algorithm from a XML document.
331 /// @param document TinyXML document containing the error term members.
332
from_XML(const tinyxml2::XMLDocument & document)333 void OptimizationAlgorithm::from_XML(const tinyxml2::XMLDocument& document)
334 {
335 const tinyxml2::XMLElement* root_element = document.FirstChildElement("OptimizationAlgorithm");
336
337 if(!root_element)
338 {
339 ostringstream buffer;
340
341 buffer << "OpenNN Exception: OptimizationAlgorithm class.\n"
342 << "void from_XML(const tinyxml2::XMLDocument&) method.\n"
343 << "Optimization algorithm element is nullptr.\n";
344
345 throw logic_error(buffer.str());
346 }
347
348 // Display
349 {
350 const tinyxml2::XMLElement* display_element = root_element->FirstChildElement("Display");
351
352 if(display_element)
353 {
354 const string new_display_string = display_element->GetText();
355
356 try
357 {
358 set_display(new_display_string != "0");
359 }
360 catch(const logic_error& e)
361 {
362 cerr << e.what() << endl;
363 }
364 }
365 }
366 }
367
368
369 /// Returns a default(empty) string matrix containing the members
370 /// of the optimization algorithm object.
371
to_string_matrix() const372 Tensor<string, 2> OptimizationAlgorithm::to_string_matrix() const
373 {
374 Tensor<string, 2> string_matrix;
375
376 return string_matrix;
377 }
378
379
380 /// Prints to the screen the XML-type representation of the optimization algorithm object.
381
print() const382 void OptimizationAlgorithm::print() const
383 {
384
385
386 }
387
388
389 /// Saves to a XML-type file the members of the optimization algorithm object.
390 /// @param file_name Name of optimization algorithm XML-type file.
391
save(const string & file_name) const392 void OptimizationAlgorithm::save(const string& file_name) const
393 {
394 // tinyxml2::XMLDocument* document = to_XML();
395
396 // document->SaveFile(file_name.c_str());
397
398 // delete document;
399 }
400
401
402 /// Loads a gradient descent object from a XML-type file.
403 /// Please mind about the file format, wich is specified in the User's Guide.
404 /// @param file_name Name of optimization algorithm XML-type file.
405
load(const string & file_name)406 void OptimizationAlgorithm::load(const string& file_name)
407 {
408 set_default();
409
410 tinyxml2::XMLDocument document;
411
412 if(document.LoadFile(file_name.c_str()))
413 {
414 ostringstream buffer;
415
416 buffer << "OpenNN Exception: OptimizationAlgorithm class.\n"
417 << "void load(const string&) method.\n"
418 << "Cannot load XML file " << file_name << ".\n";
419
420 throw logic_error(buffer.str());
421 }
422
423 from_XML(document);
424 }
425
426
427 /// Return a string with the stopping condition of the Results
428
write_stopping_condition() const429 string OptimizationAlgorithm::Results::write_stopping_condition() const
430 {
431 switch(stopping_condition)
432 {
433 case MinimumParametersIncrementNorm:
434 return "Minimum parameters increment norm";
435
436 case MinimumLossDecrease:
437 return "Minimum loss decrease";
438
439 case LossGoal:
440 return "Loss goal";
441
442 case GradientNormGoal:
443 return "Gradient norm goal";
444
445 case MaximumSelectionErrorIncreases:
446 return "Maximum selection error increases";
447
448 case MaximumEpochsNumber:
449 return "Maximum number of epochs";
450
451 case MaximumTime:
452 return "Maximum training time";
453 }
454
455 return string();
456 }
457
458
459 /// Resizes all the training history variables.
460 /// @param new_size Size of training history variables.
461
resize_training_history(const Index & new_size)462 void OptimizationAlgorithm::Results::resize_training_history(const Index& new_size)
463 {
464 training_error_history.resize(new_size);
465 }
466
467
468 /// Resizes all the selection history variables.
469 /// @param new_size Size of selection history variables.
470
resize_selection_history(const Index & new_size)471 void OptimizationAlgorithm::Results::resize_selection_history(const Index& new_size)
472 {
473 selection_error_history.resize(new_size);
474 }
475
476
477 /// Resizes the training error history keeping the values.
478 /// @param new_size Size of training history variables.
479
resize_training_error_history(const Index & new_size)480 void OptimizationAlgorithm::Results::resize_training_error_history(const Index& new_size)
481 {
482 const Tensor<type, 1> old_training_error_history = training_error_history;
483
484 training_error_history.resize(new_size);
485
486 for(Index i = 0; i < new_size; i++)
487 {
488 training_error_history(i) = old_training_error_history(i);
489 }
490 }
491
492
493 /// Resizes the training error history keeping the values.
494 /// @param new_size Size of training history variables.
495
resize_selection_error_history(const Index & new_size)496 void OptimizationAlgorithm::Results::resize_selection_error_history(const Index& new_size)
497 {
498 const Tensor<type, 1> old_selection_error_history = selection_error_history;
499
500 selection_error_history.resize(new_size);
501
502 for(Index i = 0; i < new_size; i++)
503 {
504 selection_error_history(i) = old_selection_error_history(i);
505 }
506 }
507
508
509 /// Writes the time from seconds in format HH:mm:ss.
510
write_elapsed_time(const type & time) const511 const string OptimizationAlgorithm::write_elapsed_time(const type& time) const
512 {
513
514 #ifdef __OPENNN_DEBUG__
515
516 if(time > static_cast<type>(3600e5))
517 {
518 ostringstream buffer;
519
520 buffer << "OpenNN Exception: OptimizationAlgorithm class.\n"
521 << "const string write_elapsed_time(const type& time) const method.\n"
522 << "Time must be lower than 10e5 seconds.\n";
523
524 throw logic_error(buffer.str());
525 }
526
527 if(time < static_cast<type>(0))
528 {
529 ostringstream buffer;
530
531 buffer << "OpenNN Exception: OptimizationAlgorithm class.\n"
532 << "const string write_elapsed_time(const type& time) const method.\n"
533 << "Time must be greater than 0.\n";
534
535 throw logic_error(buffer.str());
536 }
537 #endif
538
539 int hours = static_cast<int>(time) / 3600;
540 int seconds = static_cast<int>(time) % 3600;
541 int minutes = seconds / 60;
542 seconds = seconds % 60;
543
544 ostringstream elapsed_time;
545
546 elapsed_time << setfill('0') << setw(2) << hours << ":"
547 << setfill('0') << setw(2) << minutes << ":"
548 << setfill('0') << setw(2) << seconds;
549
550 return elapsed_time.str();
551 }
552
553
554 /// @todo
555
save(const string &) const556 void OptimizationAlgorithm::Results::save(const string&) const
557 {
558
559 }
560
561
562
write_final_results(const Index & precision) const563 Tensor<string, 2> OptimizationAlgorithm::Results::write_final_results(const Index& precision) const
564 {
565 ostringstream buffer;
566
567 Tensor<string, 2> final_results(7, 2);
568
569 // Final parameters norm
570
571 final_results(0,0) = "Final parameters norm";
572
573 buffer.str("");
574 buffer << setprecision(precision) << final_parameters_norm;
575
576 final_results(0,1) = buffer.str();
577
578 // Final loss
579
580 final_results(1,0) = "Final training error";
581
582 buffer.str("");
583 buffer << setprecision(precision) << final_training_error;
584
585 final_results(1,1) = buffer.str();
586
587 // Final selection error
588
589 final_results(2,0) = "Final selection error";
590
591 buffer.str("");
592 buffer << setprecision(precision) << final_selection_error;
593
594 final_results(2,1) = buffer.str();
595
596 // Final gradient norm
597
598 final_results(3,0) = "Final gradient norm";
599
600 buffer.str("");
601 buffer << setprecision(precision) << final_gradient_norm;
602
603 final_results(3,1) = buffer.str();
604
605 // Final learning rate
606
607 // names.push_back("Final learning rate");
608
609 // buffer.str("");
610 // buffer << setprecision(precision) << final_learning_rate;
611
612 // values.push_back(buffer.str());
613
614 // Epochs number
615
616 final_results(4,0) = "Epochs number";
617
618 buffer.str("");
619 buffer << epochs_number;
620
621 final_results(4,1) = buffer.str();
622
623 // Elapsed time
624
625 final_results(5,0) = "Elapsed time";
626
627 buffer.str("");
628 buffer << setprecision(precision) << elapsed_time;
629
630 final_results(5,1) = buffer.str();
631
632 // Stopping criteria
633
634 final_results(6,0) = "Stopping criterion";
635
636 final_results(6,1) = write_stopping_condition();
637
638 return final_results;
639 }
640
641 }
642
643
644 // OpenNN: Open Neural Networks Library.
645 // Copyright(C) 2005-2020 Artificial Intelligence Techniques, SL.
646 //
647 // This library is free software; you can redistribute it and/or
648 // modify it under the terms of the GNU Lesser General Public
649 // License as published by the Free Software Foundation; either
650 // version 2.1 of the License, or any later version.
651 //
652 // This library is distributed in the hope that it will be useful,
653 // but WITHOUT ANY WARRANTY; without even the implied warranty of
654 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
655 // Lesser General Public License for more details.
656
657 // You should have received a copy of the GNU Lesser General Public
658 // License along with this library; if not, write to the Free Software
659 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
660