1 // Copyright (C) 2011 Davis E. King (davis@dlib.net)
2 // License: Boost Software License See LICENSE.txt for the full license.
3
4 #include "metadata_editor.h"
5 #include <dlib/array.h>
6 #include <dlib/queue.h>
7 #include <dlib/static_set.h>
8 #include <dlib/misc_api.h>
9 #include <dlib/image_io.h>
10 #include <dlib/array2d.h>
11 #include <dlib/pixel.h>
12 #include <dlib/image_transforms.h>
13 #include <dlib/image_processing.h>
14 #include <sstream>
15 #include <ctime>
16
17 using namespace std;
18 using namespace dlib;
19
20 extern const char* VERSION;
21
22 // ----------------------------------------------------------------------------------------
23
24 metadata_editor::
metadata_editor(const std::string & filename_)25 metadata_editor(
26 const std::string& filename_
27 ) :
28 mbar(*this),
29 lb_images(*this),
30 image_pos(0),
31 display(*this),
32 overlay_label_name(*this),
33 overlay_label(*this),
34 keyboard_jump_pos(0),
35 last_keyboard_jump_pos_update(0)
36 {
37 file metadata_file(filename_);
38 filename = metadata_file.full_name();
39 // Make our current directory be the one that contains the metadata file. We
40 // do this because that file might contain relative paths to the image files
41 // we are supposed to be loading.
42 set_current_dir(get_parent_directory(metadata_file).full_name());
43
44 load_image_dataset_metadata(metadata, filename);
45
46 dlib::array<std::string>::expand_1a files;
47 files.resize(metadata.images.size());
48 for (unsigned long i = 0; i < metadata.images.size(); ++i)
49 {
50 files[i] = metadata.images[i].filename;
51 }
52 lb_images.load(files);
53 lb_images.enable_multiple_select();
54
55 lb_images.set_click_handler(*this, &metadata_editor::on_lb_images_clicked);
56
57 overlay_label_name.set_text("Next Label: ");
58 overlay_label.set_width(200);
59
60 display.set_image_clicked_handler(*this, &metadata_editor::on_image_clicked);
61 display.set_overlay_rects_changed_handler(*this, &metadata_editor::on_overlay_rects_changed);
62 display.set_overlay_rect_selected_handler(*this, &metadata_editor::on_overlay_rect_selected);
63 overlay_label.set_text_modified_handler(*this, &metadata_editor::on_overlay_label_changed);
64
65 mbar.set_number_of_menus(2);
66 mbar.set_menu_name(0,"File",'F');
67 mbar.set_menu_name(1,"Help",'H');
68
69
70 mbar.menu(0).add_menu_item(menu_item_text("Save",*this,&metadata_editor::file_save,'S'));
71 mbar.menu(0).add_menu_item(menu_item_text("Save As",*this,&metadata_editor::file_save_as,'A'));
72 mbar.menu(0).add_menu_item(menu_item_separator());
73 mbar.menu(0).add_menu_item(menu_item_text("Remove Selected Images",*this,&metadata_editor::remove_selected_images,'R'));
74 mbar.menu(0).add_menu_item(menu_item_separator());
75 mbar.menu(0).add_menu_item(menu_item_text("Exit",static_cast<base_window&>(*this),&drawable_window::close_window,'x'));
76
77 mbar.menu(1).add_menu_item(menu_item_text("About",*this,&metadata_editor::display_about,'A'));
78
79 // set the size of this window.
80 on_window_resized();
81 load_image_and_set_size(0);
82 on_window_resized();
83 if (image_pos < lb_images.size() )
84 lb_images.select(image_pos);
85
86 // make sure the window is centered on the screen.
87 unsigned long width, height;
88 get_size(width, height);
89 unsigned long screen_width, screen_height;
90 get_display_size(screen_width, screen_height);
91 set_pos((screen_width-width)/2, (screen_height-height)/2);
92
93 show();
94 }
95
96 // ----------------------------------------------------------------------------------------
97
98 metadata_editor::
~metadata_editor()99 ~metadata_editor(
100 )
101 {
102 close_window();
103 }
104
105 // ----------------------------------------------------------------------------------------
106
107 void metadata_editor::
add_labelable_part_name(const std::string & name)108 add_labelable_part_name (
109 const std::string& name
110 )
111 {
112 display.add_labelable_part_name(name);
113 }
114
115 // ----------------------------------------------------------------------------------------
116
117 void metadata_editor::
file_save()118 file_save()
119 {
120 save_metadata_to_file(filename);
121 }
122
123 // ----------------------------------------------------------------------------------------
124
125 void metadata_editor::
save_metadata_to_file(const std::string & file)126 save_metadata_to_file (
127 const std::string& file
128 )
129 {
130 try
131 {
132 save_image_dataset_metadata(metadata, file);
133 }
134 catch (dlib::error& e)
135 {
136 message_box("Error saving file", e.what());
137 }
138 }
139
140 // ----------------------------------------------------------------------------------------
141
142 void metadata_editor::
file_save_as()143 file_save_as()
144 {
145 save_file_box(*this, &metadata_editor::save_metadata_to_file);
146 }
147
148 // ----------------------------------------------------------------------------------------
149
150 void metadata_editor::
remove_selected_images()151 remove_selected_images()
152 {
153 dlib::queue<unsigned long>::kernel_1a list;
154 lb_images.get_selected(list);
155 list.reset();
156 unsigned long min_idx = lb_images.size();
157 while (list.move_next())
158 {
159 lb_images.unselect(list.element());
160 min_idx = std::min(min_idx, list.element());
161 }
162
163
164 // remove all the selected items from metadata.images
165 dlib::static_set<unsigned long>::kernel_1a to_remove;
166 to_remove.load(list);
167 std::vector<dlib::image_dataset_metadata::image> images;
168 for (unsigned long i = 0; i < metadata.images.size(); ++i)
169 {
170 if (to_remove.is_member(i) == false)
171 {
172 images.push_back(metadata.images[i]);
173 }
174 }
175 images.swap(metadata.images);
176
177
178 // reload metadata into lb_images
179 dlib::array<std::string>::expand_1a files;
180 files.resize(metadata.images.size());
181 for (unsigned long i = 0; i < metadata.images.size(); ++i)
182 {
183 files[i] = metadata.images[i].filename;
184 }
185 lb_images.load(files);
186
187
188 if (min_idx != 0)
189 min_idx--;
190 select_image(min_idx);
191 }
192
193 // ----------------------------------------------------------------------------------------
194
195 void metadata_editor::
on_window_resized()196 on_window_resized(
197 )
198 {
199 drawable_window::on_window_resized();
200
201 unsigned long width, height;
202 get_size(width, height);
203
204 lb_images.set_pos(0,mbar.bottom()+1);
205 lb_images.set_size(180, height - mbar.height());
206
207 overlay_label_name.set_pos(lb_images.right()+10, mbar.bottom() + (overlay_label.height()-overlay_label_name.height())/2+1);
208 overlay_label.set_pos(overlay_label_name.right(), mbar.bottom()+1);
209 display.set_pos(lb_images.right(), overlay_label.bottom()+3);
210
211 display.set_size(width - display.left(), height - display.top());
212 }
213
214 // ----------------------------------------------------------------------------------------
215
propagate_boxes(dlib::image_dataset_metadata::dataset & data,unsigned long prev,unsigned long next)216 void propagate_boxes(
217 dlib::image_dataset_metadata::dataset& data,
218 unsigned long prev,
219 unsigned long next
220 )
221 {
222 if (prev == next || next >= data.images.size())
223 return;
224
225 array2d<rgb_pixel> img1, img2;
226 dlib::load_image(img1, data.images[prev].filename);
227 dlib::load_image(img2, data.images[next].filename);
228 for (unsigned long i = 0; i < data.images[prev].boxes.size(); ++i)
229 {
230 correlation_tracker tracker;
231 tracker.start_track(img1, data.images[prev].boxes[i].rect);
232 tracker.update(img2);
233 dlib::image_dataset_metadata::box box = data.images[prev].boxes[i];
234 box.rect = tracker.get_position();
235 data.images[next].boxes.push_back(box);
236 }
237 }
238
239 // ----------------------------------------------------------------------------------------
240
propagate_labels(const std::string & label,dlib::image_dataset_metadata::dataset & data,unsigned long prev,unsigned long next)241 void propagate_labels(
242 const std::string& label,
243 dlib::image_dataset_metadata::dataset& data,
244 unsigned long prev,
245 unsigned long next
246 )
247 {
248 if (prev == next || next >= data.images.size())
249 return;
250
251
252 for (unsigned long i = 0; i < data.images[prev].boxes.size(); ++i)
253 {
254 if (data.images[prev].boxes[i].label != label)
255 continue;
256
257 // figure out which box in the next image matches the current one the best
258 const rectangle cur = data.images[prev].boxes[i].rect;
259 double best_overlap = 0;
260 unsigned long best_idx = 0;
261 for (unsigned long j = 0; j < data.images[next].boxes.size(); ++j)
262 {
263 const rectangle next_box = data.images[next].boxes[j].rect;
264 const double overlap = cur.intersect(next_box).area()/(double)(cur+next_box).area();
265 if (overlap > best_overlap)
266 {
267 best_overlap = overlap;
268 best_idx = j;
269 }
270 }
271
272 // If we found a matching rectangle in the next image and the best match doesn't
273 // already have a label.
274 if (best_overlap > 0.5 && data.images[next].boxes[best_idx].label == "")
275 {
276 data.images[next].boxes[best_idx].label = label;
277 }
278 }
279
280 }
281
282 // ----------------------------------------------------------------------------------------
283
has_label_or_all_boxes_labeled(const std::string & label,const dlib::image_dataset_metadata::image & img)284 bool has_label_or_all_boxes_labeled (
285 const std::string& label,
286 const dlib::image_dataset_metadata::image& img
287 )
288 {
289 if (label.size() == 0)
290 return true;
291
292 bool all_boxes_labeled = true;
293 for (unsigned long i = 0; i < img.boxes.size(); ++i)
294 {
295 if (img.boxes[i].label == label)
296 return true;
297 if (img.boxes[i].label.size() == 0)
298 all_boxes_labeled = false;
299 }
300
301 return all_boxes_labeled;
302 }
303
304 // ----------------------------------------------------------------------------------------
305
306 void metadata_editor::
on_keydown(unsigned long key,bool is_printable,unsigned long state)307 on_keydown (
308 unsigned long key,
309 bool is_printable,
310 unsigned long state
311 )
312 {
313 drawable_window::on_keydown(key, is_printable, state);
314
315 if (is_printable)
316 {
317 if (key == '\t')
318 {
319 overlay_label.give_input_focus();
320 overlay_label.select_all_text();
321 }
322
323 // If the user types a number then jump to that image.
324 if ('0' <= key && key <= '9' && metadata.images.size() != 0 && !overlay_label.has_input_focus())
325 {
326 time_t curtime = time(0);
327 // If it's been a while since the user typed numbers then forget the last jump
328 // position and start accumulating numbers over again.
329 if (curtime-last_keyboard_jump_pos_update >= 2)
330 keyboard_jump_pos = 0;
331 last_keyboard_jump_pos_update = curtime;
332
333 keyboard_jump_pos *= 10;
334 keyboard_jump_pos += key-'0';
335 if (keyboard_jump_pos >= metadata.images.size())
336 keyboard_jump_pos = metadata.images.size()-1;
337
338 image_pos = keyboard_jump_pos;
339 select_image(image_pos);
340 }
341 else
342 {
343 last_keyboard_jump_pos_update = 0;
344 }
345
346 if (key == '=')
347 {
348 display.zoom_in();
349 }
350
351 if (key == '-')
352 {
353 display.zoom_out();
354 }
355
356 if (key == 'd' && (state&base_window::KBD_MOD_ALT))
357 {
358 remove_selected_images();
359 }
360
361 if (key == 'e' && !overlay_label.has_input_focus())
362 {
363 display_equialized_image = !display_equialized_image;
364 select_image(image_pos);
365 }
366
367 // Make 'w' and 's' act like KEY_UP and KEY_DOWN
368 if ((key == 'w' || key == 'W') && !overlay_label.has_input_focus())
369 {
370 key = base_window::KEY_UP;
371 }
372 else if ((key == 's' || key == 'S') && !overlay_label.has_input_focus())
373 {
374 key = base_window::KEY_DOWN;
375 }
376 else
377 {
378 return;
379 }
380 }
381
382 if (key == base_window::KEY_UP)
383 {
384 if ((state&KBD_MOD_CONTROL) && (state&KBD_MOD_SHIFT))
385 {
386 // Don't do anything if there are no boxes in the current image.
387 if (metadata.images[image_pos].boxes.size() == 0)
388 return;
389 // Also don't do anything if there *are* boxes in the next image.
390 if (image_pos > 1 && metadata.images[image_pos-1].boxes.size() != 0)
391 return;
392
393 propagate_boxes(metadata, image_pos, image_pos-1);
394 }
395 else if (state&base_window::KBD_MOD_CONTROL)
396 {
397 // If the label we are supposed to propagate doesn't exist in the current image
398 // then don't advance.
399 if (!has_label_or_all_boxes_labeled(display.get_default_overlay_rect_label(),metadata.images[image_pos]))
400 return;
401
402 // if the next image is going to be empty then fast forward to the next one
403 while (image_pos > 1 && metadata.images[image_pos-1].boxes.size() == 0)
404 --image_pos;
405
406 propagate_labels(display.get_default_overlay_rect_label(), metadata, image_pos, image_pos-1);
407 }
408 select_image(image_pos-1);
409 }
410 else if (key == base_window::KEY_DOWN)
411 {
412 if ((state&KBD_MOD_CONTROL) && (state&KBD_MOD_SHIFT))
413 {
414 // Don't do anything if there are no boxes in the current image.
415 if (metadata.images[image_pos].boxes.size() == 0)
416 return;
417 // Also don't do anything if there *are* boxes in the next image.
418 if (image_pos+1 < metadata.images.size() && metadata.images[image_pos+1].boxes.size() != 0)
419 return;
420
421 propagate_boxes(metadata, image_pos, image_pos+1);
422 }
423 else if (state&base_window::KBD_MOD_CONTROL)
424 {
425 // If the label we are supposed to propagate doesn't exist in the current image
426 // then don't advance.
427 if (!has_label_or_all_boxes_labeled(display.get_default_overlay_rect_label(),metadata.images[image_pos]))
428 return;
429
430 // if the next image is going to be empty then fast forward to the next one
431 while (image_pos+1 < metadata.images.size() && metadata.images[image_pos+1].boxes.size() == 0)
432 ++image_pos;
433
434 propagate_labels(display.get_default_overlay_rect_label(), metadata, image_pos, image_pos+1);
435 }
436 select_image(image_pos+1);
437 }
438 }
439
440 // ----------------------------------------------------------------------------------------
441
442 void metadata_editor::
select_image(unsigned long idx)443 select_image(
444 unsigned long idx
445 )
446 {
447 if (idx < lb_images.size())
448 {
449 // unselect all currently selected images
450 dlib::queue<unsigned long>::kernel_1a list;
451 lb_images.get_selected(list);
452 list.reset();
453 while (list.move_next())
454 {
455 lb_images.unselect(list.element());
456 }
457
458
459 lb_images.select(idx);
460 load_image(idx);
461 }
462 else if (lb_images.size() == 0)
463 {
464 display.clear_overlay();
465 array2d<unsigned char> empty_img;
466 display.set_image(empty_img);
467 }
468 }
469
470 // ----------------------------------------------------------------------------------------
471
472 void metadata_editor::
on_lb_images_clicked(unsigned long idx)473 on_lb_images_clicked(
474 unsigned long idx
475 )
476 {
477 load_image(idx);
478 }
479
480 // ----------------------------------------------------------------------------------------
481
get_overlays(const dlib::image_dataset_metadata::image & data,color_mapper & string_to_color)482 std::vector<dlib::image_display::overlay_rect> get_overlays (
483 const dlib::image_dataset_metadata::image& data,
484 color_mapper& string_to_color
485 )
486 {
487 std::vector<dlib::image_display::overlay_rect> temp(data.boxes.size());
488 for (unsigned long i = 0; i < temp.size(); ++i)
489 {
490 temp[i].rect = data.boxes[i].rect;
491 temp[i].label = data.boxes[i].label;
492 temp[i].parts = data.boxes[i].parts;
493 temp[i].crossed_out = data.boxes[i].ignore;
494 temp[i].color = string_to_color(data.boxes[i].label);
495 }
496 return temp;
497 }
498
499 // ----------------------------------------------------------------------------------------
500
501 void metadata_editor::
load_image(unsigned long idx)502 load_image(
503 unsigned long idx
504 )
505 {
506 if (idx >= metadata.images.size())
507 return;
508
509 image_pos = idx;
510
511 array2d<rgb_pixel> img;
512 display.clear_overlay();
513 try
514 {
515 dlib::load_image(img, metadata.images[idx].filename);
516 set_title(metadata.name + " #"+cast_to_string(idx)+": " +metadata.images[idx].filename);
517 }
518 catch (exception& e)
519 {
520 message_box("Error loading image", e.what());
521 }
522
523 if (display_equialized_image)
524 equalize_histogram(img);
525 display.set_image(img);
526 display.add_overlay(get_overlays(metadata.images[idx], string_to_color));
527 }
528
529 // ----------------------------------------------------------------------------------------
530
531 void metadata_editor::
load_image_and_set_size(unsigned long idx)532 load_image_and_set_size(
533 unsigned long idx
534 )
535 {
536 if (idx >= metadata.images.size())
537 return;
538
539 image_pos = idx;
540
541 array2d<rgb_pixel> img;
542 display.clear_overlay();
543 try
544 {
545 dlib::load_image(img, metadata.images[idx].filename);
546 set_title(metadata.name + " #"+cast_to_string(idx)+": " +metadata.images[idx].filename);
547 }
548 catch (exception& e)
549 {
550 message_box("Error loading image", e.what());
551 }
552
553
554 unsigned long screen_width, screen_height;
555 get_display_size(screen_width, screen_height);
556
557
558 unsigned long needed_width = display.left() + img.nc() + 4;
559 unsigned long needed_height = display.top() + img.nr() + 4;
560 if (needed_width < 300) needed_width = 300;
561 if (needed_height < 300) needed_height = 300;
562
563 if (needed_width > 100 + screen_width)
564 needed_width = screen_width - 100;
565 if (needed_height > 100 + screen_height)
566 needed_height = screen_height - 100;
567
568 set_size(needed_width, needed_height);
569
570
571 if (display_equialized_image)
572 equalize_histogram(img);
573 display.set_image(img);
574 display.add_overlay(get_overlays(metadata.images[idx], string_to_color));
575 }
576
577 // ----------------------------------------------------------------------------------------
578
579 void metadata_editor::
on_overlay_rects_changed()580 on_overlay_rects_changed(
581 )
582 {
583 using namespace dlib::image_dataset_metadata;
584 if (image_pos < metadata.images.size())
585 {
586 const std::vector<image_display::overlay_rect>& rects = display.get_overlay_rects();
587
588 std::vector<box>& boxes = metadata.images[image_pos].boxes;
589
590 boxes.clear();
591 for (unsigned long i = 0; i < rects.size(); ++i)
592 {
593 box temp;
594 temp.label = rects[i].label;
595 temp.rect = rects[i].rect;
596 temp.parts = rects[i].parts;
597 temp.ignore = rects[i].crossed_out;
598 boxes.push_back(temp);
599 }
600 }
601 }
602
603 // ----------------------------------------------------------------------------------------
604
605 void metadata_editor::
on_image_clicked(const point &,bool,unsigned long)606 on_image_clicked(
607 const point& /*p*/, bool /*is_double_click*/, unsigned long /*btn*/
608 )
609 {
610 display.set_default_overlay_rect_color(string_to_color(trim(overlay_label.text())));
611 }
612
613 // ----------------------------------------------------------------------------------------
614
615 void metadata_editor::
on_overlay_label_changed()616 on_overlay_label_changed(
617 )
618 {
619 display.set_default_overlay_rect_label(trim(overlay_label.text()));
620 }
621
622 // ----------------------------------------------------------------------------------------
623
624 void metadata_editor::
on_overlay_rect_selected(const image_display::overlay_rect & orect)625 on_overlay_rect_selected(
626 const image_display::overlay_rect& orect
627 )
628 {
629 overlay_label.set_text(orect.label);
630 display.set_default_overlay_rect_label(orect.label);
631 display.set_default_overlay_rect_color(string_to_color(orect.label));
632 }
633
634 // ----------------------------------------------------------------------------------------
635
636 void metadata_editor::
display_about()637 display_about(
638 )
639 {
640 std::ostringstream sout;
641 sout << wrap_string("Image Labeler v" + string(VERSION) + "." ,0,0) << endl << endl;
642 sout << wrap_string("This program is a tool for labeling images with rectangles. " ,0,0) << endl << endl;
643
644 sout << wrap_string("You can add a new rectangle by holding the shift key, left clicking "
645 "the mouse, and dragging it. New rectangles are given the label from the \"Next Label\" "
646 "field at the top of the application. You can quickly edit the contents of the Next Label field "
647 "by hitting the tab key. Double clicking "
648 "a rectangle selects it and the delete key removes it. You can also mark "
649 "a rectangle as ignored by hitting the i or END keys when it is selected. Ignored "
650 "rectangles are visually displayed with an X through them. You can remove an image "
651 "entirely by selecting it in the list on the left and pressing alt+d."
652 ,0,0) << endl << endl;
653
654 sout << wrap_string("It is also possible to label object parts by selecting a rectangle and "
655 "then right clicking. A popup menu will appear and you can select a part label. "
656 "Note that you must define the allowable part labels by giving --parts on the "
657 "command line. An example would be '--parts \"leye reye nose mouth\"'. "
658 "Alternatively, if you don't give --parts you can simply select a rectangle and shift+left "
659 "click to add parts. Parts added this way will be labeled with integer labels starting from 0. "
660 "You can only use this simpler part adding mode if all the parts in a rectangle are already "
661 "labeled with integer labels or the rectangle has no parts at all."
662 ,0,0) << endl << endl;
663
664 sout << wrap_string("Press the down or s key to select the next image in the list and the up or w "
665 "key to select the previous one.",0,0) << endl << endl;
666
667 sout << wrap_string("Additionally, you can hold ctrl and then scroll the mouse wheel to zoom. A normal left click "
668 "and drag allows you to navigate around the image. Holding ctrl and "
669 "left clicking a rectangle will give it the label from the Next Label field. "
670 "Holding shift + right click and then dragging allows you to move things around. "
671 "Holding ctrl and pressing the up or down keyboard keys will propagate "
672 "rectangle labels from one image to the next and also skip empty images. "
673 "Similarly, holding ctrl+shift will propagate entire boxes via a visual tracking "
674 "algorithm from one image to the next. "
675 "Finally, typing a number on the keyboard will jump you to a specific image.",0,0) << endl << endl;
676
677 sout << wrap_string("You can also toggle image histogram equalization by pressing the e key."
678 ,0,0) << endl;
679
680
681 message_box("About Image Labeler",sout.str());
682 }
683
684 // ----------------------------------------------------------------------------------------
685
686