1 #ifndef bwm_observer_video_h_
2 #define bwm_observer_video_h_
3 //:
4 // \file
5 
6 #include <iostream>
7 #include <map>
8 #include "bwm_observer_cam.h"
9 #include "video/bwm_video_corr_sptr.h"
10 #ifdef _MSC_VER
11 #  include <vcl_msvc_warnings.h>
12 #endif
13 #include <vgl/vgl_plane_3d.h>
14 #include <vgui/vgui_style.h>
15 #include <vpgl/vpgl_camera.h>
16 #include <vidl/vidl_istream_sptr.h>
17 #include <bwm/video/bwm_video_cam_istream_sptr.h>
18 #include <bsta/bsta_histogram.h>
19 
20 class vgui_soview2D_point;
21 class bwm_soview2D_cross;
22 
23 class bwm_observer_video : public bwm_observer_cam
24 {
25  public:
26 
bwm_observer_video(bgui_image_tableau_sptr const & img,vpgl_camera<double> * camera,std::string cam_path)27   bwm_observer_video(bgui_image_tableau_sptr const& img,
28                      vpgl_camera<double> *camera, std::string cam_path)
29   : bwm_observer_cam(img, camera, cam_path), display_corrs_(false),
30     display_world_pts_(false), play_video_(false), time_interval_(0.0f),
31     video_istr_(nullptr), cam_istr_(nullptr), tracked_corr_(nullptr)
32     {init();}
33 
bwm_observer_video(bgui_image_tableau_sptr const & img)34   bwm_observer_video(bgui_image_tableau_sptr const& img)
35   : bwm_observer_cam(img), display_corrs_(false),
36     display_world_pts_(false), play_video_(false),
37     time_interval_(0.0f), video_istr_(nullptr), cam_istr_(nullptr), tracked_corr_(nullptr)
38     {init();}
39 
~bwm_observer_video()40   virtual ~bwm_observer_video(){this->clear_video_corrs_display();}
41 
42   bool handle(const vgui_event &e);
43 
type_name()44   virtual std::string type_name() const { return "bwm_observer_video"; }
45   // these are syncronized streams
46   // currently the camera stream is implemented as a set of files,
47   // one for each video frame
48   bool open_video_stream(std::string const& video_path);
49   bool open_camera_stream(std::string const& camera_path);
50   // virtual methods for projecting 3-d objects
51   virtual void proj_point(vgl_point_3d<double> world_pt,
52                           vgl_point_2d<double> &image_pt);
53 
54   virtual void proj_line(vsol_line_3d_sptr line_3d,
55                          vsol_line_2d_sptr &line_2d);
56 
57   virtual void proj_poly(vsol_polygon_3d_sptr poly3d,
58                          vsol_polygon_2d_sptr& poly2d);
59 
60   virtual bool intersect_ray_and_plane(vgl_point_2d<double> img_point,
61                                        vgl_plane_3d<double> plane,
62                                        vgl_point_3d<double> &world_point);
63   // standard video display functions
64   void display_current_frame();
65   void next_frame();
66   void previous_frame();
67   void seek(unsigned frame_index);
68   void play();
69   void stop();
70   void pause();
71 
72   // Save a video as image list
73   bool save_as_image_list(std::string const& path);
74 
75   //correspondence edit methods
76   void set_world_pt(double lat, double lon, double elev);
77 
78   //: add a match to an existing correspondence in the current frame
79   bool add_match();
80 
81   //: removes a match to an existing correspondence in the current frame
82   bool remove_match();
83 
84   //: updatesa match to an existing correspondence in the current frame
85   bool update_match(vgl_point_2d<double> &pt);
86 
87   //: add video corr at a selected vertex
88   bool add_match_at_vertex();
89 
90   //: remove the selected correspondence match
91   void remove_selected_corr_match();
92 
93   //: remove the selected correspondence
94   void remove_selected_corr();
95 
96   //: select corr for tracking
97   void set_selected_corr_for_tracking();
98 
99   //: select corr for tracking
100   void unset_selected_corr_for_tracking();
101 
102   // --- correspondence display methods ---------------------------------------
103 
104   //: display the correspondences for a given frame on the current frame
105   void display_video_corrs(unsigned frame_index);
106 
107   //: display the current correspondences
108   void display_current_video_corrs();
109 
110   //: clear the display
111   void clear_video_corrs_display();
112 
113   //: access the current set of correspondences
114   std::vector<bwm_video_corr_sptr> corrs();
115 
116 
117   //: set the correspondences
118   void set_corrs(std::vector<bwm_video_corr_sptr> const& corrs);
119 
120   //: turn on/off world point display
toggle_corr_display()121   void toggle_corr_display(){display_corrs_ = !display_corrs_;
122   if (!display_corrs_) this->clear_video_corrs_display();}
123 
124   //: turn on/off world point display
toggle_world_pt_display()125   void toggle_world_pt_display(){display_world_pts_ = !display_world_pts_;}
126 
127   void display_selected_world_pt();
128 
129   //: extract a 3-d plane from three selected correspondences with world pts
130   bool extract_world_plane(vgl_plane_3d<double>&  plane);
131 
132   //: get the video stream
video_stream()133   vidl_istream_sptr video_stream() {return  video_istr_;}
134 
135   //: get the camera stream
camera_stream()136   bwm_video_cam_istream_sptr camera_stream() {return  cam_istr_;}
137 
138   //: extract neighborhoods around correspondences for the stream
139   bool extract_neighborhoods(unsigned nbd_radius_x,unsigned nbd_radius_y,
140                              std::vector<std::vector<vnl_matrix<float> > >& nhds);
141 
142   //: extract histograms of each frame
143   bool extract_histograms(std::vector<bsta_histogram<float> >& hists);
144 
frame_polygon_map()145   std::map<unsigned, std::vector<vsol_polygon_2d_sptr> > frame_polygon_map()
146   { return this->frame_polygon_map_; }
147 
148   virtual unsigned create_polygon(vsol_polygon_2d_sptr);
149 
150   //: display the polygons mapped to a specific frame
151   void display_polygons_frame();
152 
153   virtual void delete_polygon( vgui_soview* obj );
154 
155   virtual void delete_all();
156 
157   virtual void delete_all_frames();
158 
159   // mask operations
160 
161   //: initialize frame/change type/polygon map
162   virtual void init_mask();
163 
164   //: associate a polygon with a change type
165   virtual void add_poly_to_mask();
166 
167   //: remove the polygon's associated change type
168   virtual void remove_poly_from_mask();
169 
170   //: save the frame/change type/polygon relation to a binary file
171   virtual bool save_changes_binary();
172 
173   //: read /frame/change type/polygon relation binary file
174   virtual bool load_changes_binary();
175 
176   //Internals --------------------------------------- -------------------------
177 
178  protected:
179   vgui_style_sptr EDIT_STYLE;
180   vgui_style_sptr CORR_STYLE;
181   vgui_style_sptr MATCHED_STYLE;
182   vgui_style_sptr PREV_STYLE;
183   vgui_style_sptr POINT_3D_STYLE;
184 
185   void init();
186 
187   //: this function is called when a correspondence is set by observer_vgui.
188   virtual void correspondence_action();
189 
190   //: add the current correspondence on bwm_observer_vgui as a video_corr
191   bool add_video_corr();
192 
193 
194   //: find the currently selected correspondence
195   bool find_selected_video_corr(unsigned& frame, unsigned& corr_index,
196                                 bwm_soview2D_cross*& cross);
197   //: find the soview for the correspondence at current frame with corr_index
198   bwm_soview2D_cross* corr_soview(unsigned corr_index);
199 
200     //: display a single video correspondence
201   void display_video_corr(bwm_video_corr_sptr const& corr,
202                           unsigned frame_index,
203                           vgui_style_sptr const& style);
204 
205   //: display the projected 3-d world point in the current frame
206   void display_projected_3d_point(bwm_video_corr_sptr const& corr);
207 
208   //: is the correspondence displayed?
209   bool is_displayed(bwm_video_corr_sptr const& corr, unsigned frame_index);
210 
211   //: select the correspondence from a different frame that is closest in time and position to the new correspondence
212   void select_closest_match();
213 
214   //: clear the entire corr display map and remove corrs from display
215   void clear_corr_display_map();
216 
217   //: clear the entire corr display map and remove corrs from display
218   void clear_world_pt_display_map();
219 
220   //: display the current frame matches and those closest in time to the current frame
221   void display_corr_track();
222 
223   //: display the correspondence index number as text
224   void display_corr_index();
225 
226   //: display any 3-d objects
227   void display_3d_objects();
228 
229   //: should the correspondences be displayed
230   bool display_corrs_;
231   //: should the world points be displayed? (requires cameras)
232   bool display_world_pts_;
233   //:the video play state - if true the video is playing
234   bool play_video_;
235   //:the delay before displaying a new frame - default is 0
236   float time_interval_;
237   //: the video input stream - currently only image list is supported
238   vidl_istream_sptr video_istr_;
239   //: the camera input stream - currently only camera list is supported
240   bwm_video_cam_istream_sptr cam_istr_;
241 
242   //: a correspondence being tracked
243   bwm_video_corr_sptr tracked_corr_;
244 
245   //: the video frame-to-frame correspondences, over all frames
246   std::map<unsigned, bwm_video_corr_sptr> video_corrs_;
247 
248   //: relation between correspondence views and correspondences and frame
249   std::map<unsigned, std::map<unsigned, bwm_soview2D_cross*> > corr_soview_map_;
250 
251   //: relation between 3-d point display and corr id
252   std::map<unsigned, vgui_soview2D_point*> world_pt_map_;
253 
254   //: relation frames/polygons
255   std::map<unsigned, std::vector<vsol_polygon_2d_sptr> > frame_polygon_map_;
256 
257   //: relation frames/change type
258   std::map<unsigned, std::vector< std::string > > frame_change_map_;
259 
260   //: relation soviewID and vector position
261   std::map<unsigned, unsigned> id_pos_map_;
262 };
263 
264 #endif
265