1 // This is brl/bseg/boxm2/cpp/pro/processes/boxm2_cpp_batch_update_opt2_processes.cxx
2 #include <iostream>
3 #include <fstream>
4 #include <bprb/bprb_func_process.h>
5 //:
6 // \file
7 // \brief  Processes to update the scene using a set of data blocks in a batch mode, migrated from the version in boxm to boxm2
8 //
9 // \author Ozge C. Ozcanli
10 // \date May 12, 2011
11 
12 #ifdef _MSC_VER
13 #  include "vcl_msvc_warnings.h"
14 #endif
15 #include <boxm2/io/boxm2_stream_cache.h>
16 #include <boxm2/io/boxm2_cache.h>
17 #include <boxm2/boxm2_scene.h>
18 #include <boxm2/boxm2_block.h>
19 #include <boxm2/boxm2_data_base.h>
20 #
21 //brdb stuff
22 #include <brdb/brdb_value.h>
23 #include <boxm2/boxm2_util.h>
24 
25 #include <boxm2/cpp/algo/boxm2_batch_functors.h>
26 #include <boxm2/cpp/algo/boxm2_batch_opt2_functors.h>
27 #include <boxm2/cpp/algo/boxm2_data_serial_iterator.h>
28 
29 #include "vil/vil_math.h"
30 #include <bsta/bsta_gauss_sf1.h>
31 
32 #include "vil/vil_save.h"
33 
34 //: create a normalized intensity value in each cell using the segment lengths of all the rays that intersect the cell
35 namespace boxm2_cpp_create_aux_data_opt2_process_globals
36 {
37   constexpr unsigned n_inputs_ = 7;
38   constexpr unsigned n_outputs_ = 0;
39 }
40 
boxm2_cpp_create_aux_data_opt2_process_cons(bprb_func_process & pro)41 bool boxm2_cpp_create_aux_data_opt2_process_cons(bprb_func_process& pro)
42 {
43   using namespace boxm2_cpp_create_aux_data_opt2_process_globals;
44 
45   //process takes 7 inputs
46   // 0) scene
47   // 1) cache
48   // 2) camera
49   // 3) image
50   // 4) image identifier
51   std::vector<std::string> input_types_(n_inputs_);
52   input_types_[0] = "boxm2_scene_sptr";
53   input_types_[1] = "boxm2_cache_sptr";
54   input_types_[2] = "vpgl_camera_double_sptr";
55   input_types_[3] = "vil_image_view_base_sptr";
56   input_types_[4] = "vcl_string";  //image identifier
57   input_types_[5] = "float"; // alternative appearance model of shadow, prior
58   input_types_[6] = "float"; // alternative appearance model of shadow, sigma
59   // process has 0 output:
60   std::vector<std::string>  output_types_(n_outputs_);
61 
62   bool good = pro.set_input_types(input_types_) && pro.set_output_types(output_types_);
63 
64   brdb_value_sptr pp = new brdb_value_t<float>(0.0f);
65   pro.set_input(5, pp);
66   brdb_value_sptr ps = new brdb_value_t<float>(0.0f);
67   pro.set_input(6, ps);
68   return good;
69 }
70 
boxm2_cpp_create_aux_data_opt2_process(bprb_func_process & pro)71 bool boxm2_cpp_create_aux_data_opt2_process(bprb_func_process& pro)
72 {
73   using namespace boxm2_cpp_create_aux_data_opt2_process_globals;
74 
75   if ( pro.n_inputs() < n_inputs_ ){
76     std::cout << pro.name() << ": The number of inputs should be " << n_inputs_<< std::endl;
77     return false;
78   }
79   //get the inputs
80   unsigned i = 0;
81   boxm2_scene_sptr scene = pro.get_input<boxm2_scene_sptr>(i++);
82   boxm2_cache_sptr cache = pro.get_input<boxm2_cache_sptr>(i++);
83   vpgl_camera_double_sptr cam = pro.get_input<vpgl_camera_double_sptr>(i++);
84   vil_image_view_base_sptr in_img = pro.get_input<vil_image_view_base_sptr>(i++);
85   vil_image_view_base_sptr float_image = boxm2_util::prepare_input_image(in_img, true);
86   std::string identifier = pro.get_input<std::string>(i++);
87   auto shadow_prior = pro.get_input<float>(i++);
88   auto shadow_sigma = pro.get_input<float>(i++);
89 
90   if (auto * input_image=dynamic_cast<vil_image_view<float> * > (float_image.ptr()))
91   {
92     std::vector<boxm2_block_id> vis_order=scene->get_vis_blocks(reinterpret_cast<vpgl_generic_camera<double>*>(cam.ptr()));
93     if (vis_order.empty())
94     {
95       std::cout<<" None of the blocks are visible from this viewpoint"<<std::endl;
96       return true;
97     }
98 
99     bool foundDataType = false;
100 
101     std::string data_type;
102     std::string num_obs_type;
103     std::vector<std::string> apps = scene->appearances();
104     for (const auto & app : apps) {
105       if ( app == boxm2_data_traits<BOXM2_MOG3_GREY>::prefix() )
106       {
107           data_type = app;
108           foundDataType = true;
109           // boxm2_data_info::datasize(boxm2_data_traits<BOXM2_MOG3_GREY>::prefix());
110       }
111       else if ( app == boxm2_data_traits<BOXM2_MOG3_GREY_16>::prefix() )
112       {
113           data_type = app;
114           foundDataType = true;
115           // boxm2_data_info::datasize(boxm2_data_traits<BOXM2_MOG3_GREY_16>::prefix());
116       }
117       else if ( app == boxm2_data_traits<BOXM2_GAUSS_GREY>::prefix() )
118       {
119           data_type = app;
120           foundDataType = true;
121           // boxm2_data_info::datasize(boxm2_data_traits<BOXM2_GAUSS_GREY>::prefix());
122       }
123     }
124     if (!foundDataType) {
125       std::cerr<<"boxm2_cpp_create_aux_data_process ERROR: scene doesn't have BOXM2_MOG3_GREY or BOXM2_MOG3_GREY_16 data type\n";
126       return false;
127     }
128 
129     vil_image_view<float> pre_inf_img(input_image->ni(),input_image->nj());
130     vil_image_view<float> vis_inf_img(input_image->ni(),input_image->nj());
131     //initialize pre and vis images
132     pre_inf_img.fill(0.0f);
133     vis_inf_img.fill(1.0f);
134 
135     std::vector<boxm2_block_id>::iterator id;
136 
137     for (id = vis_order.begin(); id != vis_order.end(); ++id)
138     {
139       std::cout<<"Block id "<<(*id)<<' ';
140       boxm2_block *     blk = cache->get_block(scene,*id);
141       boxm2_data_base *  alph = cache->get_data_base(scene,*id,boxm2_data_traits<BOXM2_ALPHA>::prefix(),0,false);
142       boxm2_data_base *  mog = cache->get_data_base(scene,*id,data_type,0,false);
143       boxm2_data_base *aux0 = cache->get_data_base(scene,*id,boxm2_data_traits<BOXM2_AUX0>::prefix(identifier));
144       boxm2_data_base *aux1 = cache->get_data_base(scene,*id,boxm2_data_traits<BOXM2_AUX1>::prefix(identifier));
145 
146       std::vector<boxm2_data_base*> datas;
147       datas.push_back(aux0);
148       datas.push_back(aux1);
149       datas.push_back(alph);
150       datas.push_back(mog);
151       auto *scene_info_wrapper=new boxm2_scene_info_wrapper();
152       scene_info_wrapper->info=scene->get_blk_metadata(*id);
153 
154       if ( data_type == boxm2_data_traits<BOXM2_MOG3_GREY>::prefix() )
155       {
156         boxm2_batch_update_pass1_functor<BOXM2_MOG3_GREY> pass1;
157         pass1.init_data(datas,&pre_inf_img,&vis_inf_img);
158         cast_ray_per_block<boxm2_batch_update_pass1_functor<BOXM2_MOG3_GREY> >(pass1,
159                                                                                scene_info_wrapper->info,
160                                                                                blk,
161                                                                                cam,
162                                                                                input_image->ni(),
163                                                                                input_image->nj());
164       }
165       else if ( data_type == boxm2_data_traits<BOXM2_GAUSS_GREY>::prefix() )
166       {
167         boxm2_batch_update_pass1_functor<BOXM2_GAUSS_GREY> pass1;
168         pass1.init_data(datas,&pre_inf_img,&vis_inf_img);
169         cast_ray_per_block<boxm2_batch_update_pass1_functor<BOXM2_GAUSS_GREY> >(pass1,
170                                                                                 scene_info_wrapper->info,
171                                                                                 blk,
172                                                                                 cam,
173                                                                                 input_image->ni(),
174                                                                                 input_image->nj());
175       }
176     }
177     // compute beta denominator
178     vil_image_view<float> beta_denom_img(input_image->ni(), input_image->nj());
179     // pre_inf + vis * 1 // assume PI = 1 for all the colors (uniform distribution)
180     vil_math_image_sum<float,float,float>(pre_inf_img,vis_inf_img,beta_denom_img);
181 
182     // compute model prior --> different than 1 only if there are alternate appearance models like shadow appearance
183     float model_prior = 1.0f - shadow_prior;
184     // sanity check
185     if (model_prior <= 0.0f) {
186       std::cerr << "error: boxm_generate_opt2_samples : alt_appearance_priors sum to " << 1.0f - model_prior << " >= 1.0!\n";
187     }
188 
189     // compute alternate appearance probability for each pixel in the image
190     vil_image_view<float> alt_prob_img(input_image->ni(), input_image->nj());
191     alt_prob_img.fill(0.0f);
192     if (shadow_prior > 0.0f) {
193       bsta_gauss_sf1 single_mode(0.0f, shadow_sigma*shadow_sigma);
194       for (unsigned i = 0; i < input_image->ni(); i++) {
195         for (unsigned j = 0; j < input_image->nj(); j++) {
196           alt_prob_img(i,j) = single_mode.prob_density((*input_image)(i,j))*shadow_prior;
197         }
198       }
199 #if 0
200       vil_save(alt_prob_img, "shadow_density_img.tiff");
201 #endif
202     }
203 
204     // now run pass 2
205 
206     vil_image_view<float> pre_img(input_image->ni(),input_image->nj());
207     vil_image_view<float> vis_img(input_image->ni(),input_image->nj());
208     pre_img.fill(0.0f);
209     vis_img.fill(1.0f);
210 
211     for (id = vis_order.begin(); id != vis_order.end(); ++id)
212     {
213       std::cout<<"Block id "<<(*id)<<' ';
214       boxm2_block *     blk = cache->get_block(scene,*id);
215       boxm2_data_base *  alph = cache->get_data_base(scene,*id,boxm2_data_traits<BOXM2_ALPHA>::prefix(),0,false);
216       boxm2_data_base *  mog = cache->get_data_base(scene,*id,data_type,0,false);
217       boxm2_data_base *aux0 = cache->get_data_base(scene,*id,boxm2_data_traits<BOXM2_AUX0>::prefix(identifier));
218       boxm2_data_base *aux1 = cache->get_data_base(scene,*id,boxm2_data_traits<BOXM2_AUX1>::prefix(identifier));
219 
220       // generate aux in a writable mode
221       boxm2_data_base *aux = cache->get_data_base_new(scene,*id, boxm2_data_traits<BOXM2_AUX>::prefix(identifier),false);
222 
223       std::vector<boxm2_data_base*> datas;
224       datas.push_back(aux0);
225       datas.push_back(aux1);
226       datas.push_back(alph);
227       datas.push_back(mog);
228       datas.push_back(aux);
229       auto *scene_info_wrapper=new boxm2_scene_info_wrapper();
230       scene_info_wrapper->info=scene->get_blk_metadata(*id);
231 
232       if ( data_type == boxm2_data_traits<BOXM2_MOG3_GREY>::prefix() )
233       {
234         boxm2_batch_update_opt2_pass2_functor<BOXM2_MOG3_GREY> pass2;
235         pass2.init_data(datas,&pre_img,&vis_img, &beta_denom_img, model_prior, &alt_prob_img);
236         cast_ray_per_block<boxm2_batch_update_opt2_pass2_functor<BOXM2_MOG3_GREY> >(pass2,
237                                                                                     scene_info_wrapper->info,
238                                                                                     blk,
239                                                                                     cam,
240                                                                                     input_image->ni(),
241                                                                                     input_image->nj());
242       }
243       else if ( data_type == boxm2_data_traits<BOXM2_GAUSS_GREY>::prefix() )
244       {
245         boxm2_batch_update_opt2_pass2_functor<BOXM2_GAUSS_GREY> pass2;
246         pass2.init_data(datas,&pre_img,&vis_img, &beta_denom_img, model_prior, &alt_prob_img);
247         cast_ray_per_block<boxm2_batch_update_opt2_pass2_functor<BOXM2_GAUSS_GREY> >(pass2,
248                                                                                      scene_info_wrapper->info,
249                                                                                      blk,
250                                                                                      cam,
251                                                                                      input_image->ni(),
252                                                                                      input_image->nj());
253       }
254     }
255   }
256 
257   return true;
258 }
259 
260 //: run batch update
261 namespace boxm2_cpp_batch_update_opt2_process_globals
262 {
263   constexpr unsigned n_inputs_ = 4;
264   constexpr unsigned n_outputs_ = 0;
265 }
266 
boxm2_cpp_batch_update_opt2_process_cons(bprb_func_process & pro)267 bool boxm2_cpp_batch_update_opt2_process_cons(bprb_func_process& pro)
268 {
269   using namespace boxm2_cpp_batch_update_opt2_process_globals;
270 
271   //process takes 4 inputs
272   // 0) scene
273   // 1) cache
274   // 2) stream cache
275   // 3) the pre-computed sigma normalizer table, for fast access to normalizer values given number of images
276   std::vector<std::string> input_types_(n_inputs_);
277   input_types_[0] = "boxm2_scene_sptr";
278   input_types_[1] = "boxm2_cache_sptr";
279   input_types_[2] = "boxm2_stream_cache_sptr";
280   input_types_[3] = "bsta_sigma_normalizer_sptr";
281   // process has no output:
282   std::vector<std::string>  output_types_(n_outputs_);
283 
284   return pro.set_input_types(input_types_) && pro.set_output_types(output_types_);
285 }
286 
287 //: moved over from boxm
boxm2_cpp_batch_update_opt2_process(bprb_func_process & pro)288 bool boxm2_cpp_batch_update_opt2_process(bprb_func_process& pro)
289 {
290   using namespace boxm2_cpp_batch_update_opt2_process_globals;
291 
292   if ( pro.n_inputs() < n_inputs_ ){
293       std::cout << pro.name() << ": The input number should be " << n_inputs_<< std::endl;
294       return false;
295   }
296   // get the inputs
297   unsigned i = 0;
298   boxm2_scene_sptr scene =pro.get_input<boxm2_scene_sptr>(i++);
299   boxm2_cache_sptr cache= pro.get_input<boxm2_cache_sptr>(i++);
300   boxm2_stream_cache_sptr str_cache = pro.get_input<boxm2_stream_cache_sptr>(i++);
301   bsta_sigma_normalizer_sptr n_table = pro.get_input<bsta_sigma_normalizer_sptr>(i++);
302 
303   std::string data_type;
304   bool foundDataType = false;
305   std::vector<std::string> apps = scene->appearances();
306   for (const auto & app : apps) {
307     if ( app == boxm2_data_traits<BOXM2_MOG3_GREY>::prefix() )
308     {
309       data_type = app;
310       foundDataType = true;
311       // boxm2_data_info::datasize(boxm2_data_traits<BOXM2_MOG3_GREY>::prefix());
312     }
313     else if ( app == boxm2_data_traits<BOXM2_MOG3_GREY_16>::prefix() )
314     {
315       std::cout << "In boxm2_cpp_batch_update_process ERROR: datatype BOXM2_MOG3_GREY_16 not implemented!\n";
316       return false;
317     }
318     else if ( app == boxm2_data_traits<BOXM2_GAUSS_GREY>::prefix() )
319     {
320       data_type = app;
321       foundDataType = true;
322       // boxm2_data_info::datasize(boxm2_data_traits<BOXM2_GAUSS_GREY>::prefix());
323     }
324   }
325   if (!foundDataType) {
326     std::cout<<"boxm2_cpp_batch_update_process ERROR: scene doesn't have BOXM2_MOG3_GREY or BOXM2_MOG3_GREY_16 data type"<<std::endl;
327     return false;
328   }
329 
330   // assumes that the data of each image has been created in the data models previously
331   int alphaTypeSize = (int)boxm2_data_info::datasize(boxm2_data_traits<BOXM2_ALPHA>::prefix());
332   // iterate the scene block by block and write to output
333   std::vector<boxm2_block_id> blk_ids = scene->get_block_ids();
334   std::vector<boxm2_block_id>::iterator id;
335   id = blk_ids.begin();
336   for (id = blk_ids.begin(); id != blk_ids.end(); ++id) {
337     boxm2_data_base *  alph = cache->get_data_base(scene,*id,boxm2_data_traits<BOXM2_ALPHA>::prefix(),0,false);
338     boxm2_data_base *  mog = cache->get_data_base(scene,*id,data_type,0,false);
339 
340     if ( data_type.find(boxm2_data_traits<BOXM2_MOG3_GREY>::prefix()) != std::string::npos )
341     {
342       boxm2_batch_update_opt2_functor<BOXM2_MOG3_GREY> data_functor;
343       data_functor.init_data(alph, mog, str_cache, n_table);
344       int data_buff_length = (int) (alph->buffer_length()/alphaTypeSize);
345       boxm2_data_serial_iterator<boxm2_batch_update_opt2_functor<BOXM2_MOG3_GREY> >(data_buff_length,data_functor);
346     }
347     if (data_type.find(boxm2_data_traits<BOXM2_GAUSS_GREY>::prefix()) != std::string::npos )
348     {
349       boxm2_batch_update_opt2_functor<BOXM2_GAUSS_GREY> data_functor;
350       data_functor.init_data(alph, mog, str_cache, n_table);
351       int data_buff_length = (int) (alph->buffer_length()/alphaTypeSize);
352       boxm2_data_serial_iterator<boxm2_batch_update_opt2_functor<BOXM2_GAUSS_GREY> >(data_buff_length,data_functor);
353     }
354   }
355 
356   return true;
357 }
358