1 //This is brl/bbas/volm/volm_query.h 2 #ifndef volm_query_h_ 3 #define volm_query_h_ 4 //: 5 // \file 6 // \brief A class to represent a volumetric matching query through an image mark-up and estimated camera parameters 7 // The container is numbers of points distributed on a unit spherecial surface 8 // The camera parameters, heading, tilt, roll, right_fov, top_fov, have resolution one degree, and their 9 // default values are chosen based on query image category (desert/coast) 10 // Use top viewing angle to define the viewing volume 11 // 12 // \author Yi Dong 13 // \date October 23, 2012 14 // \verbatim 15 // Modifications 16 // Yi Dong Jan-2013 added functions to generate object based query infomation for object based volm_matcher 17 // Yi Dong Jan-2013 added object orientation and object NLCD land classification 18 // JLM Jan 20, 2013 Added constructor from depth_map_scene 19 // Ozge C. Ozcanli Jan 30, 2013 -- replacing old camera space construction functionality with the use of volm_camera_space class 20 // Yi Dong Feb-2013 added binary i/o 21 // \endverbatim 22 // 23 24 #include <iostream> 25 #include <set> 26 #include <vbl/vbl_ref_count.h> 27 #include <bpgl/depth_map/depth_map_scene_sptr.h> 28 #include <bpgl/depth_map/depth_map_scene.h> 29 #include <vgl/vgl_polygon.h> 30 #include <volm/volm_spherical_container_sptr.h> 31 #include <volm/volm_spherical_shell_container_sptr.h> 32 #include <volm/volm_spherical_shell_container.h> 33 #include <vpgl/vpgl_perspective_camera.h> 34 #include <vil/vil_image_view.h> 35 #ifdef _MSC_VER 36 # include <vcl_msvc_warnings.h> 37 #endif 38 #include <vpgl/io/vpgl_io_perspective_camera.h> 39 #include <volm/volm_camera_space_sptr.h> 40 #include <volm/volm_camera_space.h> 41 #include <vsl/vsl_binary_io.h> 42 #include <volm/volm_category_io.h> 43 44 class volm_query : public vbl_ref_count 45 { 46 public: 47 //: default consturctor 48 volm_query() = default; 49 //: constructor from files 50 volm_query(const volm_camera_space_sptr& cam_space, 51 std::string const& label_xml_file, 52 std::string const& category_file, 53 volm_spherical_container_sptr const& sph, 54 volm_spherical_shell_container_sptr const& sph_shell); 55 56 //: constructor from depth map scene 57 volm_query(const volm_camera_space_sptr& cam_space, 58 std::string const& depth_map_scene_file, 59 volm_spherical_shell_container_sptr const& sph_shell, 60 volm_spherical_container_sptr const& sph); 61 62 //: constructor from a binary file of data members 63 volm_query(std::string const& query_file, const volm_camera_space_sptr& cam_space, 64 std::string const& depth_map_scene_file, 65 volm_spherical_shell_container_sptr const& sph_shell, 66 volm_spherical_container_sptr const& sph); 67 68 69 // === accessors === min_dist()70 std::vector<std::vector<unsigned char> >& min_dist() { return min_dist_;} max_dist()71 std::vector<std::vector<unsigned char> >& max_dist() { return max_dist_;} order()72 std::vector<std::vector<unsigned char> >& order() { return order_; } order_set()73 std::set<unsigned>& order_set() { return order_set_; } order_index()74 std::vector<std::vector<std::vector<unsigned> > >& order_index() { return order_index_; } dist_id()75 std::vector<std::vector<std::vector<unsigned> > >& dist_id() { return dist_id_; } dist_offset()76 std::vector<unsigned>& dist_offset() { return dist_offset_; } max_obj_dist()77 std::vector<unsigned char>& max_obj_dist() { return max_obj_dist_; } min_obj_dist()78 std::vector<unsigned char>& min_obj_dist() { return min_obj_dist_; } obj_orient()79 std::vector<unsigned char>& obj_orient() { return obj_orient_; } obj_land_id()80 std::vector<std::vector<unsigned char> >& obj_land_id() { return obj_land_id_; } obj_land_wgt()81 std::vector<std::vector<float> >& obj_land_wgt() { return obj_land_wgt_; } order_obj()82 std::vector<unsigned char>& order_obj() { return order_obj_; } ground_id()83 std::vector<std::vector<unsigned> >& ground_id() { return ground_id_; } ground_dist()84 std::vector<std::vector<unsigned char> >& ground_dist() { return ground_dist_; } ground_offset()85 std::vector<unsigned>& ground_offset() { return ground_offset_; } ground_land_id()86 std::vector<std::vector<std::vector<unsigned char> > >& ground_land_id() { return ground_land_id_; } ground_land_wgt()87 std::vector<std::vector<std::vector<float> > >& ground_land_wgt() { return ground_land_wgt_; } ground_orient()88 unsigned char ground_orient() { return ground_orient_; } sky_id()89 std::vector<std::vector<unsigned> >& sky_id() { return sky_id_; } sky_offset()90 std::vector<unsigned>& sky_offset() { return sky_offset_; } sky_orient()91 unsigned char sky_orient() { return sky_orient_; } depth_scene()92 depth_map_scene_sptr depth_scene() const { return dm_; } depth_regions()93 std::vector<depth_map_region_sptr>& depth_regions() { return depth_regions_; } sph_shell()94 volm_spherical_shell_container_sptr sph_shell() const { return sph_; } get_cam_num()95 unsigned get_cam_num() const { return (unsigned)cam_space_->valid_indices().size(); } get_obj_order_num()96 unsigned get_obj_order_num() const { return (unsigned)order_index_[0].size(); } get_query_size()97 unsigned get_query_size() const { return query_size_; } 98 99 //: return number of voxels having ground properties get_ground_id_size()100 unsigned get_ground_id_size() const { return ground_offset_[ground_offset_.size()-1]; } 101 102 //: return stored distance for all ground voxels get_ground_dist_size()103 unsigned get_ground_dist_size() const { return ground_offset_[ground_offset_.size()-1]; } 104 105 //: return number of voxels having non-ground, non-sky properties get_dist_id_size()106 unsigned get_dist_id_size() const { return dist_offset_[dist_offset_.size()-1]; } 107 108 //: return number of voxels having sky properties get_sky_id_size()109 unsigned get_sky_id_size() const { return sky_offset_[sky_offset_.size()-1]; } 110 111 //: return number of voxels for all non-ground objects (order_index) 112 unsigned get_order_size() const; 113 114 //: return the total query size in byte(object based) 115 unsigned obj_based_query_size_byte() const; 116 117 //: write vrml for spherical container and camera hypothesis 118 void draw_template(std::string const& vrml_fname); 119 120 //: write query image showing the depth map geometry and the penetrating ray 121 void draw_query_images(std::string const& out_dir); 122 void draw_query_image(unsigned i, std::string const& out_name); 123 124 //: get camera string get_cam_string(unsigned i)125 std::string get_cam_string(unsigned i) const 126 { 127 return cam_space_->camera_angles(i).get_string(); 128 //return cam_space_->get_string(cam_space_->valid_indices()[i]); 129 } 130 131 //: get the number of camera having the input top_fov value 132 unsigned get_num_top_fov(double const& top_fov) const; 133 134 //: extract the top_fov value from cam_id 135 double get_top_fov(unsigned const& i) const; 136 137 //: return valid top_fov from camera vector 138 std::vector<double> get_valid_top_fov() const; 139 140 //: visualized the query camera using the spherical shell geometry 141 void visualize_query(std::string const& prefix); 142 143 //: generate rgb depth image for given camera id and given depth value 144 void depth_rgb_image(std::vector<unsigned char> const& values, unsigned const& cam_id, vil_image_view<vil_rgb<vxl_byte> >& out_img, const std::string& value_type = "depth"); 145 146 //: draw the polygons of regions on top of an rgb image 147 void draw_depth_map_regions(vil_image_view<vil_rgb<vxl_byte> >& out_img); 148 void draw_query_regions(std::string const& out_name); 149 150 // =========== binary I/O ================ 151 152 //: version version()153 unsigned version() const {return 1;} 154 155 //: binary IO write -- does not write all of self, not a regular b_write 156 void write_data(vsl_b_ostream& os); 157 158 //: binary IO read -- does not read all of self, not a regular b_read 159 void read_data(vsl_b_istream& is); 160 161 //: CAUTION: not all fields are checked for equality, only the fields which are saved in write_data() are checked 162 bool operator == (const volm_query &other) const; 163 164 static void draw_polygon(vil_image_view<vil_rgb<vxl_byte> >& img, vgl_polygon<double> const& poly, unsigned char const& depth); 165 166 #if NO_CAM_SPACE 167 //: initial camera parameters read from camera kml 168 double init_focal_; 169 double head_, head_d_, head_inc_; 170 double tilt_, tilt_d_, tilt_inc_; 171 double roll_, roll_d_, roll_inc_; 172 double tfov_, tfov_d_, tfov_inc_; 173 #endif 174 double altitude_; 175 176 protected: 177 #if NO_CAM_SPACE_CLASS 178 //: a check whether use the viewing volume values provided by camera kml 179 bool use_default_; 180 #endif 181 volm_camera_space_sptr cam_space_; 182 183 //: 184 unsigned char invalid_; 185 //: image size 186 unsigned ni_, nj_; 187 unsigned log_downsample_ratio_; // 0,1,2 or 3 (ni-->ni/2^ratio_), to generate downsampled depth maps for ground regions 188 189 //: depth map scene 190 depth_map_scene_sptr dm_; 191 //: voxel array used to get voxel index 192 volm_spherical_container_sptr sph_depth_; 193 //: a unit sphere 194 volm_spherical_shell_container_sptr sph_; 195 //: upper bound on depth 196 double d_threshold_; 197 //: vector of depth_map_region sorted by depth order 198 std::vector<depth_map_region_sptr> depth_regions_; 199 200 // === camera parameters --- use even number later to ensure the init_value and init_value +/- conf_value is covered === 201 202 #if 0 203 //: vectors store the space of camera hypotheses 204 std::vector<double> top_fov_; 205 std::vector<double> headings_; 206 std::vector<double> tilts_; 207 std::vector<double> rolls_; 208 #endif 209 std::vector<vpgl_perspective_camera<double> > cameras_; 210 std::vector<std::string> camera_strings_; 211 //: ingested query information 212 std::vector<std::vector<unsigned char> > min_dist_; 213 std::vector<std::vector<unsigned char> > max_dist_; 214 std::vector<std::vector<unsigned char> > order_; 215 //: number of spherical shell rays 216 unsigned query_size_; 217 //: the order index assigned to sky 218 unsigned order_sky_; 219 //: the set of Cartesian points on the unit sphere 220 std::vector<vgl_point_3d<double> > query_points_; 221 //: order vector to store the index id associated with object order 222 std::set<unsigned> order_set_; // store the non-ground order, using set to ensure objects having same order are put together 223 std::vector<std::vector<std::vector<unsigned> > > order_index_; 224 //: ground plane distance, id, and fallback land category 225 std::vector<std::vector<unsigned> > ground_id_; 226 std::vector<std::vector<unsigned char> > ground_dist_; 227 std::vector<std::vector<std::vector<unsigned char> > > ground_land_id_; 228 std::vector<std::vector<std::vector<float> > > ground_land_wgt_; 229 std::vector<unsigned> ground_offset_; 230 unsigned char ground_orient_; // always horizontal 231 //: sky distance 232 std::vector<std::vector<unsigned> > sky_id_; 233 std::vector<unsigned> sky_offset_; 234 unsigned char sky_orient_; // always 100 (100 is the label for uncertain or ambiguous cells) 235 //: object id based on min_dist (since objects may have different min_dist but same order) 236 std::vector<std::vector<std::vector<unsigned> > > dist_id_; 237 std::vector<unsigned> dist_offset_; 238 //: min and max distance, object orders, orientation and land clarifications for different objects, based on object orders 239 std::vector<unsigned char> min_obj_dist_; 240 std::vector<unsigned char> max_obj_dist_; 241 std::vector<unsigned char> order_obj_; 242 std::vector<unsigned char> obj_orient_; 243 std::vector<std::vector<unsigned char> > obj_land_id_; 244 std::vector<std::vector<float> > obj_land_wgt_; 245 #if 0 246 //: weight parameters 247 std::vector<float> weight_obj_; 248 float weight_grd_; 249 float weight_sky_; 250 #endif 251 //: functions 252 bool query_ingest(); 253 bool offset_ingest(); 254 bool order_ingest(); 255 bool weight_ingest(); 256 unsigned char fetch_depth(double const& u, 257 double const& v, 258 unsigned char& order, 259 unsigned char& max_dist, 260 unsigned& object_id, 261 std::vector<unsigned char>& grd_fallback_id, 262 std::vector<float>& grd_fallback_wgt, 263 bool& is_ground, 264 bool& is_sky, 265 bool& is_object, 266 vil_image_view<float> const& depth_img); 267 void create_cameras(); 268 void generate_regions(); 269 void draw_viewing_volume(std::string const& fname, 270 vpgl_perspective_camera<double> cam, 271 float r, 272 float g, 273 float b); 274 void draw_rays(std::string const& fname); 275 276 void draw_dot(vil_image_view<vil_rgb<vxl_byte> >& img, 277 vgl_point_3d<double> const& world_point, 278 vil_rgb<vxl_byte>, 279 vpgl_perspective_camera<double> const& cam); 280 }; 281 282 #endif // volm_query_h_ 283