1 // This is brl/bseg/brec/brec_part_hierarchy_detector.h 2 #ifndef brec_part_hierarchy_detector_h_ 3 #define brec_part_hierarchy_detector_h_ 4 //: 5 // \file 6 // \brief class to detect instances of a hierarchy of composable parts in images 7 // 8 // \author Ozge C Ozcanli (ozge@lems.brown.edu) 9 // \date Oct 16, 2008 10 // 11 // \verbatim 12 // Modifications 13 // <none yet> 14 // \endverbatim 15 16 #include "brec_part_hierarchy.h" 17 #include "brec_part_hierarchy_sptr.h" 18 19 #include <vbl/vbl_ref_count.h> 20 21 #include <vil/vil_image_resource_sptr.h> 22 #include <vil/vil_image_view.h> 23 24 #include <vgl/vgl_polygon.h> 25 #include <vgl/vgl_area.h> 26 #include <vgl/algo/vgl_rtree.h> 27 #include <vgl/algo/vgl_rtree_c.h> 28 #include <vgl/vgl_intersection.h> 29 30 // C must have the following (static method) signatures : 31 // \code 32 // void C::init (B &, V const &); 33 // void C::update(B &, V const &); 34 // void C::update(B &, B const &); 35 // bool C::meet (B const &, V const &); 36 // bool C::meet (B const &, B const &); 37 // float C::volume(B const &); 38 // \endcode 39 template <class T> 40 class rtree_brec_instance_box_2d 41 { 42 // only static methods 43 rtree_brec_instance_box_2d() = delete; 44 ~rtree_brec_instance_box_2d() = delete; 45 46 public: 47 typedef brec_part_instance_sptr v_type; 48 typedef vgl_box_2d<T> b_type; 49 typedef T t_type; 50 51 // Operations------ init(vgl_box_2d<T> & b,brec_part_instance_sptr const & pi)52 static void init (vgl_box_2d<T>& b, brec_part_instance_sptr const& pi) 53 { vgl_point_2d<T> p(pi->x_, pi->y_); b = vgl_box_2d<T>(); b.add(p); } 54 update(vgl_box_2d<T> & b,brec_part_instance_sptr const & pi)55 static void update(vgl_box_2d<T>& b, brec_part_instance_sptr const& pi) 56 { vgl_point_2d<T> p(pi->x_, pi->y_); b.add(p); } 57 update(vgl_box_2d<T> & b0,vgl_box_2d<T> const & b1)58 static void update(vgl_box_2d<T>& b0, vgl_box_2d<T> const &b1) 59 { b0.add(b1.min_point()); b0.add(b1.max_point()); } 60 meet(vgl_box_2d<T> const & b,brec_part_instance_sptr const & pi)61 static bool meet(vgl_box_2d<T> const& b, brec_part_instance_sptr const& pi) 62 { vgl_point_2d<T> p(pi->x_, pi->y_); return b.contains(p); } 63 meet(vgl_box_2d<T> const & b0,vgl_box_2d<T> const & b1)64 static bool meet(vgl_box_2d<T> const& b0, vgl_box_2d<T> const& b1) { 65 vgl_box_2d<T> bint = vgl_intersection<T>(b0, b1); 66 return !bint.is_empty(); 67 } volume(vgl_box_2d<T> const & b)68 static float volume(vgl_box_2d<T> const& b) 69 { return static_cast<float>(vgl_area(b)); } 70 71 // point meets for a polygon, used by generic rtree probe meets(brec_part_instance_sptr const & pi,vgl_polygon<T> poly)72 static bool meets(brec_part_instance_sptr const& pi, vgl_polygon<T> poly) 73 { vgl_point_2d<T> p(pi->x_, pi->y_); return poly.contains(p); } 74 75 // box meets for a polygon, used by generic rtree probe meets(vgl_box_2d<T> const & b,vgl_polygon<T> poly)76 static bool meets(vgl_box_2d<T> const& b, vgl_polygon<T> poly) 77 { return vgl_intersection<T>(b, poly); } 78 }; 79 80 // rtree definitions 81 typedef brec_part_instance_sptr V_type; // the contained object type 82 typedef vgl_box_2d<float> B_type; // the bounding object type 83 typedef rtree_brec_instance_box_2d<float> C_type; // the helper class 84 typedef vgl_rtree<V_type, B_type, C_type> Rtree_type; 85 typedef vgl_rtree_polygon_probe<V_type, B_type, C_type> Probe_type; 86 87 class brec_detector_methods 88 { 89 public: 90 enum possible_methods { 91 POSTERIOR_NUMERATOR, // uses posterior for the primitives (given fg and bg app models) but does not divide posterior with a denominator 92 DENSITY_FOR_TRAINING, // uses the density given by the appearance models of primitives p(d,angle | ci) 93 POSTERIOR, // calculates the posterior probability, the denominator is calculated using other class hierarchies 94 // background geometry model is also required for the denominator, we assume uniform distributions for the distance and angle 95 }; 96 }; 97 98 class brec_part_hierarchy_detector : public vbl_ref_count 99 { 100 public: 101 brec_part_hierarchy_detector(brec_part_hierarchy_sptr h)102 brec_part_hierarchy_detector(brec_part_hierarchy_sptr h) : h_(h), prior_c_f_(0.15f), prior_non_c_f_(0.15f), prior_c_b_(0.15f) {} 103 ~brec_part_hierarchy_detector() override; 104 105 //: check for existence of upper_p with central_p as its central part and map will tell if all the other parts exist 106 brec_part_instance_sptr exists(const brec_part_base_sptr& upper_p, const brec_part_instance_sptr& central_p, unsigned ni, unsigned nj, Rtree_type* lower_rtree, float det_threshold); 107 108 //: check for existence of upper_p with central_p as its central part and map will tell if all the other parts exist 109 // No thresholding, \return a probabilistic score 110 brec_part_instance_sptr exists(const brec_part_base_sptr& upper_p, const brec_part_instance_sptr& central_p, Rtree_type* lower_rtree); 111 112 brec_part_instance_sptr exists_for_training(const brec_part_base_sptr& upper_p, const brec_part_instance_sptr& central_p, Rtree_type* lower_rtree); 113 114 brec_part_instance_sptr exists_using_hierarchies(const brec_part_base_sptr& upper_p, const brec_part_instance_sptr& central_p, Rtree_type* lower_rtree, double radius); 115 116 //: given a set of detected lower level parts, create a set of instance detections for one layer above in the hierarchy 117 void extract_upper_layer(std::vector<brec_part_instance_sptr>& extracted_parts, unsigned ni, unsigned nj, Rtree_type* extracted_parts_rtree, 118 std::vector<brec_part_instance_sptr>& extracted_upper_parts); 119 120 //: given a set of detected lower level parts, create a set of instance detections for one layer above in the hierarchy 121 // No thresholding, \return a probabilistic score 122 // rho_calculation_method = 0 if probabilistic score 123 // rho_calculation_method = 1 if training 124 // rho_calculation_method = 2 if using other hierarchies to compute a posterior 125 void extract_upper_layer(std::vector<brec_part_instance_sptr>& extracted_parts, Rtree_type* extracted_parts_rtree, 126 std::vector<brec_part_instance_sptr>& extracted_upper_parts, unsigned rho_calculation_method = 0, double radius = 10.0); 127 128 //: extracts instances of each layer in the given image 129 bool detect(const vil_image_resource_sptr& img); 130 131 //: extracts instances of each layer in the given image, by rotating the detector with the given amount 132 bool detect(const vil_image_resource_sptr& img, float angle); 133 134 //: extracts instances of each layer in the given image, by rotating the detector with the given amount 135 // Assumes that training is complete and a training directory path is set accordingly in h_ 136 // Reads response model parameters for each primitive from this training directory 137 bool detect_primitives_using_trained_response_models(vil_image_view<float>& img, vil_image_view<float>& fg_prob_img, float angle, float prior_class); 138 bool detect(vil_image_view<float>& img, vil_image_view<float>& fg_prob_img, float angle = 0.0f, unsigned rho_calculation_method = brec_detector_methods::POSTERIOR_NUMERATOR, double radius = 10.0, float prior_class = 0.1f, unsigned layer_id = 0); 139 140 //: extracts instances of each layer in the given image, by rotating the detector with the given amount 141 // Sets rho parameter of the primitives differently during training 142 bool detect_primitives_for_training(vil_image_view<float>& inp, vil_image_view<float>& fg_prob_img, float angle); 143 get_parts(unsigned layer)144 std::vector<brec_part_instance_sptr>& get_parts(unsigned layer) { return map_instance_[layer]; } get_tree(unsigned layer)145 Rtree_type* get_tree(unsigned layer) { return map_rtree_[layer]; } 146 get_hierarchy()147 brec_part_hierarchy_sptr get_hierarchy() { return h_; } 148 add_to_class_hierarchies(brec_part_hierarchy_sptr h)149 void add_to_class_hierarchies(brec_part_hierarchy_sptr h) { class_hierarchies_.push_back(h); } 150 151 protected: 152 brec_part_hierarchy_sptr h_; 153 154 //: map each layer to a vector of instances of it 155 std::map<unsigned, std::vector<brec_part_instance_sptr> > map_instance_; 156 157 //: map each layer to an rtree of its instances 158 std::map<unsigned, Rtree_type*> map_rtree_; 159 160 //: hierarchies of other classes if any 161 // Required for posterior computation for compositions during testing 162 std::vector<brec_part_hierarchy_sptr> class_hierarchies_; 163 164 double radius_; 165 166 public: 167 float prior_c_f_; 168 float prior_non_c_f_; 169 float prior_c_b_; // prior_non_c_b_ = 1.0f - (prior_c_f_ + prior_non_c_f_ + prior_c_b); 170 }; 171 172 // Binary io, NOT IMPLEMENTED, signatures defined to use brec_part_hierarchy_detector as a brdb_value 173 void vsl_b_write(vsl_b_ostream & os, brec_part_hierarchy_detector const &hl); 174 void vsl_b_read(vsl_b_istream & is, brec_part_hierarchy_detector &hl); 175 void vsl_b_read(vsl_b_istream& is, brec_part_hierarchy_detector* hl); 176 void vsl_b_write(vsl_b_ostream& os, const brec_part_hierarchy_detector* &hl); 177 178 #endif //brec_part_hierarchy_detector_h_ 179