1 // This is brl/bseg/brec/brec_part_hierarchy_detector.h
2 #ifndef brec_part_hierarchy_detector_h_
3 #define brec_part_hierarchy_detector_h_
4 //:
5 // \file
6 // \brief class to detect instances of a hierarchy of composable parts in images
7 //
8 // \author Ozge C Ozcanli (ozge@lems.brown.edu)
9 // \date Oct 16, 2008
10 //
11 // \verbatim
12 //  Modifications
13 //   <none yet>
14 // \endverbatim
15 
16 #include "brec_part_hierarchy.h"
17 #include "brec_part_hierarchy_sptr.h"
18 
19 #include <vbl/vbl_ref_count.h>
20 
21 #include <vil/vil_image_resource_sptr.h>
22 #include <vil/vil_image_view.h>
23 
24 #include <vgl/vgl_polygon.h>
25 #include <vgl/vgl_area.h>
26 #include <vgl/algo/vgl_rtree.h>
27 #include <vgl/algo/vgl_rtree_c.h>
28 #include <vgl/vgl_intersection.h>
29 
30 // C must have the following (static method) signatures :
31 // \code
32 //   void  C::init  (B &, V const &);
33 //   void  C::update(B &, V const &);
34 //   void  C::update(B &, B const &);
35 //   bool  C::meet  (B const &, V const &);
36 //   bool  C::meet  (B const &, B const &);
37 //   float C::volume(B const &);
38 // \endcode
39 template <class T>
40 class rtree_brec_instance_box_2d
41 {
42   // only static methods
43   rtree_brec_instance_box_2d() = delete;
44   ~rtree_brec_instance_box_2d() = delete;
45 
46  public:
47   typedef brec_part_instance_sptr v_type;
48   typedef vgl_box_2d<T> b_type;
49   typedef T t_type;
50 
51   // Operations------
init(vgl_box_2d<T> & b,brec_part_instance_sptr const & pi)52   static void  init  (vgl_box_2d<T>& b, brec_part_instance_sptr const& pi)
53   { vgl_point_2d<T> p(pi->x_, pi->y_); b = vgl_box_2d<T>();  b.add(p); }
54 
update(vgl_box_2d<T> & b,brec_part_instance_sptr const & pi)55   static void  update(vgl_box_2d<T>& b, brec_part_instance_sptr const& pi)
56   { vgl_point_2d<T> p(pi->x_, pi->y_); b.add(p); }
57 
update(vgl_box_2d<T> & b0,vgl_box_2d<T> const & b1)58   static void  update(vgl_box_2d<T>& b0, vgl_box_2d<T> const &b1)
59   { b0.add(b1.min_point());  b0.add(b1.max_point()); }
60 
meet(vgl_box_2d<T> const & b,brec_part_instance_sptr const & pi)61   static bool  meet(vgl_box_2d<T> const& b, brec_part_instance_sptr const& pi)
62   { vgl_point_2d<T> p(pi->x_, pi->y_);  return b.contains(p); }
63 
meet(vgl_box_2d<T> const & b0,vgl_box_2d<T> const & b1)64   static bool  meet(vgl_box_2d<T> const& b0, vgl_box_2d<T> const& b1) {
65     vgl_box_2d<T> bint = vgl_intersection<T>(b0, b1);
66     return !bint.is_empty();
67   }
volume(vgl_box_2d<T> const & b)68   static float volume(vgl_box_2d<T> const& b)
69   { return static_cast<float>(vgl_area(b)); }
70 
71   // point meets for a polygon, used by generic rtree probe
meets(brec_part_instance_sptr const & pi,vgl_polygon<T> poly)72   static bool meets(brec_part_instance_sptr const& pi, vgl_polygon<T> poly)
73   { vgl_point_2d<T> p(pi->x_, pi->y_); return poly.contains(p); }
74 
75   // box meets for a polygon, used by generic rtree probe
meets(vgl_box_2d<T> const & b,vgl_polygon<T> poly)76   static bool meets(vgl_box_2d<T> const& b, vgl_polygon<T> poly)
77   { return vgl_intersection<T>(b, poly); }
78 };
79 
80 // rtree definitions
81 typedef brec_part_instance_sptr V_type;  // the contained object type
82 typedef vgl_box_2d<float> B_type;    // the bounding object type
83 typedef rtree_brec_instance_box_2d<float> C_type; // the helper class
84 typedef vgl_rtree<V_type, B_type, C_type> Rtree_type;
85 typedef vgl_rtree_polygon_probe<V_type, B_type, C_type> Probe_type;
86 
87 class brec_detector_methods
88 {
89  public:
90   enum possible_methods {
91     POSTERIOR_NUMERATOR,   // uses posterior for the primitives (given fg and bg app models) but does not divide posterior with a denominator
92     DENSITY_FOR_TRAINING,        // uses the density given by the appearance models of primitives  p(d,angle | ci)
93     POSTERIOR,   // calculates the posterior probability, the denominator is calculated using other class hierarchies
94                  // background geometry model is also required for the denominator, we assume uniform distributions for the distance and angle
95   };
96 };
97 
98 class brec_part_hierarchy_detector : public vbl_ref_count
99 {
100  public:
101 
brec_part_hierarchy_detector(brec_part_hierarchy_sptr h)102   brec_part_hierarchy_detector(brec_part_hierarchy_sptr h) : h_(h), prior_c_f_(0.15f), prior_non_c_f_(0.15f), prior_c_b_(0.15f) {}
103   ~brec_part_hierarchy_detector() override;
104 
105   //: check for existence of upper_p with central_p as its central part and map will tell if all the other parts exist
106   brec_part_instance_sptr exists(const brec_part_base_sptr& upper_p, const brec_part_instance_sptr& central_p, unsigned ni, unsigned nj, Rtree_type* lower_rtree, float det_threshold);
107 
108   //: check for existence of upper_p with central_p as its central part and map will tell if all the other parts exist
109   //  No thresholding, \return a probabilistic score
110   brec_part_instance_sptr exists(const brec_part_base_sptr& upper_p, const brec_part_instance_sptr& central_p, Rtree_type* lower_rtree);
111 
112   brec_part_instance_sptr exists_for_training(const brec_part_base_sptr& upper_p, const brec_part_instance_sptr& central_p, Rtree_type* lower_rtree);
113 
114   brec_part_instance_sptr exists_using_hierarchies(const brec_part_base_sptr& upper_p, const brec_part_instance_sptr& central_p, Rtree_type* lower_rtree, double radius);
115 
116   //: given a set of detected lower level parts, create a set of instance detections for one layer above in the hierarchy
117   void extract_upper_layer(std::vector<brec_part_instance_sptr>& extracted_parts, unsigned ni, unsigned nj, Rtree_type* extracted_parts_rtree,
118                            std::vector<brec_part_instance_sptr>& extracted_upper_parts);
119 
120   //: given a set of detected lower level parts, create a set of instance detections for one layer above in the hierarchy
121   //  No thresholding, \return a probabilistic score
122   //  rho_calculation_method = 0 if probabilistic score
123   //  rho_calculation_method = 1 if training
124   //  rho_calculation_method = 2 if using other hierarchies to compute a posterior
125   void extract_upper_layer(std::vector<brec_part_instance_sptr>& extracted_parts, Rtree_type* extracted_parts_rtree,
126                            std::vector<brec_part_instance_sptr>& extracted_upper_parts, unsigned rho_calculation_method = 0, double radius = 10.0);
127 
128   //: extracts instances of each layer in the given image
129   bool detect(const vil_image_resource_sptr& img);
130 
131   //: extracts instances of each layer in the given image, by rotating the detector with the given amount
132   bool detect(const vil_image_resource_sptr& img, float angle);
133 
134   //: extracts instances of each layer in the given image, by rotating the detector with the given amount
135   //  Assumes that training is complete and a training directory path is set accordingly in h_
136   //  Reads response model parameters for each primitive from this training directory
137   bool detect_primitives_using_trained_response_models(vil_image_view<float>& img, vil_image_view<float>& fg_prob_img, float angle, float prior_class);
138   bool detect(vil_image_view<float>& img, vil_image_view<float>& fg_prob_img, float angle = 0.0f, unsigned rho_calculation_method = brec_detector_methods::POSTERIOR_NUMERATOR, double radius = 10.0, float prior_class = 0.1f, unsigned layer_id = 0);
139 
140   //: extracts instances of each layer in the given image, by rotating the detector with the given amount
141   //  Sets rho parameter of the primitives differently during training
142   bool detect_primitives_for_training(vil_image_view<float>& inp, vil_image_view<float>& fg_prob_img, float angle);
143 
get_parts(unsigned layer)144   std::vector<brec_part_instance_sptr>& get_parts(unsigned layer) { return map_instance_[layer]; }
get_tree(unsigned layer)145   Rtree_type* get_tree(unsigned layer) { return map_rtree_[layer]; }
146 
get_hierarchy()147   brec_part_hierarchy_sptr get_hierarchy() { return h_; }
148 
add_to_class_hierarchies(brec_part_hierarchy_sptr h)149   void add_to_class_hierarchies(brec_part_hierarchy_sptr h) { class_hierarchies_.push_back(h); }
150 
151  protected:
152   brec_part_hierarchy_sptr h_;
153 
154   //: map each layer to a vector of instances of it
155   std::map<unsigned, std::vector<brec_part_instance_sptr> > map_instance_;
156 
157   //: map each layer to an rtree of its instances
158   std::map<unsigned, Rtree_type*> map_rtree_;
159 
160   //: hierarchies of other classes if any
161   //  Required for posterior computation for compositions during testing
162   std::vector<brec_part_hierarchy_sptr> class_hierarchies_;
163 
164   double radius_;
165 
166  public:
167   float prior_c_f_;
168   float prior_non_c_f_;
169   float prior_c_b_;     // prior_non_c_b_ = 1.0f - (prior_c_f_ + prior_non_c_f_ + prior_c_b);
170 };
171 
172 // Binary io, NOT IMPLEMENTED, signatures defined to use brec_part_hierarchy_detector as a brdb_value
173 void vsl_b_write(vsl_b_ostream & os, brec_part_hierarchy_detector const &hl);
174 void vsl_b_read(vsl_b_istream & is, brec_part_hierarchy_detector &hl);
175 void vsl_b_read(vsl_b_istream& is, brec_part_hierarchy_detector* hl);
176 void vsl_b_write(vsl_b_ostream& os, const brec_part_hierarchy_detector* &hl);
177 
178 #endif  //brec_part_hierarchy_detector_h_
179