1 // This is brl/bseg/brec/brec_part_hierarchy_learner.h 2 #ifndef brec_part_hierarchy_learner_h_ 3 #define brec_part_hierarchy_learner_h_ 4 //: 5 // \file 6 // \brief class to build hierarchies for a given number of classes by estimating parameter distributions from labeled imgs and their background models 7 // 8 // The classes to be learned are assumed to be "foreground" regions wrt the supplied background models 9 // (or they will only be recognized when they are "foreground" after this learning stage) 10 // 11 // Layer 0: primitive parts. This layer is common to all classes and learned on the sole basis of being foreground or background 12 // The parts with high rho = foreground_posterior/background_posterior on average are kept as layer 0 13 // ranked from highest to lowest rho 14 // Initialize a number of primitive part types, make two passes on the training set: 15 // 1st pass: collect strength statistics for each part 16 // fit parametric distributions for each type 17 // 2nd pass: measure rho's (posterior ratios for being foreground and background) of each instance of each type 18 // measure expected value of log of rho's of each type 19 // Rank parts based on measure value and pick top N 20 // 21 // Starting from layer 1, upper layers are constructed recursively 22 // Layer n: stages: 23 // 1) construction of parts as compositions of two layer_n-1 parts 24 // 2) construction of parts as compositions of one layer_n-1 part and one layer_n-2 parts 25 // 26 // Implementation 27 // 1) a) Initialize N*N pairs for each class 28 // b) collect stats for angle and distance distributions for each pair of each class 29 // c) find peaks at angle and distance distributions via mean-shift 30 // d) if pa angle peaks and pd distance peaks, find data likelihood for pa*pd models 31 // e) pick top M models of pairs for each class 32 // 33 // \author Ozge C Ozcanli (ozge@lems.brown.edu) 34 // \date Jan 19, 2009 35 // 36 // \verbatim 37 // Modifications 38 // <none yet> 39 // \endverbatim 40 41 #include <vector> 42 #include <iostream> 43 #include <utility> 44 #include <vbl/vbl_ref_count.h> 45 #ifdef _MSC_VER 46 # include <vcl_msvc_warnings.h> 47 #endif 48 #include <vsl/vsl_binary_io.h> 49 50 #include <vil/vil_image_view.h> 51 #include <bsta/bsta_histogram.h> 52 #include <bsta/bsta_joint_histogram.h> 53 #include <bsta/algo/bsta_mean_shift.h> 54 55 #include <brec/brec_part_base_sptr.h> 56 #include <brec/brec_part_hierarchy_sptr.h> 57 #include <brec/brec_part_hierarchy_detector_sptr.h> 58 59 60 class brec_part_hierarchy_learner : public vbl_ref_count 61 { 62 public: 63 64 typedef bsta_sample_set<double,2>* sample_set_ptr; 65 typedef bsta_sample_set<double,2> sample_set; 66 typedef bsta_histogram<double>* hist_ptr; 67 typedef bsta_histogram<double> hist; 68 //typedef std::pair<sample_set_ptr,sample_set_ptr> sample_set_ptr_pair; 69 typedef std::pair<hist_ptr, hist_ptr> hist_ptr_pair; 70 //typedef std::map<unsigned, std::map<std::pair<unsigned, unsigned>, std::pair<brec_part_instance_sptr, std::pair<hist_ptr_pair, sample_set_ptr_pair> > >* > layer_n_map; 71 typedef std::map<unsigned, std::map<std::pair<unsigned, unsigned>, std::pair<brec_part_instance_sptr, std::pair<hist_ptr_pair, sample_set_ptr> > >* > layer_n_map; 72 typedef std::map<std::pair<unsigned, unsigned>, std::pair<brec_part_instance_sptr, std::pair<hist_ptr_pair, sample_set_ptr> > > class_map; 73 brec_part_hierarchy_learner()74 brec_part_hierarchy_learner() : n_(10) {} 75 76 void initialize_layer0_as_gaussians(int ndirs, float lambda_range, float lambda_inc, int n); 77 ~brec_part_hierarchy_learner()78 ~brec_part_hierarchy_learner() override { 79 for (auto & i : stats_layer0_) { 80 delete i.second; 81 } 82 stats_layer0_.clear(); 83 84 layer_n_map::iterator it; 85 for (it = stats_layer_n_.begin(); it != stats_layer_n_.end(); it++) { 86 class_map* v = it->second; 87 for (auto & itm : *v) 88 { 89 delete (itm.second).second.first.first; 90 delete (itm.second).second.first.second; 91 delete (itm.second).second.second; 92 (itm.second).first = nullptr; 93 } 94 v->clear(); 95 delete v; 96 } 97 stats_layer_n_.clear(); 98 } 99 100 // assumes float img with values in [0,1] range 101 // collect stats for foreground and background regions in the image using the fg_prob_img 102 void layer0_collect_stats(vil_image_view<float>& inp, vil_image_view<float>& fg_prob_img, vil_image_view<bool>& mask); 103 void layer0_collect_stats(vil_image_view<float>& inp, vil_image_view<float>& fg_prob_img); 104 105 void layer0_fit_parametric_dist(); 106 107 void layer0_collect_posterior_stats(vil_image_view<float>& inp, vil_image_view<float>& fg_prob_img, vil_image_view<bool>& mask, 108 vil_image_view<float>& mean_img, vil_image_view<float>& std_dev_img); 109 void layer0_collect_posterior_stats(vil_image_view<float>& inp, vil_image_view<float>& fg_prob_img, 110 vil_image_view<float>& mean_img, vil_image_view<float>& std_dev_img); 111 112 //: create a part hierarchy of primitive parts which are added with respect to their average rho_ (posterior ratios) 113 // This will be used to construct layers 1 and above 114 // Clears \p stats_layer0_ and initializes \p stats_layer_n_ 115 brec_part_hierarchy_sptr layer0_rank_and_create_hierarchy(int N); 116 117 //: initialize learner to construct layer_n as pairs of layer_n-1 of the given hierarchy 118 // Radius is used to initialize the histograms 119 // We use 8 bins for angle in [0, 2*pi] range and 8 bins for distance in [0,radius] range 120 bool initialize_layer_n_as_pairs(const brec_part_hierarchy_sptr& h, unsigned layer_id, unsigned nclasses, float radius); 121 122 //: collect stats to construct parts of layer with layer_id using detected parts of layer_id-1 123 // Collect stats for a pair if they exist within radius pixels of each other 124 bool layer_n_collect_stats(const brec_part_hierarchy_detector_sptr& hd, unsigned layer_id, unsigned class_id); 125 126 //: uses the joint histograms to fit gaussian distributions to distance for 8 orientations 127 // Replaces the histograms with the fitted distributions' histograms 128 bool layer_n_fit_distributions(unsigned class_id, unsigned layer_id, unsigned M); 129 stats_layer0()130 std::vector<std::pair<brec_part_instance_sptr, bsta_histogram<float>*> >& stats_layer0() { return stats_layer0_; } 131 132 void print_to_m_file_layer0(const std::string& file_name); 133 void print_to_m_file_layer0_fitted_dists(const std::string& file_name); 134 void print_layer0(); 135 136 void print_to_m_file_layer_n(const std::string& file_name, unsigned class_id, bool print_set); 137 138 public: 139 140 // collect stats for each type of primitive part types 141 std::vector<std::pair<brec_part_instance_sptr, bsta_histogram<float>*> > stats_layer0_; 142 143 //: for each class, collect stats 144 layer_n_map stats_layer_n_; 145 float radius_; // radius to search for pair compositions for layer_n 146 float d_bandwidth_; // bandwidth to run mean-shift on 1D distance sample set 147 float a_bandwidth_; // bandwidth to run mean-shift on 1D angle sample set 148 149 unsigned type_cnt_; 150 151 //: a visualization parameter, set by initialize method according to the type of primitive initialized 152 int n_; 153 154 brec_part_hierarchy_sptr h_; // keep the current hierarchy for visualization purposes. 155 156 //: create a hierarchy instance for each class 157 std::map<unsigned, brec_part_hierarchy_sptr> h_map_; 158 }; 159 160 // Binary io, NOT IMPLEMENTED, signatures defined to use brec_part_hierarchy_learner as a brdb_value 161 void vsl_b_write(vsl_b_ostream & os, brec_part_hierarchy_learner const &hl); 162 void vsl_b_read(vsl_b_istream & is, brec_part_hierarchy_learner &hl); 163 void vsl_b_read(vsl_b_istream& is, brec_part_hierarchy_learner* hl); 164 void vsl_b_write(vsl_b_ostream& os, const brec_part_hierarchy_learner* &hl); 165 166 #endif // brec_part_hierarchy_learner_h_ 167