227 void reset_source(vw& all, size_t numbits)  in reset_source()  argument
229   io_buf* input = all.p->input;  in reset_source()
231   if (all.p->write_cache)  in reset_source()
233       all.p->output->flush();  in reset_source()
234       all.p->write_cache = false;  in reset_source()
235       all.p->output->close_file();  in reset_source()
236       remove(all.p->output->finalname.begin);  in reset_source()
237       rename(all.p->output->currentname.begin, all.p->output->finalname.begin);  in reset_source()
244 	    if (!member(all.final_prediction_sink, (size_t) fd))  in reset_source()
247 …input->open_file(all.p->output->finalname.begin, all.stdin_off, io_buf::READ); //pushing is merged…  in reset_source()
248       all.p->reader = read_cached_features;  in reset_source()
250   if ( all.p->resettable == true )  in reset_source()
252       if (all.daemon)  in reset_source()
255 	  mutex_lock(&all.p->output_lock);  in reset_source()
256 	  while (all.p->local_example_number != all.p->end_parsed_examples)  in reset_source()
257 	    condition_variable_wait(&all.p->output_done, &all.p->output_lock);  in reset_source()
258 	  mutex_unlock(&all.p->output_lock);  in reset_source()
261 	  io_buf::close_file_or_socket(all.p->input->files[0]);  in reset_source()
262 	  all.final_prediction_sink.erase();  in reset_source()
263 	  all.p->input->files.erase();  in reset_source()
267 	  int f = (int)accept(all.p->bound_sock,(sockaddr*)&client_address,&size);  in reset_source()
276 	  all.final_prediction_sink.push_back((size_t) f);  in reset_source()
277 	  all.p->input->files.push_back(f);  in reset_source()
279 	  if (isbinary(*(all.p->input))) {  in reset_source()
280 	    all.p->reader = read_cached_features;  in reset_source()
281 	    all.print = binary_print_result;  in reset_source()
283 	    all.p->reader = read_features;  in reset_source()
284 	    all.print = print_result;  in reset_source()
316 void make_write_cache(vw& all, string &newname, bool quiet)  in make_write_cache()  argument
318   io_buf* output = all.p->output;  in make_write_cache()
327   int f = output->open_file(temp.c_str(), all.stdin_off, io_buf::WRITE);  in make_write_cache()
338   output->write_file(f, &all.num_bits, sizeof(all.num_bits));  in make_write_cache()
341   all.p->write_cache = true;  in make_write_cache()
346 void parse_cache(vw& all, po::variables_map &vm, string source,  in parse_cache()  argument
355   all.p->write_cache = false;  in parse_cache()
362         f = all.p->input->open_file(caches[i].c_str(), all.stdin_off, io_buf::READ);  in parse_cache()
366 	make_write_cache(all, caches[i], quiet);  in parse_cache()
368 	uint32_t c = cache_numbits(all.p->input, f);  in parse_cache()
369 	if (c < all.num_bits) {  in parse_cache()
370           all.p->input->close_file();            in parse_cache()
371 	  make_write_cache(all, caches[i], quiet);  in parse_cache()
376 	  all.p->reader = read_cached_features;  in parse_cache()
377 	  if (c == all.num_bits)  in parse_cache()
378 	    all.p->sorted_cache = true;  in parse_cache()
380 	    all.p->sorted_cache = false;  in parse_cache()
381 	  all.p->resettable = true;  in parse_cache()
386   all.parse_mask = (1 << all.num_bits) - 1;  in parse_cache()
391       all.p->output->space.delete_v();  in parse_cache()
400 void enable_sources(vw& all, bool quiet, size_t passes)  in enable_sources()  argument
402   all.p->input->current = 0;  in enable_sources()
403   parse_cache(all, all.vm, all.data_filename, quiet);  in enable_sources()
405   if (all.daemon || all.active)  in enable_sources()
412       all.p->bound_sock = (int)socket(PF_INET, SOCK_STREAM, 0);  in enable_sources()
413       if (all.p->bound_sock < 0) {  in enable_sources()
419       if (setsockopt(all.p->bound_sock, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(on)) < 0)  in enable_sources()
424 …if (setsockopt(all.p->bound_sock, SOL_SOCKET, SO_KEEPALIVE, (char*)&enableTKA, sizeof(enableTKA)) …  in enable_sources()
431       if (all.vm.count("port"))  in enable_sources()
432 	port = (uint16_t)all.vm["port"].as<size_t>();  in enable_sources()
436       if ( ::bind(all.p->bound_sock,(sockaddr*)&address, sizeof(address)) < 0 )  in enable_sources()
443       if (listen(all.p->bound_sock, 1) < 0) {  in enable_sources()
449       if (all.vm.count("port_file"))  in enable_sources()
452           if (getsockname(all.p->bound_sock, (sockaddr*)&address, &address_size) < 0)  in enable_sources()
457 	  port_file.open(all.vm["port_file"].as<string>().c_str());  in enable_sources()
468       if (!all.active && daemon(1,1))  in enable_sources()
474       if (all.vm.count("pid_file"))  in enable_sources()
477 	  pid_file.open(all.vm["pid_file"].as<string>().c_str());  in enable_sources()
487       if (all.daemon && !all.active)  in enable_sources()
494 	    (float*)mmap(0,(all.length() << all.reg.stride_shift) * sizeof(float),  in enable_sources()
497 	  size_t float_count = all.length() << all.reg.stride_shift;  in enable_sources()
499 	  memcpy(dest, all.reg.weight_vector, float_count*sizeof(float));  in enable_sources()
500 	  free(all.reg.weight_vector);  in enable_sources()
501 	  all.reg.weight_vector = dest;  in enable_sources()
506 	  memcpy(sd, all.sd, sizeof(shared_data));  in enable_sources()
507 	  free(all.sd);  in enable_sources()
508 	  all.sd = sd;  in enable_sources()
511 	  size_t num_children = all.num_children;  in enable_sources()
541                   VW::finish(all);  in enable_sources()
563       all.p->max_fd = 0;  in enable_sources()
564       if (!all.quiet)  in enable_sources()
566       int f = (int)accept(all.p->bound_sock,(sockaddr*)&client_address,&size);  in enable_sources()
573       all.p->label_sock = f;  in enable_sources()
574       all.print = print_result;  in enable_sources()
576       all.final_prediction_sink.push_back((size_t) f);  in enable_sources()
578       all.p->input->files.push_back(f);  in enable_sources()
579       all.p->max_fd = max(f, all.p->max_fd);  in enable_sources()
580       if (!all.quiet)  in enable_sources()
583       all.p->max_fd++;  in enable_sources()
584       if(all.active)  in enable_sources()
585 	all.p->reader = read_features;  in enable_sources()
587 	if (isbinary(*(all.p->input))) {  in enable_sources()
588 	  all.p->reader = read_cached_features;  in enable_sources()
589 	  all.print = binary_print_result;  in enable_sources()
591 	  all.p->reader = read_features;  in enable_sources()
593 	all.p->sorted_cache = true;  in enable_sources()
595       all.p->resettable = all.p->write_cache || all.daemon;  in enable_sources()
599       if (all.p->input->files.size() > 0)  in enable_sources()
606 	  string temp = all.data_filename;  in enable_sources()
609 	  int f = all.p->input->open_file(temp.c_str(), all.stdin_off, io_buf::READ);  in enable_sources()
614 	  all.p->reader = read_features;  in enable_sources()
615 	  all.p->resettable = all.p->write_cache;  in enable_sources()
619   if (passes > 1 && !all.p->resettable)  in enable_sources()
621 …cerr << all.program_name << ": need a cache file for multiple passes: try using --cache_file" << e…  in enable_sources()
624   all.p->input->count = all.p->input->files.size();  in enable_sources()
626     cerr << "num sources = " << all.p->input->files.size() << endl;  in enable_sources()
640 void set_done(vw& all)  in set_done()  argument
642   all.early_terminate = true;  in set_done()
643   mutex_lock(&all.p->examples_lock);  in set_done()
644   all.p->done = true;  in set_done()
645   mutex_unlock(&all.p->examples_lock);  in set_done()
648 void addgrams(vw& all, size_t ngram, size_t skip_gram, v_array<feature>& atomics, v_array<audit_dat…  in addgrams()  argument
661 	  if ((all.audit || all.hash_inv) && audits.size() >= initial_length)  in addgrams()
683       addgrams(all, ngram-1, skip_gram, atomics, audits, initial_length, gram_mask, 0);  in addgrams()
687     addgrams(all, ngram, skip_gram-1, atomics, audits, initial_length, gram_mask, skips+1);  in addgrams()
701 void generateGrams(vw& all, example* &ex) {  in generateGrams()  argument
705       for (size_t n = 1; n < all.ngram[*index]; n++)  in generateGrams()
707 	  all.p->gram_mask.erase();  in generateGrams()
708 	  all.p->gram_mask.push_back((size_t)0);  in generateGrams()
709 	  addgrams(all, n, all.skips[*index], ex->atomics[*index],   in generateGrams()
711 		   length, all.p->gram_mask, 0);  in generateGrams()
716 example* get_unused_example(vw& all)  in get_unused_example()  argument
720       mutex_lock(&all.p->examples_lock);  in get_unused_example()
721       if (all.p->examples[all.p->begin_parsed_examples % all.p->ring_size].in_use == false)  in get_unused_example()
723 	  example& ret = all.p->examples[all.p->begin_parsed_examples++ % all.p->ring_size];  in get_unused_example()
725 	  mutex_unlock(&all.p->examples_lock);  in get_unused_example()
729 	condition_variable_wait(&all.p->example_unused, &all.p->examples_lock);  in get_unused_example()
730       mutex_unlock(&all.p->examples_lock);  in get_unused_example()
734 bool parse_atomic_example(vw& all, example* ae, bool do_read = true)  in parse_atomic_example()  argument
736   if (do_read && all.p->reader(&all, ae) <= 0)  in parse_atomic_example()
739   if(all.p->sort_features && ae->sorted == false)  in parse_atomic_example()
740     unique_sort_features(all.audit, (uint32_t)all.parse_mask, ae);  in parse_atomic_example()
742   if (all.p->write_cache)   in parse_atomic_example()
744       all.p->lp.cache_label(&ae->l,*(all.p->output));  in parse_atomic_example()
745       cache_features(*(all.p->output), ae, (uint32_t)all.parse_mask);  in parse_atomic_example()
750 void end_pass_example(vw& all, example* ae)  in end_pass_example()  argument
752   all.p->lp.default_label(&ae->l);  in end_pass_example()
754   all.p->in_pass_counter = 0;  in end_pass_example()
757 void feature_limit(vw& all, example* ex)  in feature_limit()  argument
760     if (all.limit[*index] < ex->atomics[*index].size())  in feature_limit()
766 	unique_features(features, all.limit[*index]);  in feature_limit()
771 void setup_example(vw& all, example* ae)  in setup_example()  argument
778   ae->example_counter = (size_t)(all.p->end_parsed_examples);  in setup_example()
779   if (!all.p->emptylines_separate_examples)  in setup_example()
780     all.p->in_pass_counter++;  in setup_example()
782 …is_test_only(all.p->in_pass_counter, all.holdout_period, all.holdout_after, all.holdout_set_off, a…  in setup_example()
784   if (all.p->emptylines_separate_examples && example_is_newline(*ae))  in setup_example()
785     all.p->in_pass_counter++;  in setup_example()
787   all.sd->t += all.p->lp.get_weight(&ae->l);  in setup_example()
788   ae->example_t = (float)all.sd->t;  in setup_example()
791   if (all.ignore_some)  in setup_example()
793       if (all.audit || all.hash_inv)  in setup_example()
795 	  if (all.ignore[*i])  in setup_example()
799 	if (all.ignore[*i])  in setup_example()
808   if(all.ngram_strings.size() > 0)  in setup_example()
809     generateGrams(all, ae);      in setup_example()
811   if (all.add_constant) {  in setup_example()
819   if(all.limit_strings.size() > 0)  in setup_example()
820     feature_limit(all,ae);  in setup_example()
822   uint32_t multiplier = all.wpp << all.reg.stride_shift;  in setup_example()
828       if (all.audit || all.hash_inv)  in setup_example()
840   for (vector<string>::iterator i = all.pairs.begin(); i != all.pairs.end();i++)  in setup_example()
848   for (vector<string>::iterator i = all.triples.begin(); i != all.triples.end();i++)  in setup_example()
860   example* new_unused_example(vw& all) {   in new_unused_example()  argument
861     example* ec = get_unused_example(all);  in new_unused_example()
862     all.p->lp.default_label(&ec->l);  in new_unused_example()
863     all.p->begin_parsed_examples++;  in new_unused_example()
864     ec->example_counter = all.p->begin_parsed_examples;  in new_unused_example()
867   example* read_example(vw& all, char* example_line)  in read_example()  argument
869     example* ret = get_unused_example(all);  in read_example()
871     read_line(all, ret, example_line);  in read_example()
872 	parse_atomic_example(all,ret,false);  in read_example()
873     setup_example(all, ret);  in read_example()
874     all.p->end_parsed_examples++;  in read_example()
879 …example* read_example(vw& all, string example_line) { return read_example(all, (char*)example_line…  in read_example()  argument
897   example* import_example(vw& all, vector<feature_space> vf)  in import_example()  argument
899     example* ret = get_unused_example(all);  in import_example()
900     all.p->lp.default_label(&ret->l);  in import_example()
911 	parse_atomic_example(all,ret,false);  in import_example()
912     setup_example(all, ret);  in import_example()
913     all.p->end_parsed_examples++;  in import_example()
917   example* import_example(vw& all, primitive_feature_space* features, size_t len)  in import_example()  argument
919     example* ret = get_unused_example(all);  in import_example()
920     all.p->lp.default_label(&ret->l);  in import_example()
931     parse_atomic_example(all,ret,false); // all.p->parsed_examples++;  in import_example()
932     setup_example(all, ret);  in import_example()
937   primitive_feature_space* export_example(vw& all, example* ec, size_t& len)  in export_example()  argument
953 			t.weight_index >>= all.reg.stride_shift;  in export_example()
969   void parse_example_label(vw& all, example&ec, string label) {  in parse_example_label()  argument
974     all.p->lp.parse_label(all.p, all.sd, &ec.l, words);  in parse_example_label()
979   void empty_example(vw& all, example& ec)  in empty_example()  argument
981 	if (all.audit || all.hash_inv)  in empty_example()
1010   void finish_example(vw& all, example* ec)  in finish_example()  argument
1012     mutex_lock(&all.p->output_lock);  in finish_example()
1013     all.p->local_example_number++;  in finish_example()
1014     condition_variable_signal(&all.p->output_done);  in finish_example()
1015     mutex_unlock(&all.p->output_lock);  in finish_example()
1017     empty_example(all, *ec);  in finish_example()
1019     mutex_lock(&all.p->examples_lock);  in finish_example()
1022     condition_variable_signal(&all.p->example_unused);  in finish_example()
1023     if (all.p->done)  in finish_example()
1024       condition_variable_signal_all(&all.p->example_available);  in finish_example()
1025     mutex_unlock(&all.p->examples_lock);  in finish_example()
1035 	vw* all = (vw*) in;  in main_parse_loop()  local
1039 	while(!all->p->done)  in main_parse_loop()
1041             example* ae = get_unused_example(*all);  in main_parse_loop()
1042 …if (!all->do_reset_source && example_number != all->pass_length && all->max_examples > example_num…  in main_parse_loop()
1043 		   && parse_atomic_example(*all, ae) )  in main_parse_loop()
1045 	       VW::setup_example(*all, ae);  in main_parse_loop()
1050 	       reset_source(*all, all->num_bits);  in main_parse_loop()
1051 	       all->do_reset_source = false;  in main_parse_loop()
1052 	       all->passes_complete++;  in main_parse_loop()
1053 	       end_pass_example(*all, ae);  in main_parse_loop()
1054 	       if (all->passes_complete == all->numpasses && example_number == all->pass_length)  in main_parse_loop()
1056 			   all->passes_complete = 0;  in main_parse_loop()
1057 			   all->pass_length = all->pass_length*2+1;  in main_parse_loop()
1059 	       if (all->passes_complete >= all->numpasses && all->max_examples >= example_number)  in main_parse_loop()
1061 			   mutex_lock(&all->p->examples_lock);  in main_parse_loop()
1062 			   all->p->done = true;  in main_parse_loop()
1063 			   mutex_unlock(&all->p->examples_lock);  in main_parse_loop()
1067 	   mutex_lock(&all->p->examples_lock);  in main_parse_loop()
1068 	   all->p->end_parsed_examples++;  in main_parse_loop()
1069 	   condition_variable_signal_all(&all->p->example_available);  in main_parse_loop()
1070 	   mutex_unlock(&all->p->examples_lock);  in main_parse_loop()
1148 void initialize_examples(vw& all)  in initialize_examples()  argument
1150   all.p->used_index = 0;  in initialize_examples()
1151   all.p->begin_parsed_examples = 0;  in initialize_examples()
1152   all.p->end_parsed_examples = 0;  in initialize_examples()
1153   all.p->done = false;  in initialize_examples()
1155   all.p->examples = calloc_or_die<example>(all.p->ring_size);  in initialize_examples()
1157   for (size_t i = 0; i < all.p->ring_size; i++)  in initialize_examples()
1159       memset(&all.p->examples[i].l, 0, sizeof(polylabel));  in initialize_examples()
1160       all.p->examples[i].in_use = false;  in initialize_examples()
1164 void adjust_used_index(vw& all)  in adjust_used_index()  argument
1166 	all.p->used_index=all.p->begin_parsed_examples;  in adjust_used_index()
1169 void initialize_parser_datastructures(vw& all)  in initialize_parser_datastructures()  argument
1171   initialize_examples(all);  in initialize_parser_datastructures()
1172   initialize_mutex(&all.p->examples_lock);  in initialize_parser_datastructures()
1173   initialize_condition_variable(&all.p->example_available);  in initialize_parser_datastructures()
1174   initialize_condition_variable(&all.p->example_unused);  in initialize_parser_datastructures()
1175   initialize_mutex(&all.p->output_lock);  in initialize_parser_datastructures()
1176   initialize_condition_variable(&all.p->output_done);  in initialize_parser_datastructures()
1180 void start_parser(vw& all, bool init_structures)  in start_parser()  argument
1183 	initialize_parser_datastructures(all);  in start_parser()
1185   pthread_create(&all.parse_thread, NULL, main_parse_loop, &all);  in start_parser()
1187 …all.parse_thread = ::CreateThread(NULL, 0, static_cast<LPTHREAD_START_ROUTINE>(main_parse_loop), &…  in start_parser()
1191 void free_parser(vw& all)  in free_parser()  argument
1193   all.p->channels.delete_v();  in free_parser()
1194   all.p->words.delete_v();  in free_parser()
1195   all.p->name.delete_v();  in free_parser()
1197   if(all.ngram_strings.size() > 0)  in free_parser()
1198     all.p->gram_mask.delete_v();  in free_parser()
1200   for (size_t i = 0; i < all.p->ring_size; i++)   in free_parser()
1202       dealloc_example(all.p->lp.delete_label, all.p->examples[i]);  in free_parser()
1204   free(all.p->examples);  in free_parser()
1206   io_buf* output = all.p->output;  in free_parser()
1213   all.p->counts.delete_v();  in free_parser()
1216 void release_parser_datastructures(vw& all)  in release_parser_datastructures()  argument
1218   delete_mutex(&all.p->examples_lock);  in release_parser_datastructures()
1219   delete_mutex(&all.p->output_lock);  in release_parser_datastructures()
1223 void end_parser(vw& all)  in end_parser()  argument
1226   pthread_join(all.parse_thread, NULL);  in end_parser()
1228   ::WaitForSingleObject(all.parse_thread, INFINITE);  in end_parser()
1229   ::CloseHandle(all.parse_thread);  in end_parser()
1231   release_parser_datastructures(all);  in end_parser()