1 /*
2 # This file is part of the Astrometry.net suite.
3 # Licensed under a 3-clause BSD style license - see LICENSE
4 */
5
6 #include <stdio.h>
7 #include <limits.h>
8 #include <string.h>
9 #include <assert.h>
10 #include <math.h>
11
12 #include "os-features.h"
13 #include "starutil.h"
14 #include "mathutil.h"
15 #include "bl.h"
16 #include "matchobj.h"
17 #include "matchfile.h"
18 #include "solvedclient.h"
19 #include "solvedfile.h"
20 #include "boilerplate.h"
21
22 char* OPTIONS = "hA:B:I:J:L:M:r:f:s:S:Fa";
23
printHelp(char * progname)24 void printHelp(char* progname) {
25 BOILERPLATE_HELP_HEADER(stderr);
26 fprintf(stderr, "Usage: %s [options]\n"
27 " [-A first-field]\n"
28 " [-B last-field]\n"
29 " [-I first-field-filenum]\n"
30 " [-J last-field-filenum]\n"
31 " [-L write-leftover-matches-file]\n"
32 " [-M write-successful-matches-file]\n"
33 " [-r ratio-needed-to-solve]\n"
34 " [-f minimum-field-objects-needed-to-solve] (default: no minimum)\n"
35 " ( [-F]: write out the first sufficient match to surpass the solve threshold.\n"
36 " or [-a]: write out all matches passing the solve threshold.\n"
37 " (default is to write out the single best match (largest ratio))\n"
38 " )\n"
39 " [-S <solved-file-template>]\n"
40 " <input-match-file> ...\n"
41 "\n", progname);
42 }
43
44 static void write_field(pl* agreeing,
45 pl* leftover,
46 int fieldfile,
47 int fieldnum);
48
49 char* leftoverfname = NULL;
50 matchfile* leftovermf = NULL;
51 char* agreefname = NULL;
52 matchfile* agreemf = NULL;
53 il* solved;
54 il* unsolved;
55 char* solvedfile = NULL;
56 double ratio_tosolve = 0.0;
57 int ninfield_tosolve = 0;
58
59
60 enum modes {
61 MODE_BEST,
62 MODE_FIRST,
63 MODE_ALL
64 };
65
main(int argc,char * argv[])66 int main(int argc, char *argv[]) {
67 int argchar;
68 char* progname = argv[0];
69 char** inputfiles = NULL;
70 int ninputfiles = 0;
71 int i;
72 int firstfield=0, lastfield=INT_MAX-1;
73 int firstfieldfile=1, lastfieldfile=INT_MAX-1;
74 matchfile** mfs;
75 MatchObj** mos;
76 anbool* eofs;
77 anbool* eofieldfile;
78 int nread = 0;
79 int f;
80 int fieldfile;
81 int totalsolved, totalunsolved;
82 int mode = MODE_BEST;
83 double logodds_tosolve = -HUGE_VAL;
84 anbool agree = FALSE;
85
86 MatchObj* bestmo;
87 bl* keepers;
88 bl* leftovers = NULL;
89
90 while ((argchar = getopt (argc, argv, OPTIONS)) != -1) {
91 switch (argchar) {
92 case 'S':
93 solvedfile = optarg;
94 break;
95 case 'F':
96 mode = MODE_FIRST;
97 break;
98 case 'a':
99 mode = MODE_ALL;
100 break;
101 case 'r':
102 ratio_tosolve = atof(optarg);
103 logodds_tosolve = log(ratio_tosolve);
104 break;
105 case 'f':
106 ninfield_tosolve = atoi(optarg);
107 break;
108 case 'M':
109 agreefname = optarg;
110 break;
111 case 'L':
112 leftoverfname = optarg;
113 break;
114 case 'I':
115 firstfieldfile = atoi(optarg);
116 break;
117 case 'J':
118 lastfieldfile = atoi(optarg);
119 break;
120 case 'A':
121 firstfield = atoi(optarg);
122 break;
123 case 'B':
124 lastfield = atoi(optarg);
125 break;
126 case 'h':
127 default:
128 printHelp(progname);
129 exit(-1);
130 }
131 }
132 if (optind < argc) {
133 ninputfiles = argc - optind;
134 inputfiles = argv + optind;
135 } else {
136 printHelp(progname);
137 exit(-1);
138 }
139
140 if (lastfield < firstfield) {
141 fprintf(stderr, "Last field (-B) must be at least as big as first field (-A)\n");
142 exit(-1);
143 }
144
145 if (leftoverfname) {
146 leftovermf = matchfile_open_for_writing(leftoverfname);
147 if (!leftovermf) {
148 fprintf(stderr, "Failed to open file %s to write leftover matches.\n", leftoverfname);
149 exit(-1);
150 }
151 BOILERPLATE_ADD_FITS_HEADERS(leftovermf->header);
152 qfits_header_add(leftovermf->header, "HISTORY", "This file was created by the program \"agreeable\".", NULL, NULL);
153 if (matchfile_write_headers(leftovermf)) {
154 fprintf(stderr, "Failed to write leftovers matchfile header.\n");
155 exit(-1);
156 }
157 leftovers = bl_new(256, sizeof(MatchObj));
158 }
159 if (agreefname) {
160 agreemf = matchfile_open_for_writing(agreefname);
161 if (!agreemf) {
162 fprintf(stderr, "Failed to open file %s to write agreeing matches.\n", agreefname);
163 exit(-1);
164 }
165 BOILERPLATE_ADD_FITS_HEADERS(agreemf->header);
166 qfits_header_add(agreemf->header, "HISTORY", "This file was created by the program \"agreeable\".", NULL, NULL);
167 if (matchfile_write_headers(agreemf)) {
168 fprintf(stderr, "Failed to write agreeing matchfile header.\n");
169 exit(-1);
170 }
171 agree = TRUE;
172 }
173
174 solved = il_new(256);
175 unsolved = il_new(256);
176
177 keepers = bl_new(256, sizeof(MatchObj));
178
179 totalsolved = totalunsolved = 0;
180
181 mos = calloc(ninputfiles, sizeof(MatchObj*));
182 eofs = calloc(ninputfiles, sizeof(anbool));
183 eofieldfile = malloc(ninputfiles * sizeof(anbool));
184 mfs = malloc(ninputfiles * sizeof(matchfile*));
185
186 for (i=0; i<ninputfiles; i++) {
187 fprintf(stderr, "Opening file %s...\n", inputfiles[i]);
188 mfs[i] = matchfile_open(inputfiles[i]);
189 if (!mfs[i]) {
190 fprintf(stderr, "Failed to open matchfile %s.\n", inputfiles[i]);
191 exit(-1);
192 }
193 }
194
195 // we assume the matchfiles are sorted by field id and number.
196 for (fieldfile=firstfieldfile; fieldfile<=lastfieldfile; fieldfile++) {
197 anbool alldone = TRUE;
198
199 memset(eofieldfile, 0, ninputfiles * sizeof(anbool));
200
201 for (f=firstfield; f<=lastfield; f++) {
202 int fieldnum = f;
203 anbool donefieldfile;
204 anbool solved_it;
205 bl* writematches = NULL;
206
207 // quit if we've reached the end of all the input files.
208 alldone = TRUE;
209 for (i=0; i<ninputfiles; i++)
210 if (!eofs[i]) {
211 alldone = FALSE;
212 break;
213 }
214 if (alldone)
215 break;
216
217 // move on to the next fieldfile if all the input files have been
218 // exhausted.
219 donefieldfile = TRUE;
220 for (i=0; i<ninputfiles; i++)
221 if (!eofieldfile[i] && !eofs[i]) {
222 donefieldfile = FALSE;
223 break;
224 }
225 if (donefieldfile)
226 break;
227
228 // start a new field.
229 fprintf(stderr, "File %i, Field %i.\n", fieldfile, f);
230 solved_it = FALSE;
231 bestmo = NULL;
232
233 for (i=0; i<ninputfiles; i++) {
234 int nr = 0;
235 int ns = 0;
236
237 while (1) {
238 if (eofs[i])
239 break;
240 if (!mos[i])
241 mos[i] = matchfile_read_match(mfs[i]);
242 if (unlikely(!mos[i])) {
243 eofs[i] = TRUE;
244 break;
245 }
246
247 // skip past entries that are out of range...
248 if ((mos[i]->fieldfile < firstfieldfile) ||
249 (mos[i]->fieldfile > lastfieldfile) ||
250 (mos[i]->fieldnum < firstfield) ||
251 (mos[i]->fieldnum > lastfield)) {
252 mos[i] = NULL;
253 ns++;
254 continue;
255 }
256
257 if (mos[i]->fieldfile > fieldfile)
258 eofieldfile[i] = TRUE;
259
260 if (mos[i]->fieldfile != fieldfile)
261 break;
262
263 assert(mos[i]->fieldnum >= fieldnum);
264 if (mos[i]->fieldnum != fieldnum)
265 break;
266 nread++;
267 if (nread % 10000 == 9999) {
268 fprintf(stderr, ".");
269 fflush(stderr);
270 }
271
272 // if we've already found a solution, skip past the
273 // remaining matches in this file...
274 if (solved_it && (mode == MODE_FIRST)) {
275 ns++;
276 mos[i] = NULL;
277 continue;
278 }
279
280 nr++;
281
282 if ((mos[i]->logodds >= logodds_tosolve) &&
283 (mos[i]->nindex >= ninfield_tosolve)) {
284 solved_it = TRUE;
285 // (note, we get a pointer to its position in the list)
286 mos[i] = bl_append(keepers, mos[i]);
287 if (!bestmo || mos[i]->logodds > bestmo->logodds)
288 bestmo = mos[i];
289 } else if (leftovers) {
290 bl_append(leftovers, mos[i]);
291 }
292
293 mos[i] = NULL;
294
295 }
296 if (nr || ns)
297 fprintf(stderr, "File %s: read %i matches, skipped %i matches.\n", inputfiles[i], nr, ns);
298 }
299
300 // which matches do we want to write out?
301 if (agree) {
302 writematches = bl_new(256, sizeof(MatchObj));
303
304 switch (mode) {
305 case MODE_BEST:
306 case MODE_FIRST:
307 if (bestmo)
308 bl_append(writematches, bestmo);
309 break;
310 case MODE_ALL:
311 bl_append_list(writematches, keepers);
312 break;
313 }
314 }
315
316 write_field(writematches, leftovers, fieldfile, fieldnum);
317
318 if (agree)
319 bl_free(writematches);
320
321 if (leftovers)
322 bl_remove_all(leftovers);
323 if (keepers)
324 bl_remove_all(keepers);
325
326 fprintf(stderr, "This file: %i fields solved, %i unsolved.\n", il_size(solved), il_size(unsolved));
327 fprintf(stderr, "Grand total: %i solved, %i unsolved.\n", totalsolved + il_size(solved), totalunsolved + il_size(unsolved));
328 }
329 totalsolved += il_size(solved);
330 totalunsolved += il_size(unsolved);
331
332 il_remove_all(solved);
333 il_remove_all(unsolved);
334
335 if (alldone)
336 break;
337 }
338
339 for (i=0; i<ninputfiles; i++)
340 matchfile_close(mfs[i]);
341 free(mfs);
342 free(mos);
343 free(eofs);
344
345 fprintf(stderr, "\nRead %i matches.\n", nread);
346 fflush(stderr);
347
348 if (keepers)
349 bl_free(keepers);
350 if (leftovers)
351 bl_free(leftovers);
352
353 il_free(solved);
354 il_free(unsolved);
355
356 if (leftovermf) {
357 matchfile_fix_headers(leftovermf);
358 matchfile_close(leftovermf);
359 }
360
361 if (agreemf) {
362 matchfile_fix_headers(agreemf);
363 matchfile_close(agreemf);
364 }
365
366 return 0;
367 }
368
write_field(bl * agreeing,bl * leftover,int fieldfile,int fieldnum)369 static void write_field(bl* agreeing,
370 bl* leftover,
371 int fieldfile,
372 int fieldnum) {
373 int i;
374
375 if (!bl_size(agreeing))
376 il_append(unsolved, fieldnum);
377 else {
378 il_append(solved, fieldnum);
379 if (solvedfile) {
380 char fn[256];
381 sprintf(fn, solvedfile, fieldfile);
382 solvedfile_set(fn, fieldnum);
383 }
384 }
385
386 for (i=0; agreeing && i<bl_size(agreeing); i++) {
387 MatchObj* mo = bl_access(agreeing, i);
388 if (matchfile_write_match(agreemf, mo))
389 fprintf(stderr, "Error writing an agreeing match.");
390 fprintf(stderr, "Field %i: Logodds %g (%g)\n", fieldnum, mo->logodds, exp(mo->logodds));
391 }
392
393 if (leftover && bl_size(leftover)) {
394 fprintf(stderr, "Field %i: writing %i leftovers...\n", fieldnum,
395 bl_size(leftover));
396 for (i=0; i<bl_size(leftover); i++) {
397 MatchObj* mo = bl_access(leftover, i);
398 if (matchfile_write_match(leftovermf, mo))
399 fprintf(stderr, "Error writing a leftover match.");
400 }
401 }
402 }
403