1 /*
2 SPDX-FileCopyrightText: 2008 Akarsh Simha <akarshsimha@gmail.com>
3
4 SPDX-License-Identifier: GPL-2.0-or-later
5 */
6
7 // Program to convert star data stored in a MySQL database into
8 // the binary file format used by KStars
9
10 #define DB_TBL "tycho2"
11 #define DB_DB "stardb"
12 #define VERBOSE 0
13 #define LONG_NAME_LIMIT 32
14 #define BAYER_LIMIT 8
15 #define HTM_LEVEL 3
16 #define NTRIXELS 512 // TODO: Change if HTM Level Changes
17 #define INDEX_ENTRY_SIZE 12
18 #define GLOBAL_MAG_LIMIT 8.00
19 #define MYSQL_STARS_PER_QUERY 1000000
20
21 #include "binfile.h"
22 #include <mysql/mysql.h>
23 #include <sys/types.h>
24 #include <stdlib.h>
25 #include <stdio.h>
26 #include <math.h>
27 #include <string.h>
28
29 /*
30 * struct to store star data, to be written in this format, into the binary file.
31 */
32
33 typedef struct starData
34 {
35 int32_t RA;
36 int32_t Dec;
37 int32_t dRA;
38 int32_t dDec;
39 int32_t parallax;
40 int32_t HD;
41 int16_t mag;
42 int16_t bv_index;
43 char spec_type[2];
44 char flags;
45 char unused;
46 } starData;
47
48 /*
49 * struct to store star name data, to be written in this format, into the star name binary file.
50 */
51
52 typedef struct starName
53 {
54 char bayerName[BAYER_LIMIT];
55 char longName[LONG_NAME_LIMIT];
56 } starName;
57
58 /*
59 * Dump the data file header.
60 *
61 * WARNING: Must edit every time the definition of the starData structures changes
62 *
63 * f : Data file handle
64 */
65
writeDataFileHeader(FILE * f)66 int writeDataFileHeader(FILE *f)
67 {
68 char ASCII_text[124];
69 u_int16_t nfields;
70 u_int32_t nindexes;
71 int16_t endian_id = 0x4B53;
72 u_int8_t version_no = 1;
73
74 if (f == NULL)
75 return 0;
76
77 nfields = 11;
78
79 str2charv(ASCII_text, "KStars Star Data v1.0. To be read using the 32-bit starData structure only.", 124);
80 fwrite(ASCII_text, 124, 1, f);
81 fwrite(&endian_id, 2, 1, f);
82 fwrite(&version_no, 1, 1, f);
83 fwrite(&nfields, sizeof(u_int16_t), 1, f);
84
85 writeDataElementDescription(f, "RA", 4, DT_INT32, 1000000);
86 writeDataElementDescription(f, "Dec", 4, DT_INT32, 100000);
87 writeDataElementDescription(f, "dRA", 4, DT_INT32, 10);
88 writeDataElementDescription(f, "dDec", 4, DT_INT32, 10);
89 writeDataElementDescription(f, "parallax", 4, DT_INT32, 10);
90 writeDataElementDescription(f, "HD", 4, DT_INT32, 10);
91 writeDataElementDescription(f, "mag", 2, DT_INT16, 100);
92 writeDataElementDescription(f, "bv_index", 2, DT_INT16, 100);
93 writeDataElementDescription(f, "spec_type", 2, DT_CHARV, 0);
94 writeDataElementDescription(f, "flags", 1, DT_CHAR, 0);
95 writeDataElementDescription(f, "unused", 1, DT_CHAR, 100);
96
97 nindexes = NTRIXELS;
98 fwrite(&nindexes, sizeof(u_int32_t), 1, f);
99
100 return 1;
101 }
102
103 /*
104 * Dump the name file header.
105 *
106 * WARNING: Must edit every time the definition of the starName structures changes
107 *
108 * nf : Name file handle
109 */
110
writeNameFileHeader(FILE * nf)111 int writeNameFileHeader(FILE *nf)
112 {
113 char ASCII_text[124];
114 u_int16_t nfields;
115 u_int32_t nindexes;
116 int16_t endian_id = 0x4B53;
117 u_int32_t data_offset = 0;
118 u_int8_t version_no = 1;
119
120 if (nf == NULL)
121 return 0;
122
123 nfields = 2;
124
125 str2charv(ASCII_text, "KStars Star Name Data v1.0. Refer associated documentation for format description.", 124);
126 fwrite(ASCII_text, 124, 1, nf);
127 fwrite(&endian_id, 2, 1, nf);
128 fwrite(&version_no, 1, 1, nf);
129 fwrite(&nfields, sizeof(u_int16_t), 1, nf);
130
131 writeDataElementDescription(nf, "bayerName", BAYER_LIMIT, DT_STR, 0);
132 writeDataElementDescription(nf, "longName", LONG_NAME_LIMIT, DT_STR, 0);
133
134 nindexes = 1;
135 fwrite(&nindexes, sizeof(u_int32_t), 1, nf);
136
137 return 1;
138 }
139
main(int argc,char * argv[])140 int main(int argc, char *argv[])
141 {
142 /* === Declarations === */
143
144 int ret, i, exitflag;
145 long int lim;
146 char named, shallow;
147
148 unsigned long ns_header_offset, us_header_offset, ds_header_offset;
149 unsigned long nsf_trix_begin, usf_trix_begin, dsf_trix_begin;
150 unsigned long nsf_trix_count, usf_trix_count, dsf_trix_count;
151 unsigned long ntrixels;
152 unsigned long nf_header_offset;
153 unsigned int names_count;
154 int16_t maglim;
155 u_int8_t htm_level;
156 u_int16_t MSpT_named, MSpT_unnamed, MSpT_deep;
157
158 char query[512];
159 u_int32_t current_trixel, new_trixel;
160
161 /* File streams */
162 FILE *f; /* Pointer to "current" file */
163 FILE *nsf, *usf, *dsf; /* Handles of named and unnamed star data files */
164 FILE *nsfhead, *usfhead, *dsfhead; /* Handles of named and unnamed star header files */
165 FILE *namefile; /* Pointer to the name file */
166 FILE *hdindex; /* Pointer to the HD Index file */
167
168 /* starData and starName structures */
169 starData data;
170 starName name;
171
172 /* MySQL structures */
173 MYSQL link;
174 MYSQL_RES *result;
175 MYSQL_ROW row;
176
177 /* Check the number of arguments */
178 if (argc <= 9)
179 {
180 fprintf(stderr,
181 "USAGE %s DBUserName DBPassword DBName UnnamedStarDataFile UnnamedStarHeaderFile DeepStarDataFile "
182 "DeepStarHeaderFile ShallowStarDataFile ShallowStarHeaderFile StarNameFile HDIndex [TableName]\n",
183 argv[0]);
184 fprintf(stderr, "The magnitude limit for a Shallow Star is set in the program source using GLOBAL_MAG_LIMIT\n");
185 fprintf(stderr, "The database used is a MySQL DB on localhost. The default table name is `allstars`\n");
186 fprintf(stderr, "HDIndex must contain 360000 blank records of 32 bits each. You can use dd if=/dev/zero of=... "
187 "... to create it\n");
188 }
189
190 /* == Open all file streams required == */
191 /* Unnamed Star Handling */
192 usf = fopen(argv[4], "wb");
193 if (usf == NULL)
194 {
195 fprintf(stderr, "ERROR: Could not open %s [Deep Star Data File] for binary write.\n", argv[4]);
196 return 1;
197 }
198
199 usfhead = fopen(argv[5], "wb");
200 if (usfhead == NULL)
201 {
202 fprintf(stderr, "ERROR: Could not open %s [Deep Star Header File] for binary write.\n", argv[5]);
203 fcloseall();
204 return 1;
205 }
206
207 dsf = fopen(argv[6], "wb");
208 if (usfhead == NULL)
209 {
210 fprintf(stderr, "ERROR: Could not open %s [Deep Star Header File] for binary write.\n", argv[6]);
211 fcloseall();
212 return 1;
213 }
214
215 dsfhead = fopen(argv[7], "wb");
216 if (usfhead == NULL)
217 {
218 fprintf(stderr, "ERROR: Could not open %s [Deep Star Header File] for binary write.\n", argv[7]);
219 fcloseall();
220 return 1;
221 }
222
223 /* Bright / Named Star Handling */
224 nsf = fopen(argv[8], "wb");
225 if (nsf == NULL)
226 {
227 fprintf(stderr, "ERROR: Could not open %s [Shallow Star Data File] for binary write.\n", argv[8]);
228 fcloseall();
229 return 1;
230 }
231
232 nsfhead = fopen(argv[9], "wb");
233 if (nsfhead == NULL)
234 {
235 fprintf(stderr, "ERROR: Could not open %s [Shallow Star Header File] for binary write.\n", argv[9]);
236 fcloseall();
237 return 1;
238 }
239
240 namefile = fopen(argv[10], "wb");
241 if (namefile == NULL)
242 {
243 fprintf(stderr, "ERROR: Could not open %s [Star Name File] for binary write.\n", argv[7]);
244 fcloseall();
245 return 1;
246 }
247
248 hdindex = fopen(argv[11], "wb");
249 if (hdindex == NULL)
250 {
251 fprintf(stderr, "ERROR: Could not open %s [HD Index] for binary read/write.\n", argv[9]);
252 fcloseall();
253 return 1;
254 }
255 u_int32_t foo = 0;
256 for (i = 0; i < 360000; ++i)
257 fwrite(&foo, sizeof(u_int32_t), 1, hdindex);
258
259 /* MySQL connect */
260 if (mysql_init(&link) == NULL)
261 {
262 fprintf(stderr, "ERROR: Failed to initialize MySQL connection!\n");
263 return 1;
264 }
265
266 ret = mysql_real_connect(&link, "localhost", argv[1], argv[2], argv[3], 0, NULL, 0);
267
268 if (!ret)
269 {
270 fprintf(stderr, "ERROR: MySQL connect failed for the following reason: %s\n", mysql_error(&link));
271 fcloseall();
272 return 1;
273 }
274
275 if (mysql_select_db(&link, argv[3]))
276 {
277 fprintf(stderr, "ERROR: Could not select MySQL database %s. MySQL said: %s", argv[3], mysql_error(&link));
278 fcloseall();
279 mysql_close(&link);
280 return 1;
281 }
282
283 /* Write file headers */
284 writeNameFileHeader(namefile);
285 writeDataFileHeader(nsfhead);
286 writeDataFileHeader(usfhead);
287 writeDataFileHeader(dsfhead);
288 ns_header_offset = ftell(nsfhead);
289 us_header_offset = ftell(usfhead);
290 nf_header_offset = ftell(namefile);
291 ds_header_offset = ftell(dsfhead);
292
293 /* Write a bogus index entry into the namefile, to be filled with correct data later */
294 writeIndexEntry(namefile, 0, ftell(namefile) + INDEX_ENTRY_SIZE, 0);
295
296 /* Leave space for / write a deep magnitude limit specification in the data files */
297 maglim = GLOBAL_MAG_LIMIT * 100;
298 fwrite(&maglim, 2, 1, nsf); // This is also a bogus entry, because it will be overwritten later
299 maglim = GLOBAL_MAG_LIMIT * 100;
300 fwrite(&maglim, 2, 1, dsf); // This is also a bogus entry, because it will be overwritten later
301 maglim = (int)(-5.0 * 100);
302 fwrite(&maglim, 2, 1, usf); // Bogus entry
303
304 /* Write a HTM level specification in the data file */
305 htm_level = HTM_LEVEL;
306 fwrite(&htm_level, 1, 1, nsf);
307 fwrite(&htm_level, 1, 1, usf);
308 fwrite(&htm_level, 1, 1, dsf);
309
310 /* Leave space for a specification of MSpT (Maximum Stars per Trixel) in the data files */
311 MSpT_named = MSpT_unnamed = 0;
312 fwrite(&MSpT_named, 2, 1, nsf); // Bogus entry
313 fwrite(&MSpT_deep, 2, 1, dsf); // Bogus entry
314 fwrite(&MSpT_unnamed, 2, 1, usf); // Bogus entry
315
316 /* Initialize some variables */
317 lim = 0;
318 exitflag = 0;
319 current_trixel = 0;
320 nsf_trix_count = usf_trix_count = dsf_trix_count = 0;
321 nsf_trix_begin = usf_trix_begin = dsf_trix_begin =
322 2 + 1 +
323 2; // The 2+1+2 is to leave space for deep magnitude limit, HTM Level and MSpT specification. TODO: Change this if things change.
324 ntrixels = 0;
325 names_count = 0;
326
327 /* Recurse over every MYSQL_STARS_PER_QUERY DB entries */
328 while (!exitflag)
329 {
330 /* Build MySQL query for next MYSQL_STARS_PER_QUERY stars */
331 sprintf(query,
332 "SELECT `Trixel`, `RA`, `Dec`, `dRA`, `dDec`, `Parallax`, `Mag`, `BV_Index`, `Spec_Type`, `Mult`, "
333 "`Var`, `HD`, `UID`, `Name`, `GName` FROM `%s` ORDER BY `trixel`, `mag` ASC LIMIT %ld, %d",
334 (argc >= 13) ? argv[12] : DB_TBL, lim, MYSQL_STARS_PER_QUERY);
335
336 if (VERBOSE)
337 {
338 fprintf(stderr, "SQL Query: %s\n", query);
339 }
340
341 /* Execute MySQL query and get the result */
342 mysql_real_query(&link, query, (unsigned int)strlen(query));
343 result = mysql_use_result(&link);
344
345 if (!result)
346 {
347 fprintf(stderr, "MySQL returned NULL result: %s\n", mysql_error(&link));
348 fcloseall();
349 mysql_close(&link);
350 return 1;
351 }
352
353 exitflag = -1;
354
355 /* Process the result row by row */
356 while (row = mysql_fetch_row(result))
357 {
358 /* Very verbose details */
359 if (VERBOSE > 1)
360 {
361 fprintf(stderr, "UID = %s\n", row[12]);
362 for (i = 0; i <= 14; ++i)
363 fprintf(stderr, "\tField #%d = %s\n", i, row[i]);
364 }
365
366 if (exitflag == -1)
367 exitflag = 0;
368
369 /* Write index entries if we've changed trixel */
370 new_trixel = atoi(row[0]);
371 if (new_trixel != current_trixel)
372 {
373 if (VERBOSE)
374 {
375 fprintf(stderr, "Trixel Changed from %d to %d!\n", current_trixel, new_trixel);
376 }
377 while (new_trixel != current_trixel)
378 {
379 writeIndexEntry(nsfhead, current_trixel,
380 ns_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + nsf_trix_begin, nsf_trix_count);
381 writeIndexEntry(dsfhead, current_trixel,
382 ds_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + dsf_trix_begin, dsf_trix_count);
383 writeIndexEntry(usfhead, current_trixel,
384 us_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + usf_trix_begin, usf_trix_count);
385 nsf_trix_begin = ftell(nsf);
386 dsf_trix_begin = ftell(dsf);
387 usf_trix_begin = ftell(usf);
388 if (nsf_trix_count > MSpT_named)
389 MSpT_named = nsf_trix_count;
390 if (dsf_trix_count > MSpT_deep)
391 MSpT_deep = dsf_trix_count;
392 if (usf_trix_count > MSpT_unnamed)
393 MSpT_unnamed = usf_trix_count;
394 nsf_trix_count = dsf_trix_count = usf_trix_count = 0;
395 current_trixel++;
396 ntrixels++;
397 }
398 }
399
400 /* ==== Set up the starData structure ==== */
401
402 /* Initializations */
403 data.flags = 0;
404
405 /* This data is required early! */
406 str2int16(&data.mag, row[6], 2);
407
408 /* Are we looking at a named star */
409 named = 0;
410 if (!isblank(row[13]) || !isblank(row[14]))
411 {
412 named = 1;
413
414 /* Print out messages */
415 if (VERBOSE)
416 fprintf(stderr, "Named Star!\n");
417 if (VERBOSE > 1)
418 fprintf(stderr, "Bayer Name = %s, Long Name = %s\n", row[14], row[13]);
419
420 /* Check for overflows */
421 if (strlen(row[13]) > LONG_NAME_LIMIT)
422 fprintf(stderr, "ERROR: Long Name %s with length %d exceeds LONG_NAME_LIMIT = %d\n", row[13],
423 strlen(row[13]), LONG_NAME_LIMIT);
424 if (strlen(row[14]) > BAYER_LIMIT)
425 fprintf(stderr, "ERROR: Bayer designation %s with length %d exceeds BAYER_LIMIT = %d\n", row[14],
426 strlen(row[14]), BAYER_LIMIT);
427
428 /* Set up the starName structure */
429 str2charv(name.bayerName, row[14], BAYER_LIMIT);
430 str2charv(name.longName, row[13], LONG_NAME_LIMIT);
431 data.flags = data.flags | 0x01; /* Switch on the 'named' bit */
432 }
433
434 /* Are we looking at a 'global' star [always in memory] or dynamically loaded star? */
435 if (named)
436 {
437 f = nsf;
438 nsf_trix_count++;
439 shallow = 1;
440 }
441 else if ((data.mag / 100.0) <= GLOBAL_MAG_LIMIT)
442 {
443 f = usf;
444 usf_trix_count++;
445 shallow = 1;
446 }
447 else
448 {
449 dsf_trix_count++;
450 f = dsf;
451 if (maglim < data.mag)
452 maglim = data.mag;
453 shallow = 0;
454 }
455
456 /* Convert various fields and make entries into the starData structure */
457 str2int32(&data.RA, row[1], 6);
458 str2int32(&data.Dec, row[2], 5);
459 str2int32(&data.dRA, row[3], 1);
460 str2int32(&data.dDec, row[4], 1);
461 str2int32(&data.parallax, row[5], 1);
462 str2int32(&data.HD, row[11], 0);
463 str2int16(&data.bv_index, row[7], 2);
464 if (str2charv(data.spec_type, row[8], 2) < 0)
465 fprintf(stderr, "Spectral type entry %s in DB is possibly invalid for UID = %s\n", row[8], row[12]);
466 if (row[9][0] != '0' && row[9][0] != '\0')
467 data.flags = data.flags | 0x02;
468 if (row[10][0] != '0' && row[10][0] != '\0')
469 data.flags = data.flags | 0x04;
470
471 if (data.HD != 0 && !shallow)
472 {
473 // Use this information to generate the HD index we want to have.
474 int32_t off;
475 if (fseek(hdindex, (data.HD - 1) * sizeof(int32_t), SEEK_SET))
476 {
477 fprintf(stderr, "Invalid seek ( to %X ) on HD Index file. Index will be corrupt.\n",
478 (data.HD - 1) * sizeof(u_int32_t));
479 }
480 off = ftell(f) + ds_header_offset + NTRIXELS * INDEX_ENTRY_SIZE;
481 fprintf(stderr, "DEBUG: HD %d at %X. Writing at %X\n", data.HD, off, ftell(hdindex));
482 if (!fwrite(&off, sizeof(int32_t), 1, hdindex))
483 fprintf(stderr, "fwrite() on HD Index file failed. Index will be empty or incomplete.\n");
484 }
485
486 /* Write the data into the appropriate data file and any names into the name file */
487 if (VERBOSE)
488 fprintf(stderr, "Writing UID = %s...", row[12]);
489 fwrite(&data, sizeof(starData), 1, f);
490 if (named)
491 {
492 fwrite(&name.bayerName, BAYER_LIMIT, 1, namefile);
493 fwrite(&name.longName, LONG_NAME_LIMIT, 1, namefile);
494 names_count++;
495 if (VERBOSE)
496 fprintf(stderr, "Named star count = %ul", names_count);
497 }
498 if (VERBOSE)
499 fprintf(stderr, "Done.\n");
500 }
501 mysql_free_result(result);
502 lim += MYSQL_STARS_PER_QUERY;
503 }
504
505 do
506 {
507 writeIndexEntry(nsfhead, current_trixel, ns_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + nsf_trix_begin,
508 nsf_trix_count);
509 writeIndexEntry(dsfhead, current_trixel, ds_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + dsf_trix_begin,
510 dsf_trix_count);
511 writeIndexEntry(usfhead, current_trixel, us_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + usf_trix_begin,
512 usf_trix_count);
513 nsf_trix_begin = ftell(nsf);
514 dsf_trix_begin = ftell(dsf);
515 usf_trix_begin = ftell(usf);
516 nsf_trix_count = usf_trix_count = dsf_trix_count = 0;
517 current_trixel++;
518 ntrixels++;
519 } while (current_trixel != NTRIXELS);
520
521 fseek(namefile, nf_header_offset, SEEK_SET);
522 writeIndexEntry(namefile, 0, nf_header_offset + INDEX_ENTRY_SIZE, names_count);
523
524 if (ntrixels != NTRIXELS)
525 {
526 fprintf(stderr,
527 "ERROR: Expected %u trixels, but found %u instead. Please redefine NTRIXELS in this program, or check "
528 "the source database for bogus trixels\n",
529 NTRIXELS, ntrixels);
530 }
531
532 rewind(usf);
533 rewind(dsf);
534 rewind(nsf);
535 fwrite(&maglim, 2, 1, dsf);
536 maglim = GLOBAL_MAG_LIMIT * 100;
537 fwrite(&maglim, 2, 1, nsf);
538 fwrite(&maglim, 2, 1, usf);
539 fwrite(&htm_level, 1, 1, usf);
540 fwrite(&htm_level, 1, 1, nsf);
541 fwrite(&htm_level, 1, 1, dsf);
542 fwrite(&MSpT_unnamed, 2, 1, usf);
543 fwrite(&MSpT_deep, 2, 1, dsf);
544 fwrite(&MSpT_named, 2, 1, nsf);
545
546 fcloseall();
547 mysql_close(&link);
548
549 return 0;
550 }
551