1 /* $Id: tdx-data.c 9659 2014-08-30 08:08:11Z iulius $
2 **
3 ** Overview data file handling for the tradindexed overview method.
4 **
5 ** Implements the handling of the .IDX and .DAT files for the tradindexed
6 ** overview method. The .IDX files are flat arrays of binary structs
7 ** specifying the offset in the data file of the overview data for a given
8 ** article as well as the length of that data and some additional meta-data
9 ** about that article. The .DAT files contain all of the overview data for
10 ** that group in wire format.
11 **
12 ** Externally visible functions have a tdx_ prefix; internal functions do
13 ** not. (Externally visible unfortunately means everything that needs to be
14 ** visible outside of this object file, not just interfaces exported to
15 ** consumers of the overview API.)
16 */
17
18 #include "config.h"
19 #include "clibrary.h"
20 #include "portable/mmap.h"
21 #include <errno.h>
22 #include <fcntl.h>
23 #include <sys/stat.h>
24
25 #include "inn/fdflag.h"
26 #include "inn/history.h"
27 #include "inn/innconf.h"
28 #include "inn/messages.h"
29 #include "inn/mmap.h"
30 #include "inn/libinn.h"
31 #include "inn/ov.h"
32 #include "ovinterface.h"
33 #include "inn/storage.h"
34 #include "tdx-private.h"
35 #include "tdx-structure.h"
36
37 /* Returned to callers as an opaque data type, this holds the information
38 needed to manage a search in progress. */
39 struct search {
40 ARTNUM limit;
41 ARTNUM current;
42 struct group_data *data;
43 };
44
45 /* Internal prototypes. */
46 static char *group_path(const char *group);
47 static int file_open(const char *base, const char *suffix, bool writable,
48 bool append);
49 static bool file_open_index(struct group_data *, const char *suffix);
50 static bool file_open_data(struct group_data *, const char *suffix);
51 static void *map_file(int fd, size_t length, const char *base,
52 const char *suffix);
53 static bool map_index(struct group_data *data);
54 static bool map_data(struct group_data *data);
55 static void unmap_index(struct group_data *data);
56 static void unmap_data(struct group_data *data);
57 static ARTNUM index_base(ARTNUM artnum);
58
59
60 /*
61 ** Determine the path to the data files for a particular group and return
62 ** it. Allocates memory which the caller is responsible for freeing.
63 */
64 static char *
group_path(const char * group)65 group_path(const char *group)
66 {
67 char *path, *p;
68 size_t length;
69 const char *gp;
70
71 /* The path of the data files for news.groups is dir/n/g/news.groups. In
72 other words, the first letter of each component becomes a directory.
73 The length of the path is therefore the length of the base overview
74 directory path, one character for the slash, two characters for the
75 first letter and initial slash, two characters for each hierarchical
76 level of the group, and then the length of the group name.
77
78 For robustness, we want to handle leading or multiple consecutive
79 periods. We only recognize a new hierarchical level after a string of
80 periods (which doesn't end the group name). */
81 length = strlen(innconf->pathoverview);
82 for (gp = group; *gp != '\0'; gp++)
83 if (*gp == '.') {
84 if (gp[1] == '.' || gp[0] == '\0')
85 continue;
86 length += 2;
87 }
88 length += 1 + 2 + strlen(group) + 1;
89 path = xmalloc(length);
90 strlcpy(path, innconf->pathoverview, length);
91 p = path + strlen(innconf->pathoverview);
92
93 /* Generate the hierarchical directories. */
94 if (*group != '.' && *group != '\0') {
95 *p++ = '/';
96 *p++ = *group;
97 }
98 for (gp = strchr(group, '.'); gp != NULL; gp = strchr(gp, '.')) {
99 gp++;
100 if (gp == group + 1)
101 continue;
102 if (*gp != '\0' && *gp != '.' && *gp != '/') {
103 *p++ = '/';
104 *p++ = *gp;
105 }
106 }
107 *p++ = '/';
108
109 /* Finally, append the group name to the generated path and then replace
110 all slashes with commas. Commas have the advantage of being clearly
111 illegal in newsgroup names because of the syntax of the Newsgroups
112 header, but aren't shell metacharacters. */
113 strlcpy(p, group, length - (p - path));
114 for (; *p != '\0'; p++)
115 if (*p == '/')
116 *p = ',';
117 return path;
118 }
119
120
121 /*
122 ** Open a data file for a group. Takes the base portion of the file, the
123 ** suffix, a bool saying whether or not the file is being opened for write,
124 ** and a bool saying whether to open it for append. Returns the file
125 ** descriptor.
126 */
127 static int
file_open(const char * base,const char * suffix,bool writable,bool append)128 file_open(const char *base, const char *suffix, bool writable, bool append)
129 {
130 char *file;
131 int flags, fd;
132
133 file = concat(base, ".", suffix, (char *) 0);
134 flags = writable ? (O_RDWR | O_CREAT) : O_RDONLY;
135 if (append)
136 flags |= O_APPEND;
137 fd = open(file, flags, ARTFILE_MODE);
138 if (fd < 0 && writable && errno == ENOENT) {
139 char *p = strrchr(file, '/');
140
141 *p = '\0';
142 if (!MakeDirectory(file, true)) {
143 syswarn("tradindexed: cannot create directory %s", file);
144 free(file);
145 return -1;
146 }
147 *p = '/';
148 fd = open(file, flags, ARTFILE_MODE);
149 }
150 if (fd < 0) {
151 if (errno != ENOENT)
152 syswarn("tradindexed: cannot open %s", file);
153 free(file);
154 return -1;
155 }
156 free(file);
157 return fd;
158 }
159
160
161 /*
162 ** Open the index file for a group. Takes an optional suffix to use instead
163 ** of IDX (used primarily for expiring).
164 */
165 static bool
file_open_index(struct group_data * data,const char * suffix)166 file_open_index(struct group_data *data, const char *suffix)
167 {
168 struct stat st;
169
170 if (suffix == NULL)
171 suffix = "IDX";
172 if (data->indexfd >= 0)
173 close(data->indexfd);
174 data->indexfd = file_open(data->path, suffix, data->writable, false);
175 if (data->indexfd < 0)
176 return false;
177 if (fstat(data->indexfd, &st) < 0) {
178 syswarn("tradindexed: cannot stat %s.%s", data->path, suffix);
179 close(data->indexfd);
180 return false;
181 }
182 data->indexinode = st.st_ino;
183 fdflag_close_exec(data->indexfd, true);
184 return true;
185 }
186
187
188 /*
189 ** Open the data file for a group. Takes an optional suffix to use instead
190 ** of DAT (used primarily for expiring).
191 */
192 static bool
file_open_data(struct group_data * data,const char * suffix)193 file_open_data(struct group_data *data, const char *suffix)
194 {
195 if (suffix == NULL)
196 suffix = "DAT";
197 if (data->datafd >= 0)
198 close(data->datafd);
199 data->datafd = file_open(data->path, suffix, data->writable, true);
200 if (data->datafd < 0)
201 return false;
202 fdflag_close_exec(data->datafd, true);
203 return true;
204 }
205
206
207 /*
208 ** Open a particular group. Allocates a new struct group_data that should be
209 ** passed to tdx_data_close() when the caller is done with it.
210 */
211 struct group_data *
tdx_data_new(const char * group,bool writable)212 tdx_data_new(const char *group, bool writable)
213 {
214 struct group_data *data;
215
216 data = xmalloc(sizeof(struct group_data));
217 data->path = group_path(group);
218 data->writable = writable;
219 data->remapoutoforder = false;
220 data->high = 0;
221 data->base = 0;
222 data->indexfd = -1;
223 data->datafd = -1;
224 data->index = NULL;
225 data->data = NULL;
226 data->indexlen = 0;
227 data->datalen = 0;
228 data->indexinode = 0;
229 data->refcount = 0;
230
231 return data;
232 }
233
234
235 /*
236 ** Open the index and data files for a group.
237 */
238 bool
tdx_data_open_files(struct group_data * data)239 tdx_data_open_files(struct group_data *data)
240 {
241 unmap_index(data);
242 unmap_data(data);
243 data->index = NULL;
244 data->data = NULL;
245 if (!file_open_index(data, NULL))
246 goto fail;
247 if (!file_open_data(data, NULL))
248 goto fail;
249 return true;
250
251 fail:
252 if (data->indexfd >= 0)
253 close(data->indexfd);
254 if (data->datafd >= 0)
255 close(data->datafd);
256 return false;
257 }
258
259
260 /*
261 ** Map a data file (either index or data), or read in all of the data in the
262 ** file if we're avoiding mmap. Takes the base and suffix of the file for
263 ** error reporting.
264 */
265 static void *
map_file(int fd,size_t length,const char * base,const char * suffix)266 map_file(int fd, size_t length, const char *base, const char *suffix)
267 {
268 char *data;
269
270 if (length == 0)
271 return NULL;
272
273 if (!innconf->tradindexedmmap) {
274 ssize_t status;
275
276 data = xmalloc(length);
277 status = read(fd, data, length);
278 if ((size_t) status != length) {
279 syswarn("tradindexed: cannot read data file %s.%s", base, suffix);
280 free(data);
281 return NULL;
282 }
283 } else {
284 data = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0);
285 if (data == MAP_FAILED) {
286 syswarn("tradindexed: cannot mmap %s.%s", base, suffix);
287 return NULL;
288 }
289 }
290 return data;
291 }
292
293
294 /*
295 ** Memory map the index file.
296 */
297 static bool
map_index(struct group_data * data)298 map_index(struct group_data *data)
299 {
300 struct stat st;
301 int r;
302
303 r = fstat(data->indexfd, &st);
304 if (r == -1) {
305 if (errno == ESTALE) {
306 r = file_open_index(data, NULL);
307 } else {
308 syswarn("tradindexed: cannot stat %s.IDX", data->path);
309 }
310 }
311 if (r == -1)
312 return false;
313 data->indexlen = st.st_size;
314 data->index = map_file(data->indexfd, data->indexlen, data->path, "IDX");
315 return (data->index == NULL && data->indexlen > 0) ? false : true;
316 }
317
318
319 /*
320 ** Memory map the data file.
321 */
322 static bool
map_data(struct group_data * data)323 map_data(struct group_data *data)
324 {
325 struct stat st;
326 int r;
327
328 r = fstat(data->datafd, &st);
329 if (r == -1) {
330 if (errno == ESTALE) {
331 r = file_open_data(data, NULL);
332 } else {
333 syswarn("tradindexed: cannot stat %s.DAT", data->path);
334 }
335 }
336 if (r == -1)
337 return false;
338 data->datalen = st.st_size;
339 data->data = map_file(data->datafd, data->datalen, data->path, "DAT");
340 return (data->data == NULL && data->indexlen > 0) ? false : true;
341 }
342
343
344 /*
345 ** Unmap a data file or free the memory copy if we're not using mmap. Takes
346 ** the memory to free or unmap, the length for munmap, and the name base and
347 ** suffix for error reporting.
348 */
349 static void
unmap_file(void * data,off_t length,const char * base,const char * suffix)350 unmap_file(void *data, off_t length, const char *base, const char *suffix)
351 {
352 if (data == NULL)
353 return;
354 if (!innconf->tradindexedmmap)
355 free(data);
356 else {
357 if (munmap(data, length) < 0)
358 syswarn("tradindexed: cannot munmap %s.%s", base, suffix);
359 }
360 return;
361 }
362
363 /*
364 ** Unmap the index file.
365 */
366 static void
unmap_index(struct group_data * data)367 unmap_index(struct group_data *data)
368 {
369 unmap_file(data->index, data->indexlen, data->path, "IDX");
370 data->index = NULL;
371 }
372
373
374 /*
375 ** Unmap the data file.
376 */
377 static void
unmap_data(struct group_data * data)378 unmap_data(struct group_data *data)
379 {
380 unmap_file(data->data, data->datalen, data->path, "DAT");
381 data->data = NULL;
382 }
383
384 /*
385 ** Determine if the file handle associated with the index table is stale
386 */
387 static bool
stale_index(struct group_data * data)388 stale_index(struct group_data *data)
389 {
390 struct stat st;
391 int r;
392
393 r = fstat(data->indexfd, &st);
394 return r == -1 && errno == ESTALE;
395 }
396
397
398 /*
399 ** Determine if the file handle associated with the data table is stale
400 */
401 static bool
stale_data(struct group_data * data)402 stale_data(struct group_data *data)
403 {
404 struct stat st;
405 int r;
406
407 r = fstat(data->datafd, &st);
408 return r == -1 && errno == ESTALE;
409 }
410
411
412 /*
413 ** Retrieves the article metainformation stored in the index table (all the
414 ** stuff we can return without opening the data file). Takes the article
415 ** number and returns a pointer to the index entry. Also takes the high
416 ** water mark from the group index; this is used to decide whether to attempt
417 ** remapping of the index file if the current high water mark is too low.
418 */
419 const struct index_entry *
tdx_article_entry(struct group_data * data,ARTNUM article,ARTNUM high)420 tdx_article_entry(struct group_data *data, ARTNUM article, ARTNUM high)
421 {
422 struct index_entry *entry;
423 ARTNUM offset;
424
425 if (article > data->high && high > data->high) {
426 unmap_index(data);
427 map_index(data);
428 data->high = high;
429 /* Mark the data file to be remapped for the next opensearch. */
430 data->remapoutoforder = true;
431 } else if (innconf->nfsreader && stale_index(data))
432 unmap_index(data);
433 if (data->index == NULL)
434 if (!map_index(data))
435 return NULL;
436
437 if (article < data->base)
438 return NULL;
439 offset = article - data->base;
440 if (offset >= data->indexlen / sizeof(struct index_entry))
441 return NULL;
442 entry = data->index + offset;
443 if (entry->length == 0)
444 return NULL;
445 return entry;
446 }
447
448
449 /*
450 ** Begin an overview search. In addition to the bounds of the search, we
451 ** also take the high water mark from the group index; this is used to decide
452 ** whether or not to attempt remapping of the index file if the current high
453 ** water mark is too low.
454 */
455 struct search *
tdx_search_open(struct group_data * data,ARTNUM start,ARTNUM end,ARTNUM high)456 tdx_search_open(struct group_data *data, ARTNUM start, ARTNUM end, ARTNUM high)
457 {
458 struct search *search;
459
460 if (end < data->base)
461 return NULL;
462 if (end < start)
463 return NULL;
464
465 if ((end > data->high && high > data->high) || data->remapoutoforder) {
466 data->remapoutoforder = false;
467 unmap_data(data);
468 unmap_index(data);
469 map_index(data);
470 data->high = high;
471 }
472
473 if (start > data->high)
474 return NULL;
475
476 if (innconf->nfsreader && stale_index(data))
477 unmap_index(data);
478 if (data->index == NULL)
479 if (!map_index(data))
480 return NULL;
481 if (innconf->nfsreader && stale_data(data))
482 unmap_data(data);
483 if (data->data == NULL)
484 if (!map_data(data))
485 return NULL;
486
487 search = xmalloc(sizeof(struct search));
488 search->limit = end - data->base;
489 search->current = (start < data->base) ? 0 : start - data->base;
490 search->data = data;
491 search->data->refcount++;
492
493 return search;
494 }
495
496
497 /*
498 ** Return the next record in a search.
499 */
500 bool
tdx_search(struct search * search,struct article * artdata)501 tdx_search(struct search *search, struct article *artdata)
502 {
503 struct index_entry *entry;
504 size_t max;
505
506 if (search == NULL || search->data == NULL)
507 return false;
508 if (search->data->index == NULL || search->data->data == NULL)
509 return false;
510
511 max = (search->data->indexlen / sizeof(struct index_entry)) - 1;
512 entry = search->data->index + search->current;
513 while (search->current <= search->limit && search->current <= max) {
514 if (entry->length != 0)
515 break;
516 search->current++;
517 entry++;
518 }
519 if (search->current > search->limit || search->current > max)
520 return false;
521
522 /* There is a small chance that remapping the data file could make this
523 offset accessible, but changing the memory location in the middle of
524 a search conflicts with what we return for OVSTATICSEARCH. And it
525 seems not to be an issue in limited testing, although write caching
526 that leads to on-disk IDX and DAT being out of sync could trigger a
527 problem here. */
528 if (entry->offset + entry->length > search->data->datalen) {
529 search->data->remapoutoforder = true;
530 warn("Invalid or inaccessible entry for article %lu in %s.IDX:"
531 " offset %lu length %lu datalength %lu",
532 search->current + search->data->base, search->data->path,
533 (unsigned long) entry->offset, (unsigned long) entry->length,
534 (unsigned long) search->data->datalen);
535 return false;
536 }
537
538 artdata->number = search->current + search->data->base;
539 artdata->overview = search->data->data + entry->offset;
540 artdata->overlen = entry->length;
541 artdata->token = entry->token;
542 artdata->arrived = entry->arrived;
543 artdata->expires = entry->expires;
544
545 search->current++;
546 return true;
547 }
548
549
550 /*
551 ** End an overview search.
552 */
553 void
tdx_search_close(struct search * search)554 tdx_search_close(struct search *search)
555 {
556 if (search->data != NULL) {
557 search->data->refcount--;
558 if (search->data->refcount == 0)
559 tdx_data_close(search->data);
560 }
561 free(search);
562 }
563
564
565 /*
566 ** Given an article number, return an index base appropriate for that article
567 ** number. This includes a degree of slop so that we don't have to
568 ** constantly repack if the article numbers are clustered around a particular
569 ** value but don't come in order.
570 */
571 static ARTNUM
index_base(ARTNUM artnum)572 index_base(ARTNUM artnum)
573 {
574 return (artnum > 128) ? (artnum - 128) : 1;
575 }
576
577
578 /*
579 ** Store the data for a single article into the overview files for a group.
580 ** Assumes any necessary repacking has already been done. If the base value
581 ** in the group_data structure is 0, assumes this is the first time we've
582 ** written overview information to this group and sets it appropriately.
583 */
584 bool
tdx_data_store(struct group_data * data,const struct article * article)585 tdx_data_store(struct group_data *data, const struct article *article)
586 {
587 struct index_entry entry;
588 off_t offset;
589
590 if (!data->writable)
591 return false;
592 if (data->base == 0)
593 data->base = index_base(article->number);
594 if (data->base > article->number) {
595 warn("tradindexed: cannot add %lu to %s.IDX, base == %lu",
596 article->number, data->path, data->base);
597 return false;
598 }
599
600 /* Write out the data and fill in the index entry. */
601 memset(&entry, 0, sizeof(entry));
602 if (xwrite(data->datafd, article->overview, article->overlen) < 0) {
603 syswarn("tradindexed: cannot append %lu of data for %lu to %s.DAT",
604 (unsigned long) article->overlen, article->number,
605 data->path);
606 return false;
607 }
608 entry.offset = lseek(data->datafd, 0, SEEK_CUR);
609 if (entry.offset < 0) {
610 syswarn("tradindexed: cannot get offset for article %lu in %s.DAT",
611 article->number, data->path);
612 return false;
613 }
614 entry.length = article->overlen;
615 entry.offset -= entry.length;
616 entry.arrived = article->arrived;
617 entry.expires = article->expires;
618 entry.token = article->token;
619
620 /* Write out the index entry. */
621 offset = (article->number - data->base) * sizeof(struct index_entry);
622 if (xpwrite(data->indexfd, &entry, sizeof(entry), offset) < 0) {
623 syswarn("tradindexed: cannot write index record for %lu in %s.IDX",
624 article->number, data->path);
625 return false;
626 }
627 return true;
628 }
629
630
631 /*
632 ** Cancels a particular article by removing its index entry. The data will
633 ** still be present in the data file until the next expire run, but it won't
634 ** be returned by searches.
635 */
636 bool
tdx_data_cancel(struct group_data * data,ARTNUM artnum)637 tdx_data_cancel(struct group_data *data, ARTNUM artnum)
638 {
639 static const struct index_entry empty;
640 off_t offset;
641
642 if (!data->writable)
643 return false;
644 if (data->base == 0 || artnum < data->base || artnum > data->high)
645 return false;
646 offset = (artnum - data->base) * sizeof(struct index_entry);
647 if (xpwrite(data->indexfd, &empty, sizeof(empty), offset) < 0) {
648 syswarn("tradindexed: cannot cancel index record for %lu in %s.IDX",
649 artnum, data->path);
650 return false;
651 }
652 return true;
653 }
654
655
656 /*
657 ** Start the process of packing a group (rewriting its index file so that it
658 ** uses a different article base). Takes the article number of an article
659 ** that needs to be written to the index file and is below the current base.
660 ** Returns the true success and false on failure, and sets data->base to the
661 ** new article base and data->indexinode to the new inode number. At the
662 ** conclusion of this routine, the new index file has been created, but it
663 ** has not yet been moved into place; that is done by tdx_data_pack_finish.
664 */
665 bool
tdx_data_pack_start(struct group_data * data,ARTNUM artnum)666 tdx_data_pack_start(struct group_data *data, ARTNUM artnum)
667 {
668 ARTNUM base;
669 unsigned long delta;
670 int fd;
671 char *idxfile;
672 struct stat st;
673
674 if (!data->writable)
675 return false;
676 if (data->base <= artnum) {
677 warn("tradindexed: tdx_data_pack_start called unnecessarily");
678 return false;
679 }
680
681 /* Open the new index file. */
682 base = index_base(artnum);
683 delta = data->base - base;
684 fd = file_open(data->path, "IDX-NEW", true, false);
685 if (fd < 0)
686 return false;
687 if (fstat(fd, &st) < 0) {
688 warn("tradindexed: cannot stat %s.IDX-NEW", data->path);
689 goto fail;
690 }
691
692 /* For convenience, memory map the old index file. */
693 unmap_index(data);
694 if (!map_index(data))
695 goto fail;
696
697 /* Write the contents of the old index file to the new index file. */
698 if (lseek(fd, delta * sizeof(struct index_entry), SEEK_SET) < 0) {
699 syswarn("tradindexed: cannot seek in %s.IDX-NEW", data->path);
700 goto fail;
701 }
702 if (xwrite(fd, data->index, data->indexlen) < 0) {
703 syswarn("tradindexed: cannot write to %s.IDX-NEW", data->path);
704 goto fail;
705 }
706 if (close(fd) < 0) {
707 syswarn("tradindexed: cannot close %s.IDX-NEW", data->path);
708 goto fail;
709 }
710 data->base = base;
711 data->indexinode = st.st_ino;
712 return true;
713
714 fail:
715 if (fd >= 0) {
716 close(fd);
717 idxfile = concat(data->path, ".IDX-NEW", (char *) 0);
718 if (unlink(idxfile) < 0)
719 syswarn("tradindexed: cannot unlink %s", idxfile);
720 free(idxfile);
721 }
722 return false;
723 }
724
725
726 /*
727 ** Finish the process of packing a group by replacing the new index with the
728 ** old index. Also reopen the index file and update indexinode to keep our
729 ** caller from having to close and reopen the index file themselves.
730 */
731 bool
tdx_data_pack_finish(struct group_data * data)732 tdx_data_pack_finish(struct group_data *data)
733 {
734 char *newidx, *idx;
735
736 if (!data->writable)
737 return false;
738 newidx = concat(data->path, ".IDX-NEW", (char *) 0);
739 idx = concat(data->path, ".IDX", (char *) 0);
740 if (rename(newidx, idx) < 0) {
741 syswarn("tradindexed: cannot rename %s to %s", newidx, idx);
742 unlink(newidx);
743 free(newidx);
744 free(idx);
745 return false;
746 } else {
747 free(newidx);
748 free(idx);
749 if (!file_open_index(data, NULL))
750 return false;
751 return true;
752 }
753 }
754
755
756 /*
757 ** Open the data files for a group data rebuild, and return a struct
758 ** group_data for the new files. Calling this function doesn't interfere
759 ** with the existing data for the group. Either tdx_data_rebuild_abort or
760 ** tdx_data_rebuild_finish should be called on the returned struct group_data
761 ** when the caller is done.
762 */
763 struct group_data *
tdx_data_rebuild_start(const char * group)764 tdx_data_rebuild_start(const char *group)
765 {
766 struct group_data *data;
767
768 data = tdx_data_new(group, true);
769 tdx_data_delete(group, "-NEW");
770 if (!file_open_index(data, "IDX-NEW"))
771 goto fail;
772 if (!file_open_data(data, "DAT-NEW"))
773 goto fail;
774 return data;
775
776 fail:
777 tdx_data_delete(group, "-NEW");
778 tdx_data_close(data);
779 return NULL;
780 }
781
782
783 /*
784 ** Finish a rebuild by renaming the new index and data files to their
785 ** permanent names.
786 */
787 bool
tdx_data_rebuild_finish(const char * group)788 tdx_data_rebuild_finish(const char *group)
789 {
790 char *base, *newidx, *bakidx, *idx, *newdat, *dat;
791 bool saved = false;
792
793 base = group_path(group);
794 idx = concat(base, ".IDX", (char *) 0);
795 newidx = concat(base, ".IDX-NEW", (char *) 0);
796 bakidx = concat(base, ".IDX-BAK", (char *) 0);
797 dat = concat(base, ".DAT", (char *) 0);
798 newdat = concat(base, ".DAT-NEW", (char *) 0);
799 free(base);
800 if (rename(idx, bakidx) < 0) {
801 syswarn("tradindexed: cannot rename %s to %s", idx, bakidx);
802 goto fail;
803 } else {
804 saved = true;
805 }
806 if (rename(newidx, idx) < 0) {
807 syswarn("tradindexed: cannot rename %s to %s", newidx, idx);
808 goto fail;
809 }
810 if (rename(newdat, dat) < 0) {
811 syswarn("tradindexed: cannot rename %s to %s", newdat, dat);
812 goto fail;
813 }
814 if (unlink(bakidx) < 0)
815 syswarn("tradindexed: cannot remove backup %s", bakidx);
816 free(idx);
817 free(newidx);
818 free(bakidx);
819 free(dat);
820 free(newdat);
821 return true;
822
823 fail:
824 if (saved && rename(bakidx, idx) < 0)
825 syswarn("tradindexed: cannot restore old index %s", bakidx);
826 free(idx);
827 free(newidx);
828 free(bakidx);
829 free(dat);
830 free(newdat);
831 return false;
832 }
833
834
835 /*
836 ** Do the main work of expiring a group. Step through each article in the
837 ** group, only writing the unexpired entries out to the new group. There's
838 ** probably some room for optimization here for newsgroups that don't expire
839 ** so that the files don't have to be rewritten, or newsgroups where all the
840 ** data at the end of the file is still good and just needs to be moved
841 ** as-is.
842 */
843 bool
tdx_data_expire_start(const char * group,struct group_data * data,struct group_entry * index,struct history * history)844 tdx_data_expire_start(const char *group, struct group_data *data,
845 struct group_entry *index, struct history *history)
846 {
847 struct group_data *new_data;
848 struct search *search;
849 struct article article;
850 ARTNUM high;
851
852 new_data = tdx_data_rebuild_start(group);
853 if (new_data == NULL)
854 return false;
855 index->indexinode = new_data->indexinode;
856
857 /* Try to make sure that the search range is okay for even an empty group
858 so that we can treat all errors on opening a search as errors. */
859 high = index->high > 0 ? index->high : data->base;
860 new_data->high = high;
861 search = tdx_search_open(data, data->base, high, high);
862 if (search == NULL)
863 goto fail;
864
865 /* Loop through all of the articles in the group, adding the ones that are
866 still valid to the new index. */
867 while (tdx_search(search, &article)) {
868 ARTHANDLE *ah;
869
870 if (!SMprobe(EXPENSIVESTAT, &article.token, NULL) || OVstatall) {
871 ah = SMretrieve(article.token, RETR_STAT);
872 if (ah == NULL)
873 continue;
874 SMfreearticle(ah);
875 } else {
876 if (!OVhisthasmsgid(history, article.overview))
877 continue;
878 }
879 if (innconf->groupbaseexpiry)
880 if (OVgroupbasedexpire(article.token, group, article.overview,
881 article.overlen, article.arrived,
882 article.expires))
883 continue;
884 if (!tdx_data_store(new_data, &article))
885 goto fail;
886 if (index->base == 0) {
887 index->base = new_data->base;
888 index->low = article.number;
889 }
890 if (article.number > index->high)
891 index->high = article.number;
892 index->count++;
893 }
894
895 /* Done; the rest happens in tdx_data_rebuild_finish. */
896 tdx_data_close(new_data);
897 return true;
898
899 fail:
900 tdx_data_delete(group, "-NEW");
901 tdx_data_close(new_data);
902 return false;
903 }
904
905
906 /*
907 ** Close the data files for a group and free the data structure.
908 */
909 void
tdx_data_close(struct group_data * data)910 tdx_data_close(struct group_data *data)
911 {
912 unmap_index(data);
913 unmap_data(data);
914 if (data->indexfd >= 0)
915 close(data->indexfd);
916 if (data->datafd >= 0)
917 close(data->datafd);
918 free(data->path);
919 free(data);
920 }
921
922
923 /*
924 ** Delete the data files for a particular group, called when that group is
925 ** deleted from the server. Takes an optional suffix, which if present is
926 ** appended to the ends of the file names (used by expire to delete the -NEW
927 ** versions of the files).
928 */
929 void
tdx_data_delete(const char * group,const char * suffix)930 tdx_data_delete(const char *group, const char *suffix)
931 {
932 char *path, *idx, *dat;
933
934 path = group_path(group);
935 idx = concat(path, ".IDX", suffix, (char *) 0);
936 dat = concat(path, ".DAT", suffix, (char *) 0);
937 if (unlink(idx) < 0 && errno != ENOENT)
938 syswarn("tradindexed: cannot unlink %s", idx);
939 if (unlink(dat) < 0 && errno != ENOENT)
940 syswarn("tradindexed: cannot unlink %s", dat);
941 free(dat);
942 free(idx);
943 free(path);
944 }
945
946
947 /*
948 ** RECOVERY AND AUDITING
949 **
950 ** All code below this point is not used in the normal operations of the
951 ** overview method. Instead, it's code to dump various data structures or
952 ** audit them for consistency, used by recovery tools and inspection tools.
953 */
954
955 /*
956 ** Dump the index file for a given group in human-readable format.
957 */
958 void
tdx_data_index_dump(struct group_data * data,FILE * output)959 tdx_data_index_dump(struct group_data *data, FILE *output)
960 {
961 ARTNUM current;
962 struct index_entry *entry, *end;
963
964 if (data->index == NULL)
965 if (!map_index(data))
966 return;
967
968 current = data->base;
969 end = data->index + (data->indexlen / sizeof(struct index_entry));
970 for (entry = data->index; entry < end; entry++) {
971 fprintf(output, "%lu %lu %lu %lu %lu %s\n", current,
972 (unsigned long) entry->offset, (unsigned long) entry->length,
973 (unsigned long) entry->arrived,
974 (unsigned long) entry->expires, TokenToText(entry->token));
975 current++;
976 }
977 }
978
979
980 /*
981 ** Audit a specific index entry for a particular article. If there's
982 ** anything wrong with it, we delete it; to repair a particular group, it's
983 ** best to just regenerate it from scratch.
984 */
985 static void
entry_audit(struct group_data * data,struct index_entry * entry,const char * group,ARTNUM article,bool fix)986 entry_audit(struct group_data *data, struct index_entry *entry,
987 const char *group, ARTNUM article, bool fix)
988 {
989 struct index_entry new_entry;
990 off_t offset;
991
992 if (entry->length < 0) {
993 warn("tradindexed: negative length %d in %s:%lu", entry->length,
994 group, article);
995 if (fix)
996 goto clear;
997 return;
998 }
999 if (entry->offset > data->datalen || entry->length > data->datalen) {
1000 warn("tradindexed: offset %lu or length %lu out of bounds for %s:%lu",
1001 (unsigned long) entry->offset, (unsigned long) entry->length,
1002 group, article);
1003 if (fix)
1004 goto clear;
1005 return;
1006 }
1007 if (entry->offset + entry->length > data->datalen) {
1008 warn("tradindexed: offset %lu plus length %lu out of bounds for"
1009 " %s:%lu", (unsigned long) entry->offset,
1010 (unsigned long) entry->length, group, article);
1011 if (fix)
1012 goto clear;
1013 return;
1014 }
1015 if (!overview_check(data->data + entry->offset, entry->length, article)) {
1016 warn("tradindexed: malformed overview data for %s:%lu", group,
1017 article);
1018 if (fix)
1019 goto clear;
1020 }
1021 return;
1022
1023 clear:
1024 new_entry = *entry;
1025 new_entry.offset = 0;
1026 new_entry.length = 0;
1027 offset = (entry - data->index) * sizeof(struct index_entry);
1028 if (xpwrite(data->indexfd, &new_entry, sizeof(new_entry), offset) != 0)
1029 warn("tradindexed: unable to repair %s:%lu", group, article);
1030 }
1031
1032
1033 /*
1034 ** Audit the data for a particular group. Takes the index entry from the
1035 ** group.index file and optionally corrects any problems with the data or the
1036 ** index entry based on the contents of the data.
1037 */
1038 void
tdx_data_audit(const char * group,struct group_entry * index,bool fix)1039 tdx_data_audit(const char *group, struct group_entry *index, bool fix)
1040 {
1041 struct group_data *data;
1042 struct index_entry *entry;
1043 long count;
1044 off_t expected;
1045 unsigned long entries, current;
1046 ARTNUM low = 0;
1047 bool changed = false;
1048
1049 data = tdx_data_new(group, true);
1050 if (!tdx_data_open_files(data))
1051 return;
1052 if (!map_index(data))
1053 goto end;
1054 if (!map_data(data))
1055 goto end;
1056
1057 /* Check the inode of the index. */
1058 if (data->indexinode != index->indexinode) {
1059 warn("tradindexed: index inode mismatch for %s: %lu != %lu", group,
1060 (unsigned long) data->indexinode,
1061 (unsigned long) index->indexinode);
1062 if (fix) {
1063 index->indexinode = data->indexinode;
1064 changed = true;
1065 }
1066 }
1067
1068 /* Check the index size. */
1069 entries = data->indexlen / sizeof(struct index_entry);
1070 expected = entries * sizeof(struct index_entry);
1071 if (data->indexlen != expected) {
1072 warn("tradindexed: %lu bytes of trailing trash in %s.IDX",
1073 (unsigned long)(data->indexlen - expected), data->path);
1074 if (fix) {
1075 unmap_index(data);
1076 if (ftruncate(data->indexfd, expected) < 0)
1077 syswarn("tradindexed: cannot truncate %s.IDX", data->path);
1078 if (!map_index(data))
1079 goto end;
1080 }
1081 }
1082
1083 /* Now iterate through all of the index entries. In addition to checking
1084 each one individually, also count the number of valid entries to check
1085 the count in the index and verify that the low water mark is
1086 correct. */
1087 for (current = 0, count = 0; current < entries; current++) {
1088 entry = &data->index[current];
1089 if (entry->length == 0)
1090 continue;
1091 entry_audit(data, entry, group, index->base + current, fix);
1092 if (entry->length != 0) {
1093 if (low == 0)
1094 low = index->base + current;
1095 count++;
1096 }
1097 }
1098 if (index->low != low && entries != 0) {
1099 warn("tradindexed: low water mark incorrect for %s: %lu != %lu",
1100 group, low, index->low);
1101 if (fix) {
1102 index->low = low;
1103 changed = true;
1104 }
1105 }
1106 if (index->count != count) {
1107 warn("tradindexed: count incorrect for %s: %lu != %lu", group,
1108 (unsigned long) count, (unsigned long) index->count);
1109 if (fix) {
1110 index->count = count;
1111 changed = true;
1112 }
1113 }
1114
1115 /* All done. Close things down and flush the data we changed, if
1116 necessary. */
1117 if (changed)
1118 inn_msync_page(index, sizeof(*index), MS_ASYNC);
1119
1120 end:
1121 tdx_data_close(data);
1122 }
1123