1 /*
2 * (c) Copyright 1990, Kim Fabricius Storm. All rights reserved.
3 * Copyright (c) 1996-2005 Michael T Pins. All rights reserved.
4 *
5 * Expire will remove all entries in the index and data files
6 * corresponding to the articles before the first article registered
7 * in the active file. No attempt is made to eliminate other
8 * expired articles.
9 */
10
11 #include <stdlib.h>
12 #include <ctype.h>
13 #include "config.h"
14 #include "global.h"
15 #include "db.h"
16 #include "master.h"
17 #include "nntp.h"
18 #include "nn_term.h"
19
20 /* expire.c */
21
22 static int sort_art_list(register article_number * f1, register article_number * f2);
23 static article_number *get_article_list(char *dir);
24 static long expire_in_database(register group_header * gh);
25 static long expire_sliding(register group_header * gh);
26 static void block_group(register group_header * gh);
27 static void unblock_group(register group_header * gh);
28
29
30 extern int trace, debug_mode;
31 extern int nntp_failed;
32
33 /*
34 * Expire methods:
35 * 1: read directory and reuse database info.
36 * 2: "slide" index and datafiles (may leave unexpired art. in database)
37 * 3: recollect group to expire (also if "min" still exists)
38 */
39
40 int expire_method = 1; /* expire method */
41 int recollect_method = 1; /* recollection method -- see
42 * do_expire */
43 int expire_level = 0; /* automatic expiration detection */
44
45 #ifdef HAVE_DIRECTORY
46 static article_number *article_list = NULL;
47 static long art_list_length = 0;
48
49 static int
sort_art_list(register article_number * f1,register article_number * f2)50 sort_art_list(register article_number * f1, register article_number * f2)
51 {
52 return (*f1 < *f2) ? -1 : (*f1 == *f2) ? 0 : 1;
53 }
54
55 static article_number *
get_article_list(char * dir)56 get_article_list(char *dir)
57 {
58 DIR *dirp;
59 register Direntry *dp;
60 register char c, *pp, *cp;
61 register article_number *art;
62 register long count; /* No. of completions plus one */
63
64 if ((dirp = opendir(dir)) == NULL)
65 return NULL; /* tough luck */
66
67 art = article_list;
68 count = 0;
69
70 while ((dp = readdir(dirp)) != NULL) {
71 cp = dp->d_name;
72
73 #ifdef FAKED_DIRECTORY
74 if (dp->d_ino == 0)
75 continue;
76 cp[14] = NUL;
77 #endif
78
79 for (pp = cp; (c = *pp++);)
80 if (!isascii(c) || !isdigit(c))
81 break;
82 if (c)
83 continue;
84
85 if (count == art_list_length) {
86 art_list_length += 250;
87 article_list = resizeobj(article_list, article_number, art_list_length + 1);
88 art = article_list + count;
89 }
90 *art++ = atol(cp);
91 count++;
92 }
93 closedir(dirp);
94
95 if (article_list != NULL) {
96 *art = 0;
97 if (count > 1)
98 quicksort(article_list, count, article_number, sort_art_list);
99 }
100 return article_list;
101 }
102
103 static long
expire_in_database(register group_header * gh)104 expire_in_database(register group_header * gh)
105 {
106 FILE *old, *data = NULL, *ix = NULL;
107 off_t old_max_offset;
108 register article_number *list;
109 article_number old_last_article;
110 long count;
111
112 if (gh->first_db_article > gh->last_db_article)
113 return 0;
114
115 if (gh->last_db_article == 0)
116 return 0;
117
118 if (!init_group(gh))
119 return 0;
120
121 if (debug_mode == 1) {
122 printf("\t\tExp %s (%ld..%ld)\r",
123 gh->group_name, gh->first_db_article, gh->last_db_article);
124 fl;
125 }
126 count = 0;
127
128 /* get list of currently available articles in the group */
129
130 #ifdef NNTP
131 if (use_nntp)
132 list = nntp_get_article_list(gh);
133 else
134 #endif
135
136 list = get_article_list(".");
137
138 if (list == NULL || *list == 0) {
139
140 #ifdef NNTP
141 if (nntp_failed == 2) {
142 log_entry('N', "NNTP server supports neither LISTGROUP nor XHDR");
143 sys_error("Cannot use specified expire method (NNTP limitations)");
144 }
145 if (nntp_failed)
146 return -1;
147 #endif
148
149 if (debug_mode == 1) {
150 printf("\rempty");
151 fl;
152 }
153 /* group is empty - clean it */
154 count = gh->last_db_article - gh->first_db_article + 1;
155 clean_group(gh);
156 gh->last_db_article = gh->last_a_article;
157 gh->first_db_article = gh->last_db_article + 1;
158 db_write_group(gh);
159 return count;
160 }
161
162 /*
163 * Clean & block the group while expire is working on it.
164 */
165
166 gh->first_db_article = 0;
167 old_last_article = gh->last_db_article;
168 gh->last_db_article = 0;
169
170 gh->index_write_offset = 0;
171 old_max_offset = gh->data_write_offset;
172 gh->data_write_offset = 0;
173
174 gh->master_flag &= ~M_EXPIRE;
175 gh->master_flag |= M_BLOCKED;
176
177 db_write_group(gh);
178
179 /*
180 * We ignore the old index file, and we unlink the data file immediately
181 * after open because we want to write a new.
182 */
183
184 (void) open_data_file(gh, 'x', -1);
185 old = open_data_file(gh, 'd', OPEN_READ | OPEN_UNLINK);
186 if (old == NULL)
187 goto out;
188
189 data = open_data_file(gh, 'd', OPEN_CREATE | MUST_EXIST);
190 ix = open_data_file(gh, 'x', OPEN_CREATE | MUST_EXIST);
191
192 while (ftell(old) < old_max_offset) {
193 if (s_hangup) {
194 /* ok, this is what we got -- let collect get the rest */
195 old_last_article = gh->last_db_article;
196 break;
197 }
198
199 /*
200 * maybe not enough articles, or last one is incomplete we take what
201 * there is, and leave the rest to do_collect(void) It may actually
202 * be ok if the last articles have been expired/ cancelled!
203 */
204
205 if (db_read_art(old) <= 0) {
206 break;
207 }
208 if (debug_mode == 1) {
209 printf("\r%ld", (long) db_hdr.dh_number);
210 fl;
211 }
212 /* check whether we want this article */
213
214 while (*list && db_hdr.dh_number > *list) {
215 /* potentially, we have a problem here: there might be an */
216 /* article in the directory which is not in the database! */
217 /* For the moment we just ignore this - it might be a bad */
218 /* article which has been rejected!! */
219 list++;
220 }
221
222 if (*list == 0) {
223 /* no more articles in the directory - the rest must be */
224 /* expired. So we ignore the rest of the data file */
225 break;
226 }
227 if (db_hdr.dh_number < *list) {
228 /* the current article from the data file isn't in the */
229 /* article list, so it must be expired! */
230 count++;
231 if (debug_mode == 1) {
232 printf("\t%ld", count);
233 fl;
234 }
235 continue;
236 }
237 if (gh->first_db_article == 0) {
238 gh->first_db_article = db_hdr.dh_number;
239 gh->last_db_article = db_hdr.dh_number - 1;
240 }
241 if (gh->last_db_article < db_hdr.dh_number) {
242 gh->data_write_offset = ftell(data);
243
244 /* must fill gab between last index and current article */
245 while (gh->last_db_article < db_hdr.dh_number) {
246 if (!db_write_offset(ix, &(gh->data_write_offset)))
247 write_error();
248 gh->last_db_article++;
249 }
250 }
251 if (db_write_art(data) < 0)
252 write_error();
253 }
254
255 if (gh->first_db_article == 0) {
256 gh->first_db_article = old_last_article + 1;
257 gh->last_db_article = old_last_article;
258 } else {
259 gh->data_write_offset = ftell(data);
260 while (gh->last_db_article < old_last_article) {
261 /* must fill gab between last index and last article */
262 ++gh->last_db_article;
263 if (!db_write_offset(ix, &(gh->data_write_offset)))
264 write_error();
265 }
266 gh->index_write_offset = ftell(ix);
267 }
268
269 gh->master_flag &= ~M_BLOCKED;
270
271 db_write_group(gh);
272
273 out:
274 if (old)
275 fclose(old);
276 if (data)
277 fclose(data);
278 if (ix)
279 fclose(ix);
280
281 if (debug_mode)
282 putchar(NL);
283
284 return count;
285 }
286
287 #else
288 #define expire_in_database expire_sliding
289 #endif
290
291 static long
expire_sliding(register group_header * gh)292 expire_sliding(register group_header * gh)
293 {
294 FILE *old_x, *old_d;
295 FILE *new;
296 long data_offset, new_offset;
297 long count, expire_count, index_offset;
298 char *err_message;
299
300 #define expire_error(msg) { \
301 err_message = msg; \
302 goto error_handler; \
303 }
304
305 if (!init_group(gh))
306 return 0;
307
308 #ifdef RENUMBER_DANGER
309
310 /*
311 * check whether new first article is collected
312 */
313
314 if (!art_collected(gh, gh->first_a_article)) {
315 expire_count = gh->first_db_article - gh->last_db_article + 1;
316 err_message = NULL;
317 goto error_handler; /* renumbering, collect from start */
318 }
319 #else
320 if (gh->first_a_article <= gh->first_db_article)
321 return 0;
322 #endif
323
324 expire_count = gh->first_a_article - gh->first_db_article;
325
326 new = NULL;
327
328 /*
329 * Open old files, unlink after open
330 */
331
332 old_x = open_data_file(gh, 'x', OPEN_READ | OPEN_UNLINK);
333 old_d = open_data_file(gh, 'd', OPEN_READ | OPEN_UNLINK);
334
335 if (old_x == NULL || old_d == NULL)
336 expire_error("INDEX or DATA file missing");
337
338 /*
339 * Create new index file; copy from old
340 */
341
342 new = open_data_file(gh, 'x', OPEN_CREATE);
343 if (new == NULL)
344 expire_error("INDEX: cannot create");
345
346 /*
347 * index_offset is the offset into the old index file for the first entry
348 * in the new index file
349 */
350
351 index_offset = get_index_offset(gh, gh->first_a_article);
352
353 /*
354 * adjust the group's index write offset (the next free entry)
355 */
356
357 gh->index_write_offset -= index_offset;
358
359 /*
360 * calculate the number of entries to copy
361 */
362
363 count = gh->index_write_offset / sizeof(long);
364
365 /*
366 * data offset is the offset into the old data file for the first byte in
367 * the new data file; it is initialized in the loop below, by reading the
368 * entry in the old index file at offset 'index_offset'.
369 */
370
371 data_offset = (off_t) 0;
372
373 /*
374 * read 'count' entries from the old index file starting from
375 * index_offset, subtract the 'data_offset', and output the new offset to
376 * the new index file.
377 */
378
379 fseek(old_x, index_offset, 0);
380
381 while (--count >= 0) {
382 if (!db_read_offset(old_x, &new_offset))
383 expire_error("INDEX: too short");
384
385 if (data_offset == 0)
386 data_offset = new_offset;
387
388 new_offset -= data_offset;
389 if (!db_write_offset(new, &new_offset))
390 expire_error("NEW INDEX: cannot write");
391 }
392
393 fclose(new);
394 fclose(old_x);
395 old_x = NULL;
396
397 /*
398 * copy from old data file to new data file
399 */
400
401 new = open_data_file(gh, 'd', OPEN_CREATE);
402 if (new == NULL)
403 expire_error("DATA: cannot create");
404
405 /*
406 * calculate offset for next free entry in the new data file
407 */
408
409 gh->data_write_offset -= data_offset;
410
411 /*
412 * calculate number of bytes to copy (piece of cake)
413 */
414
415 count = gh->data_write_offset;
416
417 /*
418 * copy 'count' bytes from the old data file, starting at offset
419 * 'data_offset', to the new data file
420 */
421
422 fseek(old_d, data_offset, 0);
423 while (count > 0) {
424 char block[1024];
425 int count1;
426
427 count1 = fread(block, sizeof(char), 1024, old_d);
428 if (count1 <= 0)
429 expire_error("DATA: read error");
430
431 if (fwrite(block, sizeof(char), count1, new) != count1)
432 expire_error("DATA: write error");
433
434 count -= count1;
435 }
436
437 fclose(new);
438 fclose(old_d);
439
440 /*
441 * Update group entry
442 */
443
444 gh->first_db_article = gh->first_a_article;
445
446 /*
447 * Return number of expired articles
448 */
449
450 return expire_count;
451
452 /*
453 * Errors end up here. We simply recollect the whole group once more.
454 */
455
456 error_handler:
457
458 if (new)
459 fclose(new);
460 if (old_x)
461 fclose(old_x);
462 if (old_d)
463 fclose(old_d);
464
465 if (err_message)
466 log_entry('E', "Expire Error (%s): %s", gh->group_name, err_message);
467
468 clean_group(gh);
469
470 /* will be saved & unblocked later */
471
472 /*
473 * We cannot say whether any articles actually had to be expired, but
474 * then we must guess...
475 */
476
477 return expire_count;
478 }
479
480 static void
block_group(register group_header * gh)481 block_group(register group_header * gh)
482 {
483 if ((gh->master_flag & M_BLOCKED) == 0) {
484 gh->master_flag |= M_BLOCKED;
485 db_write_group(gh);
486 }
487 }
488
489 static void
unblock_group(register group_header * gh)490 unblock_group(register group_header * gh)
491 {
492 if (gh->master_flag & M_BLOCKED) {
493 gh->master_flag &= ~(M_BLOCKED | M_EXPIRE);
494 db_write_group(gh);
495 }
496 }
497
498 int
do_expire(void)499 do_expire(void)
500 {
501 register group_header *gh;
502 long exp_article_count, temp;
503 int exp_group_count, must_expire;
504 time_t start_time;
505
506 must_expire = 0;
507
508 Loop_Groups_Header(gh) {
509 if (s_hangup)
510 break;
511 if (gh->master_flag & M_IGNORE_GROUP)
512 continue;
513
514 if ((gh->master_flag & M_VALID) == 0) {
515 log_entry('X', "Group %s removed", gh->group_name);
516 gh->master_flag |= M_IGNORE_A;
517 clean_group(gh);
518 continue;
519 }
520
521 #ifdef RENUMBER_DANGER
522 if (gh->last_db_article > gh->last_a_article ||
523 gh->first_db_article > gh->first_a_article) {
524 log_entry('X', "group %s renumbered", gh->group_name);
525 clean_group(gh);
526 continue;
527 }
528 #endif
529
530 if (gh->master_flag & M_AUTO_RECOLLECT) {
531 switch (recollect_method) {
532 case 1: /* expire when new articles arrive */
533 if (gh->last_a_article <= gh->last_db_article)
534 break;
535 /* FALLTHRU */
536 case 2: /* expire unconditionally */
537 gh->master_flag |= M_EXPIRE;
538 must_expire = 1;
539 continue;
540
541 case 3: /* clean when new articles arrive */
542 if (gh->last_a_article <= gh->last_db_article)
543 break;
544 /* FALLTHRU */
545 case 4: /* clean unconditionally */
546 gh->master_flag |= M_MUST_CLEAN;
547 continue;
548 default: /* ignore auto-recollect */
549 break;
550 }
551 }
552 if (gh->index_write_offset > 0) {
553 if (gh->first_a_article > gh->last_db_article) {
554 if (trace)
555 log_entry('T', "%s expire void", gh->group_name);
556 if (debug_mode)
557 printf("%s expire void\n", gh->group_name);
558 clean_group(gh);
559 continue;
560 }
561 }
562 if (gh->master_flag & M_EXPIRE) {
563 must_expire = 1;
564 continue;
565 }
566 if (expire_level > 0 &&
567 (gh->first_db_article + expire_level) <= gh->first_a_article) {
568 if (trace)
569 log_entry('T', "%s expire level", gh->group_name);
570 if (debug_mode)
571 printf("Expire level: %s\n", gh->group_name);
572 gh->master_flag |= M_EXPIRE;
573 must_expire = 1;
574 continue;
575 }
576 }
577
578 if (!must_expire)
579 return 1;
580
581 start_time = cur_time();
582 exp_article_count = exp_group_count = 0;
583 temp = 0;
584
585 Loop_Groups_Header(gh) {
586 if (s_hangup) {
587 temp = -1;
588 break;
589 }
590 if (gh->master_flag & M_IGNORE_GROUP)
591 continue;
592 if ((gh->master_flag & M_EXPIRE) == 0)
593 continue;
594 if (gh->master_flag & M_MUST_CLEAN)
595 continue;
596
597 if (trace)
598 log_entry('T', "Exp %s (%ld -> %ld)", gh->group_name,
599 (long) gh->first_db_article, (long) gh->first_a_article);
600
601 switch (expire_method) {
602 case 1:
603 block_group(gh);
604 temp = expire_in_database(gh);
605 unblock_group(gh);
606 break;
607
608 case 2:
609 block_group(gh);
610 temp = expire_sliding(gh);
611 unblock_group(gh);
612 break;
613
614 case 4:
615 temp = gh->first_a_article - gh->first_db_article;
616 if (temp <= 0)
617 break;
618 /* FALLTHRU */
619 case 3:
620 gh->master_flag |= M_MUST_CLEAN;
621 break;
622
623 }
624
625 #ifdef NNTP
626 if (nntp_failed) {
627 /* connection broken while reading article list */
628 /* so no harm was done - leave the group to be expired */
629 /* again on next expire */
630 break;
631 }
632 #endif
633
634 if (temp > 0) {
635 exp_article_count += temp;
636 exp_group_count++;
637 }
638 }
639
640 if (exp_article_count > 0)
641 log_entry('X', "Expire: %ld art, %d gr, %ld s",
642 exp_article_count, exp_group_count,
643 (long) (cur_time() - start_time));
644
645 return temp >= 0;
646 }
647