1 /*
2 **  Expire news articles.
3 */
4 
5 #include "portable/system.h"
6 
7 #include <ctype.h>
8 #include <errno.h>
9 #include <sys/stat.h>
10 #include <syslog.h>
11 #include <time.h>
12 
13 #include "inn/history.h"
14 #include "inn/innconf.h"
15 #include "inn/inndcomm.h"
16 #include "inn/libinn.h"
17 #include "inn/messages.h"
18 #include "inn/newsuser.h"
19 #include "inn/paths.h"
20 #include "inn/storage.h"
21 
22 
23 typedef struct _EXPIRECLASS {
24     time_t Keep;
25     time_t Default;
26     time_t Purge;
27     bool Missing;
28     bool ReportedMissing;
29 } EXPIRECLASS;
30 
31 /*
32 **  Expire-specific stuff.
33 */
34 #define MAGIC_TIME 49710.
35 
36 static bool EXPtracing;
37 static bool EXPusepost;
38 static bool Ignoreselfexpire = false;
39 static FILE *EXPunlinkfile;
40 static EXPIRECLASS EXPclasses[NUM_STORAGE_CLASSES + 1];
41 static char *EXPreason;
42 static time_t EXPremember;
43 static time_t Now;
44 static time_t RealNow;
45 
46 /* Statistics; for -v flag. */
47 static char *EXPgraph;
48 static int EXPverbose;
49 static long EXPprocessed;
50 static long EXPunlinked;
51 static long EXPallgone;
52 static long EXPstillhere;
53 static struct history *History;
54 static char *NHistory;
55 
56 static void CleanupAndExit(bool Server, bool Paused, int x)
57     __attribute__((__noreturn__));
58 static void Usage(void) __attribute__((__noreturn__));
59 
60 static int EXPsplit(char *p, char sep, char **argv, int count);
61 
62 enum KR
63 {
64     Keep,
65     Remove
66 };
67 
68 
69 /*
70 **  Open a file or give up.
71 */
72 static FILE *
EXPfopen(bool Unlink,const char * Name,const char * Mode,bool Needclean,bool Server,bool Paused)73 EXPfopen(bool Unlink, const char *Name, const char *Mode, bool Needclean,
74          bool Server, bool Paused)
75 {
76     FILE *F;
77 
78     if (Unlink && unlink(Name) < 0 && errno != ENOENT)
79         syswarn("cannot remove %s", Name);
80     if ((F = fopen(Name, Mode)) == NULL) {
81         syswarn("cannot open %s in %s mode", Name, Mode);
82         if (Needclean)
83             CleanupAndExit(Server, Paused, 1);
84         else
85             exit(1);
86     }
87     return F;
88 }
89 
90 
91 /*
92 **  Split a line at a specified field separator into a vector and return
93 **  the number of fields found, or -1 on error.
94 */
95 static int
EXPsplit(char * p,char sep,char ** argv,int count)96 EXPsplit(char *p, char sep, char **argv, int count)
97 {
98     int i;
99 
100     if (!p)
101         return 0;
102 
103     while (*p == sep)
104         ++p;
105 
106     if (*p == '\0')
107         return 0;
108 
109     for (i = 1, *argv++ = p; *p;)
110         if (*p++ == sep) {
111             p[-1] = '\0';
112             for (; *p == sep; p++)
113                 ;
114             if (!*p)
115                 return i;
116             if (++i == count)
117                 /* Overflow. */
118                 return -1;
119             *argv++ = p;
120         }
121     return i;
122 }
123 
124 
125 /*
126 **  Parse a number field converting it into a "when did this start?".
127 **  This makes the "keep it" tests fast, but inverts the logic of
128 **  just about everything you expect.  Print a message and return false
129 **  on error.
130 */
131 static bool
EXPgetnum(int line,char * word,time_t * v,const char * name)132 EXPgetnum(int line, char *word, time_t *v, const char *name)
133 {
134     char *p;
135     bool SawDot;
136     double d;
137 
138     if (strcasecmp(word, "never") == 0) {
139         *v = (time_t) 0;
140         return true;
141     }
142 
143     /* Check the number.  We don't have strtod yet. */
144     for (p = word; ISWHITE(*p); p++)
145         ;
146     if (*p == '+' || *p == '-')
147         p++;
148     for (SawDot = false; *p; p++)
149         if (*p == '.') {
150             if (SawDot)
151                 break;
152             SawDot = true;
153         } else if (!isdigit((unsigned char) *p))
154             break;
155     if (*p) {
156         warn("bad '%c' character in %s field on line %d", *p, name, line);
157         return false;
158     }
159     d = atof(word);
160     if (d > MAGIC_TIME)
161         *v = (time_t) 0;
162     else
163         *v = Now - (time_t)(d * 86400.);
164     return true;
165 }
166 
167 
168 /*
169 **  Parse the expiration control file.  Return true if okay.
170 */
171 static bool
EXPreadfile(FILE * F)172 EXPreadfile(FILE *F)
173 {
174     char *p;
175     int i;
176     int j;
177     bool SawDefault;
178     char buff[BUFSIZ];
179     char *fields[7];
180 
181     /* Scan all lines. */
182     EXPremember = -1;
183     SawDefault = false;
184 
185     for (i = 0; i <= NUM_STORAGE_CLASSES; i++) {
186         EXPclasses[i].ReportedMissing = false;
187         EXPclasses[i].Missing = true;
188     }
189 
190     for (i = 1; fgets(buff, sizeof buff, F) != NULL; i++) {
191         if ((p = strchr(buff, '\n')) == NULL) {
192             warn("line %d too long", i);
193             return false;
194         }
195         *p = '\0';
196         p = strchr(buff, '#');
197         if (p)
198             *p = '\0';
199         else
200             p = buff + strlen(buff);
201         while (--p >= buff) {
202             if (isspace((unsigned char) *p))
203                 *p = '\0';
204             else
205                 break;
206         }
207         if (buff[0] == '\0')
208             continue;
209         if ((j = EXPsplit(buff, ':', fields, ARRAY_SIZE(fields))) == -1) {
210             warn("too many fields on line %d", i);
211             return false;
212         }
213 
214         /* Expired-article remember line? */
215         if (strcmp(fields[0], "/remember/") == 0) {
216             if (j != 2) {
217                 warn("invalid format on line %d", i);
218                 return false;
219             }
220             if (EXPremember != -1) {
221                 warn("duplicate /remember/ on line %d", i);
222                 return false;
223             }
224             if (!EXPgetnum(i, fields[1], &EXPremember, "remember"))
225                 return false;
226             continue;
227         }
228 
229         /* Storage class line? */
230         if (j == 4) {
231             /* Is this the default line? */
232             if (fields[0][0] == '*' && fields[0][1] == '\0') {
233                 if (SawDefault) {
234                     warn("duplicate default on line %d", i);
235                     return false;
236                 }
237                 j = NUM_STORAGE_CLASSES;
238                 SawDefault = true;
239             } else {
240                 j = atoi(fields[0]);
241                 if ((j < 0) || (j >= NUM_STORAGE_CLASSES))
242                     warn("bad storage class %d on line %d", j, i);
243             }
244 
245             if (!EXPgetnum(i, fields[1], &EXPclasses[j].Keep, "keep")
246                 || !EXPgetnum(i, fields[2], &EXPclasses[j].Default, "default")
247                 || !EXPgetnum(i, fields[3], &EXPclasses[j].Purge, "purge"))
248                 return false;
249             /* These were turned into offsets, so the test is the opposite
250              * of what you think it should be.  If Purge isn't forever,
251              * make sure it's greater than the other two fields. */
252             if (EXPclasses[j].Purge) {
253                 /* Some value not forever; make sure other values are in range.
254                  */
255                 if (EXPclasses[j].Keep
256                     && EXPclasses[j].Keep < EXPclasses[j].Purge) {
257                     warn("keep time longer than purge time on line %d", i);
258                     return false;
259                 }
260                 if (EXPclasses[j].Default
261                     && EXPclasses[j].Default < EXPclasses[j].Purge) {
262                     warn("default time longer than purge time on line %d", i);
263                     return false;
264                 }
265             }
266             EXPclasses[j].Missing = false;
267             continue;
268         }
269 
270         /* Regular expiration line -- right number of fields? */
271         if (j != 5) {
272             warn("bad format on line %d", i);
273             return false;
274         }
275         continue; /* don't process this line--per-group expiry is done by
276                      expireover */
277     }
278 
279     return true;
280 }
281 
282 /*
283 **  Should we keep the specified article?
284 */
285 static enum KR
EXPkeepit(const TOKEN * token,time_t when,time_t Expires)286 EXPkeepit(const TOKEN *token, time_t when, time_t Expires)
287 {
288     EXPIRECLASS class;
289 
290     class = EXPclasses[token->class];
291     if (class.Missing) {
292         if (EXPclasses[NUM_STORAGE_CLASSES].Missing) {
293             /* no default */
294             if (!class.ReportedMissing) {
295                 warn("class definition for %d missing from control file,"
296                      " assuming it should never expire",
297                      token->class);
298                 EXPclasses[token->class].ReportedMissing = true;
299             }
300             return Keep;
301         } else {
302             /* use the default */
303             class = EXPclasses[NUM_STORAGE_CLASSES];
304             EXPclasses[token->class] = class;
305         }
306     }
307     /* Bad posting date? */
308     if (when > (RealNow + 86400)) {
309         /* Yes -- force the article to go to right now */
310         when = Expires ? class.Purge : class.Default;
311     }
312     if (EXPverbose > 2) {
313         if (EXPverbose > 3)
314             printf("%s age = %0.2f\n", TokenToText(*token),
315                    (double) (Now - when) / 86400.);
316         if (Expires == 0) {
317             if (when <= class.Default)
318                 printf("%s too old (no exp)\n", TokenToText(*token));
319         } else {
320             if (when <= class.Purge)
321                 printf("%s later than purge\n", TokenToText(*token));
322             if (when >= class.Keep)
323                 printf("%s earlier than min\n", TokenToText(*token));
324             if (Now >= Expires)
325                 printf("%s later than header\n", TokenToText(*token));
326         }
327     }
328 
329     /* If no expiration, make sure it wasn't posted before the default. */
330     if (Expires == 0) {
331         if (when >= class.Default)
332             return Keep;
333 
334         /* Make sure it's not posted before the purge cut-off and
335          * that it's not due to expire. */
336     } else {
337         if (when >= class.Purge && (Expires >= Now || when >= class.Keep))
338             return Keep;
339     }
340     return Remove;
341 }
342 
343 
344 /*
345 **  An article can be removed.  Either print a note, or actually remove it.
346 **  Also fill in the article size.
347 */
348 static void
EXPremove(const TOKEN * token)349 EXPremove(const TOKEN *token)
350 {
351     /* Turn into a filename and get the size if we need it. */
352     if (EXPverbose > 1)
353         printf("\tunlink %s\n", TokenToText(*token));
354 
355     if (EXPtracing) {
356         EXPunlinked++;
357         printf("%s\n", TokenToText(*token));
358         return;
359     }
360 
361     EXPunlinked++;
362     if (EXPunlinkfile) {
363         fprintf(EXPunlinkfile, "%s\n", TokenToText(*token));
364         if (!ferror(EXPunlinkfile))
365             return;
366         syswarn("cannot write to -z file (will ignore it for rest of run)");
367         fclose(EXPunlinkfile);
368         EXPunlinkfile = NULL;
369     }
370     if (!SMcancel(*token) && SMerrno != SMERR_NOENT && SMerrno != SMERR_UNINIT)
371         warn("cannot unlink %s", TokenToText(*token));
372 }
373 
374 /*
375 **  Do the work of expiring one line.
376 */
377 static bool
EXPdoline(void * cookie UNUSED,time_t arrived,time_t posted,time_t expires,TOKEN * token)378 EXPdoline(void *cookie UNUSED, time_t arrived, time_t posted, time_t expires,
379           TOKEN *token)
380 {
381     time_t when;
382     bool HasSelfexpire = false;
383     bool Selfexpired = false;
384     ARTHANDLE *article;
385     enum KR kr;
386     bool r;
387 
388     if (innconf->groupbaseexpiry || SMprobe(SELFEXPIRE, token, NULL)) {
389         if ((article = SMretrieve(*token, RETR_STAT)) == (ARTHANDLE *) NULL) {
390             HasSelfexpire = true;
391             Selfexpired = true;
392         } else {
393             /* the article is still alive */
394             SMfreearticle(article);
395             if (innconf->groupbaseexpiry || !Ignoreselfexpire)
396                 HasSelfexpire = true;
397         }
398     }
399     if (EXPusepost && posted != 0)
400         when = posted;
401     else
402         when = arrived;
403     EXPprocessed++;
404 
405     if (HasSelfexpire) {
406         if (Selfexpired || token->type == TOKEN_EMPTY) {
407             EXPallgone++;
408             r = false;
409         } else {
410             EXPstillhere++;
411             r = true;
412         }
413     } else {
414         kr = EXPkeepit(token, when, expires);
415         if (kr == Remove) {
416             EXPremove(token);
417             EXPallgone++;
418             r = false;
419         } else {
420             EXPstillhere++;
421             r = true;
422         }
423     }
424 
425     return r;
426 }
427 
428 
429 /*
430 **  Clean up link with the server and exit.
431 */
432 static void
CleanupAndExit(bool Server,bool Paused,int x)433 CleanupAndExit(bool Server, bool Paused, int x)
434 {
435     FILE *F;
436 
437     if (Server) {
438         ICCreserve("");
439         if (Paused && ICCgo(EXPreason) != 0) {
440             syswarn("cannot unpause server");
441             x = 1;
442         }
443     }
444     if (Server && ICCclose() < 0) {
445         syswarn("cannot close communication link to server");
446         x = 1;
447     }
448     if (EXPunlinkfile && fclose(EXPunlinkfile) == EOF) {
449         syswarn("cannot close -z file");
450         x = 1;
451     }
452 
453     /* Report stats. */
454     if (EXPverbose) {
455         printf("Article lines processed %8ld\n", EXPprocessed);
456         printf("Articles retained       %8ld\n", EXPstillhere);
457         printf("Entries expired         %8ld\n", EXPallgone);
458         if (!innconf->groupbaseexpiry)
459             printf("Articles dropped        %8ld\n", EXPunlinked);
460     }
461 
462     /* Append statistics to a summary file */
463     if (EXPgraph) {
464         F = EXPfopen(false, EXPgraph, "a", false, false, false);
465         fprintf(F, "%ld %ld %ld %ld %ld\n", (long) Now, EXPprocessed,
466                 EXPstillhere, EXPallgone, EXPunlinked);
467         fclose(F);
468     }
469 
470     SMshutdown();
471     HISclose(History);
472     if (EXPreason != NULL)
473         free(EXPreason);
474 
475     if (NHistory != NULL)
476         free(NHistory);
477     closelog();
478     exit(x);
479 }
480 
481 /*
482 **  Print a usage message and exit.
483 */
484 static void
Usage(void)485 Usage(void)
486 {
487     fprintf(stderr, "Usage: expire [flags] [expire.ctl]\n");
488     exit(1);
489 }
490 
491 
492 int
main(int ac,char * av[])493 main(int ac, char *av[])
494 {
495     int i;
496     char *p;
497     FILE *F;
498     char *HistoryText;
499     const char *NHistoryPath = NULL;
500     const char *NHistoryText = NULL;
501     char *EXPhistdir;
502     char buff[SMBUF];
503     bool Server;
504     bool Bad;
505     bool IgnoreOld;
506     bool Writing;
507     bool UnlinkFile;
508     bool val;
509     time_t TimeWarp;
510     size_t Size = 0;
511 
512     /* First thing, set up logging and our identity. */
513     openlog("expire", L_OPENLOG_FLAGS | LOG_PID, LOG_INN_PROG);
514     message_program_name = "expire";
515 
516     /* Set defaults. */
517     Server = true;
518     IgnoreOld = false;
519     Writing = true;
520     TimeWarp = 0;
521     UnlinkFile = false;
522 
523     if (!innconf_read(NULL))
524         exit(1);
525 
526     HistoryText = concatpath(innconf->pathdb, INN_PATH_HISTORY);
527 
528     umask(NEWSUMASK);
529 
530     /* find the default history file directory */
531     EXPhistdir = xstrdup(HistoryText);
532     p = strrchr(EXPhistdir, '/');
533     if (p != NULL) {
534         *p = '\0';
535     }
536 
537     /* Parse JCL. */
538     while ((i = getopt(ac, av, "d:f:g:h:iNnpr:s:tv:w:xz:")) != EOF)
539         switch (i) {
540         default:
541             Usage();
542             /* NOTREACHED */
543         case 'd':
544             NHistoryPath = optarg;
545             break;
546         case 'f':
547             NHistoryText = optarg;
548             break;
549         case 'g':
550             EXPgraph = optarg;
551             break;
552         case 'h':
553             HistoryText = optarg;
554             break;
555         case 'i':
556             IgnoreOld = true;
557             break;
558         case 'N':
559             Ignoreselfexpire = true;
560             break;
561         case 'n':
562             Server = false;
563             break;
564         case 'p':
565             EXPusepost = true;
566             break;
567         case 'r':
568             EXPreason = xstrdup(optarg);
569             break;
570         case 's':
571             Size = atoi(optarg);
572             break;
573         case 't':
574             EXPtracing = true;
575             break;
576         case 'v':
577             EXPverbose = atoi(optarg);
578             break;
579         case 'w':
580             TimeWarp = (time_t)(atof(optarg) * 86400.);
581             break;
582         case 'x':
583             Writing = false;
584             break;
585         case 'z':
586             EXPunlinkfile = EXPfopen(true, optarg, "a", false, false, false);
587             UnlinkFile = true;
588             break;
589         }
590     ac -= optind;
591     av += optind;
592     if ((ac != 0 && ac != 1))
593         Usage();
594 
595     /* if EXPtracing is set, then pass in a path, this ensures we
596      * don't replace the existing history files */
597     if (EXPtracing || NHistoryText || NHistoryPath) {
598         if (NHistoryPath == NULL)
599             NHistoryPath = innconf->pathdb;
600         if (NHistoryText == NULL)
601             NHistoryText = INN_PATH_HISTORY;
602         NHistory = concatpath(NHistoryPath, NHistoryText);
603     } else {
604         NHistory = NULL;
605     }
606 
607     time(&Now);
608     RealNow = Now;
609     Now += TimeWarp;
610 
611     /* Change to the runasuser user and runasgroup group if necessary. */
612     ensure_news_user_grp(true, true);
613 
614     /* Parse the control file. */
615     if (av[0]) {
616         if (strcmp(av[0], "-") == 0)
617             F = stdin;
618         else
619             F = EXPfopen(false, av[0], "r", false, false, false);
620     } else {
621         char *path;
622 
623         path = concatpath(innconf->pathetc, INN_PATH_EXPIRECTL);
624         F = EXPfopen(false, path, "r", false, false, false);
625         free(path);
626     }
627     if (!EXPreadfile(F)) {
628         fclose(F);
629         die("format error in expire.ctl");
630     }
631     fclose(F);
632 
633     /* Set up the link, reserve the lock. */
634     if (Server) {
635         if (EXPreason == NULL) {
636             snprintf(buff, sizeof(buff), "Expiring process %ld",
637                      (long) getpid());
638             EXPreason = xstrdup(buff);
639         }
640     } else {
641         EXPreason = NULL;
642     }
643 
644     if (Server) {
645         /* If we fail, leave evidence behind. */
646         if (ICCopen() < 0) {
647             syswarn("cannot open channel to server");
648             CleanupAndExit(false, false, 1);
649         }
650         if (ICCreserve((char *) EXPreason) != 0) {
651             warn("cannot reserve server");
652             CleanupAndExit(false, false, 1);
653         }
654     }
655 
656     History = HISopen(HistoryText, innconf->hismethod, HIS_RDONLY);
657     if (!History) {
658         warn("cannot open history");
659         CleanupAndExit(Server, false, 1);
660     }
661 
662     /* Ignore failure on the HISctl()s, if the underlying history
663      * manager doesn't implement them its not a disaster */
664     HISctl(History, HISCTLS_IGNOREOLD, &IgnoreOld);
665     if (Size != 0) {
666         HISctl(History, HISCTLS_NPAIRS, &Size);
667     }
668 
669     val = true;
670     if (!SMsetup(SM_RDWR, (void *) &val)
671         || !SMsetup(SM_PREOPEN, (void *) &val)) {
672         warn("cannot set up storage manager");
673         CleanupAndExit(Server, false, 1);
674     }
675     if (!SMinit()) {
676         warn("cannot initialize storage manager: %s", SMerrorstr);
677         CleanupAndExit(Server, false, 1);
678     }
679     if (chdir(EXPhistdir) < 0) {
680         syswarn("cannot chdir to %s", EXPhistdir);
681         CleanupAndExit(Server, false, 1);
682     }
683 
684     Bad = HISexpire(History, NHistory, EXPreason, Writing, NULL, EXPremember,
685                     EXPdoline)
686           == false;
687 
688     if (UnlinkFile && EXPunlinkfile == NULL)
689         /* Got -z but file was closed; oops. */
690         Bad = true;
691 
692     /* If we're done okay, and we're not tracing, slip in the new files. */
693     if (EXPverbose) {
694         if (Bad)
695             printf("Expire errors: history files not updated.\n");
696         if (EXPtracing)
697             printf("Expire tracing: history files not updated.\n");
698     }
699 
700     if (!Bad && NHistory != NULL) {
701         snprintf(buff, sizeof(buff), "%s.n.done", NHistory);
702         fclose(EXPfopen(false, buff, "w", true, Server, false));
703         CleanupAndExit(Server, false, Bad ? 1 : 0);
704     }
705 
706     CleanupAndExit(Server, !Bad, Bad ? 1 : 0);
707     /* NOTREACHED */
708     abort();
709 }
710