1 /*
2   client.c
3 
4   $Id: client.c,v 1.44 2003/06/05 17:55:44 godisch Exp $
5 */
6 
7 #if HAVE_CONFIG_H
8 #include <config.h>
9 #endif
10 
11 #include <sys/types.h>
12 #include <stdio.h>
13 #include <arpa/inet.h>
14 #include <ctype.h>
15 #include <netdb.h>
16 #include <netinet/in.h>
17 #include <signal.h>
18 #include <stdarg.h>
19 #include <sys/socket.h>
20 #include <unistd.h>
21 #include "client.h"
22 #include "configfile.h"
23 #include "content.h"
24 #include "control.h"
25 #include "dynamicstring.h"
26 #include "filter.h"
27 #include "group.h"
28 #include "itemlist.h"
29 #include "lock.h"
30 #include "log.h"
31 #include "over.h"
32 #include "protocol.h"
33 #include "pseudo.h"
34 #include "request.h"
35 #include "util.h"
36 #include "wildmat.h"
37 #include "portable.h"
38 
39 struct
40 {
41     FILE* in;       /* Receiving socket from server */
42     FILE* out;      /* Sending socket to server */
43     Str lastCmd;    /* Last command line */
44     Str lastStat;   /* Response from server to last command */
45     Str grp;        /* Selected group */
46     int rmtFirst;   /* First article of current group at server */
47     int rmtLast;    /* Last article of current group at server */
48     Bool auth;      /* Authentication already done? */
49     Str serv;       /* Remote server name */
50 } client = { NULL, NULL, "", "", "", 1, 0, FALSE, "" };
51 
52 static void
logBreakDown(void)53 logBreakDown( void )
54 {
55     Log_err( "Connection to remote server lost "
56              "(article numbers could be inconsistent)" );
57 }
58 
59 static Bool
getLn(Str line)60 getLn( Str line )
61 {
62     Bool r;
63 
64     r = Prt_getLn( line, client.in, Cfg_connectTimeout() );
65     if ( ! r )
66         logBreakDown();
67     return r;
68 }
69 
70 static Bool
getTxtLn(Str line,Bool * err)71 getTxtLn( Str line, Bool *err )
72 {
73     Bool r;
74 
75     r = Prt_getTxtLn( line, err, client.in, Cfg_connectTimeout() );
76     if ( *err )
77         logBreakDown();
78     return r;
79 }
80 
81 static void
putTxtBuf(const char * buf)82 putTxtBuf( const char *buf )
83 {
84     Prt_putTxtBuf( buf, client.out );
85     fflush( client.out );
86     Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
87 }
88 
89 static void
putEndOfTxt(void)90 putEndOfTxt( void )
91 {
92     Prt_putEndOfTxt( client.out );
93     fflush( client.out );
94     Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
95 }
96 
97 static Bool
putCmdLn(const char * line)98 putCmdLn( const char *line )
99 {
100     Bool err;
101     unsigned int n;
102 
103     Utl_cpyStr( client.lastCmd, line );
104     Utl_cpyStr( client.lastStat, "[no status available]" );
105     Log_dbg( LOG_DBG_PROTOCOL, "[S] %s", line );
106     n = fprintf( client.out, "%s\r\n", line );
107     err = ( n != strlen( line ) + 2 );
108     if ( err )
109         logBreakDown();;
110     return ! err;
111 }
112 
113 static Bool
putCmd(const char * fmt,...)114 putCmd( const char *fmt, ... )
115 {
116     Str line;
117     va_list ap;
118 
119     va_start( ap, fmt );
120     vsnprintf( line, MAXCHAR, fmt, ap );
121     va_end( ap );
122     if ( ! putCmdLn( line ) )
123         return FALSE;
124     fflush( client.out );
125     Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
126     return TRUE;
127 }
128 
129 static Bool
putCmdNoFlush(const char * fmt,...)130 putCmdNoFlush( const char *fmt, ... )
131 {
132     Str line;
133     va_list ap;
134 
135     va_start( ap, fmt );
136     vsnprintf( line, MAXCHAR, fmt, ap );
137     va_end( ap );
138     return putCmdLn( line );
139 }
140 
141 static int getStat( void );
142 
143 static int
performAuth(void)144 performAuth( void )
145 {
146     int stat;
147     Str user, pass;
148 
149     Cfg_authInfo( client.serv, user, pass );
150     if ( strcmp( user, "" ) == 0 )
151     {
152         Log_err( "No username for authentication set" );
153         return STAT_AUTH_REQUIRED;
154     }
155     putCmd( "AUTHINFO USER %s", user );
156     stat = getStat();
157     if ( stat == STAT_AUTH_ACCEPTED
158          || stat == STAT_AUTH_ACCEPTED_DEPREC )
159         return stat;
160     else if ( stat != STAT_MORE_AUTH_REQUIRED_DEPREC
161               && stat != STAT_MORE_AUTH_REQUIRED )
162     {
163         Log_err( "Username rejected. Server stat: %s", client.lastStat );
164         return stat;
165     }
166     if ( strcmp( pass, "" ) == 0 )
167     {
168         Log_err( "No password for authentication set" );
169         return STAT_AUTH_REQUIRED;
170     }
171     putCmd( "AUTHINFO PASS %s", pass );
172     stat = getStat();
173     if ( stat != STAT_AUTH_ACCEPTED
174          && stat != STAT_AUTH_ACCEPTED_DEPREC)
175         Log_err( "Password rejected. Server status: %s", client.lastStat );
176     return stat;
177 }
178 
179 static int
getStat(void)180 getStat( void )
181 {
182     int result;
183     Str lastCmd;
184 
185     if ( ! getLn( client.lastStat ) )
186         result = STAT_CONNECTION_LOST;
187     else if ( sscanf( client.lastStat, "%d", &result ) != 1 )
188     {
189         Log_err( "Invalid server status: %s", client.lastStat );
190         result = STAT_PROGRAM_FAULT;
191     }
192     if ( ( result == STAT_AUTH_REQUIRED
193            || result == STAT_AUTH_REQUIRED_DEPREC )
194          && ! client.auth )
195     {
196         client.auth = TRUE;
197         Utl_cpyStr( lastCmd, client.lastCmd );
198 	result = performAuth();
199         if ( result == STAT_AUTH_ACCEPTED
200              || result == STAT_AUTH_ACCEPTED_DEPREC )
201         {
202             putCmd( lastCmd );
203             return getStat();
204         }
205     }
206     return result;
207 }
208 
209 static void
connectAlarm(int sig)210 connectAlarm( int sig )
211 {
212     UNUSED( sig );
213 
214     return;
215 }
216 
217 static Bool
connectWithTimeout(int sock,const struct sockaddr * servAddr,socklen_t addrLen)218 connectWithTimeout( int sock, const struct sockaddr *servAddr,
219                     socklen_t addrLen )
220 {
221     SignalHandler oldHandler;
222     int r, to;
223 
224     oldHandler = Utl_installSignalHandler( SIGALRM, connectAlarm );
225     if ( oldHandler == (SignalHandler) SIG_ERR )
226     {
227         Log_err( "client.c:connectWithTimeout: signal failed." );
228         return FALSE;
229     }
230     to = Cfg_connectTimeout();
231     if ( alarm( ( unsigned int ) to ) != 0 )
232         Log_err( "client.c:connectWithTimeout: Alarm was already set." );
233     r = connect( sock, servAddr, addrLen );
234     alarm( 0 );
235     Utl_installSignalHandler( SIGALRM, oldHandler );
236     return ( r >= 0 );
237 }
238 
239 static DynStr *
collectTxt(void)240 collectTxt( void )
241 {
242     DynStr *res;
243     Str line;
244     Bool err;
245 
246     res = new_DynStr( MAXCHAR );
247     if ( res == NULL )
248 	return NULL;
249 
250     while ( getTxtLn( line, &err ) && ! err )
251 	DynStr_appLn( res, line );
252 
253     if ( err )
254     {
255 	del_DynStr( res );
256 	return NULL;
257     }
258     else
259 	return res;
260 }
261 
262 Bool
Client_connect(const char * serv)263 Client_connect( const char *serv )
264 {
265     long port;
266     int sock, i;
267     unsigned int stat;
268     struct hostent *hp;
269     char *pStart, *pColon;
270     Str host, s;
271     Str user, pass;
272     struct sockaddr_in sIn;
273 
274     ASSERT( client.in == NULL && client.out == NULL );
275     client.auth = FALSE;
276     Utl_cpyStr( s, serv );
277     pStart = Utl_stripWhiteSpace( s );
278     pColon = strstr( pStart, ":" );
279     if ( pColon == NULL )
280     {
281         Utl_cpyStr( host, pStart );
282         port = 119;
283     }
284     else
285     {
286         *pColon = '\0';
287         Utl_cpyStr( host, pStart );
288         if ( sscanf( pColon + 1, "%li", &port ) != 1 )
289         {
290             Log_err( "Syntax error in server name: '%s'", serv );
291             return FALSE;;
292         }
293         if ( port <= 0 || port > 65535 )
294         {
295             Log_err( "Invalid port number %hu. Must be in [1, 65535]", port );
296             return FALSE;;
297         }
298     }
299     memset( (void *)&sIn, 0, sizeof( sIn ) );
300     hp = gethostbyname( host );
301     if ( hp )
302     {
303         for ( i = 0; (hp->h_addr_list)[ i ]; ++i )
304         {
305             sIn.sin_family = hp->h_addrtype;
306             sIn.sin_port = htons( port );
307             sIn.sin_addr = *( (struct in_addr *)hp->h_addr_list[ i ] );
308             sock = socket( AF_INET, SOCK_STREAM, 0 );
309             if ( sock < 0 )
310                 break;
311             if ( ! connectWithTimeout( sock, (struct sockaddr *)&sIn,
312                                        sizeof( sIn ) ) )
313             {
314                 close( sock );
315                 break;
316             }
317             if ( ! ( client.out = fdopen( sock, "w" ) )
318                  || ! ( client.in  = fdopen( dup( sock ), "r" ) ) )
319             {
320 		if ( client.out != NULL )
321 		    fclose( client.out );
322                 close( sock );
323 		client.in = client.out = NULL;
324                 break;
325             }
326             Utl_cpyStr( client.serv, serv );
327             stat = getStat();
328             switch( stat )
329 	    {
330             case STAT_READY_POST_ALLOW:
331             case STAT_READY_NO_POST_ALLOW:
332                 Log_inf( "Connected to %s:%d",
333                          inet_ntoa( sIn.sin_addr ), port );
334 		/* INN needs a MODE READER before it will permit POST. */
335 		putCmd( "MODE READER" );
336 		getStat();
337                 Cfg_authInfo( client.serv, user, pass );
338                 if ( strcmp( user, "" ) != 0 )
339                     performAuth();
340                 return TRUE;
341             default:
342                 Log_err( "Bad server stat %d", stat );
343             }
344             shutdown( fileno( client.out ), 0 );
345 	    fclose( client.in );
346 	    fclose( client.out );
347 	    close( sock );
348 	    client.in = client.out = NULL;
349         }
350     }
351     return FALSE;
352 }
353 
354 static Bool
isGetGroup(const char * name)355 isGetGroup( const char *name )
356 {
357     GroupEnum *ge;
358     Bool emptyList;
359     const char *pattern;
360 
361     emptyList = TRUE;
362     ge = new_GetGrEn( client.serv );
363     while ( ( pattern = GrEn_next( ge ) ) != NULL )
364     {
365 	emptyList = FALSE;
366 	if ( Wld_match( name, pattern ) )
367 	{
368 	    del_GrEn( ge );
369 	    return TRUE;
370 	}
371     }
372 
373     del_GrEn( ge );
374     return emptyList;
375 }
376 
377 static Bool
isOmitGroup(const char * name)378 isOmitGroup( const char *name )
379 {
380     GroupEnum *ge;
381     const char *pattern;
382 
383     ge = new_OmitGrEn( client.serv );
384     while ( ( pattern = GrEn_next( ge ) ) != NULL )
385 	if ( Wld_match( name, pattern ) )
386 	{
387 	    del_GrEn( ge );
388 	    return TRUE;
389 	}
390 
391     del_GrEn( ge );
392     return FALSE;
393 }
394 
395 static Bool
processGrps(Bool noServerPattern)396 processGrps( Bool noServerPattern )
397 {
398     char postAllow;
399     Bool groupupdate;
400     Bool err;
401     int first, last;
402     Str grp, file;
403     Str line;
404 
405     ASSERT( ! Lock_gotLock() );
406     if ( ! Lock_openDatabases() )
407 	return TRUE;    /* silently ignore */
408 
409     groupupdate = FALSE;
410 
411     while ( getTxtLn( line, &err ) && !err )
412     {
413         if ( sscanf( line, MAXCHAR_FMT " %d %d %c",
414                      grp, &last, &first, &postAllow ) != 4 )
415         {
416             Log_err( "Unknown reply to LIST or NEWGROUPS: %s", line );
417             continue;
418         }
419         if ( ! Grp_isValidName( grp ) )
420         {
421             Log_inf( "Group name %s invalid", grp );
422             continue;
423         }
424 	if ( Grp_isForbiddenName( grp ) )
425 	{
426 	    Log_inf( "Group %s forbidden", grp );
427 	    continue;
428 	}
429 	if ( noServerPattern && ! isGetGroup( grp ) )
430 	    continue;
431 	if ( isOmitGroup( grp ) )
432 	    continue;
433         if ( ! Grp_exists( grp ) )
434         {
435             Log_inf( "Registering new group '%s'", grp );
436             Grp_create( grp );
437             /* Start local numbering with remote first number to avoid
438                new numbering at the readers if noffle is re-installed */
439             if ( first != 0 )
440                 Grp_setFirstLast( grp, first, first - 1 );
441             else
442                 Grp_setFirstLast( grp, 1, 0 );
443             Grp_setServ( grp, client.serv );
444 	    Grp_setPostAllow( grp, postAllow );
445 	    groupupdate = TRUE;
446         }
447         else
448         {
449             if ( ! Grp_local( grp ) && \
450                 Cfg_servIsPreferential( client.serv, Grp_server( grp ) ) )
451             {
452                 Log_inf( "Changing server for '%s': '%s'->'%s'",
453                          grp, Grp_server( grp ), client.serv );
454                 Grp_setServ( grp, client.serv );
455                 Grp_setRmtNext( grp, first );
456 		Grp_setPostAllow( grp, postAllow );
457 		groupupdate = TRUE;
458             }
459             else
460                 Log_dbg( LOG_DBG_FETCH,
461 			 "Group %s is already fetched from %s",
462 			 grp, Grp_server( grp ) );
463         }
464     }
465 
466     snprintf( file, MAXCHAR, "%s/lastupdate.%s",
467 	      Cfg_spoolDir(), client.serv );
468     Utl_stamp( file );
469     if ( groupupdate )
470     {
471 	snprintf( file, MAXCHAR, "%s/groupinfo.lastupdate",
472 		  Cfg_spoolDir() );
473 	Utl_stamp( file );
474     }
475 
476     /* I'm absolutely not sure about this. */
477     if ( err && groupupdate )
478             Log_err( "Group list may be corrupted with bogus data." );
479 
480     Lock_closeDatabases();
481     return !err;
482 }
483 
484 void
Client_disconnect(void)485 Client_disconnect( void )
486 {
487     if ( putCmd( "QUIT" ) )
488         getStat();
489     fclose( client.in );
490     fclose( client.out );
491     client.in = client.out = NULL;
492 }
493 
494 static int
doGetGrps(const char * pattern,Bool * noServerPattern)495 doGetGrps( const char *pattern, Bool *noServerPattern )
496 {
497     Str cmd;
498     int stat;
499 
500     Utl_cpyStr( cmd, "LIST ACTIVE" );
501     if ( pattern[ 0 ] != '\0' )
502     {
503 	Utl_catStr( cmd, " " );
504 	Utl_catStr( cmd, pattern );
505     }
506 
507     *noServerPattern = FALSE;
508     if ( ! putCmd( cmd ) )
509         return STAT_CONNECTION_LOST;
510     stat = getStat();
511     if ( IS_FATAL( stat ) )
512         return stat;
513 
514     /*
515      * Try LIST instead of LIST ACTIVE in case server doesn't
516      * support LIST ACTIVE.
517      */
518     if ( stat != STAT_GRPS_FOLLOW )
519     {
520 	if ( pattern[ 0 ] != '\0' )
521 	    *noServerPattern = TRUE;
522 	if ( ! putCmd( "LIST" ) )
523 	    return STAT_CONNECTION_LOST;
524 	stat = getStat();
525     }
526     if ( stat != STAT_GRPS_FOLLOW )
527     {
528 	Log_err( "%s failed: %s", cmd, client.lastStat );
529 	return stat;
530     }
531 
532 
533     if ( processGrps( *noServerPattern ) == FALSE )
534 	return STAT_CONNECTION_LOST;
535 
536     return STAT_OK;
537 }
538 
539 int
Client_getGrps(void)540 Client_getGrps( void )
541 {
542     GroupEnum *ge;
543     const char *pattern;
544     Bool doneOne, noServerPattern;
545     int res;
546 
547     Log_inf( "Getting groups" );
548 
549     doneOne = FALSE;
550     res = STAT_OK;
551     ge = new_GetGrEn( client.serv );
552     while ( res == STAT_OK && ( pattern = GrEn_next( ge ) ) != NULL )
553     {
554 	res = doGetGrps( pattern, &noServerPattern );
555 	doneOne = TRUE;
556 	if ( noServerPattern )
557 	    break;
558     }
559 
560     if ( ! doneOne )
561 	res = doGetGrps( "", &noServerPattern );
562 
563     del_GrEn( ge );
564     return res;
565 }
566 
567 static int
doGetDsc(const char * pattern,Bool * noServerPattern)568 doGetDsc( const char *pattern, Bool *noServerPattern )
569 {
570     Str name, line, dsc, cmd;
571     int stat;
572     DynStr *response;
573     const char *lines;
574     Bool result;
575 
576     ASSERT( ! Lock_gotLock() );
577     Utl_cpyStr( cmd, "LIST NEWSGROUPS" );
578     if ( pattern[ 0 ] != '\0' )
579     {
580 	Utl_catStr( cmd, " " );
581 	Utl_catStr( cmd, pattern );
582     }
583 
584     *noServerPattern = FALSE;
585     if ( ! putCmd( cmd ) )
586         return STAT_CONNECTION_LOST;
587     stat = getStat();
588     if ( IS_FATAL( stat ) )
589         return stat;
590 
591     /* Try without pattern in case server doesn't support patterns. */
592     if ( pattern[ 0 ] != '\0' && stat != STAT_GRPS_FOLLOW )
593     {
594 	*noServerPattern = TRUE;
595 	if ( !putCmd( "LIST NEWSGROUPS" ) )
596 	     return STAT_CONNECTION_LOST;
597 	stat = getStat();
598     }
599     if ( stat != STAT_GRPS_FOLLOW )
600     {
601         Log_err( "%s failed: %s", cmd, client.lastStat );
602         return stat;
603     }
604 
605     response = collectTxt();
606     if ( response == NULL )
607 	return STAT_CONNECTION_LOST;
608 
609     if ( ! Lock_openDatabases() )
610 	return STAT_NEWSBASE_FATAL;
611 
612     lines = DynStr_str( response );
613     result = STAT_OK;
614     while ( ( lines = Utl_getLn( line, lines) ) != NULL )
615     {
616         if ( sscanf( line, MAXCHAR_FMT, name ) != 1 )
617         {
618             Log_err( "Unknown reply to LIST NEWSGROUPS: %s", line );
619 	    result = STAT_PROGRAM_FAULT;
620 	    break;
621         }
622 	if ( *noServerPattern && ! isGetGroup( name ) )
623 	    continue;
624         Utl_cpyStr( dsc, Utl_restOfLn( line, 1 ) );
625         if ( Grp_exists( name ) )
626         {
627             Log_dbg( LOG_DBG_FETCH, "Description of %s: %s", name, dsc );
628             Grp_setDsc( name, dsc );
629         }
630     }
631     Lock_closeDatabases();
632     del_DynStr( response );
633     return result;
634 }
635 
636 int
Client_getDsc(void)637 Client_getDsc( void )
638 {
639     GroupEnum *ge;
640     const char *pattern;
641     Bool doneOne, noServerPattern;
642     int res;
643 
644     Log_inf( "Querying group descriptions" );
645 
646     doneOne = FALSE;
647     res = STAT_OK;
648     ge = new_GetGrEn( client.serv );
649     while ( res == STAT_OK && ( pattern = GrEn_next( ge ) ) != NULL )
650     {
651 	res = doGetDsc( pattern, &noServerPattern );
652 	doneOne = TRUE;
653 	if ( noServerPattern )
654 	    break;
655     }
656 
657     if ( ! doneOne )
658 	res = doGetDsc( "", &noServerPattern );
659 
660     del_GrEn( ge );
661     return res;
662 }
663 
664 int
Client_getNewgrps(const time_t * lastTime)665 Client_getNewgrps( const time_t *lastTime )
666 {
667     Str s;
668     const char *p;
669     int stat;
670 
671     ASSERT( *lastTime > 0 );
672     strftime( s, MAXCHAR, "%Y%m%d %H%M00", gmtime( lastTime ) );
673     /*
674       Do not use century for working with old server software until 2000.
675       According to newest IETF draft, this is still valid after 2000.
676       (directly using %y in fmt string causes a Y2K compiler warning)
677     */
678     p = s + 2;
679     if ( ! putCmd( "NEWGROUPS %s GMT", p ) )
680         return STAT_CONNECTION_LOST;
681     stat = getStat();
682     if ( stat != STAT_NEW_GRP_FOLLOW )
683     {
684         Log_err( "NEWGROUPS command failed: %s", client.lastStat );
685         return stat;
686     }
687 
688     if( processGrps( TRUE ) == FALSE )
689 	return STAT_CONNECTION_LOST;
690 
691     return STAT_OK;
692 }
693 
694 static const char *
readField(Str result,const char * p)695 readField( Str result, const char *p )
696 {
697     int len;
698     char *r;
699 
700     if ( ! p )
701         return NULL;
702     r = result;
703     *r = '\0';
704     len = 0;
705     while ( *p != '\t' && *p != '\n' )
706     {
707         if ( ! *p )
708 	{
709 	    *r = '\0';
710             return p;
711 	}
712         *(r++) = *(p++);
713         ++len;
714         if ( len >= MAXCHAR - 1 )
715         {
716             *r = '\0';
717             Log_err( "Field in overview too long: %s", r );
718             return ++p;
719         }
720     }
721     *r = '\0';
722     return ++p;
723 }
724 
725 static Bool
parseOvLn(Str line,int * numb,Str subj,Str from,Str date,Str msgId,Str ref,unsigned long * bytes,unsigned long * lines)726 parseOvLn( Str line, int *numb, Str subj, Str from,
727            Str date, Str msgId, Str ref,
728 	   unsigned long *bytes, unsigned long *lines )
729 {
730     const char *p;
731     Str t;
732 
733     p = readField( t, line );
734     if ( sscanf( t, "%d", numb ) != 1 )
735         return FALSE;
736     p = readField( subj, p );
737     p = readField( from, p );
738     p = readField( date, p );
739     p = readField( msgId, p );
740     p = readField( ref, p );
741     p = readField( t, p );
742     *bytes = 0;
743     *lines = 0;
744     if ( sscanf( t, "%lu", bytes ) != 1 )
745         return TRUE;
746     p = readField( t, p );
747     if ( sscanf( t, "%lu", lines ) != 1 )
748         return TRUE;
749     return TRUE;
750 }
751 
752 static const char*
nextXref(const char * pXref,Str grp,int * numb)753 nextXref( const char *pXref, Str grp, int *numb )
754 {
755     Str s;
756     const char *pColon, *src;
757     char *dst;
758 
759     src = pXref;
760     while ( *src && isspace( *src ) )
761         ++src;
762     dst = s;
763     while ( *src && ! isspace( *src ) )
764         *(dst++) = *(src++);
765     *dst = '\0';
766     if ( strlen( s ) == 0 )
767         return NULL;
768     pColon = strstr( s, ":" );
769     if ( ! pColon || sscanf( pColon + 1, "%d", numb ) != 1 )
770     {
771         Log_err( "Corrupt Xref at position '%s'", pXref );
772         return NULL;
773     }
774     Utl_cpyStrN( grp, s, pColon - s );
775     Log_dbg( LOG_DBG_FETCH,
776 	     "client.c: nextXref: grp '%s' numb %lu",
777 	     grp, numb );
778     return src;
779 }
780 
781 static Bool
needsMark(const char * ref)782 needsMark( const char *ref )
783 {
784     Bool interesting, result;
785     const char *msgId;
786     unsigned status;
787     time_t lastAccess, nowTime;
788     double threadFollowTime, maxTime, timeSinceLastAccess;
789     ItemList *itl;
790     const double secPerDay = 24.0 * 3600.0;
791 
792     ASSERT( Lock_gotLock() );
793     Log_dbg( LOG_DBG_FETCH, "Checking references '%s' for thread mode", ref );
794     result = FALSE;
795     itl = new_Itl( ref, " \t" );
796     nowTime = time( NULL );
797     threadFollowTime = (double)Cfg_threadFollowTime();
798     maxTime = threadFollowTime * secPerDay;
799     Log_dbg( LOG_DBG_FETCH, "Max time = %.0f", maxTime );
800     for ( msgId = Itl_first( itl ); msgId != NULL; msgId = Itl_next( itl ) )
801     {
802         /*
803           References does not have to contain only Message IDs,
804           but often it does, so we look up every item in the database.
805         */
806         if ( Db_contains( msgId ) )
807         {
808             status = Db_status( msgId );
809             lastAccess = Db_lastAccess( msgId );
810             interesting = ( status & DB_INTERESTING );
811             timeSinceLastAccess = difftime( nowTime, lastAccess );
812             Log_dbg( LOG_DBG_FETCH,
813 		     "Msg ID '%s': since last access = %.0f, interesting = %s",
814                      msgId, timeSinceLastAccess, ( interesting ? "y" : "n" ) );
815             if ( interesting && timeSinceLastAccess <= maxTime )
816             {
817                 result = TRUE;
818                 break;
819             }
820         }
821         else
822         {
823             Log_dbg( LOG_DBG_FETCH, "MsgID '%s': not in database.", msgId );
824         }
825     }
826     del_Itl( itl );
827     Log_dbg( LOG_DBG_FETCH,
828 	     "Article %s marking for download.",
829              ( result ? "needs" : "doesn't need" ) );
830     return result;
831 }
832 
833 static void
prepareEntry(Over * ov)834 prepareEntry( Over *ov )
835 {
836     Str g, t;
837     const char *msgId, *p, *xref;
838     int n;
839 
840     ASSERT( Lock_gotLock() );
841     msgId = Ov_msgId( ov );
842     if ( Pseudo_isGeneralInfo( msgId ) )
843         Log_dbg( LOG_DBG_FETCH, "Skipping general info '%s'", msgId );
844     else if ( Db_contains( msgId ) )
845     {
846         xref = Db_xref( msgId );
847         Log_dbg( LOG_DBG_FETCH,
848 		 "Entry '%s' already in db with Xref '%s'",
849 		 msgId, xref );
850         p = nextXref( xref, g, &n );
851         if ( p == NULL )
852             Log_err( "Overview with no group in Xref '%s'", msgId );
853         else
854         {
855             /* TODO: This code block seems unnessesary. Can we remove it? */
856             if ( Cfg_servIsPreferential( client.serv, Grp_server( g ) ) )
857             {
858                 Log_dbg( LOG_DBG_FETCH,
859 			 "Changing first server for '%s' from '%s' to '%s'",
860                          msgId, Grp_server( g ), client.serv );
861                 snprintf( t, MAXCHAR, "%s:%d %s",
862                           client.grp, Ov_numb( ov ), xref );
863                 Db_setXref( msgId, t );
864             }
865             else
866             {
867                 Log_dbg( LOG_DBG_FETCH,
868 			 "Adding '%s' to Xref of '%s'", g, msgId );
869                 snprintf( t, MAXCHAR, "%s %s:%d",
870                           xref, client.grp, Ov_numb( ov ) );
871                 Db_setXref( msgId, t );
872             }
873         }
874     }
875     else
876     {
877         Log_dbg( LOG_DBG_FETCH, "Preparing '%s' in database", msgId );
878         Db_prepareEntry( ov, client.grp, Ov_numb( ov ) );
879     }
880 }
881 
882 int
Client_getOver(const char * grp,int rmtFirst,int rmtLast,FetchMode mode)883 Client_getOver( const char *grp, int rmtFirst, int rmtLast, FetchMode mode )
884 {
885     unsigned long nbytes, nlines;
886     int rmtNumb, groupsNumb, oldLast, cntMarked;
887     Over *ov;
888     Str line, subj, from, date, msgId, ref, groups;
889     DynStr *response, *newsgroups;
890     const char *lines, *groupLines;
891     char *p;
892     FilterAction action;
893     int stat;
894 
895     ASSERT( ! Lock_gotLock() );
896     ASSERT( strcmp( grp, "" ) != 0 );
897 
898     /* Do we need the article Newsgroups: for filtering? */
899     if ( Flt_getNewsgroups() )
900     {
901 	if ( ! putCmd( "XHDR Newsgroups %lu-%lu", rmtFirst, rmtLast ) )
902 	    return STAT_CONNECTION_LOST;
903 	stat = getStat();
904 	if ( stat != STAT_HEAD_FOLLOWS )
905 	{
906 	    Log_err( "XHDR command failed: %s", client.lastStat );
907 	    return stat;
908 	}
909 
910 	Log_dbg( LOG_DBG_FETCH,
911 		 "Requesting Newsgroups headers for remote %lu-%lu",
912 		 rmtFirst, rmtLast );
913 
914 	newsgroups = collectTxt();
915 	if ( newsgroups == NULL )
916 	    return STAT_CONNECTION_LOST;
917 
918 	groupLines = DynStr_str( newsgroups );
919     }
920     else
921     {
922 	groupLines = NULL;
923 	newsgroups = NULL;
924     }
925 
926     if ( ! putCmd( "XOVER %lu-%lu", rmtFirst, rmtLast ) )
927     {
928 	del_DynStr( newsgroups );
929         return STAT_CONNECTION_LOST;
930     }
931 
932     stat = getStat();
933     if ( stat != STAT_OVERS_FOLLOW )
934     {
935 	del_DynStr( newsgroups );
936         Log_err( "XOVER command failed: %s", client.lastStat );
937         return stat;
938     }
939     Log_dbg( LOG_DBG_FETCH,
940 	     "Requesting overview for remote %lu-%lu",
941 	     rmtFirst, rmtLast );
942 
943     response = collectTxt();
944     if ( response == NULL )
945     {
946 	del_DynStr( newsgroups );
947 	return STAT_CONNECTION_LOST;
948     }
949 
950     if ( ! Lock_openDatabases() )
951     {
952 	del_DynStr( newsgroups );
953 	del_DynStr( response );
954 	return STAT_NEWSBASE_FATAL;
955     }
956 
957     Cont_read( grp );
958     oldLast = Cont_last();
959     cntMarked = 0;
960     lines = DynStr_str( response );
961     while ( ( lines = Utl_getLn( line, lines ) ) != NULL )
962     {
963         if ( ! parseOvLn( line, &rmtNumb, subj, from, date, msgId, ref,
964                           &nbytes, &nlines ) )
965             Log_err( "Bad overview line: %s", line );
966 	else if ( Cont_find( msgId ) >= 0 )
967 	    Log_inf( "Already have '%s'", msgId );
968         else
969         {
970             ov = new_Over( subj, from, date, msgId, ref, nbytes, nlines );
971 	    groupsNumb = 0;
972 	    p = NULL;
973 	    if ( groupLines != NULL )
974 	    {
975 		do
976 		{
977 		    groupLines = Utl_getLn( groups, groupLines );
978 		    groupsNumb = strtoul( groups, &p, 10 );
979 		} while ( groupLines != NULL
980 			  && p > groups
981 			  && groupsNumb < rmtNumb );
982 		if ( groupsNumb != rmtNumb )
983 		    p = NULL;
984 	    }
985 
986 	    action = Flt_checkFilters( grp, p, ov, mode );
987 	    if ( action == FILTER_DISCARD )
988             {
989                 del_Over( ov );
990             }
991 	    else
992 	    {
993 		Cont_app( ov );     /* Cont modules owns ov after this */
994 		prepareEntry( ov );
995 		if ( action == FILTER_FULL
996 		     || ( action == FILTER_THREAD && needsMark( ref ) ) )
997 		{
998 		    Req_add( client.serv, msgId );
999 		    ++cntMarked;
1000 		}
1001 	    }
1002         }
1003         Grp_setRmtNext( client.grp, rmtNumb + 1 );
1004     }
1005     if ( oldLast != Cont_last() )
1006     {
1007         Log_inf( "Added %s %lu-%lu", client.grp, oldLast + 1, Cont_last() );
1008 	Log_inf( "%u articles marked for download in %s",
1009 		 cntMarked, client.grp  );
1010 	if ( Cont_write() )
1011             Grp_setFirstLast( grp, Cont_first(), Cont_last() );
1012 	Grp_setLastPostTime( grp );
1013     }
1014     Lock_closeDatabases();
1015     del_DynStr( response );
1016     del_DynStr( newsgroups );
1017     return STAT_OK;
1018 }
1019 
1020 static void
retrievingFailed(const char * msgId,const char * reason)1021 retrievingFailed( const char* msgId, const char *reason )
1022 {
1023     unsigned status;
1024 
1025     ASSERT( ! Lock_gotLock() );
1026     Log_err( "Retrieving of %s failed: %s", msgId, reason );
1027     if ( ! Lock_openDatabases() )
1028 	return;
1029     status = Db_status( msgId );
1030     Pseudo_retrievingFailed( msgId, reason );
1031     Db_setStatus( msgId, status | DB_RETRIEVING_FAILED );
1032     Lock_closeDatabases();
1033     return;
1034 }
1035 
1036 static int
retrieveAndStoreArt(const char * msgId,int artcnt,int artmax)1037 retrieveAndStoreArt( const char *msgId, int artcnt, int artmax )
1038 {
1039     Bool err;
1040     DynStr *s = NULL;
1041 
1042     ASSERT( ! Lock_gotLock() );
1043     Log_inf( "[%d/%d] Retrieving %s", artcnt, artmax, msgId );
1044     err = TRUE;
1045 
1046     s = collectTxt();
1047     if ( s != NULL )
1048     {
1049 	const char *txt;
1050 
1051 	txt = DynStr_str( s );
1052 	if ( ! Lock_openDatabases() )
1053 	{
1054 	    del_DynStr( s );
1055 	    retrievingFailed( msgId, "Can't open message base" );
1056 	    return STAT_NEWSBASE_FATAL;
1057 	}
1058 
1059         err = ! Db_storeArt( msgId, txt );
1060 	if ( ! err )
1061 	{
1062 	    Str supersedeIds;
1063 
1064 	    if ( Prt_searchHeader( txt, "Supersedes", supersedeIds ) )
1065 	    {
1066 		ItemList *ids;
1067 		const char *supersededMsgId;
1068 
1069 		ids = new_Itl( supersedeIds, " \n\t" );
1070 		for ( supersededMsgId = Itl_first( ids );
1071 		      supersededMsgId != NULL;
1072 		      supersededMsgId = Itl_next( ids ) )
1073 		    Ctrl_cancel( supersededMsgId );
1074 		del_Itl( ids );
1075 	    }
1076 	}
1077 	Lock_closeDatabases();
1078 	del_DynStr( s );
1079     }
1080     else
1081     {
1082         retrievingFailed( msgId, "Connection broke down" );
1083 	return STAT_CONNECTION_LOST;
1084     }
1085     return err ? STAT_NEWSBASE_FATAL : STAT_OK;
1086 }
1087 
1088 int
Client_retrieveArt(const char * msgId)1089 Client_retrieveArt( const char *msgId )
1090 {
1091     int res;
1092 
1093     ASSERT( Lock_gotLock() );
1094     if ( ! Db_contains( msgId ) )
1095     {
1096         Log_err( "Article '%s' not prepared in database. Skipping.", msgId );
1097         return STAT_PROGRAM_FAULT;
1098     }
1099     if ( ! ( Db_status( msgId ) & DB_NOT_DOWNLOADED ) )
1100     {
1101         Log_inf( "Article '%s' already retrieved. Skipping.", msgId );
1102         return STAT_OK;
1103     }
1104 
1105     Lock_closeDatabases();
1106     if ( ! putCmd( "ARTICLE %s", msgId ) )
1107     {
1108         retrievingFailed( msgId, "Connection broke down" );
1109 	res = STAT_CONNECTION_LOST;
1110     }
1111     else if ( ( res = getStat() ) != STAT_ART_FOLLOWS )
1112         retrievingFailed( msgId, client.lastStat );
1113     else
1114         res = retrieveAndStoreArt( msgId, 0, 0 );
1115     if ( ! Lock_openDatabases() )
1116 	res = STAT_NEWSBASE_FATAL;
1117     return res;
1118 }
1119 
1120 int
Client_retrieveArtList(const char * list,int * artcnt,int artmax)1121 Client_retrieveArtList( const char *list, int *artcnt, int artmax )
1122 {
1123     Str msgId;
1124     DynStr *s;
1125     const char *p;
1126     int res, msgStat;
1127 
1128     ASSERT( Lock_gotLock() );
1129     Log_inf( "Retrieving article list" );
1130     s = new_DynStr( (int)strlen( list ) );
1131     p = list;
1132     res = STAT_OK;
1133     while ( ( p = Utl_getLn( msgId, p ) ) )
1134         if ( ! Db_contains( msgId ) )
1135 	{
1136             Log_err( "[%d/%d] Skipping retrieving of %s "
1137 		     "(not prepared in database)",
1138                      ++(*artcnt), artmax, msgId );
1139 	    res = STAT_PROGRAM_FAULT;
1140 	}
1141         else if ( ! ( Db_status( msgId ) & DB_NOT_DOWNLOADED ) )
1142             Log_inf( "[%d/%d] Skipping %s (already retrieved)",
1143 		     ++(*artcnt), artmax, msgId );
1144         else if ( ! putCmdNoFlush( "ARTICLE %s", msgId ) )
1145         {
1146             retrievingFailed( msgId, "Connection broke down" );
1147             del_DynStr( s );
1148             return STAT_CONNECTION_LOST;
1149         }
1150         else
1151             DynStr_appLn( s, msgId );
1152 
1153     Lock_closeDatabases();
1154     fflush( client.out );
1155     Log_dbg( LOG_DBG_PROTOCOL, "[S FLUSH]" );
1156 
1157     /*
1158      * We got something. Try to process all messages and return the
1159      * 'worst' error encountered (note we may have already hit a
1160      * STAT_PROGRAM_FAULT).
1161      */
1162     p = DynStr_str( s );
1163     while ( ! IS_FATAL( res ) && ( p = Utl_getLn( msgId, p ) ) )
1164     {
1165 	msgStat = getStat();
1166 	if ( msgStat == STAT_ART_FOLLOWS )
1167 	    msgStat = retrieveAndStoreArt( msgId, ++(*artcnt), artmax );
1168 	else
1169             retrievingFailed( msgId, client.lastStat );
1170 
1171 	if ( res == STAT_OK || ( ! IS_FATAL( res ) && IS_FATAL( msgStat ) ) )
1172 	    res = msgStat;
1173     }
1174     del_DynStr( s );
1175     if ( ! Lock_openDatabases() && ! IS_FATAL( res ) )
1176 	res = STAT_NEWSBASE_FATAL;
1177     return res;
1178 }
1179 
1180 int
Client_changeToGrp(const char * name)1181 Client_changeToGrp( const char* name )
1182 {
1183     unsigned int stat;
1184     int estimatedNumb, first, last, res;
1185 
1186     ASSERT( Lock_gotLock() );
1187     if ( ! Grp_exists( name ) )
1188         return STAT_NEWSBASE_FATAL;
1189     Lock_closeDatabases();
1190     stat = STAT_OK;
1191     if ( ! putCmd( "GROUP %s", name ) )
1192 	res = STAT_CONNECTION_LOST;
1193     if ( stat == STAT_OK )
1194 	stat = getStat();
1195     if ( ! Lock_openDatabases() )
1196 	return STAT_NEWSBASE_FATAL;
1197     if ( stat != STAT_GRP_SELECTED )
1198 	return stat;
1199     if ( sscanf( client.lastStat, "%u %d %d %d",
1200                  &stat, &estimatedNumb, &first, &last ) != 4 )
1201     {
1202         Log_err( "Bad server response to GROUP: %s", client.lastStat );
1203         return STAT_PROGRAM_FAULT;
1204     }
1205     Utl_cpyStr( client.grp, name );
1206     client.rmtFirst = first;
1207     client.rmtLast = last;
1208     return STAT_OK;
1209 }
1210 
1211 void
Client_rmtFirstLast(int * first,int * last)1212 Client_rmtFirstLast( int *first, int *last )
1213 {
1214     ASSERT( Lock_gotLock() );
1215     *first = client.rmtFirst;
1216     *last = client.rmtLast;
1217 }
1218 
1219 /**
1220  * Post an article.
1221  *
1222  * Return status if there's a connection problem. Otherwise return
1223  * STAT_OK. If there's an error in posting, put the error into
1224  * errStr, and return STAT_OK. That is, the return value indicates if
1225  * a proper transaction happened, and errStr indicates if that
1226  * transaction contained a posting error.
1227  */
1228 int
Client_postArt(const char * msgId,const char * artTxt,Str errStr)1229 Client_postArt( const char *msgId, const char *artTxt, Str errStr )
1230 {
1231     int stat;
1232 
1233     errStr[0] = '\0';
1234 
1235     if ( ! putCmd( "POST" ) )
1236         return STAT_CONNECTION_LOST;
1237     stat = getStat();
1238     if ( IS_FATAL( stat ) )
1239 	return stat;
1240     else if ( stat != STAT_SEND_ART )
1241     {
1242         Log_err( "Posting of %s not allowed: %s", msgId, client.lastStat );
1243         Utl_cpyStr( errStr, client.lastStat );
1244         return STAT_OK;
1245     }
1246     putTxtBuf( artTxt );
1247     putEndOfTxt();
1248     stat = getStat();
1249     if ( IS_FATAL( stat ) )
1250 	return stat;
1251     else if ( stat != STAT_POST_OK )
1252     {
1253         Log_err( "Posting of %s failed: %s", msgId, client.lastStat );
1254         Utl_cpyStr( errStr, client.lastStat );
1255         return STAT_OK;
1256     }
1257     Log_inf( "Posted %s (Status: %s)", msgId, client.lastStat );
1258     return STAT_OK;
1259 }
1260