1 /*
2  *  Project   : tin - a Usenet reader
3  *  Module    : art.c
4  *  Author    : I.Lea & R.Skrenta
5  *  Created   : 1991-04-01
6  *  Updated   : 2020-07-08
7  *  Notes     :
8  *
9  * Copyright (c) 1991-2021 Iain Lea <iain@bricbrac.de>, Rich Skrenta <skrenta@pbm.com>
10  * All rights reserved.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  *
16  * 1. Redistributions of source code must retain the above copyright notice,
17  *    this list of conditions and the following disclaimer.
18  *
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in the
21  *    documentation and/or other materials provided with the distribution.
22  *
23  * 3. Neither the name of the copyright holder nor the names of its
24  *    contributors may be used to endorse or promote products derived from
25  *    this software without specific prior written permission.
26  *
27  * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
31  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37  * POSSIBILITY OF SUCH DAMAGE.
38  */
39 
40 
41 #ifndef TIN_H
42 #	include "tin.h"
43 #endif /* !TIN_H */
44 #ifndef TCURSES_H
45 #	include "tcurses.h"
46 #endif /* !TCURSES_H */
47 #ifndef NEWSRC_H
48 #	include "newsrc.h"
49 #endif /* !NEWSRC_H */
50 
51 #ifndef STPWATCH_H
52 #	include "stpwatch.h"
53 #endif /* !STPWATCH_H */
54 
55 /*
56  * TODO: fixup to remove CURR_GROUP dependency in all sort funcs
57  */
58 #define SortBy(func)	tin_sort(arts, (size_t) top_art, sizeof(struct t_article), func);
59 
60 int top_art = 0;				/* # of articles in arts[] */
61 
62 /*
63  * Local prototypes
64  */
65 static FILE *open_art_header(char *groupname, t_artnum art, t_artnum *next);
66 static FILE *open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local);
67 static char *find_nov_file(struct t_group *group, int mode);
68 static char *print_date(time_t secs);
69 static char *print_from(struct t_group *group, struct t_article *article, int charset);
70 static int artnum_comp(t_comptype p1, t_comptype p2);
71 static int base_comp(t_comptype p1, t_comptype p2);
72 static int date_comp_asc(t_comptype p1, t_comptype p2);
73 static int date_comp_desc(t_comptype p1, t_comptype p2);
74 static int from_comp_asc(t_comptype p1, t_comptype p2);
75 static int from_comp_desc(t_comptype p1, t_comptype p2);
76 static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset);
77 static int last_date_comp_base_asc(t_comptype p1, t_comptype p2);
78 static int last_date_comp_base_desc(t_comptype p1, t_comptype p2);
79 static int lines_comp_asc(t_comptype p1, t_comptype p2);
80 static int lines_comp_desc(t_comptype p1, t_comptype p2);
81 static int read_art_headers(struct t_group *group, int total, t_artnum top);
82 static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache);
83 static int score_comp_asc(t_comptype p1, t_comptype p2);
84 static int score_comp_desc(t_comptype p1, t_comptype p2);
85 static int score_comp_base(t_comptype p1, t_comptype p2);
86 static int subj_comp_asc(t_comptype p1, t_comptype p2);
87 static int subj_comp_desc(t_comptype p1, t_comptype p2);
88 static int valid_artnum(t_artnum art);
89 static t_artnum find_first_unread(struct t_group *group);
90 static t_artnum setup_hard_base(struct t_group *group);
91 static t_bool parse_headers(FILE *fp, struct t_article *h);
92 static t_compfunc eval_sort_arts_func(unsigned int sort_art_type);
93 static time_t get_last_posting_date(long n);
94 static void sort_base(unsigned int sort_threads_type);
95 static void thread_by_multipart(void);
96 static void thread_by_percentage(unsigned int percentage);
97 static void thread_by_subject(void);
98 static void write_overview(struct t_group *group);
99 #ifdef NNTP_ABLE
100 	static struct t_article_range *build_range_list(t_artnum min, t_artnum max, int *range_cnt);
101 	static t_bool get_path_header(int cur, int cnt, struct t_group *group, t_artnum min, t_artnum max);
102 #endif /* NNTP_ABLE */
103 
104 
105 /*
106  * Display a suitable 'entering group' message if screen needs redrawing
107  * Allow for the non-printing %s, and the %-age counter
108  */
109 void
show_art_msg(const char * group)110 show_art_msg(
111 	const char *group)
112 {
113 	wait_message(0, _(txt_group), cCOLS - (strwidth(_(txt_group)) > cCOLS ? cCOLS : strwidth(_(txt_group)) + 2 - 3), group);
114 }
115 
116 
117 /*
118  * Construct the pointers to the first (base) article in each thread.
119  * If we are showing only unread, then point to the first unread. I have
120  * no idea why this should be so, it causes problems elsewhere [which_response]
121  */
122 void
find_base(struct t_group * group)123 find_base(
124 	struct t_group *group)
125 {
126 	int i, j;
127 
128 	grpmenu.max = 0;
129 
130 #ifdef DEBUG
131 	if (debug & DEBUG_FILTER)
132 		debug_print_arts();
133 #endif /* DEBUG */
134 
135 	for_each_art(i) {
136 		/*
137 		 * .prev will be set on each article that is after the first article in
138 		 * the thread. Invalid articles which have been expired will have
139 		 * .thread set to ART_EXPIRED
140 		 */
141 		if (arts[i].prev >= 0 || arts[i].thread == ART_EXPIRED || (arts[i].killed && tinrc.kill_level == KILL_NOTHREAD))
142 			continue;
143 
144 		if (grpmenu.max >= max_base)
145 			expand_base();
146 
147 		if (group->attribute->show_only_unread_arts) {
148 			if (arts[i].status != ART_READ || arts[i].keep_in_base)
149 				base[grpmenu.max++] = i;
150 			else {
151 				/* Find 1st unread art in thread */
152 				for (j = i; j >= 0; j = arts[j].thread) {
153 					if (arts[j].status != ART_READ || arts[j].keep_in_base) {
154 						base[grpmenu.max++] = i;
155 						break;
156 					}
157 				}
158 			}
159 		} else
160 			base[grpmenu.max++] = i;
161 	}
162 	/* sort base[] */
163 	if (group->attribute->sort_threads_type > SORT_THREADS_BY_NOTHING)
164 		sort_base(group->attribute->sort_threads_type);
165 }
166 
167 
168 /*
169  * Longword comparison routine for the tin_sort()
170  */
171 static int
base_comp(t_comptype p1,t_comptype p2)172 base_comp(
173 	t_comptype p1,
174 	t_comptype p2)
175 {
176 	const t_artnum *a = (const t_artnum *) p1;
177 	const t_artnum *b = (const t_artnum *) p2;
178 
179 	if (*a < *b)
180 		return -1;
181 
182 	if (*a > *b)
183 		return 1;
184 
185 	return 0;
186 }
187 
188 
189 /*
190  * via NNTP:
191  *   Issue a LISTGROUP command
192  *   Read the article numbers existing in the group into base[]
193  *   If the LISTGROUP failed, issue a GROUP command. Use the results to
194  *   create a less accurate version of base[]
195  *   This data will already be sorted
196  *
197  * on local spool:
198  *   Read the spool dir to populate base[] as above. Sort it.
199  *
200  * Grow the arts[] and bitmaps as needed.
201  * NB: the output will be sorted on artnum
202  *
203  * grpmenu.max is one past top.
204  * Returns total number of articles in group, or -1 on error
205  */
206 static t_artnum
setup_hard_base(struct t_group * group)207 setup_hard_base(
208 	struct t_group *group)
209 {
210 	t_artnum total = 0;
211 
212 	grpmenu.max = 0;
213 
214 	/*
215 	 * If reading with NNTP, issue a LISTGROUP
216 	 */
217 	if (read_news_via_nntp && !read_saved_news && group->type == GROUP_TYPE_NEWS) {
218 #ifdef NNTP_ABLE
219 		char buf[NNTP_STRLEN];
220 		char line[NNTP_STRLEN];
221 		int getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
222 		FILE *fp;
223 		t_artnum last, start, count = 0, j = 0;
224 		static t_bool skip_listgroup = FALSE;
225 
226 		/*
227 		 * Some nntp servers are broken and need an extra GROUP command
228 		 * (reported by reorx@irc.pl). This affects (old?) versions of
229 		 * nntpcache, leafnode and SurgeNews. Usually this should not be
230 		 * needed.
231 		 *
232 		 * For getart_limit recheck lowwatermark as at least giganews gives
233 		 * very different results for LIST ACTIVE (3 year retention for all)
234 		 * and GROUP (based on the clients contract).
235 		 * Calculate range and prepare base[] not to lose unread arts.
236 		 */
237 		if (nntp_caps.broken_listgroup || (!skip_listgroup && getart_limit && nntp_caps.type == CAPABILITIES && nntp_caps.reader)) {
238 			snprintf(buf, sizeof(buf), "GROUP %s", group->name);
239 			if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
240 				return -1;
241 
242 			if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start) != 2)
243 				return -1;
244 
245 			if (getart_limit > 0) {
246 				j = group->xmax - getart_limit;
247 				count = MAX(find_first_unread(group), start);
248 			}
249 			if (getart_limit < 0) {
250 				j = getart_limit + find_first_unread(group);
251 				count = group->xmin;
252 			}
253 			if (j < group->xmin)
254 				j = group->xmin;
255 
256 			for (; count < j; count++) {
257 				if (grpmenu.max >= max_base)
258 					expand_base();
259 				base[grpmenu.max++] = count;
260 			}
261 		}
262 
263 		/*
264 		 * See if LISTGROUP works
265 		 */
266 		if (!skip_listgroup && getart_limit != 0) { /* try to avoid traffic */
267 			if (nntp_caps.type == CAPABILITIES && nntp_caps.reader) {
268 				/* RFC 3977 allows ranges in LISTGROUP */
269 				if (getart_limit > 0)
270 					snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT"", group->name, j, group->xmax);
271 				else /* getart_limit < 0; fetch till newest art */
272 					snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-", group->name, j);
273 
274 			} else /* for RFC 977 just use GROUP */
275 				skip_listgroup = TRUE;
276 
277 		} else /* get all article numbers */
278 			snprintf(buf, sizeof(buf), "LISTGROUP %s", group->name);
279 
280 		if (!skip_listgroup) {
281 			if ((fp = nntp_command(buf, OK_GROUP, NULL, 0)) != NULL) {
282 				char *ptr;
283 
284 #	ifdef DEBUG
285 				if ((debug & DEBUG_NNTP) && verbose > 1)
286 					debug_print_file("NNTP", "setup_hard_base(%s)", buf);
287 #	endif /* DEBUG */
288 
289 				while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
290 					if (grpmenu.max >= max_base)
291 						expand_base();
292 					base[grpmenu.max++] = atoartnum(ptr);
293 					total++;
294 				}
295 
296 				if (tin_errno)
297 					return -1;
298 			} else
299 				skip_listgroup = TRUE;
300 		}
301 
302 		if (skip_listgroup) { /* LISTGROUP was skipped or failed */
303 			/*
304 			 * Handle the obscure case that the user aborted before the LISTGROUP
305 			 * had a chance to respond
306 			 */
307 			if (tin_errno)
308 				return -1;
309 
310 			snprintf(buf, sizeof(buf), "GROUP %s", group->name);
311 			if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
312 				return -1;
313 
314 			if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start, &last) != 3)
315 				return -1;
316 
317 #	ifdef DEBUG
318 			if ((debug & DEBUG_NNTP) && verbose > 1)
319 				debug_print_file("NNTP", "setup_hard_base(%s)", buf);
320 #	endif /* DEBUG */
321 			total = count;
322 			grpmenu.max = 0;
323 			if (getart_limit > 0) {
324 				if ((j = find_first_unread(group)) > start) {
325 					if (group->xmax > getart_limit) {
326 						start = MIN(j, group->xmax - getart_limit);
327 						total = getart_limit;
328 					} else
329 						start = j;
330 				}
331 			}
332 			if (getart_limit < 0) {
333 				if ((j = (getart_limit + find_first_unread(group))) > start)
334 					start = j;
335 			}
336 			while (start <= last) {
337 				if (grpmenu.max >= max_base)
338 					expand_base();
339 				base[grpmenu.max++] = start++;
340 			}
341 		}
342 #endif /* NNTP_ABLE */
343 	/*
344 	 * Reading off local spool, read the directory files
345 	 */
346 	} else {
347 		DIR *d;
348 		DIR_BUF *e;
349 		char group_path[PATH_LEN];
350 		t_artnum art;
351 
352 		make_base_group_path(group->spooldir, group->name, group_path, sizeof(group_path));
353 
354 		if ((d = opendir(group_path)) != NULL) {
355 			while ((e = readdir(d)) != NULL) {
356 				art = atoartnum(e->d_name);
357 				if (art >= 1) {
358 					total++;
359 					if (grpmenu.max >= max_base)
360 						expand_base();
361 					base[grpmenu.max++] = art;
362 				}
363 			}
364 			CLOSEDIR(d);
365 			tin_sort((char *) base, (size_t) grpmenu.max, sizeof(t_artnum), base_comp);
366 		} else {
367 			perror_message(_(txt_cannot_open), group_path);
368 #if 0
369 			if (access(group_path, R_OK) != 0)
370 				error_message(2, _(txt_not_exist));
371 #endif /* 0 */
372 			return -1;
373 		}
374 	}
375 
376 	if (grpmenu.max) {
377 		if (base[grpmenu.max - 1] > group->xmax)
378 			group->xmax = base[grpmenu.max - 1];
379 		expand_bitmap(group, base[0]);
380 	}
381 
382 	return total;
383 }
384 
385 
386 /*
387  * Main group indexing routine.
388  *
389  * Will read any existing index, create or incrementally update
390  * the index by looking at the articles in the spool directory,
391  * and attempt to write a new index if necessary.
392  *
393  * Returns FALSE if the user aborted the indexing, otherwise TRUE
394  */
395 t_bool
index_group(struct t_group * group)396 index_group(
397 	struct t_group *group)
398 {
399 	int i;
400 	int changed;				/* Count of articles whose overview has changed */
401 	int getart_limit;
402 	int respnum;
403 	int total;
404 	t_artnum last_read_article;
405 	t_artnum min, new_min, max;
406 	t_bool caching_xover;
407 	t_bool filtered;
408 	t_bool path_in_nov = FALSE;
409 	t_bool rebuild_cache = FALSE;
410 
411 	if (group == NULL)
412 		return TRUE;
413 
414 	if (!batch_mode)
415 		show_art_msg(group->name);
416 	else {
417 		if (verbose) /* -> lang.c */
418 			wait_message(0, _("Reading %s\n"), group->name);
419 	}
420 
421 	signal_context = cArt;			/* Set this only once curr_group is valid */
422 
423 	hash_reclaim();
424 	free_art_array();
425 	free_msgids();
426 
427 	BegStopWatch("setup_hard_base()");
428 
429 	/*
430 	 * Get list of valid article numbers
431 	 */
432 	if (setup_hard_base(group) < 0)
433 		return FALSE;
434 
435 	EndStopWatch();
436 	PrintStopWatch();
437 
438 #ifdef DEBUG
439 	if (debug & DEBUG_NEWSRC) {
440 		debug_print_comment("Before read_overview");
441 		debug_print_bitmap(group, NULL);
442 	}
443 #endif /* DEBUG */
444 
445 	min = grpmenu.max ? base[0] : group->xmin;
446 	max = grpmenu.max ? base[grpmenu.max - 1] : min - 1;
447 
448 	getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
449 
450 	if (getart_limit > 0) {
451 		if (grpmenu.max && (grpmenu.max > getart_limit))
452 			min = base[grpmenu.max - getart_limit];
453 		else
454 			getart_limit = 0;
455 	} else if (getart_limit < 0) {
456 		t_artnum first_unread = find_first_unread(group);
457 
458 		if (min - first_unread < getart_limit)
459 			min = first_unread + getart_limit;
460 		else
461 			getart_limit = 0;
462 	}
463 
464 	/*
465 	 * Quit now if no articles
466 	 */
467 	if (max < 0)
468 		return FALSE;
469 
470 	top_art = 0;
471 	last_read_article = T_ARTNUM_CONST(0);
472 
473 	/*
474 	 * Read in the existing overview data for min..max
475 	 * This read has local=TRUE set if locally caching XOVER records to ensure
476 	 * we pull in any private overview caches in preference to using OVER
477 	 *
478 	 * When reading local spool, this will pull in the system wide overview
479 	 * cache (if found) otherwise the private overview cache will be read
480 	 */
481 	caching_xover = (tinrc.cache_overview_files && nntp_caps.over_cmd && group->type == GROUP_TYPE_NEWS);
482 	if ((changed = read_overview(group, min, max, &last_read_article, caching_xover, &rebuild_cache)) == -1)
483 		return FALSE;	/* user aborted indexing */
484 
485 	/*
486 	 * Fill in the range last_read_article...max using XOVER
487 	 * Only do this if the previous read_overview() was against private cache
488 	 */
489 	if ((last_read_article < max) && caching_xover) {
490 		new_min = (last_read_article >= min) ? last_read_article + 1 : min;
491 
492 		if ((i = read_overview(group, new_min, max, &last_read_article, FALSE, &rebuild_cache)) == -1)
493 			return FALSE;	/* user aborted indexing */
494 		else
495 			changed += i;
496 	} else
497 		caching_xover = FALSE;
498 
499 	/*
500 	 * Mark as UNTHREADED all articles that have been verified as valid
501 	 * Get num of new arts to index so the user will have an idea of index time
502 	 */
503 	for (i = 0, total = 0; i < grpmenu.max; i++) {
504 		if ((respnum = valid_artnum(base[i])) >= 0) {
505 			arts[respnum].thread = ART_UNTHREADED;
506 			continue;
507 		}
508 		if (base[i] <= last_read_article)		/* It is vital this test be done last */
509 			continue;
510 		total++;
511 	}
512 
513 	/*
514 	 * Add any articles to arts[] that are new or were killed
515 	 */
516 	if (total > 0) {
517 		new_min = (getart_limit != 0 && last_read_article < min) ? min - 1 : last_read_article;
518 
519 		if ((i = read_art_headers(group, total, new_min)) == -1)
520 			return FALSE;		/* user aborted indexing */
521 		else
522 			changed += i;
523 	}
524 
525 #ifdef DEBUG
526 	if (debug & DEBUG_NEWSRC) {
527 		debug_print_comment("Before parse_unread_arts()");
528 		debug_print_bitmap(group, NULL);
529 	}
530 #endif /* DEBUG */
531 	/*
532 	 * Do this before calling art_mark(,, ART_READ) if you want
533 	 * the unread count to be correct.
534 	 */
535 	min = getart_limit > 0 ? min : T_ARTNUM_CONST(0);
536 	parse_unread_arts(group, min);
537 #ifdef DEBUG
538 	if (debug & DEBUG_NEWSRC) {
539 		debug_print_comment("After parse_unread_arts()");
540 		debug_print_bitmap(group, NULL);
541 	}
542 #endif /* DEBUG */
543 
544 	/*
545 	 * Stat all articles to see if any have expired
546 	 */
547 	for_each_art(i) {
548 		if (arts[i].thread == ART_EXPIRED) {
549 			changed++;
550 #ifdef DEBUG
551 			if (debug & DEBUG_NEWSRC)
552 				debug_print_comment("art.c: index_group() purging...");
553 #endif /* DEBUG */
554 			art_mark(group, &arts[i], ART_READ);
555 			if (group->attribute->show_only_unread_arts)
556 				arts[i].keep_in_base = FALSE;
557 		}
558 		if (!path_in_nov && arts[i].path && *arts[i].path != '\0')
559 			path_in_nov = TRUE;
560 	}
561 
562 	/*
563 	 * Only rewrite the index if it has changed
564 	 * TODO review the exact logic behind "|| caching_xover"
565 	 */
566 	if (changed || caching_xover || rebuild_cache)
567 		write_overview(group);
568 
569 	/*
570 	 * Create the reference tree. The msgid and ref ptrs will
571 	 * be free()d now that the NovFile has been written.
572 	 */
573 	build_references(group);
574 
575 	/*
576 	 * Needs access to the reference tree
577 	 */
578 	filtered = filter_articles(group);
579 
580 	BegStopWatch("make_threads()");
581 
582 	/*
583 	 * Thread the group
584 	 */
585 	make_threads(group, FALSE);
586 
587 	EndStopWatch();
588 	PrintStopWatch();
589 
590 	if ((changed > 0 || filtered) && !batch_mode)
591 		clear_message();
592 
593 	return TRUE;
594 }
595 
596 
597 /*
598  * Returns number of first unread article
599  */
600 static t_artnum
find_first_unread(struct t_group * group)601 find_first_unread(
602 	struct t_group *group)
603 {
604 	unsigned char *p;
605 	unsigned char *end = group->newsrc.xbitmap;
606 	t_artnum first = group->newsrc.xmin; /* initial value */
607 
608 	if ((p = group->newsrc.xbitmap)) {
609 		end += group->newsrc.xbitlen / NBITS;
610 		for (; *p == '\0' && p < end; p++, first += NBITS)
611 			;
612 	}
613 	return first;
614 }
615 
616 
617 /*
618  * Open an article for reading just the header
619  * 'next' is used/updated with the next article number
620  * to optimise the number of 'HEAD' commands issued on
621  * groups with holes.
622  */
623 static FILE *
open_art_header(char * groupname,t_artnum art,t_artnum * next)624 open_art_header(
625 	char *groupname,
626 	t_artnum art,
627 	t_artnum *next)
628 {
629 	char buf[NNTP_STRLEN];
630 #ifdef NNTP_ABLE
631 	FILE *fp;
632 	int i;
633 
634 	if (read_news_via_nntp && CURR_GROUP.type == GROUP_TYPE_NEWS) {
635 		/*
636 		 * Don't bother requesting if we have not got there yet.
637 		 * This is a big win if the group has got holes in it (ie. if 000's
638 		 * of articles have expired between active files min & max values).
639 		 */
640 		if (art < *next)
641 			return NULL;
642 
643 		snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
644 		if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
645 			return fp;
646 
647 		/*
648 		 * TODO:
649 		 * shall we stop on 5xx?, i.e JamNNTPd/2 1.3 responds with
650 		 * "503 Access denied" instead of 480 but NEXT still works,
651 		 * so tin loops over all articles without getting useful data
652 		 */
653 
654 		/*
655 		 * HEAD failed, try to find NEXT
656 		 * Should return "223 artno message-id more text...."
657 		 */
658 		i = new_nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf));
659 		switch (i) {
660 			case OK_NOTEXT:
661 				*next = atoartnum(buf);		/* Set next art number */
662 				break;
663 
664 #	ifndef BROKEN_LISTGROUP
665 			/*
666 			 * might happen if LISTGROUP doesn't select group, but
667 			 * we are not -DBROKEN_LISTGROUP
668 			 */
669 			case ERR_NCING:
670 				nntp_caps.broken_listgroup = TRUE;
671 				snprintf(buf, sizeof(buf), "GROUP %s", groupname);
672 				if (nntp_command(buf, OK_GROUP, NULL, 0) == NULL)
673 					return NULL;
674 				snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
675 				if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
676 					return fp;
677 				if (nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf)))
678 					*next = atoartnum(buf);
679 				break;
680 #	endif /* !BROKEN_LISTGROUP */
681 
682 			default:
683 				/*
684 				 * TODO: abort loop over all arts on ERR_NONEXT
685 				 */
686 #	ifndef BROKEN_LISTGROUP
687 				/*
688 				 * to avoid out of sync responses
689 				 * (listgroup seems to work, but didn't select new group,
690 				 *  so xover seems to work but returns old data)
691 				 * we set listgroup_broken = TRUE; once we saw a
692 				 * ERR_NOARTIG / ERR_NONEXT or the like - even if
693 				 * ERR_NOARTIG may occur on servers where listgroup
694 				 * isn't broken...
695 				 */
696 				nntp_caps.broken_listgroup = TRUE;
697 #	endif /* !BROKEN_LISTGROUP */
698 				break;
699 		}
700 
701 		return NULL;
702 	}
703 #endif /* NNTP_ABLE */
704 
705 	snprintf(buf, sizeof(buf), "%"T_ARTNUM_PFMT, art);
706 	return (fopen(buf, "r"));
707 }
708 
709 
710 /*
711  * Called after XOVER/local/private overview databases have been loaded
712  * Read and parse in headers for any arts not already found (usually
713  * new articles that have not been indexed yet)
714  * Any new articles that are added have ->thread set to ART_UNTHREADED
715  * 'top' is the current highest artnum read
716  *
717  * Return values are:
718  *   >0   Number of additional articles read in
719  *    0   No additional (new) articles were found
720  *   -1   user aborted during read
721  */
722 static int
read_art_headers(struct t_group * group,int total,t_artnum top)723 read_art_headers(
724 	struct t_group *group,
725 	int total,
726 	t_artnum top)
727 {
728 	FILE *fp;
729 	char dir[PATH_LEN];
730 	char group_msg[LEN];
731 	int i;
732 	int modified = 0;
733 	t_artnum art;
734 	t_artnum head_next = -1; /* Reset the next article number index (for when HEAD fails) */
735 	t_bool res;
736 
737 	/*
738 	 * Change to groups spooldir to optimize fopen()'s on local articles
739 	 * NB open_art_header() requires this
740 	 */
741 	if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
742 		char buf[PATH_LEN];
743 
744 		get_cwd(dir);
745 		make_base_group_path(group->spooldir, group->name, buf, sizeof(buf));
746 		if (chdir(buf) != 0) {
747 #ifdef DEBUG
748 			if (debug & DEBUG_MISC)
749 				fprintf(stderr, "read_art_headers(chdir(%s))", strerror(errno));
750 #endif /* DEBUG */
751 			return -1;
752 		}
753 	}
754 
755 	snprintf(group_msg, sizeof(group_msg), _(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
756 
757 	for (i = 0; i < grpmenu.max; i++) {	/* for each article number */
758 		art = base[i];
759 
760 		/*
761 		 * Skip articles that are below the low water mark or are
762 		 * already present
763 		 */
764 		if (valid_artnum(art) >= 0)
765 			continue;
766 		if (art <= top)
767 			continue;
768 
769 		/*
770 		 * Try and open the article
771 		 */
772 		if ((fp = open_art_header(group->name, art, &head_next)) == NULL)
773 			continue;
774 
775 		/*
776 		 * Add article to arts[]
777 		 */
778 		if (top_art >= max_art)
779 			expand_art();
780 
781 		set_article(&arts[top_art]);
782 		arts[top_art].artnum = art;
783 		arts[top_art].thread = ART_UNTHREADED;
784 
785 		res = parse_headers(fp, &arts[top_art]);
786 
787 		TIN_FCLOSE(fp);
788 		if (tin_errno) {
789 			modified = -1;
790 			break;
791 		}
792 
793 		if (!res) {
794 #ifdef DEBUG
795 			if (debug & DEBUG_FILTER) { /* we currently have no "local spool" debug level */
796 				char buf[PATH_LEN];
797 
798 				snprintf(buf, sizeof(buf), "FAILED parse_headers(%"T_ARTNUM_PFMT")", art);
799 				debug_print_file("ARTS", "read_art_headers() %s", buf);
800 			}
801 #endif /* DEBUG */
802 			arts[top_art].artnum = T_ARTNUM_CONST(0);
803 			arts[top_art].date = (time_t) 0;
804 			FreeAndNull(arts[top_art].xref);
805 			FreeAndNull(arts[top_art].path);
806 			FreeAndNull(arts[top_art].refs);
807 			FreeAndNull(arts[top_art].msgid);
808 			if (arts[top_art].archive) {
809 				FreeAndNull(arts[top_art].archive->partnum);
810 				FreeAndNull(arts[top_art].archive);
811 			}
812 			arts[top_art].tagged = 0;
813 			arts[top_art].thread = ART_EXPIRED;
814 			arts[top_art].prev = ART_NORMAL;
815 			arts[top_art].status = ART_UNREAD;
816 			arts[top_art].killed = ART_NOTKILLED;
817 			arts[top_art].selected = FALSE;
818 			continue;
819 		}
820 
821 		top = arts[top_art].artnum;	/* used if arts are killed */
822 		top_art++;
823 
824 		if (++modified % (MODULO_COUNT_NUM * 20) == 0)
825 			show_progress(group_msg, modified, total);
826 	}
827 
828 	/*
829 	 * Change back to previous dir before indexing started
830 	 */
831 	if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS)
832 		chdir(dir);
833 
834 	return modified;
835 }
836 
837 
838 /*
839  * The algorithm is elegant, using the fact that identical Subject lines
840  * are hashed to the same node in table[] (see hashstr.c)
841  *
842  * Mark i as being in j's thread list if
843  * . The article is _not_ being ignored
844  * . The article is not already threaded
845  * . One of the following is true:
846  *    1) The subject lines are the same
847  *    2) Both are part of the same archive (name's match and arch bit set)
848  * IMHO the tests for archive name are redundant and have been for years
849  */
850 static void
thread_by_subject(void)851 thread_by_subject(
852 	void)
853 {
854 	int i, j;
855 	struct t_hashnode *h;
856 
857 	for_each_art(i) {
858 		if (IGNORE_ART_THREAD(i))
859 			continue;
860 
861 		/*
862 		 * Get the contents of the magic marker in the hashnode
863 		 */
864 		h = (struct t_hashnode *) (arts[i].subject - sizeof(int) - sizeof(void *)); /* FIXME: cast increases required alignment of target type */
865 
866 		j = h->aptr;
867 
868 		if (j != -1 && j < i) {
869 #if 1
870 			if (arts[i].prev == ART_NORMAL && (arts[i].subject == arts[j].subject))
871 #else
872 			/* see also refs.c:collate_subjects() */
873 			if (arts[i].prev == ART_NORMAL && ((arts[i].subject == arts[j].subject) || (arts[i].archive && arts[j].archive && (arts[i].archive->name == arts[j].archive->name))))
874 #endif /* 1 */
875 			{
876 				arts[j].thread = i;
877 				arts[i].prev = j;
878 			}
879 		}
880 
881 		/*
882 		 * Update the magic marker with the highest numbered mesg in
883 		 * arts[] that has been used in this thread so far
884 		 */
885 		h->aptr = i;
886 	}
887 
888 #if 0
889 	fprintf(stderr, "Subj dump\n");
890 	fprintf(stderr, "%3s %3s %3s %3s : %3s %3s\n", "#", "Par", "Sib", "Chd", "In", "Thd");
891 	for_each_art(i) {
892 		fprintf(stderr, "%3d %3d %3d %3d : %3d %3d : %.50s %s\n", i,
893 			(arts[i].refptr->parent) ? arts[i].refptr->parent->article : -2,
894 			(arts[i].refptr->sibling) ? arts[i].refptr->sibling->article : -2,
895 			(arts[i].refptr->child) ? arts[i].refptr->child->article : -2,
896 			arts[i].prev, arts[i].thread, arts[i].refptr->txt, arts[i].subject);
897 	}
898 #endif /* 0 */
899 }
900 
901 
902 /*
903  * This Threading algorithm threads articles into 'buckets' where each bucket
904  * contains all the articles which match the root article's subject line to
905  * the configured percentage. Eg, if the root article had the subject "asdf"
906  * and the match percentage was configured to be 75% then any article would
907  * match if its subject was no different in more than a single character.
908  */
909 static void
thread_by_percentage(unsigned int percentage)910 thread_by_percentage(
911 	unsigned int percentage)
912 {
913 	int i, j, k;
914 	int root_num = 0; /* The index number of the root we are currently working on. */
915 	unsigned int unmatched; /* This is the number of characters that don't match between the two strings */
916 	size_t slen;
917 
918 	/* First we need to sort art[] to simplify and speed up the matching. */
919 	SortBy(subj_comp_asc);
920 
921 	/*
922 	 * Now we put all the articles which match enough into the thread. If
923 	 * an article doesn't match enough we create a new thread and then add
924 	 * to that and so on.
925 	 */
926 	base[0] = 0;
927 	arts[0].prev = ART_NORMAL;
928 	for_each_art(i) {
929 		if (i == 0)
930 			continue;
931 
932 		/* Check each character to see if it matched enough */
933 		k = 0;
934 		unmatched = 0;
935 		for (j = 0; arts[base[root_num]].subject[j] != '\0' && arts[i].subject[k] != '\0'; j++, k++) {
936 			if (arts[base[root_num]].subject[j] != arts[i].subject[k])
937 				unmatched++;
938 		}
939 
940 		/*
941 		 * By getting here we have a number of unmatched characters
942 		 * between the two strings. We also have the length of the
943 		 * strings available to us easily.
944 		 * All we need to do is see if the match is good enough, but
945 		 * we count differences in the length of the strings against
946 		 * them matching.
947 		 */
948 		if (!(slen = strlen(arts[base[root_num]].subject)))
949 			slen++;
950 		unmatched += slen - strlen(arts[i].subject);
951 		if (unmatched * 100 / slen > percentage) {
952 			/*
953 			 * If there is less greater than percentage% different start a
954 			 * new thread.
955 			 */
956 			base[++root_num] = i;
957 			arts[i].prev = ART_NORMAL;
958 			continue;
959 		} else {
960 			/*
961 			 * The subject lines match enough to consider them part of a single
962 			 * thread, so add the current article to the thread.
963 			 */
964 			if (arts[base[root_num]].thread < 0)
965 				arts[base[root_num]].thread = i;
966 			arts[i].prev = i - 1;
967 			arts[i - 1].thread = i;
968 			continue;
969 		}
970 	}
971 }
972 
973 
974 /*
975  * Parses a subject header of the type "multipart message subject (01/42)"
976  * into a MultiPartInfo struct, or fails if the message subject isn't in the
977  * right form.
978  *
979  * @return nonzero on success
980  */
981 int
global_get_multipart_info(int aindex,MultiPartInfo * setme)982 global_get_multipart_info(
983 	int aindex,
984 	MultiPartInfo *setme)
985 {
986 	int i, j, offi, offj;
987 	MultiPartInfo setmei, setmej;
988 
989 	i = global_look_for_multipart_info(aindex, &setmei, '[', ']', &offi);
990 	j = global_look_for_multipart_info(aindex, &setmej, '(', ')', &offj);
991 
992 	/* Ok i hits first */
993 	if (offi > offj) {
994 		*setme = setmei;
995 		return i;
996 	}
997 
998 	/* Its j or they are both the same (which must be zero!) so we don't care */
999 	*setme = setmej;
1000 	return j;
1001 }
1002 
1003 
1004 static int
global_look_for_multipart_info(int aindex,MultiPartInfo * setme,char start,char stop,int * offset)1005 global_look_for_multipart_info(
1006 	int aindex,
1007 	MultiPartInfo* setme,
1008 	char start,
1009 	char stop,
1010 	int *offset)
1011 {
1012 	char *subj;
1013 	char *pch;
1014 	MultiPartInfo tmp;
1015 
1016 	*offset = 0;
1017 
1018 	/* entry assertions */
1019 	assert(0 <= aindex && aindex < top_art && "invalid index");
1020 	assert(setme != NULL && "setme must not be NULL");
1021 
1022 	/* parse the message */
1023 	subj = arts[aindex].subject;
1024 	pch = strrchr(subj, start);
1025 	if (!pch || !isdigit((int) pch[1]))
1026 		return 0;
1027 
1028 	tmp.arts_index = aindex;
1029 	tmp.subject_compare_len = pch - subj;
1030 	tmp.part_number = (int) strtol(pch + 1, &pch, 10);
1031 	if (*pch != '/' && *pch != '|')
1032 		return 0;
1033 
1034 	if (!isdigit((int) pch[1]))
1035 		return 0;
1036 
1037 	tmp.total = (int) strtol(pch + 1, &pch, 10);
1038 	if (*pch != stop)
1039 		return 0;
1040 
1041 	/*
1042 	 * skip "blah (00/102)" or "blah (103/102)" subjects
1043 	 */
1044 	if (tmp.part_number < 1 || tmp.part_number > tmp.total)
1045 		return 0;
1046 
1047 	tmp.subject = subj;
1048 	*setme = tmp;
1049 	*offset = pch - subj;
1050 	return 1;
1051 }
1052 
1053 
1054 t_bool
global_look_for_multipart(int aindex,char start,char stop)1055 global_look_for_multipart(
1056 	int aindex,
1057 	char start,
1058 	char stop)
1059 {
1060 	char *pch;
1061 
1062 	pch = strrchr(arts[aindex].subject, start);
1063 	if (!pch || !isdigit((int) pch[1]))
1064 		return FALSE;
1065 
1066 	strtol(pch + 1, &pch, 10);
1067 	if (*pch != '/' && *pch != '|')
1068 		return FALSE;
1069 
1070 	if (!isdigit((int) pch[1]))
1071 		return FALSE;
1072 
1073 	strtol(pch + 1, &pch, 10);
1074 	if (*pch != stop)
1075 		return FALSE;
1076 
1077 	arts[aindex].multipart_subj = TRUE;
1078 	return TRUE;
1079 }
1080 
1081 
1082 /*
1083  * Tries to find all the parts to the multipart message pointed to by
1084  * aindex.
1085  *
1086  * @return on success, the number of parts found. On failure, zero if not
1087  * a multipart or the negative value of the first missing part in case of
1088  * tagging.
1089  * @param aindex index pointing to one of the messages in a multipart
1090  * message.
1091  * @param malloc_and_setme_info on success, set to a malloced array the
1092  * parts found.
1093  */
1094 int
global_get_multiparts(int aindex,MultiPartInfo ** malloc_and_setme_info,t_bool tagging)1095 global_get_multiparts(
1096 	int aindex,
1097 	MultiPartInfo **malloc_and_setme_info,
1098 	t_bool tagging)
1099 {
1100 	int i, part_index, part_cnt = 0;
1101 	MultiPartInfo tmp, tmp2;
1102 	MultiPartInfo *info = NULL;
1103 
1104 	/* entry assertions */
1105 	assert(0 <= aindex && aindex < top_art && "Invalid index");
1106 	assert(malloc_and_setme_info != NULL && "malloc_and_setme_info must not be NULL");
1107 
1108 	/* make sure this is a multipart message... */
1109 	if (!global_get_multipart_info(aindex, &tmp) || tmp.total < 1)
1110 		return 0;
1111 
1112 	/* make a temporary buffer to hold the multipart info... */
1113 	info = my_malloc(sizeof(MultiPartInfo) * tmp.total);
1114 
1115 	/* zero out part-number for the repost check below */
1116 	for (i = 0; i < tmp.total; ++i) {
1117 		info[i].total = tmp.total; /* Added this for thread_by_multipart */
1118 		info[i].part_number = -1;
1119 	}
1120 
1121 	/* try to find all the multiparts... */
1122 	for (i = (tagging ? 0 : aindex); i < top_art; i++) {
1123 		if (!arts[i].multipart_subj || strncmp(arts[i].subject, tmp.subject, tmp.subject_compare_len))
1124 			continue;
1125 
1126 		if (!global_get_multipart_info(i, &tmp2))
1127 			continue;
1128 
1129 		/* 'test (1/5)' is not the same as 'test (1/15)' */
1130 		if (tmp.total != tmp2.total)
1131 			continue;
1132 
1133 		part_index = tmp2.part_number - 1;
1134 
1135 		/* repost check: do we already have this part? */
1136 		if (info[part_index].part_number != -1) {
1137 			assert(info[part_index].part_number == tmp2.part_number && "bookkeeping error");
1138 			continue;
1139 		}
1140 
1141 		/* we have a match, hooray! */
1142 		info[part_index] = tmp2;
1143 
1144 		arts[i].multipart_subj = FALSE;
1145 
1146 		/* all parts found? */
1147 		if (++part_cnt == tmp.total)
1148 			break;
1149 	}
1150 
1151 	/* see if we got them all. */
1152 	if (tagging) {
1153 		for (i = 0; i < tmp.total; ++i) {
1154 			if (info[i].part_number != i + 1) {
1155 				free(info);
1156 				return -(i + 1); /* missing part #(i+1) */
1157 			}
1158 		}
1159 	}
1160 
1161 	/* looks like a success .. */
1162 	*malloc_and_setme_info = info;
1163 	return tmp.total;
1164 }
1165 
1166 
1167 /*
1168  *	The algorithm uses the tag multipart searches to thread articles together.
1169  */
1170 static void
thread_by_multipart(void)1171 thread_by_multipart(
1172 	void)
1173 {
1174 	int i, j, threadNum;
1175 	MultiPartInfo *minfo = NULL;
1176 
1177 	for_each_art(i) {
1178 		if (!global_look_for_multipart(i, '[', ']'))
1179 			global_look_for_multipart(i, '(', ')');
1180 	}
1181 
1182 	for_each_art(i) {
1183 		if (!arts[i].multipart_subj)
1184 			continue;
1185 
1186 		if (IGNORE_ART_THREAD(i) || arts[i].prev >= 0 || !global_get_multiparts(i, &minfo, FALSE)) {
1187 			FreeAndNull(minfo);
1188 			arts[i].multipart_subj = FALSE;
1189 			continue;
1190 		}
1191 
1192 		threadNum = -1;
1193 		for (j = minfo[0].total - 1; j >= 0; j--) {
1194 			if (minfo[j].part_number != -1) {
1195 				if (threadNum != -1) {
1196 					arts[minfo[j].arts_index].thread = threadNum;
1197 					arts[threadNum].prev = minfo[j].arts_index;
1198 				}
1199 				threadNum = minfo[j].arts_index;
1200 			}
1201 		}
1202 		FreeAndNull(minfo);
1203 		arts[i].multipart_subj = FALSE;
1204 		if (i % MODULO_COUNT_NUM == 0) /* TODO: -> lang.c */
1205 			show_progress(_("Threading by multipart"), i, top_art);
1206 	}
1207 }
1208 
1209 
1210 /*
1211  * Go through the articles in arts[] and create threads. There are
1212  * 5 strategies currently defined :
1213  *
1214  *	THREAD_NONE		No threading
1215  *	THREAD_SUBJ		Threads are created using like Subject lines
1216  *	THREAD_REFS		Threads are created using the References headers
1217  *	THREAD_BOTH		Threads created using References and then Subject
1218  *	THREAD_MULTI	Threads created using Subject to search for Multiparts
1219  *	THREAD_PERC		Threads based upon a char for char match of greater than x%
1220  *
1221  * .thread and .prev are used to hold the threading information, see tin.h for
1222  * more information
1223  * Only process valid (unexpired) articles we haven't visited yet
1224  * (ie arts[].thread == ART_UNTHREADED)
1225  *
1226  * The rethread parameter is used to force the deletion of existing threading
1227  * information before threading which happens anyway expect when using
1228  * THREAD_NONE (I don't immediately see how this is useful)
1229  */
1230 /* TODO: rewrite that user can easily combine different 'threading'
1231  *       methods, i.e:
1232  *       - thread_by_multipart() + collate_subjects()
1233  */
1234 void
make_threads(struct t_group * group,t_bool rethread)1235 make_threads(
1236 	struct t_group *group,
1237 	t_bool rethread)
1238 {
1239 	if (!cmd_line && !batch_mode) {
1240 		info_message((group->attribute->thread_articles == THREAD_NONE ? _(txt_unthreading_arts) : _(txt_threading_arts)));
1241 		my_flush();
1242 	}
1243 
1244 #ifdef DEBUG
1245 	if (debug & DEBUG_MISC)
1246 		error_message(2, "rethread=[%d]  thread_articles=[%d]  attr_thread_articles=[%d]",
1247 			rethread, tinrc.thread_articles, group->attribute->thread_articles);
1248 #endif /* DEBUG */
1249 
1250 	/*
1251 	 * Sort all the articles using the preferred method
1252 	 * When find_base() is called, the bases are created ordered
1253 	 * on arts[] and so the base messages under all threading systems
1254 	 * will be sorted in this way.
1255 	 */
1256 	sort_arts(group->attribute->sort_article_type);
1257 
1258 	/*
1259 	 * Reset all the ptrs to articles following the above sort
1260 	 */
1261 	clear_art_ptrs();
1262 
1263 	/*
1264 	 * The threading pointers need to be reset if re-threading
1265 	 * If using ref threading, revector the links back to the articles
1266 	 */
1267 	if (rethread || group->attribute->thread_articles) {
1268 		int i;
1269 
1270 		for_each_art(i) {
1271 			if (arts[i].thread >= 0)
1272 				arts[i].thread = ART_UNTHREADED;
1273 
1274 			arts[i].prev = ART_NORMAL;
1275 
1276 			/* Should never happen if tree is built properly */
1277 			if (arts[i].refptr == NULL) {
1278 #ifdef DEBUG
1279 				if (debug & DEBUG_REFS) {
1280 					my_fprintf(stderr, "\nError  : art->refptr is NULL\n");
1281 					my_fprintf(stderr, "Artnum : %"T_ARTNUM_PFMT"\n", arts[i].artnum);
1282 					my_fprintf(stderr, "Subject: %s\n", arts[i].subject);
1283 					my_fprintf(stderr, "From   : %s\n", arts[i].from);
1284 					assert(arts[i].refptr != NULL);
1285 				} else
1286 #endif /* DEBUG */
1287 					continue;
1288 			}
1289 			arts[i].refptr->article = i;
1290 		}
1291 	}
1292 
1293 	/*
1294 	 * Do the right thing according to the threading strategy
1295 	 */
1296 	switch (group->attribute->thread_articles) {
1297 		case THREAD_NONE:
1298 			break;
1299 
1300 		case THREAD_SUBJ:
1301 			thread_by_subject();
1302 			break;
1303 
1304 		case THREAD_REFS:
1305 			thread_by_reference();
1306 			break;
1307 
1308 		case THREAD_BOTH:
1309 			thread_by_reference();
1310 			collate_subjects();
1311 			break;
1312 
1313 		case THREAD_MULTI:
1314 			thread_by_multipart();
1315 			break;
1316 
1317 		case THREAD_PERC:
1318 			thread_by_percentage(100 - group->attribute->thread_perc);
1319 			break;
1320 
1321 		default: /* not reached */
1322 			break;
1323 	}
1324 
1325 	/*
1326 	 * Rebuild base[]
1327 	 */
1328 	find_base(group);
1329 }
1330 
1331 
1332 static t_compfunc
eval_sort_arts_func(unsigned int sort_art_type)1333 eval_sort_arts_func(
1334 	unsigned int sort_art_type)
1335 {
1336 	switch (sort_art_type) {
1337 		case SORT_ARTICLES_BY_NOTHING:		/* don't sort at all */
1338 			return artnum_comp;
1339 
1340 		case SORT_ARTICLES_BY_SUBJ_DESCEND:
1341 			return subj_comp_desc;
1342 
1343 		case SORT_ARTICLES_BY_SUBJ_ASCEND:
1344 			return subj_comp_asc;
1345 
1346 		case SORT_ARTICLES_BY_FROM_DESCEND:
1347 			return from_comp_desc;
1348 
1349 		case SORT_ARTICLES_BY_FROM_ASCEND:
1350 			return from_comp_asc;
1351 
1352 		case SORT_ARTICLES_BY_DATE_DESCEND:
1353 			return date_comp_desc;
1354 
1355 		case SORT_ARTICLES_BY_DATE_ASCEND:
1356 			return date_comp_asc;
1357 
1358 		case SORT_ARTICLES_BY_SCORE_DESCEND:
1359 			return score_comp_desc;
1360 
1361 		case SORT_ARTICLES_BY_SCORE_ASCEND:
1362 			return score_comp_asc;
1363 
1364 		case SORT_ARTICLES_BY_LINES_DESCEND:
1365 			return lines_comp_desc;
1366 
1367 		case SORT_ARTICLES_BY_LINES_ASCEND:
1368 			return lines_comp_asc;
1369 
1370 		default:
1371 			break;
1372 	}
1373 	return NULL;
1374 }
1375 
1376 
1377 void
sort_arts(unsigned int sort_art_type)1378 sort_arts(
1379 	unsigned int sort_art_type)
1380 {
1381 	t_compfunc comp_func = eval_sort_arts_func(sort_art_type);
1382 
1383 	if (comp_func)
1384 		SortBy(comp_func);
1385 }
1386 
1387 
1388 static void
sort_base(unsigned int sort_threads_type)1389 sort_base(
1390 	unsigned int sort_threads_type)
1391 {
1392 	switch (sort_threads_type) {
1393 		case SORT_THREADS_BY_SCORE_DESCEND:
1394 		case SORT_THREADS_BY_SCORE_ASCEND:
1395 			tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), score_comp_base);
1396 			break;
1397 
1398 		case SORT_THREADS_BY_LAST_POSTING_DATE_DESCEND:
1399 			tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_desc);
1400 			break;
1401 
1402 		case SORT_THREADS_BY_LAST_POSTING_DATE_ASCEND:
1403 			tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_asc);
1404 			break;
1405 	}
1406 }
1407 
1408 
1409 /*
1410  * This is called to get header info for articles not already found in the
1411  * overview files.
1412  * Code reads (max_lineno) lines of article to catch headers like Archive-name:
1413  * which are not normally included in XOVER or even the normal block of headers.
1414  * How this is supposed to be useful when 99% of the time we'll have overview
1415  * data I don't know...
1416  * TODO: move Archive-name: parsing to article body parsing, remove the
1417  * TODO: max_lineno nonsense and parse just the hdrs. Only parse if
1418  * TODO: currgrp->auto_save is set, otherwise it is redundant info
1419  */
1420 static t_bool
parse_headers(FILE * fp,struct t_article * h)1421 parse_headers(
1422 	FILE *fp,
1423 	struct t_article *h)
1424 {
1425 	char art_from_addr[HEADER_LEN];
1426 	char art_full_name[HEADER_LEN];
1427 	char *s, *hdr, *ptr;
1428 	unsigned int lineno = 0;
1429 	unsigned int max_lineno = 25;
1430 	t_bool got_from, got_lines, got_received;
1431 
1432 	got_from = got_lines = got_received = FALSE;
1433 
1434 	while ((ptr = tin_fgets(fp, TRUE)) != NULL) {
1435 		/*
1436 		 * Look for the end of information which tin wants to get.
1437 		 * Applies when reading local spool and via NNTP.
1438 		 */
1439 
1440 		/*
1441 		 * as Archive-name: is placed in the body it's safe to exit
1442 		 * the loop if it was found. Don't mix up with Archive: from
1443 		 * RFC 5536.
1444 		 */
1445 		if (lineno++ > max_lineno || h->archive)
1446 			break;
1447 
1448 		unfold_header(ptr);
1449 		switch (my_toupper((unsigned char) *ptr)) {
1450 			case 'A':	/* Archive-name:  optional */
1451 				/*
1452 				 * Archive-name: {name}/{part|patch}{number}
1453 				 * eg, acorn/faq/part01
1454 				 */
1455 				if ((hdr = parse_header(ptr + 1, "rchive-name", FALSE, FALSE, FALSE))) {
1456 					if ((s = strrchr(hdr, '/')) != NULL) {
1457 						struct t_archive *archptr = my_malloc(sizeof(struct t_archive));
1458 
1459 						if (STRNCASECMPEQ(s + 1, "part", 4)) {
1460 							archptr->partnum = my_strdup(s + 5);
1461 							archptr->ispart = TRUE;
1462 						} else if (STRNCASECMPEQ(s + 1, "patch", 5)) {
1463 							archptr->partnum = my_strdup(s + 6);
1464 							archptr->ispart = FALSE;
1465 						} else {		/* part or patch must be present */
1466 							free(archptr);
1467 							continue;
1468 						}
1469 						strtok(archptr->partnum, "\n");
1470 						*s = '\0';
1471 						archptr->name = hash_str(hdr);
1472 						h->archive = archptr;
1473 					}
1474 				}
1475 				break;
1476 
1477 			case 'D':	/* Date:  mandatory */
1478 				if (!h->date) {
1479 					if ((hdr = parse_header(ptr + 1, "ate", FALSE, FALSE, FALSE)))
1480 						h->date = parsedate(hdr, (struct _TIMEINFO *) 0);
1481 				}
1482 				break;
1483 
1484 			case 'F':	/* From:  mandatory */
1485 				if (!got_from) {
1486 					if ((hdr = parse_header(ptr + 1, "rom", FALSE, FALSE, FALSE))) {
1487 						h->gnksa_code = parse_from(hdr, art_from_addr, art_full_name);
1488 						h->from = hash_str(buffer_to_ascii(art_from_addr));
1489 						if (*art_full_name)
1490 							h->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
1491 						got_from = TRUE;
1492 					}
1493 				}
1494 				break;
1495 
1496 			case 'L':	/* Lines:  optional */
1497 				if (!got_lines) {
1498 					if ((hdr = parse_header(ptr + 1, "ines", FALSE, FALSE, FALSE))) {
1499 						h->line_count = atoi(hdr);
1500 						got_lines = TRUE;
1501 					}
1502 				}
1503 				break;
1504 
1505 			case 'M':	/* Message-ID:  mandatory */
1506 				if (!h->msgid) {
1507 					if ((hdr = parse_header(ptr + 1, "essage-ID", FALSE, FALSE, FALSE)))
1508 						h->msgid = my_strdup(hdr);
1509 				}
1510 				break;
1511 
1512 			/* for Path:-filter when reading from local spool */
1513 			case 'P':	/* Path: */
1514 				if (!h->path) {
1515 					if ((hdr = parse_header(ptr + 1, "ath", FALSE, FALSE, FALSE)))
1516 						h->path = my_strdup(hdr);
1517 				}
1518 				break;
1519 
1520 			case 'R':	/* References:  optional */
1521 				if (!h->refs) {
1522 					if ((hdr = parse_header(ptr + 1, "eferences", FALSE, FALSE, FALSE)))
1523 						h->refs = my_strdup(hdr);
1524 				}
1525 
1526 				/* Received:  If found it's probably a mail article */
1527 				if (!got_received) {
1528 					if (parse_header(ptr + 1, "eceived", FALSE, FALSE, FALSE)) {
1529 						max_lineno <<= 1;		/* double the max number of line to read for mails */
1530 						got_received = TRUE;
1531 					}
1532 				}
1533 				break;
1534 
1535 			case 'S':	/* Subject:  mandatory */
1536 				if (!h->subject) {
1537 					if ((hdr = parse_header(ptr + 1, "ubject", FALSE, FALSE, FALSE))) {
1538 #ifdef HAVE_UNICODE_NORMALIZATION
1539 						if (IS_LOCAL_CHARSET("UTF-8"))
1540 							s = normalize(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
1541 						else
1542 #endif /* HAVE_UNICODE_NORMALIZATION */
1543 							s = my_strdup(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
1544 
1545 						h->subject = hash_str(s);
1546 						free(s);
1547 					}
1548 				}
1549 				break;
1550 
1551 			case 'X':	/* Xref:  optional */
1552 				if (!h->xref) {
1553 					if ((hdr = parse_header(ptr + 1, "ref", FALSE, FALSE, FALSE)))
1554 						h->xref = my_strdup(hdr);
1555 				}
1556 				break;
1557 
1558 			default:
1559 				break;
1560 		} /* switch */
1561 
1562 	} /* while */
1563 
1564 #ifdef NNTP_ABLE
1565 	if (ptr)
1566 		drain_buffer(fp);
1567 #endif /* NNTP_ABLE */
1568 
1569 	if (tin_errno)
1570 		return FALSE;
1571 
1572 	/*
1573 	 * The son of RFC 1036 states that the following hdrs are mandatory. It
1574 	 * also states that Subject, Newsgroups and Path are too. Ho hum.
1575 	 *
1576 	 * What about reading mail from local spool via ~/.tin/active.mail,
1577 	 * they might not have a Message-ID but got_received is very likely to
1578 	 * be true.
1579 	 */
1580 	if (got_from && h->date && h->msgid) {
1581 		if (!h->subject)
1582 			h->subject = hash_str("<No subject>");
1583 
1584 #ifdef DEBUG
1585 		if (debug & DEBUG_FILTER)
1586 			debug_print_header(h);
1587 #endif /* DEBUG */
1588 		return TRUE;
1589 	}
1590 
1591 	return FALSE;
1592 }
1593 
1594 #ifdef NNTP_ABLE
1595 /*
1596  * Loop over arts[] and find ranges without Path: header
1597  * If there are any try to optimize the ranges regarding traffic consumption
1598  * Start optimization if at least MIN_CNT ranges exist
1599  * If there are more than MAX_CNT ranges after optimization, fetch all in one
1600  * big range
1601  */
1602 #define MIN_CNT 10
1603 #define MAX_CNT 50
1604 static struct t_article_range *
build_range_list(t_artnum min,t_artnum max,int * range_cnt)1605 build_range_list(
1606 	t_artnum min,
1607 	t_artnum max,
1608 	int *range_cnt)
1609 {
1610 	int i, gap_cnt = 0;
1611 	struct t_article_range *res = NULL, *gap_list, *curr, *from;
1612 	t_artnum new_end;
1613 
1614 	new_end = T_ARTNUM_CONST(0);
1615 	gap_list = my_malloc(sizeof(struct t_article_range));
1616 	curr = gap_list;
1617 	curr->start = min;
1618 	curr->end = max;
1619 	curr->cnt = T_ARTNUM_CONST(0);
1620 	curr->next = NULL;
1621 
1622 	for_each_art(i) {
1623 		if (arts[i].artnum < min)
1624 			continue;
1625 		if (arts[i].artnum > max)
1626 			break;
1627 		if (arts[i].path) {
1628 			for (; i < top_art && arts[i].path; i++)
1629 				;
1630 			/*
1631 			 * the current art has no path -> we use this one
1632 			 * if we reached top_art all arts have path
1633 			 * so we use max
1634 			 */
1635 			curr->start = i == top_art ? max : arts[i--].artnum;
1636 		} else {
1637 			for (; i < top_art && !arts[i].path; i++)
1638 				;
1639 			/* the current art has path -> we use the last one */
1640 			new_end = curr->end = arts[--i].artnum;
1641 		}
1642 		if (new_end) {
1643 			curr->cnt = curr->end - curr->start + 1;
1644 			curr->next = my_malloc(sizeof(struct t_article_range));
1645 			curr = curr->next;
1646 			curr->start = new_end;
1647 			curr->end = max;
1648 			curr->cnt = T_ARTNUM_CONST(0);
1649 			curr->next = NULL;
1650 			new_end = T_ARTNUM_CONST(0);
1651 		}
1652 	}
1653 
1654 	curr = gap_list;
1655 	while (curr && curr->cnt) {
1656 		++gap_cnt;
1657 #	ifdef DEBUG
1658 		if ((debug & DEBUG_NNTP) && verbose > 1)
1659 			debug_print_file("NNTP", "range #%d without path in overview cache: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1660 #	endif /* DEBUG */
1661 		curr = curr->next;
1662 	}
1663 
1664 	/*
1665 	 * Optimize only if there are at least MIN_CNT ranges
1666 	 */
1667 	if (gap_cnt >= MIN_CNT) {
1668 		res = my_malloc(sizeof(struct t_article_range));
1669 		res->start = T_ARTNUM_CONST(0);
1670 		res->end = T_ARTNUM_CONST(0);
1671 		res->cnt = T_ARTNUM_CONST(0);
1672 		res->next = NULL;
1673 
1674 		from = gap_list;
1675 		curr = res;
1676 		while (from) {
1677 			curr->start = from->start;
1678 			curr->end = from->end;
1679 			curr->cnt = from->cnt;
1680 			if ((from = from->next)) {
1681 				/*
1682 				 * If the next range is grater then the gap between the current
1683 				 * one and the next one we build a new range including the
1684 				 * current one, the next one and the gap between
1685 				 */
1686 				while (from && from->cnt >= from->start - curr->end - 1) {
1687 					curr->end = from->end;
1688 					from = from->next;
1689 				}
1690 				curr->cnt = curr->end - curr->start + 1;
1691 				curr->next = my_malloc(sizeof(struct t_article_range));
1692 				curr = curr->next;
1693 				curr->start = T_ARTNUM_CONST(0);
1694 				curr->end = T_ARTNUM_CONST(0);
1695 				curr->cnt = T_ARTNUM_CONST(0);
1696 				curr->next = NULL;
1697 			}
1698 		}
1699 	}
1700 
1701 	/*
1702 	 * If there are less then MIN_CNT ranges
1703 	 * no res is build -> return the original list
1704 	 */
1705 	if (res) {
1706 		while (gap_list) {
1707 			curr = gap_list;
1708 			gap_list = curr->next;
1709 			free(curr);
1710 		}
1711 	} else
1712 		res = gap_list;
1713 
1714 	curr = res;
1715 	gap_cnt = 0;
1716 	while (curr && curr->cnt) {
1717 		++gap_cnt;
1718 #	ifdef DEBUG
1719 		if ((debug & DEBUG_NNTP) && verbose > 1)
1720 			debug_print_file("NNTP", "optimized range #%d: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1721 #	endif /* DEBUG */
1722 		curr = curr->next;
1723 	}
1724 
1725 	if (gap_cnt >= MAX_CNT) {
1726 		curr = res;
1727 		while (curr->next && curr->next->cnt) {
1728 			res->end = curr->next->end;
1729 			curr->next->cnt = 0;
1730 			curr = curr->next;
1731 		}
1732 		res->cnt = res->end - res->start + 1;
1733 		gap_cnt = 1;
1734 #	ifdef DEBUG
1735 		if ((debug & DEBUG_NNTP) && verbose > 1)
1736 			debug_print_file("NNTP", "more then %d ranges after optimization, fetch all at once instead: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", MAX_CNT, res->start, res->end, res->cnt);
1737 #	endif /* DEBUG */
1738 	}
1739 	*range_cnt = gap_cnt;
1740 
1741 	return res;
1742 }
1743 
1744 
1745 /*
1746  * Fetch the Path header in case we want to filter on that in the given group
1747  *
1748  * Try [X]HDR first, then XPAT
1749  */
1750 static t_bool
get_path_header(int cur,int cnt,struct t_group * group,t_artnum min,t_artnum max)1751 get_path_header(
1752 	int cur,
1753 	int cnt,
1754 	struct t_group *group,
1755 	t_artnum min,
1756 	t_artnum max)
1757 {
1758 	FILE *fp = NULL;
1759 	char *prep_msg;
1760 	char *buf, *ptr;
1761 	char cmd[NNTP_STRLEN];
1762 	t_artnum artnum, i;
1763 	t_bool found = FALSE;
1764 	static t_bool supported = TRUE; /* assume HDR || XPAT works */
1765 
1766 	if (!read_news_via_nntp || !supported || group->type != GROUP_TYPE_NEWS)
1767 		return FALSE;
1768 
1769 #	ifdef DEBUG
1770 	if ((debug & DEBUG_NNTP) && verbose > 1)
1771 		debug_print_file("NNTP", "%s: Filtering on Path header requested.", group->name);
1772 #	endif /* DEBUG */
1773 
1774 	if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers && !*nntp_caps.headers_range && nntp_caps.hdr_cmd[0] != 'X') {
1775 		int j = new_nntp_command("LIST HEADERS RANGE", 215, cmd, sizeof(cmd));
1776 		switch (j) {
1777 			case 215:
1778 				while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
1779 #	ifdef DEBUG
1780 					if (debug & DEBUG_NNTP)
1781 						debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
1782 #	endif /* DEBUG */
1783 					nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
1784 					strcat(nntp_caps.headers_range, ptr);
1785 					strcat(nntp_caps.headers_range, "\n");
1786 				}
1787 				break;
1788 
1789 			default:
1790 				break;
1791 		}
1792 	}
1793 
1794 	/* does HDR return Path? */
1795 	if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
1796 		do {
1797 			if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Path", 4))
1798 				found = TRUE;
1799 		} while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
1800 	}
1801 
1802 	if ((nntp_caps.hdr || nntp_caps.hdr_cmd) && (!(nntp_caps.type == CAPABILITIES) || found)) {
1803 		if (min == max)
1804 			snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min);
1805 		else
1806 			snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, max);
1807 		fp = nntp_command(cmd, nntp_caps.hdr_cmd[0] == 'X' ? OK_XHDR : OK_HDR, NULL, 0);
1808 		if (!nntp_caps.hdr && fp)
1809 			nntp_caps.hdr = TRUE;
1810 	} else if (nntp_caps.xpat) {
1811 		if (min == max)
1812 			snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT" *", min);
1813 		else
1814 			snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT" *", min, max);
1815 		fp = nntp_command(cmd, OK_XPAT, NULL, 0);
1816 	}
1817 
1818 	if (fp) {
1819 		int j = 0;
1820 
1821 		prep_msg = fmt_string(_(txt_prep_for_filter_on_path), cur, cnt);
1822 		while ((buf = tin_fgets(fp, FALSE)) != NULL && buf[0] != '.') {
1823 #	ifdef DEBUG
1824 			if ((debug & DEBUG_NNTP) && verbose)
1825 				debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1826 #	endif /* DEBUG */
1827 			if ((ptr = tin_strtok(buf, " ")) == NULL)
1828 				continue;
1829 			artnum = atoartnum(ptr);
1830 			if ((ptr = tin_strtok(NULL, " ")) == NULL)
1831 				continue;
1832 			for (i = j; i < top_art; i++) {
1833 				if (arts[i].artnum == artnum) {
1834 					FreeIfNeeded(arts[i].path);
1835 					arts[i].path = my_strdup(ptr);
1836 					j = i;
1837 					break;
1838 				}
1839 			}
1840 			if (++artnum % MODULO_COUNT_NUM == 0)
1841 				show_progress(prep_msg, artnum - min, max - min);
1842 		}
1843 		free(prep_msg);
1844 		return supported;
1845 	}
1846 
1847 	/* !fp */
1848 	supported = FALSE;
1849 	if (nntp_caps.xpat)
1850 		nntp_caps.xpat = FALSE;
1851 	/* as nntp_caps.hdr may work with other headers we don't disable it */
1852 
1853 #	ifdef DEBUG
1854 	if ((debug & DEBUG_NNTP) && verbose > 1)
1855 		debug_print_file("NNTP", "%s: Neither \"[X]HDR Path\" nor \"XPAT Path\" are supported.", group->name);
1856 #	endif /* DEBUG */
1857 	return supported;
1858 }
1859 #endif /* NNTP_ABLE */
1860 
1861 
1862 /*
1863  * Read in an overview index file. Fields are separated by TAB.
1864  * return the number of expired articles encountered or -1 if the user aborted
1865  * the read
1866  * 'top' is set to the highest artnum read
1867  * If 'local' is set then always open local overview cache in preference to
1868  * using NNTP XOVER
1869  *
1870  * Format (mandatory as far as line count [RFC2980]):
1871  *	1. article number (ie. 183)                [mandatory]
1872  *	2. Subject: line  (ie. Which newsreader?)  [mandatory]
1873  *	3. From: line     (ie. iain@ecrc.de)       [mandatory]
1874  *	4. Date: line     (rfc822 format)          [mandatory]
1875  *	5. MessageID:     (ie. <123@ether.net>)    [mandatory]
1876  *	6. References:    (ie. <message-id> ....)  [optional]
1877  *	7. Byte count     (Skipped - not used)     [mandatory]
1878  *	8. Line count     (ie. 23)                 [mandatory]
1879  *	9. Xref: line     (ie. alt.test:389)       [optional]
1880  */
1881 static int
read_overview(struct t_group * group,t_artnum min,t_artnum max,t_artnum * top,t_bool local,t_bool * rebuild_cache)1882 read_overview(
1883 	struct t_group *group,
1884 	t_artnum min,
1885 	t_artnum max,
1886 	t_artnum *top,
1887 	t_bool local,
1888 	t_bool *rebuild_cache)
1889 {
1890 	FILE *fp;
1891 	char *ptr;
1892 	char *q;
1893 	char *buf;
1894 	char *group_msg;
1895 	char art_full_name[HEADER_LEN];
1896 	char art_from_addr[HEADER_LEN];
1897 	unsigned int count;
1898 	int expired = 0;
1899 	t_artnum artnum;
1900 	t_bool path_found = FALSE, path_in_ofmt = FALSE;
1901 	struct t_article *art;
1902 	size_t over_fields = 1;
1903 
1904 	/*
1905 	 * open the overview file (whether it be local or via nntp)
1906 	 */
1907 	if ((fp = open_xover_fp(group, "r", min, max, local)) == NULL)
1908 		return expired;
1909 
1910 	if (group->xmax > max)
1911 		group->xmax = max;
1912 
1913 	group_msg = fmt_string(_(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
1914 
1915 	/* get the number of fields per over-record as announced by LIST OVERVIEW.FMT */
1916 	if (ofmt) {
1917 		for (; ofmt[over_fields].name; over_fields++) {
1918 			if (local && !path_in_ofmt && !strcasecmp(ofmt[over_fields].name, "Path:"))
1919 				path_in_ofmt = TRUE;
1920 		}
1921 	}
1922 
1923 	if (!--over_fields) { /* e.g. nntp_caps.type == CAPABILITIES && !nntp_caps.list_overview_fmt -> assume defaults */
1924 		ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (8 + 1));
1925 		ofmt[0].type = OVER_T_INT;
1926 		ofmt[0].name = my_strdup("Artnum:");
1927 		ofmt[1].type = OVER_T_STRING;
1928 		ofmt[1].name = my_strdup("Subject:");
1929 		ofmt[2].type = OVER_T_STRING;
1930 		ofmt[2].name = my_strdup("From:");
1931 		ofmt[3].type = OVER_T_STRING;
1932 		ofmt[3].name = my_strdup("Date:");
1933 		ofmt[4].type = OVER_T_STRING;
1934 		ofmt[4].name = my_strdup("Message-ID:");
1935 		ofmt[5].type = OVER_T_STRING;
1936 		ofmt[5].name = my_strdup("References:");
1937 		ofmt[6].type = OVER_T_INT;
1938 		ofmt[6].name = my_strdup("Bytes:");
1939 		ofmt[7].type = OVER_T_INT;
1940 		ofmt[7].name = my_strdup("Lines:");
1941 		ofmt[8].type = OVER_T_ERROR;
1942 		ofmt[8].name = NULL;
1943 		over_fields = 7;
1944 	}
1945 
1946 	while ((buf = tin_fgets(fp, FALSE)) != NULL) {
1947 #ifdef DEBUG
1948 		if ((debug & DEBUG_NNTP) && fp == FAKE_NNTP_FP && verbose)
1949 			debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1950 #endif /* DEBUG */
1951 
1952 		if (need_resize) {
1953 			handle_resize((need_resize == cRedraw) ? TRUE : FALSE);
1954 			need_resize = cNo;
1955 		}
1956 
1957 		/*
1958 		 * Read artnum
1959 		 */
1960 		if ((ptr = tin_strtok(buf, "\t")) == NULL)
1961 			continue;
1962 
1963 		/*
1964 		 * read the article number, guaranteed to be the first field
1965 		 */
1966 		artnum = atoartnum(ptr);
1967 
1968 		/*
1969 		 * artnum field invalid/corrupt or is 1st line of local cached overview
1970 		 * (group name)
1971 		 */
1972 		if (artnum <= 0)
1973 			continue;
1974 
1975 		/*
1976 		 * skip artnums below the given minimum (getart_limit)
1977 		 */
1978 		if (artnum < min)
1979 			continue;
1980 
1981 		/*
1982 		 * Check to make sure article in nov file has not expired in group
1983 		 */
1984 		if (artnum < group->xmin) {
1985 			expired++;
1986 			continue;
1987 		}
1988 
1989 		/*
1990 		 * artnum in overview data higher than groups high mark
1991 		 *
1992 		 * TODO: - warn user about broken overviews?
1993 		 *       - try to parse the Xref:-line to get the correct artnum
1994 		 *       - see also parse_unread_arts()
1995 		 */
1996 		if (artnum > group->xmax)
1997 			continue;
1998 
1999 		if (top_art >= max_art)
2000 			expand_art();
2001 
2002 		art = &arts[top_art];
2003 		set_article(art);
2004 		art->artnum = *top = artnum;
2005 
2006 		/*
2007 		 * Note: Fields after line count are not mandatory, use "LIST OVERVIEW.FMT"
2008 		 *       to check for additions like we do with xref_supported
2009 		 */
2010 		for (count = 1; (ptr = tin_strtok(NULL, "\t")) != NULL; count++) {
2011 			/* skip unexpected tailing fields */
2012 			if (count > over_fields) {
2013 #ifdef DEBUG
2014 				if ((debug & DEBUG_NNTP) && verbose > 1)
2015 					debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") Unexpected overview-field %d of %d: %s", nntp_caps.over_cmd, artnum, count, over_fields, ptr);
2016 #endif /* DEBUG */
2017 
2018 				/* "common error" Xref:full in overview-data but not in OVERVIEW.FMT */
2019 				if (count == over_fields + 1) {
2020 					if (!strncasecmp(ptr, "Xref: ", 6)) {
2021 #ifdef DEBUG
2022 						if ((debug & DEBUG_NNTP) && verbose > 1)
2023 							debug_print_file("NNTP", "%s: found unexpected Xref: on semi std. position", nntp_caps.over_cmd);
2024 #endif /* DEBUG */
2025 						over_fields++;
2026 						ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
2027 						ofmt[over_fields].type = OVER_T_FSTRING;
2028 						ofmt[over_fields].name = my_strdup("Xref:");
2029 						ofmt[over_fields + 1].type = OVER_T_ERROR;
2030 						ofmt[over_fields + 1].name = NULL;
2031 						xref_supported = TRUE;
2032 					} else if (local && !strncasecmp(ptr, "Path: ", 6)) {
2033 #ifdef DEBUG
2034 						if ((debug & DEBUG_NNTP) && verbose > 1)
2035 							debug_print_file("NNTP", "%s: found Path:", nntp_caps.over_cmd);
2036 #endif /* DEBUG */
2037 						over_fields++;
2038 						ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
2039 						ofmt[over_fields].type = OVER_T_FSTRING;
2040 						ofmt[over_fields].name = my_strdup("Path:");
2041 						ofmt[over_fields + 1].type = OVER_T_ERROR;
2042 						ofmt[over_fields + 1].name = NULL;
2043 						xref_supported = TRUE;
2044 					} else
2045 						continue;
2046 				} else
2047 					continue;
2048 			}
2049 
2050 			/* for duplicated headers this is last match counts, INN >= 2.5.3 does first match counts */
2051 			if (expensive_over_parse) { /* strange order */
2052 				/* mandatory fields */
2053 				if (ofmt[count].type == OVER_T_STRING) {
2054 					if (!strcasecmp(ofmt[count].name, "Subject:")) {
2055 						if (*ptr) {
2056 #ifdef HAVE_UNICODE_NORMALIZATION
2057 							if (IS_LOCAL_CHARSET("UTF-8"))
2058 								q =  normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2059 							else
2060 #endif /* HAVE_UNICODE_NORMALIZATION */
2061 								q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2062 
2063 							art->subject = hash_str(q);
2064 							free(q);
2065 						} else {
2066 							art->subject = hash_str("");
2067 #ifdef DEBUG
2068 							if ((debug & DEBUG_NNTP) && verbose > 1)
2069 								debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2070 #endif /* DEBUG */
2071 						}
2072 						continue;
2073 					}
2074 
2075 					if (!strcasecmp(ofmt[count].name, "From:")) {
2076 						if (*ptr) {
2077 							art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2078 							art->from = hash_str(buffer_to_ascii(art_from_addr));
2079 							if (*art_full_name)
2080 								art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2081 						} else {
2082 							art->from = hash_str("");
2083 #ifdef DEBUG
2084 							if ((debug & DEBUG_NNTP) && verbose > 1)
2085 								debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2086 #endif /* DEBUG */
2087 						}
2088 						continue;
2089 					}
2090 
2091 					if (!strcasecmp(ofmt[count].name, "Date:")) {
2092 						art->date = parsedate(ptr, (TIMEINFO *) 0);
2093 #ifdef DEBUG
2094 						if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2095 							debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2096 #endif /* DEBUG */
2097 						continue;
2098 					}
2099 
2100 					if (!strcasecmp(ofmt[count].name, "Message-ID:")) {
2101 						if (*ptr) {
2102 							FreeIfNeeded(art->msgid); /* if field is listed more than once in overview.fmt */
2103 							art->msgid = my_strdup(ptr);
2104 						} else {
2105 							art->msgid = NULL;
2106 #ifdef DEBUG
2107 							if ((debug & DEBUG_NNTP) && verbose > 1)
2108 								debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2109 #endif /* DEBUG */
2110 						}
2111 						continue;
2112 					}
2113 
2114 					if (!strcasecmp(ofmt[count].name, "References:")) {
2115 						if (*ptr) {
2116 							FreeIfNeeded(art->refs); /* if field is listed more than once in overview.fmt */
2117 							art->refs = my_strdup(ptr);
2118 						} else
2119 							art->refs = NULL;
2120 						continue;
2121 					}
2122 
2123 					/*
2124 					 * non std. fields when doing
2125 					 * expensive overview parsing (very
2126 					 * rare, just happens if RFC 3977
2127 					 * 8.4.2 is violated) go here
2128 					 */
2129 					/* for Path:-filter */
2130 					if (!strcasecmp(ofmt[count].name, "Path:")) {
2131 						if (!path_found)
2132 							path_found = TRUE;
2133 						if (*ptr) {
2134 							FreeIfNeeded(art->path); /* if field is listed more than once in overview.fmt */
2135 							art->path = my_strdup(ptr);
2136 						} else
2137 							art->path = NULL;
2138 						continue;
2139 					}
2140 				}
2141 				/* metadata fields */
2142 				if (ofmt[count].type == OVER_T_INT) {
2143 					if (!strcasecmp(ofmt[count].name, "Bytes:")) {
2144 						if (*ptr) {
2145 #ifdef DEBUG
2146 							if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2147 								debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2148 #endif /* DEBUG */
2149 						}
2150 						continue;
2151 					}
2152 
2153 					if (!strcasecmp(ofmt[count].name, "Lines:")) {
2154 						if (*ptr) {
2155 							if (isdigit((unsigned char) *ptr))
2156 								art->line_count = atoi(ptr);
2157 							else {
2158 								art->line_count = 0;
2159 #ifdef DEBUG
2160 								if ((debug & DEBUG_NNTP) && verbose > 1)
2161 									debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2162 #endif /* DEBUG */
2163 							}
2164 						} else
2165 							art->line_count = 0;
2166 						continue;
2167 					}
2168 				}
2169 			} else { /* first 7 fields are in RFC 3977 order */
2170 				switch (count) {
2171 					case 1: /* Subject: */
2172 						if (*ptr) {
2173 #ifdef HAVE_UNICODE_NORMALIZATION
2174 							if (IS_LOCAL_CHARSET("UTF-8"))
2175 								q =  normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2176 							else
2177 #endif /* HAVE_UNICODE_NORMALIZATION */
2178 								q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2179 
2180 							art->subject = hash_str(q);
2181 							free(q);
2182 						} else {
2183 							art->subject = hash_str("");
2184 #ifdef DEBUG
2185 							if ((debug & DEBUG_NNTP) && verbose > 1)
2186 								debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2187 #endif /* DEBUG */
2188 						}
2189 						break;
2190 
2191 					case 2:	/* From: */
2192 						if (*ptr) {
2193 							art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2194 							art->from = hash_str(buffer_to_ascii(art_from_addr));
2195 							if (*art_full_name)
2196 								art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2197 						} else {
2198 							art->from = hash_str("");
2199 #ifdef DEBUG
2200 							if ((debug & DEBUG_NNTP) && verbose > 1)
2201 								debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2202 #endif /* DEBUG */
2203 						}
2204 						break;
2205 
2206 					case 3:	/* Date: */
2207 						art->date = parsedate(ptr, (TIMEINFO *) 0);
2208 #ifdef DEBUG
2209 						if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2210 							debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2211 #endif /* DEBUG */
2212 						break;
2213 
2214 					case 4:	/* Message-ID: */
2215 						if (*ptr)
2216 							art->msgid = my_strdup(ptr);
2217 						else {
2218 							art->msgid = NULL;
2219 #ifdef DEBUG
2220 							if ((debug & DEBUG_NNTP) && verbose > 1)
2221 								debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2222 #endif /* DEBUG */
2223 						}
2224 						break;
2225 
2226 					case 5:	/* References: */
2227 						if (*ptr)
2228 							art->refs = my_strdup(ptr);
2229 						else
2230 							art->refs = NULL;
2231 						break;
2232 
2233 					case 6:	/* :bytes || Bytes: */
2234 						if (*ptr) {
2235 #ifdef DEBUG
2236 							if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2237 								debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2238 #endif /* DEBUG */
2239 						}
2240 						break;
2241 
2242 					case 7:	/* :lines || Lines: */
2243 						if (*ptr) {
2244 							if (isdigit((unsigned char) *ptr))
2245 								art->line_count = atoi(ptr);
2246 							else {
2247 								art->line_count = 0;
2248 #ifdef DEBUG
2249 								if ((debug & DEBUG_NNTP) && verbose > 1)
2250 									debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2251 #endif /* DEBUG */
2252 							}
2253 						} else
2254 							art->line_count = 0;
2255 						break;
2256 
2257 					default:
2258 						break;
2259 				}
2260 			}
2261 
2262 			/* optional fields; for duplicated headers: last match counts, INN >= 2.5.3 does first match counts */
2263 			if (ofmt[count].type == OVER_T_FSTRING) {
2264 				if (*ptr) {
2265 					if (!strcasecmp(ofmt[count].name, "Xref:")) {
2266 						if ((q = parse_header(ptr, "Xref", FALSE, FALSE, FALSE)) != NULL) {
2267 							FreeIfNeeded(art->xref); /* if field is listed more than once in overview.fmt */
2268 							art->xref = my_strdup(q);
2269 						}
2270 #ifdef DEBUG
2271 						else {
2272 							if ((debug & DEBUG_NNTP) && verbose > 1)
2273 								debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2274 						}
2275 #endif /* DEBUG */
2276 						continue;
2277 					}
2278 					/*
2279 					 * handling of addition overview fields
2280 					 * goes here
2281 					 */
2282 #ifdef DEBUG
2283 					if ((debug & DEBUG_NNTP) && verbose > 1)
2284 						debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") extra overview-field \"%s\" at position %d %s", nntp_caps.over_cmd, artnum, ofmt[count].name, count, ptr);
2285 #endif /* DEBUG */
2286 					/* if we're lucky we've Path in NOV */
2287 					/*
2288 					 * if reading locally cached overview data try
2289 					 * path regardless of the server OVERVIEW.FMT
2290 					 */
2291 					if (local || !strcasecmp(ofmt[count].name, "Path:")) {
2292 						if ((q = parse_header(ptr, "Path", FALSE, FALSE, FALSE)) != NULL) {
2293 							if (!path_found)
2294 								path_found = TRUE;
2295 							FreeIfNeeded(art->path);
2296 							art->path = my_strdup(q);
2297 #ifdef DEBUG
2298 							if ((debug & DEBUG_NNTP) && verbose > 1 && strcasecmp(ofmt[count].name, "Path:"))
2299 								debug_print_file("NNTP", "\tUsing as \"Path:\" not \"%s\"", ofmt[count].name);
2300 #endif /* DEBUG */
2301 
2302 						}
2303 						continue;
2304 					}
2305 				}
2306 				continue;
2307 			}
2308 		}
2309 
2310 		/*
2311 		 * RFC says Message-ID is mandatory in newsgroups (but not in
2312 		 * mailgroups etc..) NB. a NULL Message-ID would abort if we ever do
2313 		 * threading in mailgroups
2314 		 */
2315 		if (!art->msgid && group->type == GROUP_TYPE_NEWS)
2316 			continue;
2317 
2318 		/* we might lose accuracy here, but that shouldn't hurt */
2319 		if (artnum % (MODULO_COUNT_NUM * 20) == 0)
2320 			show_progress(group_msg, artnum - min, max - min);
2321 
2322 		top_art++;				/* Basically this statement commits the article */
2323 	}
2324 
2325 	free(group_msg);
2326 	TIN_FCLOSE(fp);
2327 
2328 	if (tin_errno)
2329 		return -1;
2330 
2331 #if defined(NNTP_ABLE) && defined(XHDR_XREF)
2332 	if (read_news_via_nntp && !read_saved_news && !xref_supported && nntp_caps.hdr_cmd) {
2333 		char cbuf[HEADER_LEN];
2334 		int i;
2335 		static t_bool found;
2336 		static t_bool first = TRUE;
2337 
2338 		if (first) {
2339 			found = TRUE;
2340 			/*
2341 			 * TODO: if "LIST HEADERS RANGE" failed try "LIST HEADERS"?
2342 			 */
2343 			if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers) {
2344 				if (!*nntp_caps.headers_range) {
2345 					i = new_nntp_command("LIST HEADERS RANGE", 215, cbuf, sizeof(cbuf));
2346 
2347 					found = FALSE;
2348 					switch (i) {
2349 						case 215:
2350 							while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
2351 #	ifdef DEBUG
2352 								if (debug & DEBUG_NNTP)
2353 									debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2354 #	endif /* DEBUG */
2355 								if (!found && ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4)))
2356 									found = TRUE;
2357 								nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
2358 								strcat(nntp_caps.headers_range, ptr);
2359 								strcat(nntp_caps.headers_range, "\n");
2360 							}
2361 							break;
2362 
2363 						default:
2364 							break;
2365 					}
2366 					first = FALSE;
2367 				} else {
2368 					found = FALSE;
2369 					if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n" )) != NULL) {
2370 						do {
2371 							if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4))
2372 								found = TRUE;
2373 						} while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
2374 					}
2375 				}
2376 			}
2377 		}
2378 
2379 		if (found) {
2380 			snprintf(cbuf, sizeof(cbuf), "%s XREF %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, MAX(min, max));
2381 			group_msg = fmt_string("%s XREF loop", nntp_caps.hdr_cmd); /* TODO: find a better message, move to lang.c */
2382 			if ((fp = nntp_command(cbuf, nntp_caps.hdr ? OK_HDR : OK_HEAD, NULL, 0)) != NULL) { /* RFC 2980 (XHDR) uses 221; RFC 3977 (HDR) uses 225 */
2383 				while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
2384 #	ifdef DEBUG
2385 					if ((debug & DEBUG_NNTP) && verbose)
2386 						debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2387 #	endif /* DEBUG */
2388 
2389 					artnum = atoartnum(ptr);
2390 					if (artnum <= 0 || artnum < group->xmin || artnum > group->xmax)
2391 						continue;
2392 					art = &arts[top_art];
2393 					set_article(art);
2394 					if (!art->xref && !strstr(ptr, "(none)")) {
2395 						if ((q = strchr(ptr, ' ')) == NULL) /* skip article number */
2396 							continue;
2397 						ptr = q;
2398 						while (*ptr && isspace((int) *ptr))
2399 							ptr++;
2400 						q = strchr(ptr, '\n');
2401 						if (q)
2402 							*q = '\0';
2403 						art->xref = my_strdup(ptr);
2404 					}
2405 					/* we might lose accuracy here, but that shouldn't hurt */
2406 					if (artnum % (MODULO_COUNT_NUM * 20) == 0)
2407 						show_progress(group_msg, artnum - min, max - min);
2408 				}
2409 			}
2410 			free(group_msg);
2411 		}
2412 	}
2413 #endif /* NNTP_ABLE && XHDR_XREF */
2414 
2415 	if (local) {
2416 #ifdef NNTP_ABLE
2417 		if (filter_on_path(group)) {
2418 			struct t_article_range *ranges, *curr;
2419 			t_bool supported = TRUE;
2420 			int curr_range, range_cnt;
2421 
2422 			/*
2423 			 * Get the ranges without Path: header and try to fetch the
2424 			 * headers
2425 			 */
2426 			if ((ranges = build_range_list(min, *top, &range_cnt))) {
2427 				curr = ranges;
2428 				curr_range = 1;
2429 				while (curr && supported) {
2430 					if (curr->cnt)
2431 						supported = get_path_header(curr_range++, range_cnt, group, curr->start, curr->end);
2432 					curr = curr->next;
2433 				}
2434 				if (!supported && path_in_ofmt) {
2435 					/*
2436 					 * fetching Path: headers via [X]HDR or XPAT has failed
2437 					 * Path: is in the servers overview so let the next
2438 					 * read_overview() fetch them
2439 					 */
2440 					free_art_array();
2441 					free_msgids();
2442 					top_art = 0;
2443 					*top = T_ARTNUM_CONST(0);
2444 					expired = 0;
2445 				}
2446 				*rebuild_cache = TRUE;
2447 				while (ranges) {
2448 					curr = ranges;
2449 					ranges = curr->next;
2450 					free(curr);
2451 				}
2452 			}
2453 		}
2454 #endif /* NNTP_ABLE */
2455 	} else
2456 		if (!path_found && filter_on_path(group)) {
2457 #ifdef NNTP_ABLE
2458 			if (!get_path_header(1, 1, group, min, *top))
2459 #endif /* NNTP_ABLE */
2460 				wait_message(2, _(txt_cannot_filter_on_path));
2461 		}
2462 	return expired;
2463 }
2464 
2465 
2466 /*
2467  * Write an Nov/Xover index file. Fields are separated by '\t'.
2468  *
2469  * Format:
2470  *	1. article number (ie. 183)                [mandatory]
2471  *	2. Subject: line  (ie. Which newsreader?)  [mandatory]
2472  *	3. From: line     (ie. iain@ecrc.de)       [mandatory]
2473  *	4. Date: line     (rfc822 format)          [mandatory]
2474  *	5. MessageID:     (ie. <123@ether.net>)    [mandatory]
2475  *	6. References:    (ie. <message-id> ....)  [optional]
2476  *	7. Byte count     (Skipped - not used)     [mandatory]
2477  *	8. Line count     (ie. 23)                 [mandatory]
2478  *	9. Xref: line     (ie. alt.test:389)       [optional]
2479  *
2480  * TODO: as we don't use the original data, we currently can't store
2481  *       the data (from/subject) in the original charset (we don't store
2482  *       that info). this has the advantage that we can avoid raw 8bit data
2483  *       in our overviews, but the disadvantage that we might store the data
2484  *       with a wrong charset and thus lose information. a similar problem
2485  *       exists with the data for the from:-line, we don't store it in the
2486  *       original format, whenever our from-parser (partially) fails we'll
2487  *       lose information in our overviews (but those couldn't be handled
2488  *       by tin anyway, so this is not a real problem).
2489  *       long-term solution: store the original data in the overview
2490  *       (tin has to handle raw 8bit data and other ugly stuff in the
2491  *       overviews anyway and thus we preserver as much info as possible)
2492  *       this would require some changes in read_overview() and
2493  *       parse_headers(): don't do the decoding/unfolding there, but in a
2494  *       second pass right after write_overview(), or two additional fields
2495  *       which hold the raw data for from/subject. the latter has the
2496  *       disadvantage that it costs (much) more memory.
2497  */
2498 static void
write_overview(struct t_group * group)2499 write_overview(
2500 	struct t_group *group)
2501 {
2502 	FILE *fp;
2503 	int i;
2504 	struct t_article *article;
2505 #ifdef CHARSET_CONVERSION
2506 	int c = -1;
2507 #endif /* CHARSET_CONVERSION */
2508 
2509 	/*
2510 	 * Can't write or caching is off or getart_limit is set
2511 	 */
2512 	if (no_write || !tinrc.cache_overview_files || ((cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit) != 0)
2513 		return;
2514 
2515 	if ((fp = open_xover_fp(group, "w", T_ARTNUM_CONST(0), T_ARTNUM_CONST(0), FALSE)) == NULL)
2516 		return;
2517 
2518 	if (group->attribute->sort_article_type != SORT_ARTICLES_BY_NOTHING)
2519 		SortBy(artnum_comp);
2520 
2521 	/*
2522 	 * Needed to preserve uniqueness in hashed private overview files
2523 	 */
2524 	fprintf(fp, "%s\n", group->name);
2525 
2526 #ifdef CHARSET_CONVERSION
2527 	/* get undeclared_charset number if required */
2528 	if (group->attribute->undeclared_charset) {
2529 		for (i = 0; txt_mime_charsets[i] != NULL; i++) {
2530 			if (!strcasecmp(group->attribute->undeclared_charset, txt_mime_charsets[i])) {
2531 				c = i;
2532 				break;
2533 			}
2534 		}
2535 	}
2536 #endif /* CHARSET_CONVERSION */
2537 
2538 	if (verbose && batch_mode) /* -> lang.c */
2539 		wait_message(0, _("Writing %s\n"), group->name);
2540 
2541 	for_each_art(i) {
2542 		char *p;
2543 		char *q, *ref;
2544 
2545 		article = &arts[i];
2546 
2547 		if (article->thread != ART_EXPIRED && article->artnum >= group->xmin) {
2548 			ref = NULL;
2549 
2550 			if (!group->attribute->post_8bit_header) { /* write encoded data */
2551 				/*
2552 				 * TODO: instead of tinrc.mm_local_charset we'd better use UTF-8
2553 				 *       here and in print_from() in the CHARSET_CONVERSION case.
2554 				 *       note that this requires something like
2555 				 *          buffer_to_network(article->subject, "UTF-8");
2556 				 *       right before the rfc1522_encode() call.
2557 				 *
2558 				 *       if we would cache the original undecoded data, we could
2559 				 *       ignore stuff like this.
2560 				 */
2561 				p = rfc1522_encode(article->subject, tinrc.mm_local_charset, FALSE);
2562 				/* as the subject might now be folded we have to unfold it */
2563 				unfold_header(p);
2564 			} else { /* raw data */
2565 				p = my_strdup(article->subject);
2566 #ifdef CHARSET_CONVERSION
2567 				if (group->attribute->undeclared_charset && c != -1) /* use undeclared_charset if set (otherwise local charset is used) */
2568 					buffer_to_network(p, c);
2569 #endif /* CHARSET_CONVERSION */
2570 			}
2571 
2572 			/*
2573 			 * replace any '\t's with ' ' in the references-data
2574 			 *
2575 			 * TODO: nntpext-draft might come up with a new scheme:
2576 			 *       For all fields, the value is processed by first
2577 			 *       removing all US-ASCII CRLF pairs and then replacing
2578 			 *       each remaining US-ASCII NUL, TAB, CR, or LF character
2579 			 *       with a single US-ASCII space (for example, CR LF LF TAB
2580 			 *       will become two spaces).
2581 			 */
2582 			if (article->refs) {
2583 				ref = q = my_strdup(article->refs);
2584 				while (*q) {
2585 					if (*q == '\t')
2586 						*q = ' ';
2587 					q++;
2588 				}
2589 			}
2590 
2591 			fprintf(fp, "%"T_ARTNUM_PFMT"\t%s\t%s\t%s\t%s\t%s\t%d\t%d",
2592 				article->artnum,
2593 				p,
2594 #ifdef CHARSET_CONVERSION
2595 				print_from(group, article, c),
2596 #else
2597 				print_from(group, article, -1),
2598 #endif /* CHARSET_CONVERSION */
2599 				print_date(article->date),
2600 				BlankIfNull(article->msgid),
2601 				BlankIfNull(ref),
2602 				0,	/* bytes */
2603 				article->line_count);
2604 
2605 			if (article->xref)
2606 				fprintf(fp, "\tXref: %s", article->xref);
2607 
2608 			if (article->path)
2609 				fprintf(fp, "\tPath: %s", article->path);
2610 
2611 			fprintf(fp, "\n");
2612 			free(p);
2613 			if (article->refs) {
2614 				FreeIfNeeded(ref);
2615 				q = NULL;
2616 			}
2617 		}
2618 		if (i % (MODULO_COUNT_NUM * 20) == 0) /* TODO: -> lang.c */
2619 			show_progress(_("Writing overview cache..."), i, top_art);
2620 	}
2621 #ifdef HAVE_FCHMOD
2622 	fchmod(fileno(fp), (mode_t) (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH));
2623 	/*
2624 	 * TODO:
2625 	 * add code for !HAVE_FCHMOD && HAVE_CHMOD
2626 	 */
2627 #endif /* HAVE_FCHMOD */
2628 	fclose(fp);
2629 }
2630 
2631 
2632 /*
2633  * A complex little function to determine the correct overview index file
2634  * according to 'mode' (read or write)
2635  * NULL is returned if the current setup dictates otherwise
2636  *
2637  * GROUP_TYPE_MAIL index files are read/written in ~/.tin/.mail
2638  * GROUP_TYPE_SAVE index files are read/written in ~/.tin/.save
2639  *
2640  * Both of these are hashed
2641  *
2642  * GROUP_TYPE_NEWS index files are a little bit more complex
2643  *
2644  * When hashing the index filename will be in format number.number.
2645  * Hashing the groupname gets a number. See if that #.1 file exists;
2646  * if so, read first line. Is this the group we want? If no, try #.2.
2647  * Repeat until no such file or we find an existing file that matches
2648  * our group. Return pointer to path or NULL if not found.
2649  */
2650 static char *
find_nov_file(struct t_group * group,int mode)2651 find_nov_file(
2652 	struct t_group *group,
2653 	int mode)
2654 {
2655 	FILE *fp;
2656 	const char *dir;
2657 	char buf[PATH_LEN];
2658 	int i;
2659 	struct stat sb;
2660 	unsigned long hash;
2661 	static char nov_file[PATH_LEN];
2662 	static t_bool once_only = FALSE;	/* Trap things that are done only 1 time */
2663 
2664 	if (group == NULL || (mode != R_OK && mode != W_OK))
2665 		return NULL;
2666 
2667 	switch (group->type) {
2668 		case GROUP_TYPE_MAIL:
2669 			dir = index_maildir;
2670 			break;
2671 
2672 		case GROUP_TYPE_SAVE:
2673 			dir = index_savedir;
2674 			break;
2675 
2676 		case GROUP_TYPE_NEWS:
2677 			/*
2678 			 * nntp.caps.over_cmd is not an issue here, any gripes and warnings
2679 			 * about [X]OVER are handled in nntp_open()
2680 			 */
2681 
2682 			/*
2683 			 * When reading via NNTP, system wide overviews are irrelevant, of
2684 			 * course, and the private overview filename will be the same for
2685 			 * both reading and writing.
2686 			 *
2687 			 * When working locally, we only use a private cache for reading
2688 			 * if requested and when system wide overviews don't already exist.
2689 			 * When writing then only private overviews can be used since
2690 			 * updating system wide overviews is not safe wrt locking etc.
2691 			 *
2692 			 * See if local overview file $SPOOLDIR/<groupname>/.overview exists
2693 			 */
2694 #ifndef NNTP_ONLY
2695 			if (!read_news_via_nntp) {
2696 				make_base_group_path(novrootdir, group->name, buf, sizeof(buf));
2697 				joinpath(nov_file, sizeof(nov_file), buf, novfilename);
2698 				if (access(nov_file, R_OK) == 0) {
2699 					if (mode == R_OK)
2700 						return nov_file;		/* Use system wide overviews */
2701 					else
2702 						return NULL;			/* Don't write cache in this case */
2703 				}
2704 			}
2705 #endif /* !NNTP_ONLY */
2706 
2707 			/*
2708 			 * We only get here when private overviews are going to be used
2709 			 * Go no further if they are explicitly turned off
2710 			 */
2711 			if (!tinrc.cache_overview_files)
2712 				return NULL;
2713 
2714 			/*
2715 			 * Append -<nntpserver> to private cache dir
2716 			 */
2717 			if (!once_only && nntp_server) {
2718 				size_t sp, ln = strlen(index_newsdir);
2719 
2720 				if ((sp = sizeof(index_newsdir) - ln - 1) >= 2) {
2721 					char *srv = my_strdup(nntp_server);
2722 
2723 					strcat(index_newsdir, "-");
2724 					sp--;
2725 					ln++;
2726 					str_lwr(srv);
2727 					my_strncpy(index_newsdir + ln, srv, sp);
2728 					free(srv);
2729 				}
2730 				once_only = TRUE;
2731 			}
2732 
2733 			/*
2734 			 * Only try to set up the private cache when writing. If it
2735 			 * doesn't exist yet, then ergo we can't read from it.
2736 			 * The cache will be checked/created on every write; a previous
2737 			 * bug report complained that this was not the case
2738 			 */
2739 			if (stat(index_newsdir, &sb) == -1) {			/* Private cache doesn't exist */
2740 				if (mode == R_OK)
2741 					return NULL;
2742 				if (my_mkdir(index_newsdir, (mode_t) S_IRWXU) != 0)
2743 					return NULL;
2744 			} else {
2745 				if (!S_ISDIR(sb.st_mode))
2746 					return NULL;
2747 			}
2748 
2749 			/*
2750 			 * Update the newsgroups cache to point to the new location
2751 			 * now that we know it is valid
2752 			 */
2753 			if (!once_only)
2754 				joinpath(local_newsgroups_file, sizeof(local_newsgroups_file), index_newsdir, NEWSGROUPS_FILE);
2755 
2756 			dir = index_newsdir;
2757 			break;
2758 
2759 		default: /* not reached */
2760 			return NULL;
2761 	}
2762 
2763 	/*
2764 	 * We only get here if writing to a private overview.
2765 	 * These always have hashed filenames.
2766 	 * Try <hash>.<seqno> and check the group name tagline until
2767 	 * matching index file is found. If not found return next unused
2768 	 * filename
2769 	 */
2770 	hash = hash_groupname(group->name);
2771 
2772 	for (i = 1; ; i++) {
2773 		char *ptr;
2774 
2775 		snprintf(buf, sizeof(buf), "%lu.%d", hash, i);
2776 		joinpath(nov_file, sizeof(nov_file), dir, buf);
2777 
2778 		if ((fp = fopen(nov_file, "r")) == NULL)
2779 			break;
2780 
2781 		/*
2782 		 * No group name header, so not a valid index file => overwrite it
2783 		 */
2784 		if (fgets(buf, (int) sizeof(buf), fp) == NULL) {
2785 			fclose(fp);
2786 			break;
2787 		}
2788 		fclose(fp);
2789 
2790 		if ((ptr = strrchr(buf, '\n')) != NULL)
2791 			*ptr = '\0';
2792 
2793 		if (strcmp(buf, group->name) == 0)
2794 			break;
2795 	}
2796 
2797 	return nov_file;
2798 }
2799 
2800 
2801 /*
2802  * Run the index file updater only for the groups we've loaded.
2803  */
2804 void
do_update(t_bool catchup)2805 do_update(
2806 	t_bool catchup)
2807 {
2808 	int i, j, k = 0;
2809 	time_t beg_epoch = 0;
2810 	struct t_article *art;
2811 	struct t_group *group;
2812 
2813 	if (verbose)
2814 		(void) time(&beg_epoch);
2815 
2816 	/*
2817 	 * loop through groups and update any required index files
2818 	 */
2819 	for (i = 0; i < selmenu.max; i++) {
2820 		group = &active[my_group[i]];
2821 		/*
2822 		 * FIXME: workaround to get a valid CURR_GROUP
2823 		 * it also points to the currently processed group so that
2824 		 * the correct attributes are used
2825 		 * The correct fix is to get rid of CURR_GROUP
2826 		 */
2827 		selmenu.curr = i;
2828 
2829 		if (group->bogus || !group->subscribed)
2830 			continue;
2831 
2832 		if (!index_group(group)) {
2833 			for_each_art(j) {
2834 				art = &arts[j];
2835 				FreeAndNull(art->refs);
2836 				FreeAndNull(art->msgid);
2837 			}
2838 			continue;
2839 		}
2840 
2841 		k++;
2842 
2843 		if (verbose) {
2844 			my_printf("%s %s\n", (catchup ? _(txt_catchup) : _(txt_updating)), group->name);
2845 			my_flush();
2846 		}
2847 
2848 		if (catchup) {
2849 			for_each_art(j)
2850 				art_mark(group, &arts[j], ART_READ);
2851 		}
2852 	}
2853 
2854 	if (verbose) {
2855 		wait_message(0, _(txt_catchup_update_info),
2856 			(catchup ? _(txt_caughtup) : _(txt_updated)), k,
2857 			PLURAL(selmenu.max, txt_group), (unsigned long int) (time(NULL) - beg_epoch));
2858 	}
2859 }
2860 
2861 
2862 static int
artnum_comp(t_comptype p1,t_comptype p2)2863 artnum_comp(
2864 	t_comptype p1,
2865 	t_comptype p2)
2866 {
2867 	const struct t_article *s1 = (const struct t_article *) p1;
2868 	const struct t_article *s2 = (const struct t_article *) p2;
2869 
2870 	/*
2871 	 * s1->artnum less than s2->artnum
2872 	 */
2873 	if (s1->artnum < s2->artnum)
2874 		return -1;
2875 
2876 	/*
2877 	 * s1->artnum greater than s2->artnum
2878 	 */
2879 	if (s1->artnum > s2->artnum)
2880 		return 1;
2881 
2882 	return 0;
2883 }
2884 
2885 
2886 /*
2887  * return result of strcmp (reversed for descending)
2888  */
2889 static int
subj_comp_asc(t_comptype p1,t_comptype p2)2890 subj_comp_asc(
2891 	t_comptype p1,
2892 	t_comptype p2)
2893 {
2894 	int retval;
2895 	const struct t_article *s1 = (const struct t_article *) p1;
2896 	const struct t_article *s2 = (const struct t_article *) p2;
2897 
2898 	if ((retval = strcasecmp(s1->subject, s2->subject))) /* != 0 */
2899 		return retval;
2900 
2901 	return s1->date - s2->date > 0 ? 1 : -1;
2902 }
2903 
2904 
2905 static int
subj_comp_desc(t_comptype p1,t_comptype p2)2906 subj_comp_desc(
2907 	t_comptype p1,
2908 	t_comptype p2)
2909 {
2910 	int retval;
2911 	const struct t_article *s1 = (const struct t_article *) p1;
2912 	const struct t_article *s2 = (const struct t_article *) p2;
2913 
2914 	if ((retval = strcasecmp(s2->subject, s1->subject))) /* != 0 */
2915 		return retval;
2916 
2917 	return s1->date - s2->date > 0 ? 1 : -1;
2918 }
2919 
2920 
2921 /*
2922  * return result of strcmp (reversed for descending)
2923  */
2924 static int
from_comp_asc(t_comptype p1,t_comptype p2)2925 from_comp_asc(
2926 	t_comptype p1,
2927 	t_comptype p2)
2928 {
2929 	int retval;
2930 	const struct t_article *s1 = (const struct t_article *) p1;
2931 	const struct t_article *s2 = (const struct t_article *) p2;
2932 
2933 	if ((retval = strcasecmp(s1->from, s2->from))) /* != 0 */
2934 		return retval;
2935 
2936 	return s1->date - s2->date > 0 ? 1 : -1;
2937 }
2938 
2939 
2940 static int
from_comp_desc(t_comptype p1,t_comptype p2)2941 from_comp_desc(
2942 	t_comptype p1,
2943 	t_comptype p2)
2944 {
2945 	int retval;
2946 	const struct t_article *s1 = (const struct t_article *) p1;
2947 	const struct t_article *s2 = (const struct t_article *) p2;
2948 
2949 	if ((retval = strcasecmp(s2->from, s1->from))) /* != 0 */
2950 		return retval;
2951 
2952 	return s1->date - s2->date > 0 ? 1 : -1;
2953 }
2954 
2955 
2956 /*
2957  * Works like strcmp() for comparing time_t type values
2958  * Return codes:
2959  *  -1:		If p1 is before p2
2960  *   0:		If they are the same time
2961  *   1:		If p1 is after p2
2962  * If the sort order is _not_ DATE_ASCEND then the sense of the above
2963  * is reversed.
2964  */
2965 static int
date_comp_asc(t_comptype p1,t_comptype p2)2966 date_comp_asc(
2967 	t_comptype p1,
2968 	t_comptype p2)
2969 {
2970 	const struct t_article *s1 = (const struct t_article *) p1;
2971 	const struct t_article *s2 = (const struct t_article *) p2;
2972 
2973 	/*
2974 	 * s1->date less than s2->date
2975 	 */
2976 	if (s1->date < s2->date)
2977 		return -1;
2978 
2979 	/*
2980 	 * s1->date greater than s2->date
2981 	 */
2982 	if (s1->date > s2->date)
2983 		return 1;
2984 
2985 	return 0;
2986 }
2987 
2988 
2989 static int
date_comp_desc(t_comptype p1,t_comptype p2)2990 date_comp_desc(
2991 	t_comptype p1,
2992 	t_comptype p2)
2993 {
2994 	const struct t_article *s1 = (const struct t_article *) p1;
2995 	const struct t_article *s2 = (const struct t_article *) p2;
2996 
2997 	/*
2998 	 * s2->date less than s1->date
2999 	 */
3000 	if (s2->date < s1->date)
3001 		return -1;
3002 
3003 	/*
3004 	 * s2->date greater than s1->date
3005 	 */
3006 	if (s2->date > s1->date)
3007 		return 1;
3008 
3009 	return 0;
3010 }
3011 
3012 
3013 /*
3014  * Same again, but for art[].score
3015  */
3016 static int
score_comp_asc(t_comptype p1,t_comptype p2)3017 score_comp_asc(
3018 	t_comptype p1,
3019 	t_comptype p2)
3020 {
3021 	const struct t_article *s1 = (const struct t_article *) p1;
3022 	const struct t_article *s2 = (const struct t_article *) p2;
3023 
3024 	if (s1->score < s2->score)
3025 		return -1;
3026 
3027 	if (s1->score > s2->score)
3028 		return 1;
3029 
3030 	return s1->date - s2->date > 0 ? 1 : -1;
3031 }
3032 
3033 
3034 static int
score_comp_desc(t_comptype p1,t_comptype p2)3035 score_comp_desc(
3036 	t_comptype p1,
3037 	t_comptype p2)
3038 {
3039 	const struct t_article *s1 = (const struct t_article *) p1;
3040 	const struct t_article *s2 = (const struct t_article *) p2;
3041 
3042 	if (s2->score < s1->score)
3043 		return -1;
3044 
3045 	if (s2->score > s1->score)
3046 		return 1;
3047 
3048 	return s1->date - s2->date > 0 ? 1 : -1;
3049 }
3050 
3051 
3052 /*
3053  * Same again, but for art[].line_count
3054  */
3055 static int
lines_comp_asc(t_comptype p1,t_comptype p2)3056 lines_comp_asc(
3057 	t_comptype p1,
3058 	t_comptype p2)
3059 {
3060 	const struct t_article *s1 = (const struct t_article *) p1;
3061 	const struct t_article *s2 = (const struct t_article *) p2;
3062 
3063 	if (s1->line_count < s2->line_count)
3064 		return -1;
3065 
3066 	if (s1->line_count > s2->line_count)
3067 		return 1;
3068 
3069 	return s1->date - s2->date > 0 ? 1 : -1;
3070 }
3071 
3072 
3073 static int
lines_comp_desc(t_comptype p1,t_comptype p2)3074 lines_comp_desc(
3075 	t_comptype p1,
3076 	t_comptype p2)
3077 {
3078 	const struct t_article *s1 = (const struct t_article *) p1;
3079 	const struct t_article *s2 = (const struct t_article *) p2;
3080 
3081 	if (s2->line_count < s1->line_count)
3082 		return -1;
3083 
3084 	if (s2->line_count > s1->line_count)
3085 		return 1;
3086 
3087 	return s1->date - s2->date > 0 ? 1 : -1;
3088 }
3089 
3090 
3091 /*
3092  * Compares the total score of two threads. Used for sorting base[].
3093  */
3094 static int
score_comp_base(t_comptype p1,t_comptype p2)3095 score_comp_base(
3096 	t_comptype p1,
3097 	t_comptype p2)
3098 {
3099 	int a = get_score_of_thread(*(const long *) p1);
3100 	int b = get_score_of_thread(*(const long *) p2);
3101 
3102 	/* If scores are equal, compare using the article sort order.
3103 	 * This determines the order in a group of equally scored threads.
3104 	 */
3105 	if (a == b) {
3106 		const struct t_article *s1 = &arts[*(const long *) p1];
3107 		const struct t_article *s2 = &arts[*(const long *) p2];
3108 		t_compfunc comp_func = eval_sort_arts_func(CURR_GROUP.attribute->sort_article_type);
3109 
3110 		if (comp_func)
3111 			return (*comp_func)(s1, s2);
3112 		return 0;
3113 	}
3114 
3115 	if (CURR_GROUP.attribute->sort_threads_type == SORT_THREADS_BY_SCORE_ASCEND)
3116 		return a > b ? 1 : -1;
3117 	return a < b ? 1 : -1;
3118 }
3119 
3120 
3121 /*
3122  * Compare the date of the last posted article of two threads.
3123  * Used for sorting base[].
3124  */
3125 static int
last_date_comp_base_desc(t_comptype p1,t_comptype p2)3126 last_date_comp_base_desc(
3127 	t_comptype p1,
3128 	t_comptype p2)
3129 {
3130 	time_t s1_last = get_last_posting_date(*(const long *) p1);
3131 	time_t s2_last = get_last_posting_date(*(const long *) p2);
3132 
3133 	if (s2_last < s1_last)
3134 		return -1;
3135 
3136 	if (s2_last > s1_last)
3137 		return 1;
3138 
3139 	return 0;
3140 }
3141 
3142 
3143 static int
last_date_comp_base_asc(t_comptype p1,t_comptype p2)3144 last_date_comp_base_asc(
3145 	t_comptype p1,
3146 	t_comptype p2)
3147 {
3148 	time_t s1_last = get_last_posting_date(*(const long *) p1);
3149 	time_t s2_last = get_last_posting_date(*(const long *) p2);
3150 
3151 	if (s2_last > s1_last)
3152 		return -1;
3153 
3154 	if (s2_last < s1_last)
3155 		return 1;
3156 
3157 	return 0;
3158 }
3159 
3160 
3161 static time_t
get_last_posting_date(long n)3162 get_last_posting_date(
3163 	long n)
3164 {
3165 	long i;
3166 	time_t last = (time_t) 0;
3167 
3168 	for (i = n; i >= 0; i = arts[i].thread) {
3169 		if (arts[i].date > last)
3170 			last = arts[i].date;
3171 	}
3172 
3173 	return last;
3174 }
3175 
3176 
3177 void
set_article(struct t_article * art)3178 set_article(
3179 	struct t_article *art)
3180 {
3181 	art->subject = NULL;
3182 	art->from = NULL;
3183 	art->name = NULL;
3184 	art->date = (time_t) 0;
3185 	art->xref = NULL;
3186 	art->msgid = NULL;
3187 	art->refs = NULL;
3188 	art->refptr = NULL;
3189 	art->line_count = -1;
3190 	art->archive = NULL;
3191 	art->tagged = 0;
3192 	art->thread = ART_EXPIRED;
3193 	art->prev = ART_NORMAL;
3194 	art->score = 0;
3195 	art->status = ART_UNREAD;
3196 	art->killed = ART_NOTKILLED;
3197 	art->zombie = FALSE;
3198 	art->delete_it = FALSE;
3199 	art->selected = FALSE;
3200 	art->inrange = FALSE;
3201 	art->matched = FALSE;
3202 	art->keep_in_base = FALSE;
3203 	art->multipart_subj = FALSE;
3204 }
3205 
3206 
3207 /*
3208  * Do a binary chop to see if 'art' (an article number) exists in arts[]
3209  * Naturally arts[] must be sorted on artnum
3210  * Return index into arts[] or -1
3211  */
3212 static int
valid_artnum(t_artnum art)3213 valid_artnum(
3214 	t_artnum art)
3215 {
3216 	int prev, range;
3217 	int dctop = top_art;
3218 	int cur = 1;
3219 
3220 	while ((dctop >>= 1))
3221 		cur <<= 1;
3222 
3223 	range = cur >> 1;
3224 	cur--;
3225 
3226 	forever {
3227 		if (arts[cur].artnum == art)
3228 			return cur;
3229 
3230 		prev = cur;
3231 		cur += ((arts[cur].artnum < art) ? range : -range);
3232 		if (prev == cur)
3233 			break;
3234 
3235 		if (cur >= top_art)
3236 			cur = top_art - 1;
3237 
3238 		range >>= 1;
3239 	}
3240 	return -1;
3241 }
3242 
3243 
3244 /*
3245  * Loop over arts[] to see if 'art' (an article number) exists in arts[]
3246  * Needed if arts[] is not sorted on artnum
3247  * Return index into arts[] or -1
3248  */
3249 int
find_artnum(t_artnum art)3250 find_artnum(
3251 	t_artnum art)
3252 {
3253 	int i;
3254 
3255 	for_each_art(i) {
3256 		if (arts[i].artnum == art)
3257 			return i;
3258 	}
3259 	return -1;
3260 }
3261 
3262 
3263 /*----------------------------- Overview handling -----------------------*/
3264 /* TODO: use
3265  *           setlocale(LC_ALL, "POSIX"); setlocale(LC_TIME, "POSIX");
3266  *           my_strftime(date, sizeof(date) -1, "%d %b %Y %H:%M:%S GMT", gmtime(&secs));
3267  *       instead?
3268  */
3269 static char *
print_date(time_t secs)3270 print_date(
3271 	time_t secs)
3272 {
3273 	static char date[25];
3274 	struct tm *tm;
3275 	static const char *const months_a[] = {
3276 		"Jan", "Feb", "Mar", "Apr", "May", "Jun",
3277 		"Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
3278 	};
3279 
3280 	if ((tm = gmtime(&secs)) != NULL)
3281 		snprintf(date, sizeof(date), "%02d %.3s %04d %02d:%02d:%02d GMT",
3282 			tm->tm_mday,
3283 			months_a[tm->tm_mon],
3284 			tm->tm_year + 1900,
3285 			tm->tm_hour, tm->tm_min, tm->tm_sec);
3286 	else
3287 		snprintf(date, sizeof(date), "01 Jan 1970 00:00:00 UTC");
3288 
3289 	return date;
3290 }
3291 
3292 
3293 static char *
print_from(struct t_group * group,struct t_article * article,int charset)3294 print_from(
3295 	struct t_group *group,
3296 	struct t_article *article,
3297 	int charset)
3298 {
3299 	char *p, *q;
3300 	static char from[PATH_LEN];
3301 
3302 	*from = '\0';
3303 
3304 	if (article->name != NULL) {
3305 		q = my_strdup(article->name);
3306 #ifdef CHARSET_CONVERSION
3307 		if (charset != -1)
3308 			buffer_to_network(q, charset);
3309 #endif /* CHARSET_CONVERSION */
3310 		p = rfc1522_encode(article->name, tinrc.mm_local_charset, FALSE);
3311 		unfold_header(p);
3312 		if (strpbrk(article->name, "\".:;<>@[]()\\") != NULL && article->name[0] != '"' && article->name[strlen(article->name)] != '"')
3313 			snprintf(from, sizeof(from), "\"%s\" <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3314 		else
3315 			snprintf(from, sizeof(from), "%s <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3316 
3317 		free(p);
3318 		free(q);
3319 	} else
3320 		snprintf(from, sizeof(from), "<%s>", article->from);
3321 
3322 	return from;
3323 }
3324 
3325 
3326 /*
3327  * Open a group news overview file
3328  * Use NNTP XOVER where possible unless 'local' is set
3329  */
3330 static FILE *
open_xover_fp(struct t_group * group,const char * mode,t_artnum min,t_artnum max,t_bool local)3331 open_xover_fp(
3332 	struct t_group *group,
3333 	const char *mode,
3334 	t_artnum min,
3335 	t_artnum max,
3336 	t_bool local)
3337 {
3338 #ifdef NNTP_ABLE
3339 	if (!local && nntp_caps.over_cmd && *mode == 'r' && group->type == GROUP_TYPE_NEWS) {
3340 		char line[NNTP_STRLEN];
3341 
3342 		if (!max)
3343 			return NULL;
3344 		if (min == max)
3345 			snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT, nntp_caps.over_cmd, min);
3346 		else
3347 			snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.over_cmd, min, MAX(min, max));
3348 		return (nntp_command(line, OK_XOVER, NULL, 0));
3349 	}
3350 #endif /* NNTP_ABLE */
3351 	{
3352 		FILE *fp;
3353 		char *nov_file = find_nov_file(group, (*mode == 'r') ? R_OK : W_OK);
3354 
3355 		if (nov_file != NULL) {
3356 			if ((fp = fopen(nov_file, mode)) != NULL)
3357 				return fp;
3358 
3359 			if (*mode != 'r')
3360 				error_message(2, _(txt_cannot_open), nov_file);
3361 		}
3362 	}
3363 	return NULL;
3364 }
3365 
3366 
3367 #ifdef USE_HEAPSORT
3368 int
tin_sort(void * sbase,size_t nel,size_t width,t_compfunc compar)3369 tin_sort(
3370 	void *sbase,
3371 	size_t nel,
3372 	size_t width,
3373 	t_compfunc compar)
3374 {
3375 	int rc;
3376 
3377 	switch (tinrc.sort_function) {
3378 		case 0:
3379 			qsort(sbase, nel, width, compar);
3380 			rc = 0;
3381 			break;
3382 
3383 		case 1:
3384 			rc = heapsort(sbase, nel, width, compar);
3385 			break;
3386 
3387 		default:
3388 			rc = -1;
3389 			break;
3390 	}
3391 	return rc;
3392 }
3393 #endif /* USE_HEAPSORT */
3394