1 /**
2 * @file
3 * Usenet network mailbox type; talk to an NNTP server
4 *
5 * @authors
6 * Copyright (C) 1998 Brandon Long <blong@fiction.net>
7 * Copyright (C) 1999 Andrej Gritsenko <andrej@lucky.net>
8 * Copyright (C) 2000-2017 Vsevolod Volkov <vvv@mutt.org.ua>
9 * Copyright (C) 2018 Richard Russon <rich@flatcap.org>
10 *
11 * @copyright
12 * This program is free software: you can redistribute it and/or modify it under
13 * the terms of the GNU General Public License as published by the Free Software
14 * Foundation, either version 2 of the License, or (at your option) any later
15 * version.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
19 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
20 * details.
21 *
22 * You should have received a copy of the GNU General Public License along with
23 * this program. If not, see <http://www.gnu.org/licenses/>.
24 */
25
26 /**
27 * @page nntp_nntp Talk to an NNTP server
28 *
29 * Usenet network mailbox type; talk to an NNTP server
30 *
31 * Implementation: #MxNntpOps
32 */
33
34 #include "config.h"
35 #include <ctype.h>
36 #include <limits.h>
37 #include <stdbool.h>
38 #include <stdint.h>
39 #include <stdio.h>
40 #include <string.h>
41 #include <strings.h>
42 #include <time.h>
43 #include <unistd.h>
44 #include "private.h"
45 #include "mutt/lib.h"
46 #include "config/lib.h"
47 #include "email/lib.h"
48 #include "core/lib.h"
49 #include "conn/lib.h"
50 #include "lib.h"
51 #include "attach/lib.h"
52 #include "bcache/lib.h"
53 #include "hcache/lib.h"
54 #include "ncrypt/lib.h"
55 #include "progress/lib.h"
56 #include "question/lib.h"
57 #include "adata.h"
58 #include "edata.h"
59 #include "hook.h"
60 #include "mdata.h"
61 #include "mutt_logging.h"
62 #include "mutt_socket.h"
63 #include "muttlib.h"
64 #include "mx.h"
65 #ifdef USE_HCACHE
66 #include "protos.h"
67 #endif
68 #ifdef USE_SASL
69 #include <sasl/sasl.h>
70 #include <sasl/saslutil.h>
71 #endif
72 #if defined(USE_SSL) || defined(USE_HCACHE)
73 #include "mutt.h"
74 #endif
75
76 struct stat;
77
78 struct NntpAccountData *CurrentNewsSrv;
79
80 const char *OverviewFmt = "Subject:\0"
81 "From:\0"
82 "Date:\0"
83 "Message-ID:\0"
84 "References:\0"
85 "Content-Length:\0"
86 "Lines:\0"
87 "\0";
88
89 /**
90 * struct FetchCtx - Keep track when getting data from a server
91 */
92 struct FetchCtx
93 {
94 struct Mailbox *mailbox;
95 anum_t first;
96 anum_t last;
97 bool restore;
98 unsigned char *messages;
99 struct Progress *progress;
100 struct HeaderCache *hc;
101 };
102
103 /**
104 * struct ChildCtx - Keep track of the children of an article
105 */
106 struct ChildCtx
107 {
108 struct Mailbox *mailbox;
109 unsigned int num;
110 unsigned int max;
111 anum_t *child;
112 };
113
114 /**
115 * nntp_hashelem_free - Free our hash table data - Implements ::hash_hdata_free_t - @ingroup hash_hdata_free_api
116 */
nntp_hashelem_free(int type,void * obj,intptr_t data)117 void nntp_hashelem_free(int type, void *obj, intptr_t data)
118 {
119 nntp_mdata_free(&obj);
120 }
121
122 /**
123 * nntp_connect_error - Signal a failed connection
124 * @param adata NNTP server
125 * @retval -1 Always
126 */
nntp_connect_error(struct NntpAccountData * adata)127 static int nntp_connect_error(struct NntpAccountData *adata)
128 {
129 adata->status = NNTP_NONE;
130 mutt_error(_("Server closed connection"));
131 return -1;
132 }
133
134 /**
135 * nntp_capabilities - Get capabilities
136 * @param adata NNTP server
137 * @retval -1 Error, connection is closed
138 * @retval 0 Mode is reader, capabilities set up
139 * @retval 1 Need to switch to reader mode
140 */
nntp_capabilities(struct NntpAccountData * adata)141 static int nntp_capabilities(struct NntpAccountData *adata)
142 {
143 struct Connection *conn = adata->conn;
144 bool mode_reader = false;
145 char buf[1024];
146 char authinfo[1024] = { 0 };
147
148 adata->hasCAPABILITIES = false;
149 adata->hasSTARTTLS = false;
150 adata->hasDATE = false;
151 adata->hasLIST_NEWSGROUPS = false;
152 adata->hasLISTGROUP = false;
153 adata->hasLISTGROUPrange = false;
154 adata->hasOVER = false;
155 FREE(&adata->authenticators);
156
157 if ((mutt_socket_send(conn, "CAPABILITIES\r\n") < 0) ||
158 (mutt_socket_readln(buf, sizeof(buf), conn) < 0))
159 {
160 return nntp_connect_error(adata);
161 }
162
163 /* no capabilities */
164 if (!mutt_str_startswith(buf, "101"))
165 return 1;
166 adata->hasCAPABILITIES = true;
167
168 /* parse capabilities */
169 do
170 {
171 size_t plen = 0;
172 if (mutt_socket_readln(buf, sizeof(buf), conn) < 0)
173 return nntp_connect_error(adata);
174 if (mutt_str_equal("STARTTLS", buf))
175 adata->hasSTARTTLS = true;
176 else if (mutt_str_equal("MODE-READER", buf))
177 mode_reader = true;
178 else if (mutt_str_equal("READER", buf))
179 {
180 adata->hasDATE = true;
181 adata->hasLISTGROUP = true;
182 adata->hasLISTGROUPrange = true;
183 }
184 else if ((plen = mutt_str_startswith(buf, "AUTHINFO ")))
185 {
186 mutt_str_cat(buf, sizeof(buf), " ");
187 mutt_str_copy(authinfo, buf + plen - 1, sizeof(authinfo));
188 }
189 #ifdef USE_SASL
190 else if ((plen = mutt_str_startswith(buf, "SASL ")))
191 {
192 char *p = buf + plen;
193 while (*p == ' ')
194 p++;
195 adata->authenticators = mutt_str_dup(p);
196 }
197 #endif
198 else if (mutt_str_equal("OVER", buf))
199 adata->hasOVER = true;
200 else if (mutt_str_startswith(buf, "LIST "))
201 {
202 char *p = strstr(buf, " NEWSGROUPS");
203 if (p)
204 {
205 p += 11;
206 if ((*p == '\0') || (*p == ' '))
207 adata->hasLIST_NEWSGROUPS = true;
208 }
209 }
210 } while (!mutt_str_equal(".", buf));
211 *buf = '\0';
212 #ifdef USE_SASL
213 if (adata->authenticators && strcasestr(authinfo, " SASL "))
214 mutt_str_copy(buf, adata->authenticators, sizeof(buf));
215 #endif
216 if (strcasestr(authinfo, " USER "))
217 {
218 if (*buf != '\0')
219 mutt_str_cat(buf, sizeof(buf), " ");
220 mutt_str_cat(buf, sizeof(buf), "USER");
221 }
222 mutt_str_replace(&adata->authenticators, buf);
223
224 /* current mode is reader */
225 if (adata->hasDATE)
226 return 0;
227
228 /* server is mode-switching, need to switch to reader mode */
229 if (mode_reader)
230 return 1;
231
232 mutt_socket_close(conn);
233 adata->status = NNTP_BYE;
234 mutt_error(_("Server doesn't support reader mode"));
235 return -1;
236 }
237
238 /**
239 * nntp_attempt_features - Detect supported commands
240 * @param adata NNTP server
241 * @retval 0 Success
242 * @retval -1 Failure
243 */
nntp_attempt_features(struct NntpAccountData * adata)244 static int nntp_attempt_features(struct NntpAccountData *adata)
245 {
246 struct Connection *conn = adata->conn;
247 char buf[1024];
248
249 /* no CAPABILITIES, trying DATE, LISTGROUP, LIST NEWSGROUPS */
250 if (!adata->hasCAPABILITIES)
251 {
252 if ((mutt_socket_send(conn, "DATE\r\n") < 0) ||
253 (mutt_socket_readln(buf, sizeof(buf), conn) < 0))
254 {
255 return nntp_connect_error(adata);
256 }
257 if (!mutt_str_startswith(buf, "500"))
258 adata->hasDATE = true;
259
260 if ((mutt_socket_send(conn, "LISTGROUP\r\n") < 0) ||
261 (mutt_socket_readln(buf, sizeof(buf), conn) < 0))
262 {
263 return nntp_connect_error(adata);
264 }
265 if (!mutt_str_startswith(buf, "500"))
266 adata->hasLISTGROUP = true;
267
268 if ((mutt_socket_send(conn, "LIST NEWSGROUPS +\r\n") < 0) ||
269 (mutt_socket_readln(buf, sizeof(buf), conn) < 0))
270 {
271 return nntp_connect_error(adata);
272 }
273 if (!mutt_str_startswith(buf, "500"))
274 adata->hasLIST_NEWSGROUPS = true;
275 if (mutt_str_startswith(buf, "215"))
276 {
277 do
278 {
279 if (mutt_socket_readln(buf, sizeof(buf), conn) < 0)
280 return nntp_connect_error(adata);
281 } while (!mutt_str_equal(".", buf));
282 }
283 }
284
285 /* no LIST NEWSGROUPS, trying XGTITLE */
286 if (!adata->hasLIST_NEWSGROUPS)
287 {
288 if ((mutt_socket_send(conn, "XGTITLE\r\n") < 0) ||
289 (mutt_socket_readln(buf, sizeof(buf), conn) < 0))
290 {
291 return nntp_connect_error(adata);
292 }
293 if (!mutt_str_startswith(buf, "500"))
294 adata->hasXGTITLE = true;
295 }
296
297 /* no OVER, trying XOVER */
298 if (!adata->hasOVER)
299 {
300 if ((mutt_socket_send(conn, "XOVER\r\n") < 0) ||
301 (mutt_socket_readln(buf, sizeof(buf), conn) < 0))
302 {
303 return nntp_connect_error(adata);
304 }
305 if (!mutt_str_startswith(buf, "500"))
306 adata->hasXOVER = true;
307 }
308
309 /* trying LIST OVERVIEW.FMT */
310 if (adata->hasOVER || adata->hasXOVER)
311 {
312 if ((mutt_socket_send(conn, "LIST OVERVIEW.FMT\r\n") < 0) ||
313 (mutt_socket_readln(buf, sizeof(buf), conn) < 0))
314 {
315 return nntp_connect_error(adata);
316 }
317 if (!mutt_str_startswith(buf, "215"))
318 adata->overview_fmt = mutt_str_dup(OverviewFmt);
319 else
320 {
321 bool cont = false;
322 size_t buflen = 2048, off = 0, b = 0;
323
324 FREE(&adata->overview_fmt);
325 adata->overview_fmt = mutt_mem_malloc(buflen);
326
327 while (true)
328 {
329 if ((buflen - off) < 1024)
330 {
331 buflen *= 2;
332 mutt_mem_realloc(&adata->overview_fmt, buflen);
333 }
334
335 const int chunk = mutt_socket_readln_d(adata->overview_fmt + off,
336 buflen - off, conn, MUTT_SOCK_LOG_HDR);
337 if (chunk < 0)
338 {
339 FREE(&adata->overview_fmt);
340 return nntp_connect_error(adata);
341 }
342
343 if (!cont && mutt_str_equal(".", adata->overview_fmt + off))
344 break;
345
346 cont = (chunk >= (buflen - off));
347 off += strlen(adata->overview_fmt + off);
348 if (!cont)
349 {
350 if (adata->overview_fmt[b] == ':')
351 {
352 memmove(adata->overview_fmt + b, adata->overview_fmt + b + 1, off - b - 1);
353 adata->overview_fmt[off - 1] = ':';
354 }
355 char *colon = strchr(adata->overview_fmt + b, ':');
356 if (!colon)
357 adata->overview_fmt[off++] = ':';
358 else if (strcmp(colon + 1, "full") != 0)
359 off = colon + 1 - adata->overview_fmt;
360 if (strcasecmp(adata->overview_fmt + b, "Bytes:") == 0)
361 {
362 size_t len = strlen(adata->overview_fmt + b);
363 mutt_str_copy(adata->overview_fmt + b, "Content-Length:", len + 1);
364 off = b + len;
365 }
366 adata->overview_fmt[off++] = '\0';
367 b = off;
368 }
369 }
370 adata->overview_fmt[off++] = '\0';
371 mutt_mem_realloc(&adata->overview_fmt, off);
372 }
373 }
374 return 0;
375 }
376
377 #ifdef USE_SASL
378 /**
379 * nntp_memchr - Look for a char in a binary buf, conveniently
380 * @param haystack [in/out] input: start here, output: store address of hit
381 * @param sentinel points just beyond (1 byte after) search area
382 * @param needle the character to search for
383 * @retval true found and updated haystack
384 * @retval false not found
385 */
nntp_memchr(char ** haystack,char * sentinel,int needle)386 static bool nntp_memchr(char **haystack, char *sentinel, int needle)
387 {
388 char *start = *haystack;
389 size_t max_offset = sentinel - start;
390 void *vp = memchr(start, max_offset, needle);
391 if (!vp)
392 return false;
393 *haystack = vp;
394 return true;
395 }
396
397 /**
398 * nntp_log_binbuf - Log a buffer possibly containing NUL bytes
399 * @param buf source buffer
400 * @param len how many bytes from buf
401 * @param pfx logging prefix (protocol etc.)
402 * @param dbg which loglevel does message belong
403 */
nntp_log_binbuf(const char * buf,size_t len,const char * pfx,int dbg)404 static void nntp_log_binbuf(const char *buf, size_t len, const char *pfx, int dbg)
405 {
406 char tmp[1024];
407 char *p = tmp;
408 char *sentinel = tmp + len;
409
410 const short c_debug_level = cs_subset_number(NeoMutt->sub, "debug_level");
411 if (c_debug_level < dbg)
412 return;
413 memcpy(tmp, buf, len);
414 tmp[len] = '\0';
415 while (nntp_memchr(&p, sentinel, '\0'))
416 *p = '.';
417 mutt_debug(dbg, "%s> %s\n", pfx, tmp);
418 }
419 #endif
420
421 /**
422 * nntp_auth - Get login, password and authenticate
423 * @param adata NNTP server
424 * @retval 0 Success
425 * @retval -1 Failure
426 */
nntp_auth(struct NntpAccountData * adata)427 static int nntp_auth(struct NntpAccountData *adata)
428 {
429 struct Connection *conn = adata->conn;
430 char buf[1024];
431 char authenticators[1024] = "USER";
432 char *method = NULL, *a = NULL, *p = NULL;
433 unsigned char flags = conn->account.flags;
434
435 while (true)
436 {
437 /* get login and password */
438 if ((mutt_account_getuser(&conn->account) < 0) || (conn->account.user[0] == '\0') ||
439 (mutt_account_getpass(&conn->account) < 0) || (conn->account.pass[0] == '\0'))
440 {
441 break;
442 }
443
444 /* get list of authenticators */
445 const char *const c_nntp_authenticators =
446 cs_subset_string(NeoMutt->sub, "nntp_authenticators");
447 if (c_nntp_authenticators)
448 mutt_str_copy(authenticators, c_nntp_authenticators, sizeof(authenticators));
449 else if (adata->hasCAPABILITIES)
450 {
451 mutt_str_copy(authenticators, adata->authenticators, sizeof(authenticators));
452 p = authenticators;
453 while (*p)
454 {
455 if (*p == ' ')
456 *p = ':';
457 p++;
458 }
459 }
460 p = authenticators;
461 while (*p)
462 {
463 *p = toupper(*p);
464 p++;
465 }
466
467 mutt_debug(LL_DEBUG1, "available methods: %s\n", adata->authenticators);
468 a = authenticators;
469 while (true)
470 {
471 if (!a)
472 {
473 mutt_error(_("No authenticators available"));
474 break;
475 }
476
477 method = a;
478 a = strchr(a, ':');
479 if (a)
480 *a++ = '\0';
481
482 /* check authenticator */
483 if (adata->hasCAPABILITIES)
484 {
485 char *m = NULL;
486
487 if (!adata->authenticators)
488 continue;
489 m = strcasestr(adata->authenticators, method);
490 if (!m)
491 continue;
492 if ((m > adata->authenticators) && (*(m - 1) != ' '))
493 continue;
494 m += strlen(method);
495 if ((*m != '\0') && (*m != ' '))
496 continue;
497 }
498 mutt_debug(LL_DEBUG1, "trying method %s\n", method);
499
500 /* AUTHINFO USER authentication */
501 if (strcmp(method, "USER") == 0)
502 {
503 // L10N: (%s) is the method name, e.g. Anonymous, CRAM-MD5, GSSAPI, SASL
504 mutt_message(_("Authenticating (%s)..."), method);
505 snprintf(buf, sizeof(buf), "AUTHINFO USER %s\r\n", conn->account.user);
506 if ((mutt_socket_send(conn, buf) < 0) ||
507 (mutt_socket_readln_d(buf, sizeof(buf), conn, MUTT_SOCK_LOG_FULL) < 0))
508 {
509 break;
510 }
511
512 /* authenticated, password is not required */
513 if (mutt_str_startswith(buf, "281"))
514 return 0;
515
516 /* username accepted, sending password */
517 if (mutt_str_startswith(buf, "381"))
518 {
519 mutt_debug(MUTT_SOCK_LOG_FULL, "%d> AUTHINFO PASS *\n", conn->fd);
520 snprintf(buf, sizeof(buf), "AUTHINFO PASS %s\r\n", conn->account.pass);
521 if ((mutt_socket_send_d(conn, buf, MUTT_SOCK_LOG_FULL) < 0) ||
522 (mutt_socket_readln_d(buf, sizeof(buf), conn, MUTT_SOCK_LOG_FULL) < 0))
523 {
524 break;
525 }
526
527 /* authenticated */
528 if (mutt_str_startswith(buf, "281"))
529 return 0;
530 }
531
532 /* server doesn't support AUTHINFO USER, trying next method */
533 if (*buf == '5')
534 continue;
535 }
536 else
537 {
538 #ifdef USE_SASL
539 sasl_conn_t *saslconn = NULL;
540 sasl_interact_t *interaction = NULL;
541 int rc;
542 char inbuf[1024] = { 0 };
543 const char *mech = NULL;
544 const char *client_out = NULL;
545 unsigned int client_len, len;
546
547 if (mutt_sasl_client_new(conn, &saslconn) < 0)
548 {
549 mutt_debug(LL_DEBUG1, "error allocating SASL connection\n");
550 continue;
551 }
552
553 while (true)
554 {
555 rc = sasl_client_start(saslconn, method, &interaction, &client_out,
556 &client_len, &mech);
557 if (rc != SASL_INTERACT)
558 break;
559 mutt_sasl_interact(interaction);
560 }
561 if ((rc != SASL_OK) && (rc != SASL_CONTINUE))
562 {
563 sasl_dispose(&saslconn);
564 mutt_debug(LL_DEBUG1,
565 "error starting SASL authentication exchange\n");
566 continue;
567 }
568
569 // L10N: (%s) is the method name, e.g. Anonymous, CRAM-MD5, GSSAPI, SASL
570 mutt_message(_("Authenticating (%s)..."), method);
571 snprintf(buf, sizeof(buf), "AUTHINFO SASL %s", method);
572
573 /* looping protocol */
574 while ((rc == SASL_CONTINUE) || ((rc == SASL_OK) && client_len))
575 {
576 /* send out client response */
577 if (client_len)
578 {
579 nntp_log_binbuf(client_out, client_len, "SASL", MUTT_SOCK_LOG_FULL);
580 if (*buf != '\0')
581 mutt_str_cat(buf, sizeof(buf), " ");
582 len = strlen(buf);
583 if (sasl_encode64(client_out, client_len, buf + len,
584 sizeof(buf) - len, &len) != SASL_OK)
585 {
586 mutt_debug(LL_DEBUG1, "error base64-encoding client response\n");
587 break;
588 }
589 }
590
591 mutt_str_cat(buf, sizeof(buf), "\r\n");
592 if (strchr(buf, ' '))
593 {
594 mutt_debug(MUTT_SOCK_LOG_CMD, "%d> AUTHINFO SASL %s%s\n", conn->fd,
595 method, client_len ? " sasl_data" : "");
596 }
597 else
598 mutt_debug(MUTT_SOCK_LOG_CMD, "%d> sasl_data\n", conn->fd);
599 client_len = 0;
600 if ((mutt_socket_send_d(conn, buf, MUTT_SOCK_LOG_FULL) < 0) ||
601 (mutt_socket_readln_d(inbuf, sizeof(inbuf), conn, MUTT_SOCK_LOG_FULL) < 0))
602 {
603 break;
604 }
605 if (!mutt_str_startswith(inbuf, "283 ") && !mutt_str_startswith(inbuf, "383 "))
606 {
607 mutt_debug(MUTT_SOCK_LOG_FULL, "%d< %s\n", conn->fd, inbuf);
608 break;
609 }
610 inbuf[3] = '\0';
611 mutt_debug(MUTT_SOCK_LOG_FULL, "%d< %s sasl_data\n", conn->fd, inbuf);
612
613 if (strcmp("=", inbuf + 4) == 0)
614 len = 0;
615 else if (sasl_decode64(inbuf + 4, strlen(inbuf + 4), buf,
616 sizeof(buf) - 1, &len) != SASL_OK)
617 {
618 mutt_debug(LL_DEBUG1, "error base64-decoding server response\n");
619 break;
620 }
621 else
622 nntp_log_binbuf(buf, len, "SASL", MUTT_SOCK_LOG_FULL);
623
624 while (true)
625 {
626 rc = sasl_client_step(saslconn, buf, len, &interaction, &client_out, &client_len);
627 if (rc != SASL_INTERACT)
628 break;
629 mutt_sasl_interact(interaction);
630 }
631 if (*inbuf != '3')
632 break;
633
634 *buf = '\0';
635 } /* looping protocol */
636
637 if ((rc == SASL_OK) && (client_len == 0) && (*inbuf == '2'))
638 {
639 mutt_sasl_setup_conn(conn, saslconn);
640 return 0;
641 }
642
643 /* terminate SASL session */
644 sasl_dispose(&saslconn);
645 if (conn->fd < 0)
646 break;
647 if (mutt_str_startswith(inbuf, "383 "))
648 {
649 if ((mutt_socket_send(conn, "*\r\n") < 0) ||
650 (mutt_socket_readln(inbuf, sizeof(inbuf), conn) < 0))
651 {
652 break;
653 }
654 }
655
656 /* server doesn't support AUTHINFO SASL, trying next method */
657 if (*inbuf == '5')
658 continue;
659 #else
660 continue;
661 #endif /* USE_SASL */
662 }
663
664 // L10N: %s is the method name, e.g. Anonymous, CRAM-MD5, GSSAPI, SASL
665 mutt_error(_("%s authentication failed"), method);
666 break;
667 }
668 break;
669 }
670
671 /* error */
672 adata->status = NNTP_BYE;
673 conn->account.flags = flags;
674 if (conn->fd < 0)
675 {
676 mutt_error(_("Server closed connection"));
677 }
678 else
679 mutt_socket_close(conn);
680 return -1;
681 }
682
683 /**
684 * nntp_query - Send data from buffer and receive answer to same buffer
685 * @param mdata NNTP Mailbox data
686 * @param line Buffer containing data
687 * @param linelen Length of buffer
688 * @retval 0 Success
689 * @retval -1 Failure
690 */
nntp_query(struct NntpMboxData * mdata,char * line,size_t linelen)691 static int nntp_query(struct NntpMboxData *mdata, char *line, size_t linelen)
692 {
693 struct NntpAccountData *adata = mdata->adata;
694 char buf[1024] = { 0 };
695
696 if (adata->status == NNTP_BYE)
697 return -1;
698
699 while (true)
700 {
701 if (adata->status == NNTP_OK)
702 {
703 int rc = 0;
704
705 if (*line)
706 rc = mutt_socket_send(adata->conn, line);
707 else if (mdata->group)
708 {
709 snprintf(buf, sizeof(buf), "GROUP %s\r\n", mdata->group);
710 rc = mutt_socket_send(adata->conn, buf);
711 }
712 if (rc >= 0)
713 rc = mutt_socket_readln(buf, sizeof(buf), adata->conn);
714 if (rc >= 0)
715 break;
716 }
717
718 /* reconnect */
719 while (true)
720 {
721 adata->status = NNTP_NONE;
722 if (nntp_open_connection(adata) == 0)
723 break;
724
725 snprintf(buf, sizeof(buf), _("Connection to %s lost. Reconnect?"),
726 adata->conn->account.host);
727 if (mutt_yesorno(buf, MUTT_YES) != MUTT_YES)
728 {
729 adata->status = NNTP_BYE;
730 return -1;
731 }
732 }
733
734 /* select newsgroup after reconnection */
735 if (mdata->group)
736 {
737 snprintf(buf, sizeof(buf), "GROUP %s\r\n", mdata->group);
738 if ((mutt_socket_send(adata->conn, buf) < 0) ||
739 (mutt_socket_readln(buf, sizeof(buf), adata->conn) < 0))
740 {
741 return nntp_connect_error(adata);
742 }
743 }
744 if (*line == '\0')
745 break;
746 }
747
748 mutt_str_copy(line, buf, linelen);
749 return 0;
750 }
751
752 /**
753 * nntp_fetch_lines - Read lines, calling a callback function for each
754 * @param mdata NNTP Mailbox data
755 * @param query Query to match
756 * @param qlen Length of query
757 * @param msg Progress message (OPTIONAL)
758 * @param func Callback function
759 * @param data Data for callback function
760 * @retval 0 Success
761 * @retval 1 Bad response (answer in query buffer)
762 * @retval -1 Connection lost
763 * @retval -2 Error in func(*line, *data)
764 *
765 * This function calls func(*line, *data) for each received line,
766 * func(NULL, *data) if rewind(*data) needs, exits when fail or done:
767 */
nntp_fetch_lines(struct NntpMboxData * mdata,char * query,size_t qlen,const char * msg,int (* func)(char *,void *),void * data)768 static int nntp_fetch_lines(struct NntpMboxData *mdata, char *query, size_t qlen,
769 const char *msg, int (*func)(char *, void *), void *data)
770 {
771 bool done = false;
772 int rc;
773
774 while (!done)
775 {
776 char buf[1024];
777 char *line = NULL;
778 unsigned int lines = 0;
779 size_t off = 0;
780 struct Progress *progress = NULL;
781
782 mutt_str_copy(buf, query, sizeof(buf));
783 if (nntp_query(mdata, buf, sizeof(buf)) < 0)
784 return -1;
785 if (buf[0] != '2')
786 {
787 mutt_str_copy(query, buf, qlen);
788 return 1;
789 }
790
791 line = mutt_mem_malloc(sizeof(buf));
792 rc = 0;
793
794 if (msg)
795 progress = progress_new(msg, MUTT_PROGRESS_READ, 0);
796
797 while (true)
798 {
799 char *p = NULL;
800 int chunk = mutt_socket_readln_d(buf, sizeof(buf), mdata->adata->conn, MUTT_SOCK_LOG_FULL);
801 if (chunk < 0)
802 {
803 mdata->adata->status = NNTP_NONE;
804 break;
805 }
806
807 p = buf;
808 if (!off && (buf[0] == '.'))
809 {
810 if (buf[1] == '\0')
811 {
812 done = true;
813 break;
814 }
815 if (buf[1] == '.')
816 p++;
817 }
818
819 mutt_str_copy(line + off, p, sizeof(buf));
820
821 if (chunk >= sizeof(buf))
822 off += strlen(p);
823 else
824 {
825 if (msg)
826 progress_update(progress, ++lines, -1);
827
828 if ((rc == 0) && (func(line, data) < 0))
829 rc = -2;
830 off = 0;
831 }
832
833 mutt_mem_realloc(&line, off + sizeof(buf));
834 }
835 FREE(&line);
836 func(NULL, data);
837 progress_free(&progress);
838 }
839
840 return rc;
841 }
842
843 /**
844 * fetch_description - Parse newsgroup description
845 * @param line String to parse
846 * @param data NNTP Server
847 * @retval 0 Always
848 */
fetch_description(char * line,void * data)849 static int fetch_description(char *line, void *data)
850 {
851 if (!line)
852 return 0;
853
854 struct NntpAccountData *adata = data;
855
856 char *desc = strpbrk(line, " \t");
857 if (desc)
858 {
859 *desc++ = '\0';
860 desc += strspn(desc, " \t");
861 }
862 else
863 desc = strchr(line, '\0');
864
865 struct NntpMboxData *mdata = mutt_hash_find(adata->groups_hash, line);
866 if (mdata && !mutt_str_equal(desc, mdata->desc))
867 {
868 mutt_str_replace(&mdata->desc, desc);
869 mutt_debug(LL_DEBUG2, "group: %s, desc: %s\n", line, desc);
870 }
871 return 0;
872 }
873
874 /**
875 * get_description - Fetch newsgroups descriptions
876 * @param mdata NNTP Mailbox data
877 * @param wildmat String to match
878 * @param msg Progress message
879 * @retval 0 Success
880 * @retval 1 Bad response (answer in query buffer)
881 * @retval -1 Connection lost
882 * @retval -2 Error
883 */
get_description(struct NntpMboxData * mdata,const char * wildmat,const char * msg)884 static int get_description(struct NntpMboxData *mdata, const char *wildmat, const char *msg)
885 {
886 char buf[256];
887 const char *cmd = NULL;
888
889 /* get newsgroup description, if possible */
890 struct NntpAccountData *adata = mdata->adata;
891 if (!wildmat)
892 wildmat = mdata->group;
893 if (adata->hasLIST_NEWSGROUPS)
894 cmd = "LIST NEWSGROUPS";
895 else if (adata->hasXGTITLE)
896 cmd = "XGTITLE";
897 else
898 return 0;
899
900 snprintf(buf, sizeof(buf), "%s %s\r\n", cmd, wildmat);
901 int rc = nntp_fetch_lines(mdata, buf, sizeof(buf), msg, fetch_description, adata);
902 if (rc > 0)
903 {
904 mutt_error("%s: %s", cmd, buf);
905 }
906 return rc;
907 }
908
909 /**
910 * nntp_parse_xref - Parse cross-reference
911 * @param m Mailbox
912 * @param e Email
913 *
914 * Update read flag and set article number if empty
915 */
nntp_parse_xref(struct Mailbox * m,struct Email * e)916 static void nntp_parse_xref(struct Mailbox *m, struct Email *e)
917 {
918 struct NntpMboxData *mdata = m->mdata;
919
920 char *buf = mutt_str_dup(e->env->xref);
921 char *p = buf;
922 while (p)
923 {
924 anum_t anum;
925
926 /* skip to next word */
927 p += strspn(p, " \t");
928 char *grp = p;
929
930 /* skip to end of word */
931 p = strpbrk(p, " \t");
932 if (p)
933 *p++ = '\0';
934
935 /* find colon */
936 char *colon = strchr(grp, ':');
937 if (!colon)
938 continue;
939 *colon++ = '\0';
940 if (sscanf(colon, ANUM, &anum) != 1)
941 continue;
942
943 nntp_article_status(m, e, grp, anum);
944 if (!nntp_edata_get(e)->article_num && mutt_str_equal(mdata->group, grp))
945 nntp_edata_get(e)->article_num = anum;
946 }
947 FREE(&buf);
948 }
949
950 /**
951 * fetch_tempfile - Write line to temporary file
952 * @param line Text to write
953 * @param data FILE pointer
954 * @retval 0 Success
955 * @retval -1 Failure
956 */
fetch_tempfile(char * line,void * data)957 static int fetch_tempfile(char *line, void *data)
958 {
959 FILE *fp = data;
960
961 if (!line)
962 rewind(fp);
963 else if ((fputs(line, fp) == EOF) || (fputc('\n', fp) == EOF))
964 return -1;
965 return 0;
966 }
967
968 /**
969 * fetch_numbers - Parse article number
970 * @param line Article number
971 * @param data FetchCtx
972 * @retval 0 Always
973 */
fetch_numbers(char * line,void * data)974 static int fetch_numbers(char *line, void *data)
975 {
976 struct FetchCtx *fc = data;
977 anum_t anum;
978
979 if (!line)
980 return 0;
981 if (sscanf(line, ANUM, &anum) != 1)
982 return 0;
983 if ((anum < fc->first) || (anum > fc->last))
984 return 0;
985 fc->messages[anum - fc->first] = 1;
986 return 0;
987 }
988
989 /**
990 * parse_overview_line - Parse overview line
991 * @param line String to parse
992 * @param data FetchCtx
993 * @retval 0 Success
994 * @retval -1 Failure
995 */
parse_overview_line(char * line,void * data)996 static int parse_overview_line(char *line, void *data)
997 {
998 if (!line || !data)
999 return 0;
1000
1001 struct FetchCtx *fc = data;
1002 struct Mailbox *m = fc->mailbox;
1003 if (!m)
1004 return -1;
1005
1006 struct NntpMboxData *mdata = m->mdata;
1007 struct Email *e = NULL;
1008 char *header = NULL, *field = NULL;
1009 bool save = true;
1010 anum_t anum;
1011
1012 /* parse article number */
1013 field = strchr(line, '\t');
1014 if (field)
1015 *field++ = '\0';
1016 if (sscanf(line, ANUM, &anum) != 1)
1017 return 0;
1018 mutt_debug(LL_DEBUG2, "" ANUM "\n", anum);
1019
1020 /* out of bounds */
1021 if ((anum < fc->first) || (anum > fc->last))
1022 return 0;
1023
1024 /* not in LISTGROUP */
1025 if (!fc->messages[anum - fc->first])
1026 {
1027 /* progress */
1028 if (m->verbose)
1029 progress_update(fc->progress, anum - fc->first + 1, -1);
1030 return 0;
1031 }
1032
1033 /* convert overview line to header */
1034 FILE *fp = mutt_file_mkstemp();
1035 if (!fp)
1036 return -1;
1037
1038 header = mdata->adata->overview_fmt;
1039 while (field)
1040 {
1041 char *b = field;
1042
1043 if (*header)
1044 {
1045 if (!strstr(header, ":full") && (fputs(header, fp) == EOF))
1046 {
1047 mutt_file_fclose(&fp);
1048 return -1;
1049 }
1050 header = strchr(header, '\0') + 1;
1051 }
1052
1053 field = strchr(field, '\t');
1054 if (field)
1055 *field++ = '\0';
1056 if ((fputs(b, fp) == EOF) || (fputc('\n', fp) == EOF))
1057 {
1058 mutt_file_fclose(&fp);
1059 return -1;
1060 }
1061 }
1062 rewind(fp);
1063
1064 /* allocate memory for headers */
1065 if (m->msg_count >= m->email_max)
1066 mx_alloc_memory(m);
1067
1068 /* parse header */
1069 m->emails[m->msg_count] = email_new();
1070 e = m->emails[m->msg_count];
1071 e->env = mutt_rfc822_read_header(fp, e, false, false);
1072 e->env->newsgroups = mutt_str_dup(mdata->group);
1073 e->received = e->date_sent;
1074 mutt_file_fclose(&fp);
1075
1076 #ifdef USE_HCACHE
1077 if (fc->hc)
1078 {
1079 char buf[16];
1080
1081 /* try to replace with header from cache */
1082 snprintf(buf, sizeof(buf), "%u", anum);
1083 struct HCacheEntry hce = mutt_hcache_fetch(fc->hc, buf, strlen(buf), 0);
1084 if (hce.email)
1085 {
1086 mutt_debug(LL_DEBUG2, "mutt_hcache_fetch %s\n", buf);
1087 email_free(&e);
1088 e = hce.email;
1089 m->emails[m->msg_count] = e;
1090 e->edata = NULL;
1091 e->read = false;
1092 e->old = false;
1093
1094 /* skip header marked as deleted in cache */
1095 if (e->deleted && !fc->restore)
1096 {
1097 if (mdata->bcache)
1098 {
1099 mutt_debug(LL_DEBUG2, "mutt_bcache_del %s\n", buf);
1100 mutt_bcache_del(mdata->bcache, buf);
1101 }
1102 save = false;
1103 }
1104 }
1105
1106 /* not cached yet, store header */
1107 else
1108 {
1109 mutt_debug(LL_DEBUG2, "mutt_hcache_store %s\n", buf);
1110 mutt_hcache_store(fc->hc, buf, strlen(buf), e, 0);
1111 }
1112 }
1113 #endif
1114
1115 if (save)
1116 {
1117 e->index = m->msg_count++;
1118 e->read = false;
1119 e->old = false;
1120 e->deleted = false;
1121 e->edata = nntp_edata_new();
1122 e->edata_free = nntp_edata_free;
1123 nntp_edata_get(e)->article_num = anum;
1124 if (fc->restore)
1125 e->changed = true;
1126 else
1127 {
1128 nntp_article_status(m, e, NULL, anum);
1129 if (!e->read)
1130 nntp_parse_xref(m, e);
1131 }
1132 if (anum > mdata->last_loaded)
1133 mdata->last_loaded = anum;
1134 }
1135 else
1136 email_free(&e);
1137
1138 /* progress */
1139 if (m->verbose)
1140 progress_update(fc->progress, anum - fc->first + 1, -1);
1141 return 0;
1142 }
1143
1144 /**
1145 * nntp_fetch_headers - Fetch headers
1146 * @param m Mailbox
1147 * @param hc Header cache
1148 * @param first Number of first header to fetch
1149 * @param last Number of last header to fetch
1150 * @param restore Restore message listed as deleted
1151 * @retval 0 Success
1152 * @retval -1 Failure
1153 */
nntp_fetch_headers(struct Mailbox * m,void * hc,anum_t first,anum_t last,bool restore)1154 static int nntp_fetch_headers(struct Mailbox *m, void *hc, anum_t first, anum_t last, bool restore)
1155 {
1156 if (!m)
1157 return -1;
1158
1159 struct NntpMboxData *mdata = m->mdata;
1160 struct FetchCtx fc = { 0 };
1161 struct Email *e = NULL;
1162 char buf[8192];
1163 int rc = 0;
1164 anum_t current;
1165 anum_t first_over = first;
1166
1167 /* if empty group or nothing to do */
1168 if (!last || (first > last))
1169 return 0;
1170
1171 /* init fetch context */
1172 fc.mailbox = m;
1173 fc.first = first;
1174 fc.last = last;
1175 fc.restore = restore;
1176 fc.messages = mutt_mem_calloc(last - first + 1, sizeof(unsigned char));
1177 if (!fc.messages)
1178 return -1;
1179 fc.hc = hc;
1180
1181 /* fetch list of articles */
1182 const bool c_nntp_listgroup = cs_subset_bool(NeoMutt->sub, "nntp_listgroup");
1183 if (c_nntp_listgroup && mdata->adata->hasLISTGROUP && !mdata->deleted)
1184 {
1185 if (m->verbose)
1186 mutt_message(_("Fetching list of articles..."));
1187 if (mdata->adata->hasLISTGROUPrange)
1188 snprintf(buf, sizeof(buf), "LISTGROUP %s %u-%u\r\n", mdata->group, first, last);
1189 else
1190 snprintf(buf, sizeof(buf), "LISTGROUP %s\r\n", mdata->group);
1191 rc = nntp_fetch_lines(mdata, buf, sizeof(buf), NULL, fetch_numbers, &fc);
1192 if (rc > 0)
1193 {
1194 mutt_error("LISTGROUP: %s", buf);
1195 }
1196 if (rc == 0)
1197 {
1198 for (current = first; (current <= last) && (rc == 0); current++)
1199 {
1200 if (fc.messages[current - first])
1201 continue;
1202
1203 snprintf(buf, sizeof(buf), "%u", current);
1204 if (mdata->bcache)
1205 {
1206 mutt_debug(LL_DEBUG2, "#1 mutt_bcache_del %s\n", buf);
1207 mutt_bcache_del(mdata->bcache, buf);
1208 }
1209
1210 #ifdef USE_HCACHE
1211 if (fc.hc)
1212 {
1213 mutt_debug(LL_DEBUG2, "mutt_hcache_delete_record %s\n", buf);
1214 mutt_hcache_delete_record(fc.hc, buf, strlen(buf));
1215 }
1216 #endif
1217 }
1218 }
1219 }
1220 else
1221 {
1222 for (current = first; current <= last; current++)
1223 fc.messages[current - first] = 1;
1224 }
1225
1226 /* fetching header from cache or server, or fallback to fetch overview */
1227 if (m->verbose)
1228 {
1229 fc.progress = progress_new(_("Fetching message headers..."),
1230 MUTT_PROGRESS_READ, last - first + 1);
1231 }
1232 for (current = first; (current <= last) && (rc == 0); current++)
1233 {
1234 if (m->verbose)
1235 progress_update(fc.progress, current - first + 1, -1);
1236
1237 #ifdef USE_HCACHE
1238 snprintf(buf, sizeof(buf), "%u", current);
1239 #endif
1240
1241 /* delete header from cache that does not exist on server */
1242 if (!fc.messages[current - first])
1243 continue;
1244
1245 /* allocate memory for headers */
1246 if (m->msg_count >= m->email_max)
1247 mx_alloc_memory(m);
1248
1249 #ifdef USE_HCACHE
1250 /* try to fetch header from cache */
1251 struct HCacheEntry hce = mutt_hcache_fetch(fc.hc, buf, strlen(buf), 0);
1252 if (hce.email)
1253 {
1254 mutt_debug(LL_DEBUG2, "mutt_hcache_fetch %s\n", buf);
1255 e = hce.email;
1256 m->emails[m->msg_count] = e;
1257 e->edata = NULL;
1258
1259 /* skip header marked as deleted in cache */
1260 if (e->deleted && !restore)
1261 {
1262 email_free(&e);
1263 if (mdata->bcache)
1264 {
1265 mutt_debug(LL_DEBUG2, "#2 mutt_bcache_del %s\n", buf);
1266 mutt_bcache_del(mdata->bcache, buf);
1267 }
1268 continue;
1269 }
1270
1271 e->read = false;
1272 e->old = false;
1273 }
1274 else
1275 #endif
1276 if (mdata->deleted)
1277 {
1278 /* don't try to fetch header from removed newsgroup */
1279 continue;
1280 }
1281
1282 /* fallback to fetch overview */
1283 else if (mdata->adata->hasOVER || mdata->adata->hasXOVER)
1284 {
1285 if (c_nntp_listgroup && mdata->adata->hasLISTGROUP)
1286 break;
1287 else
1288 continue;
1289 }
1290
1291 /* fetch header from server */
1292 else
1293 {
1294 FILE *fp = mutt_file_mkstemp();
1295 if (!fp)
1296 {
1297 mutt_perror(_("Can't create temporary file"));
1298 rc = -1;
1299 break;
1300 }
1301
1302 snprintf(buf, sizeof(buf), "HEAD %u\r\n", current);
1303 rc = nntp_fetch_lines(mdata, buf, sizeof(buf), NULL, fetch_tempfile, fp);
1304 if (rc)
1305 {
1306 mutt_file_fclose(&fp);
1307 if (rc < 0)
1308 break;
1309
1310 /* invalid response */
1311 if (!mutt_str_startswith(buf, "423"))
1312 {
1313 mutt_error("HEAD: %s", buf);
1314 break;
1315 }
1316
1317 /* no such article */
1318 if (mdata->bcache)
1319 {
1320 snprintf(buf, sizeof(buf), "%u", current);
1321 mutt_debug(LL_DEBUG2, "#3 mutt_bcache_del %s\n", buf);
1322 mutt_bcache_del(mdata->bcache, buf);
1323 }
1324 rc = 0;
1325 continue;
1326 }
1327
1328 /* parse header */
1329 m->emails[m->msg_count] = email_new();
1330 e = m->emails[m->msg_count];
1331 e->env = mutt_rfc822_read_header(fp, e, false, false);
1332 e->received = e->date_sent;
1333 mutt_file_fclose(&fp);
1334 }
1335
1336 /* save header in context */
1337 e->index = m->msg_count++;
1338 e->read = false;
1339 e->old = false;
1340 e->deleted = false;
1341 e->edata = nntp_edata_new();
1342 e->edata_free = nntp_edata_free;
1343 nntp_edata_get(e)->article_num = current;
1344 if (restore)
1345 e->changed = true;
1346 else
1347 {
1348 nntp_article_status(m, e, NULL, nntp_edata_get(e)->article_num);
1349 if (!e->read)
1350 nntp_parse_xref(m, e);
1351 }
1352 if (current > mdata->last_loaded)
1353 mdata->last_loaded = current;
1354 first_over = current + 1;
1355 }
1356
1357 if (!c_nntp_listgroup || !mdata->adata->hasLISTGROUP)
1358 current = first_over;
1359
1360 /* fetch overview information */
1361 if ((current <= last) && (rc == 0) && !mdata->deleted)
1362 {
1363 char *cmd = mdata->adata->hasOVER ? "OVER" : "XOVER";
1364 snprintf(buf, sizeof(buf), "%s %u-%u\r\n", cmd, current, last);
1365 rc = nntp_fetch_lines(mdata, buf, sizeof(buf), NULL, parse_overview_line, &fc);
1366 if (rc > 0)
1367 {
1368 mutt_error("%s: %s", cmd, buf);
1369 }
1370 }
1371
1372 FREE(&fc.messages);
1373 progress_free(&fc.progress);
1374 if (rc != 0)
1375 return -1;
1376 mutt_clear_error();
1377 return 0;
1378 }
1379
1380 /**
1381 * nntp_group_poll - Check newsgroup for new articles
1382 * @param mdata NNTP Mailbox data
1383 * @param update_stat Update the stats?
1384 * @retval 1 New articles found
1385 * @retval 0 No change
1386 * @retval -1 Lost connection
1387 */
nntp_group_poll(struct NntpMboxData * mdata,bool update_stat)1388 static int nntp_group_poll(struct NntpMboxData *mdata, bool update_stat)
1389 {
1390 char buf[1024] = { 0 };
1391 anum_t count, first, last;
1392
1393 /* use GROUP command to poll newsgroup */
1394 if (nntp_query(mdata, buf, sizeof(buf)) < 0)
1395 return -1;
1396 if (sscanf(buf, "211 " ANUM " " ANUM " " ANUM, &count, &first, &last) != 3)
1397 return 0;
1398 if ((first == mdata->first_message) && (last == mdata->last_message))
1399 return 0;
1400
1401 /* articles have been renumbered */
1402 if (last < mdata->last_message)
1403 {
1404 mdata->last_cached = 0;
1405 if (mdata->newsrc_len)
1406 {
1407 mutt_mem_realloc(&mdata->newsrc_ent, sizeof(struct NewsrcEntry));
1408 mdata->newsrc_len = 1;
1409 mdata->newsrc_ent[0].first = 1;
1410 mdata->newsrc_ent[0].last = 0;
1411 }
1412 }
1413 mdata->first_message = first;
1414 mdata->last_message = last;
1415 if (!update_stat)
1416 return 1;
1417
1418 /* update counters */
1419 else if (!last || (!mdata->newsrc_ent && !mdata->last_cached))
1420 mdata->unread = count;
1421 else
1422 nntp_group_unread_stat(mdata);
1423 return 1;
1424 }
1425
1426 /**
1427 * check_mailbox - Check current newsgroup for new articles
1428 * @param m Mailbox
1429 * @retval enum #MxStatus
1430 *
1431 * Leave newsrc locked
1432 */
check_mailbox(struct Mailbox * m)1433 static enum MxStatus check_mailbox(struct Mailbox *m)
1434 {
1435 if (!m)
1436 return MX_STATUS_ERROR;
1437
1438 struct NntpMboxData *mdata = m->mdata;
1439 struct NntpAccountData *adata = mdata->adata;
1440 time_t now = mutt_date_epoch();
1441 enum MxStatus rc = MX_STATUS_OK;
1442 void *hc = NULL;
1443
1444 const short c_nntp_poll = cs_subset_number(NeoMutt->sub, "nntp_poll");
1445 if (adata->check_time + c_nntp_poll > now)
1446 return MX_STATUS_OK;
1447
1448 mutt_message(_("Checking for new messages..."));
1449 if (nntp_newsrc_parse(adata) < 0)
1450 return MX_STATUS_ERROR;
1451
1452 adata->check_time = now;
1453 int rc2 = nntp_group_poll(mdata, false);
1454 if (rc2 < 0)
1455 {
1456 nntp_newsrc_close(adata);
1457 return -1;
1458 }
1459 if (rc2 != 0)
1460 nntp_active_save_cache(adata);
1461
1462 /* articles have been renumbered, remove all headers */
1463 if (mdata->last_message < mdata->last_loaded)
1464 {
1465 for (int i = 0; i < m->msg_count; i++)
1466 email_free(&m->emails[i]);
1467 m->msg_count = 0;
1468 m->msg_tagged = 0;
1469
1470 if (mdata->last_message < mdata->last_loaded)
1471 {
1472 mdata->last_loaded = mdata->first_message - 1;
1473 const short c_nntp_context =
1474 cs_subset_number(NeoMutt->sub, "nntp_context");
1475 if (c_nntp_context && (mdata->last_message - mdata->last_loaded > c_nntp_context))
1476 mdata->last_loaded = mdata->last_message - c_nntp_context;
1477 }
1478 rc = MX_STATUS_REOPENED;
1479 }
1480
1481 /* .newsrc has been externally modified */
1482 if (adata->newsrc_modified)
1483 {
1484 #ifdef USE_HCACHE
1485 unsigned char *messages = NULL;
1486 char buf[16];
1487 struct Email *e = NULL;
1488 anum_t first = mdata->first_message;
1489
1490 const short c_nntp_context = cs_subset_number(NeoMutt->sub, "nntp_context");
1491 if (c_nntp_context && (mdata->last_message - first + 1 > c_nntp_context))
1492 first = mdata->last_message - c_nntp_context + 1;
1493 messages = mutt_mem_calloc(mdata->last_loaded - first + 1, sizeof(unsigned char));
1494 hc = nntp_hcache_open(mdata);
1495 nntp_hcache_update(mdata, hc);
1496 #endif
1497
1498 /* update flags according to .newsrc */
1499 int j = 0;
1500 for (int i = 0; i < m->msg_count; i++)
1501 {
1502 if (!m->emails[i])
1503 continue;
1504 bool flagged = false;
1505 anum_t anum = nntp_edata_get(m->emails[i])->article_num;
1506
1507 #ifdef USE_HCACHE
1508 /* check hcache for flagged and deleted flags */
1509 if (hc)
1510 {
1511 if ((anum >= first) && (anum <= mdata->last_loaded))
1512 messages[anum - first] = 1;
1513
1514 snprintf(buf, sizeof(buf), "%u", anum);
1515 struct HCacheEntry hce = mutt_hcache_fetch(hc, buf, strlen(buf), 0);
1516 if (hce.email)
1517 {
1518 bool deleted;
1519
1520 mutt_debug(LL_DEBUG2, "#1 mutt_hcache_fetch %s\n", buf);
1521 e = hce.email;
1522 e->edata = NULL;
1523 deleted = e->deleted;
1524 flagged = e->flagged;
1525 email_free(&e);
1526
1527 /* header marked as deleted, removing from context */
1528 if (deleted)
1529 {
1530 mutt_set_flag(m, m->emails[i], MUTT_TAG, false);
1531 email_free(&m->emails[i]);
1532 continue;
1533 }
1534 }
1535 }
1536 #endif
1537
1538 if (!m->emails[i]->changed)
1539 {
1540 m->emails[i]->flagged = flagged;
1541 m->emails[i]->read = false;
1542 m->emails[i]->old = false;
1543 nntp_article_status(m, m->emails[i], NULL, anum);
1544 if (!m->emails[i]->read)
1545 nntp_parse_xref(m, m->emails[i]);
1546 }
1547 m->emails[j++] = m->emails[i];
1548 }
1549
1550 #ifdef USE_HCACHE
1551 m->msg_count = j;
1552
1553 /* restore headers without "deleted" flag */
1554 for (anum_t anum = first; anum <= mdata->last_loaded; anum++)
1555 {
1556 if (messages[anum - first])
1557 continue;
1558
1559 snprintf(buf, sizeof(buf), "%u", anum);
1560 struct HCacheEntry hce = mutt_hcache_fetch(hc, buf, strlen(buf), 0);
1561 if (hce.email)
1562 {
1563 mutt_debug(LL_DEBUG2, "#2 mutt_hcache_fetch %s\n", buf);
1564 if (m->msg_count >= m->email_max)
1565 mx_alloc_memory(m);
1566
1567 e = hce.email;
1568 m->emails[m->msg_count] = e;
1569 e->edata = NULL;
1570 if (e->deleted)
1571 {
1572 email_free(&e);
1573 if (mdata->bcache)
1574 {
1575 mutt_debug(LL_DEBUG2, "mutt_bcache_del %s\n", buf);
1576 mutt_bcache_del(mdata->bcache, buf);
1577 }
1578 continue;
1579 }
1580
1581 m->msg_count++;
1582 e->read = false;
1583 e->old = false;
1584 e->edata = nntp_edata_new();
1585 e->edata_free = nntp_edata_free;
1586 nntp_edata_get(e)->article_num = anum;
1587 nntp_article_status(m, e, NULL, anum);
1588 if (!e->read)
1589 nntp_parse_xref(m, e);
1590 }
1591 }
1592 FREE(&messages);
1593 #endif
1594
1595 adata->newsrc_modified = false;
1596 rc = MX_STATUS_REOPENED;
1597 }
1598
1599 /* some headers were removed, context must be updated */
1600 if (rc == MX_STATUS_REOPENED)
1601 mailbox_changed(m, NT_MAILBOX_INVALID);
1602
1603 /* fetch headers of new articles */
1604 if (mdata->last_message > mdata->last_loaded)
1605 {
1606 int oldmsgcount = m->msg_count;
1607 bool verbose = m->verbose;
1608 m->verbose = false;
1609 #ifdef USE_HCACHE
1610 if (!hc)
1611 {
1612 hc = nntp_hcache_open(mdata);
1613 nntp_hcache_update(mdata, hc);
1614 }
1615 #endif
1616 int old_msg_count = m->msg_count;
1617 rc2 = nntp_fetch_headers(m, hc, mdata->last_loaded + 1, mdata->last_message, false);
1618 m->verbose = verbose;
1619 if (rc2 == 0)
1620 {
1621 if (m->msg_count > old_msg_count)
1622 mailbox_changed(m, NT_MAILBOX_INVALID);
1623 mdata->last_loaded = mdata->last_message;
1624 }
1625 if ((rc == MX_STATUS_OK) && (m->msg_count > oldmsgcount))
1626 rc = MX_STATUS_NEW_MAIL;
1627 }
1628
1629 #ifdef USE_HCACHE
1630 mutt_hcache_close(hc);
1631 #endif
1632 if (rc != MX_STATUS_OK)
1633 nntp_newsrc_close(adata);
1634 mutt_clear_error();
1635 return rc;
1636 }
1637
1638 /**
1639 * nntp_date - Get date and time from server
1640 * @param adata NNTP server
1641 * @param now Server time
1642 * @retval 0 Success
1643 * @retval -1 Failure
1644 */
nntp_date(struct NntpAccountData * adata,time_t * now)1645 static int nntp_date(struct NntpAccountData *adata, time_t *now)
1646 {
1647 if (adata->hasDATE)
1648 {
1649 struct NntpMboxData mdata = { 0 };
1650 char buf[1024];
1651 struct tm tm = { 0 };
1652
1653 mdata.adata = adata;
1654 mdata.group = NULL;
1655 mutt_str_copy(buf, "DATE\r\n", sizeof(buf));
1656 if (nntp_query(&mdata, buf, sizeof(buf)) < 0)
1657 return -1;
1658
1659 if (sscanf(buf, "111 %4d%2d%2d%2d%2d%2d%*s", &tm.tm_year, &tm.tm_mon,
1660 &tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec) == 6)
1661 {
1662 tm.tm_year -= 1900;
1663 tm.tm_mon--;
1664 *now = timegm(&tm);
1665 if (*now >= 0)
1666 {
1667 mutt_debug(LL_DEBUG1, "server time is %lu\n", *now);
1668 return 0;
1669 }
1670 }
1671 }
1672 *now = mutt_date_epoch();
1673 return 0;
1674 }
1675
1676 /**
1677 * fetch_children - Parse XPAT line
1678 * @param line String to parse
1679 * @param data ChildCtx
1680 * @retval 0 Always
1681 */
fetch_children(char * line,void * data)1682 static int fetch_children(char *line, void *data)
1683 {
1684 struct ChildCtx *cc = data;
1685 anum_t anum;
1686
1687 if (!line || (sscanf(line, ANUM, &anum) != 1))
1688 return 0;
1689 for (unsigned int i = 0; i < cc->mailbox->msg_count; i++)
1690 {
1691 struct Email *e = cc->mailbox->emails[i];
1692 if (!e)
1693 break;
1694 if (nntp_edata_get(e)->article_num == anum)
1695 return 0;
1696 }
1697 if (cc->num >= cc->max)
1698 {
1699 cc->max *= 2;
1700 mutt_mem_realloc(&cc->child, sizeof(anum_t) * cc->max);
1701 }
1702 cc->child[cc->num++] = anum;
1703 return 0;
1704 }
1705
1706 /**
1707 * nntp_open_connection - Connect to server, authenticate and get capabilities
1708 * @param adata NNTP server
1709 * @retval 0 Success
1710 * @retval -1 Failure
1711 */
nntp_open_connection(struct NntpAccountData * adata)1712 int nntp_open_connection(struct NntpAccountData *adata)
1713 {
1714 struct Connection *conn = adata->conn;
1715 char buf[256];
1716 int cap;
1717 bool posting = false, auth = true;
1718
1719 if (adata->status == NNTP_OK)
1720 return 0;
1721 if (adata->status == NNTP_BYE)
1722 return -1;
1723 adata->status = NNTP_NONE;
1724
1725 if (mutt_socket_open(conn) < 0)
1726 return -1;
1727
1728 if (mutt_socket_readln(buf, sizeof(buf), conn) < 0)
1729 return nntp_connect_error(adata);
1730
1731 if (mutt_str_startswith(buf, "200"))
1732 posting = true;
1733 else if (!mutt_str_startswith(buf, "201"))
1734 {
1735 mutt_socket_close(conn);
1736 mutt_str_remove_trailing_ws(buf);
1737 mutt_error("%s", buf);
1738 return -1;
1739 }
1740
1741 /* get initial capabilities */
1742 cap = nntp_capabilities(adata);
1743 if (cap < 0)
1744 return -1;
1745
1746 /* tell news server to switch to mode reader if it isn't so */
1747 if (cap > 0)
1748 {
1749 if ((mutt_socket_send(conn, "MODE READER\r\n") < 0) ||
1750 (mutt_socket_readln(buf, sizeof(buf), conn) < 0))
1751 {
1752 return nntp_connect_error(adata);
1753 }
1754
1755 if (mutt_str_startswith(buf, "200"))
1756 posting = true;
1757 else if (mutt_str_startswith(buf, "201"))
1758 posting = false;
1759 /* error if has capabilities, ignore result if no capabilities */
1760 else if (adata->hasCAPABILITIES)
1761 {
1762 mutt_socket_close(conn);
1763 mutt_error(_("Could not switch to reader mode"));
1764 return -1;
1765 }
1766
1767 /* recheck capabilities after MODE READER */
1768 if (adata->hasCAPABILITIES)
1769 {
1770 cap = nntp_capabilities(adata);
1771 if (cap < 0)
1772 return -1;
1773 }
1774 }
1775
1776 mutt_message(_("Connected to %s. %s"), conn->account.host,
1777 posting ? _("Posting is ok") : _("Posting is NOT ok"));
1778 mutt_sleep(1);
1779
1780 #ifdef USE_SSL
1781 /* Attempt STARTTLS if available and desired. */
1782 const bool c_ssl_force_tls = cs_subset_bool(NeoMutt->sub, "ssl_force_tls");
1783 if ((adata->use_tls != 1) && (adata->hasSTARTTLS || c_ssl_force_tls))
1784 {
1785 if (adata->use_tls == 0)
1786 {
1787 const enum QuadOption c_ssl_starttls =
1788 cs_subset_quad(NeoMutt->sub, "ssl_starttls");
1789 adata->use_tls =
1790 c_ssl_force_tls ||
1791 (query_quadoption(c_ssl_starttls,
1792 _("Secure connection with TLS?")) == MUTT_YES) ?
1793 2 :
1794 1;
1795 }
1796 if (adata->use_tls == 2)
1797 {
1798 if ((mutt_socket_send(conn, "STARTTLS\r\n") < 0) ||
1799 (mutt_socket_readln(buf, sizeof(buf), conn) < 0))
1800 {
1801 return nntp_connect_error(adata);
1802 }
1803 // Clear any data after the STARTTLS acknowledgement
1804 mutt_socket_empty(conn);
1805 if (!mutt_str_startswith(buf, "382"))
1806 {
1807 adata->use_tls = 0;
1808 mutt_error("STARTTLS: %s", buf);
1809 }
1810 else if (mutt_ssl_starttls(conn))
1811 {
1812 adata->use_tls = 0;
1813 adata->status = NNTP_NONE;
1814 mutt_socket_close(adata->conn);
1815 mutt_error(_("Could not negotiate TLS connection"));
1816 return -1;
1817 }
1818 else
1819 {
1820 /* recheck capabilities after STARTTLS */
1821 cap = nntp_capabilities(adata);
1822 if (cap < 0)
1823 return -1;
1824 }
1825 }
1826 }
1827 #endif
1828
1829 /* authentication required? */
1830 if (conn->account.flags & MUTT_ACCT_USER)
1831 {
1832 if (!conn->account.user[0])
1833 auth = false;
1834 }
1835 else
1836 {
1837 if ((mutt_socket_send(conn, "STAT\r\n") < 0) ||
1838 (mutt_socket_readln(buf, sizeof(buf), conn) < 0))
1839 {
1840 return nntp_connect_error(adata);
1841 }
1842 if (!mutt_str_startswith(buf, "480"))
1843 auth = false;
1844 }
1845
1846 /* authenticate */
1847 if (auth && (nntp_auth(adata) < 0))
1848 return -1;
1849
1850 /* get final capabilities after authentication */
1851 if (adata->hasCAPABILITIES && (auth || (cap > 0)))
1852 {
1853 cap = nntp_capabilities(adata);
1854 if (cap < 0)
1855 return -1;
1856 if (cap > 0)
1857 {
1858 mutt_socket_close(conn);
1859 mutt_error(_("Could not switch to reader mode"));
1860 return -1;
1861 }
1862 }
1863
1864 /* attempt features */
1865 if (nntp_attempt_features(adata) < 0)
1866 return -1;
1867
1868 adata->status = NNTP_OK;
1869 return 0;
1870 }
1871
1872 /**
1873 * nntp_post - Post article
1874 * @param m Mailbox
1875 * @param msg Message to post
1876 * @retval 0 Success
1877 * @retval -1 Failure
1878 */
nntp_post(struct Mailbox * m,const char * msg)1879 int nntp_post(struct Mailbox *m, const char *msg)
1880 {
1881 struct NntpMboxData *mdata = NULL;
1882 struct NntpMboxData tmp_mdata = { 0 };
1883 char buf[1024];
1884
1885 if (m && (m->type == MUTT_NNTP))
1886 mdata = m->mdata;
1887 else
1888 {
1889 const char *const c_news_server =
1890 cs_subset_string(NeoMutt->sub, "news_server");
1891 CurrentNewsSrv = nntp_select_server(m, c_news_server, false);
1892 if (!CurrentNewsSrv)
1893 return -1;
1894
1895 mdata = &tmp_mdata;
1896 mdata->adata = CurrentNewsSrv;
1897 mdata->group = NULL;
1898 }
1899
1900 FILE *fp = mutt_file_fopen(msg, "r");
1901 if (!fp)
1902 {
1903 mutt_perror(msg);
1904 return -1;
1905 }
1906
1907 mutt_str_copy(buf, "POST\r\n", sizeof(buf));
1908 if (nntp_query(mdata, buf, sizeof(buf)) < 0)
1909 {
1910 mutt_file_fclose(&fp);
1911 return -1;
1912 }
1913 if (buf[0] != '3')
1914 {
1915 mutt_error(_("Can't post article: %s"), buf);
1916 mutt_file_fclose(&fp);
1917 return -1;
1918 }
1919
1920 buf[0] = '.';
1921 buf[1] = '\0';
1922 while (fgets(buf + 1, sizeof(buf) - 2, fp))
1923 {
1924 size_t len = strlen(buf);
1925 if (buf[len - 1] == '\n')
1926 {
1927 buf[len - 1] = '\r';
1928 buf[len] = '\n';
1929 len++;
1930 buf[len] = '\0';
1931 }
1932 if (mutt_socket_send_d(mdata->adata->conn, (buf[1] == '.') ? buf : buf + 1,
1933 MUTT_SOCK_LOG_FULL) < 0)
1934 {
1935 mutt_file_fclose(&fp);
1936 return nntp_connect_error(mdata->adata);
1937 }
1938 }
1939 mutt_file_fclose(&fp);
1940
1941 if (((buf[strlen(buf) - 1] != '\n') &&
1942 (mutt_socket_send_d(mdata->adata->conn, "\r\n", MUTT_SOCK_LOG_FULL) < 0)) ||
1943 (mutt_socket_send_d(mdata->adata->conn, ".\r\n", MUTT_SOCK_LOG_FULL) < 0) ||
1944 (mutt_socket_readln(buf, sizeof(buf), mdata->adata->conn) < 0))
1945 {
1946 return nntp_connect_error(mdata->adata);
1947 }
1948 if (buf[0] != '2')
1949 {
1950 mutt_error(_("Can't post article: %s"), buf);
1951 return -1;
1952 }
1953 return 0;
1954 }
1955
1956 /**
1957 * nntp_active_fetch - Fetch list of all newsgroups from server
1958 * @param adata NNTP server
1959 * @param mark_new Mark the groups as new
1960 * @retval 0 Success
1961 * @retval -1 Failure
1962 */
nntp_active_fetch(struct NntpAccountData * adata,bool mark_new)1963 int nntp_active_fetch(struct NntpAccountData *adata, bool mark_new)
1964 {
1965 struct NntpMboxData tmp_mdata = { 0 };
1966 char msg[256];
1967 char buf[1024];
1968 unsigned int i;
1969 int rc;
1970
1971 snprintf(msg, sizeof(msg), _("Loading list of groups from server %s..."),
1972 adata->conn->account.host);
1973 mutt_message(msg);
1974 if (nntp_date(adata, &adata->newgroups_time) < 0)
1975 return -1;
1976
1977 tmp_mdata.adata = adata;
1978 tmp_mdata.group = NULL;
1979 i = adata->groups_num;
1980 mutt_str_copy(buf, "LIST\r\n", sizeof(buf));
1981 rc = nntp_fetch_lines(&tmp_mdata, buf, sizeof(buf), msg, nntp_add_group, adata);
1982 if (rc)
1983 {
1984 if (rc > 0)
1985 {
1986 mutt_error("LIST: %s", buf);
1987 }
1988 return -1;
1989 }
1990
1991 if (mark_new)
1992 {
1993 for (; i < adata->groups_num; i++)
1994 {
1995 struct NntpMboxData *mdata = adata->groups_list[i];
1996 mdata->has_new_mail = true;
1997 }
1998 }
1999
2000 for (i = 0; i < adata->groups_num; i++)
2001 {
2002 struct NntpMboxData *mdata = adata->groups_list[i];
2003
2004 if (mdata && mdata->deleted && !mdata->newsrc_ent)
2005 {
2006 nntp_delete_group_cache(mdata);
2007 mutt_hash_delete(adata->groups_hash, mdata->group, NULL);
2008 adata->groups_list[i] = NULL;
2009 }
2010 }
2011
2012 const bool c_nntp_load_description =
2013 cs_subset_bool(NeoMutt->sub, "nntp_load_description");
2014 if (c_nntp_load_description)
2015 rc = get_description(&tmp_mdata, "*", _("Loading descriptions..."));
2016
2017 nntp_active_save_cache(adata);
2018 if (rc < 0)
2019 return -1;
2020 mutt_clear_error();
2021 return 0;
2022 }
2023
2024 /**
2025 * nntp_check_new_groups - Check for new groups/articles in subscribed groups
2026 * @param m Mailbox
2027 * @param adata NNTP server
2028 * @retval 1 New groups found
2029 * @retval 0 No new groups
2030 * @retval -1 Error
2031 */
nntp_check_new_groups(struct Mailbox * m,struct NntpAccountData * adata)2032 int nntp_check_new_groups(struct Mailbox *m, struct NntpAccountData *adata)
2033 {
2034 struct NntpMboxData tmp_mdata = { 0 };
2035 time_t now;
2036 char buf[1024];
2037 char *msg = _("Checking for new newsgroups...");
2038 unsigned int i;
2039 int rc, update_active = false;
2040
2041 if (!adata || !adata->newgroups_time)
2042 return -1;
2043
2044 /* check subscribed newsgroups for new articles */
2045 const bool c_show_new_news = cs_subset_bool(NeoMutt->sub, "show_new_news");
2046 if (c_show_new_news)
2047 {
2048 mutt_message(_("Checking for new messages..."));
2049 for (i = 0; i < adata->groups_num; i++)
2050 {
2051 struct NntpMboxData *mdata = adata->groups_list[i];
2052
2053 if (mdata && mdata->subscribed)
2054 {
2055 rc = nntp_group_poll(mdata, true);
2056 if (rc < 0)
2057 return -1;
2058 if (rc > 0)
2059 update_active = true;
2060 }
2061 }
2062 }
2063 else if (adata->newgroups_time)
2064 return 0;
2065
2066 /* get list of new groups */
2067 mutt_message(msg);
2068 if (nntp_date(adata, &now) < 0)
2069 return -1;
2070 tmp_mdata.adata = adata;
2071 if (m && m->mdata)
2072 tmp_mdata.group = ((struct NntpMboxData *) m->mdata)->group;
2073 else
2074 tmp_mdata.group = NULL;
2075 i = adata->groups_num;
2076 struct tm tm = mutt_date_gmtime(adata->newgroups_time);
2077 snprintf(buf, sizeof(buf), "NEWGROUPS %02d%02d%02d %02d%02d%02d GMT\r\n",
2078 tm.tm_year % 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
2079 rc = nntp_fetch_lines(&tmp_mdata, buf, sizeof(buf), msg, nntp_add_group, adata);
2080 if (rc)
2081 {
2082 if (rc > 0)
2083 {
2084 mutt_error("NEWGROUPS: %s", buf);
2085 }
2086 return -1;
2087 }
2088
2089 /* new groups found */
2090 rc = 0;
2091 if (adata->groups_num != i)
2092 {
2093 int groups_num = i;
2094
2095 adata->newgroups_time = now;
2096 for (; i < adata->groups_num; i++)
2097 {
2098 struct NntpMboxData *mdata = adata->groups_list[i];
2099 mdata->has_new_mail = true;
2100 }
2101
2102 /* loading descriptions */
2103 const bool c_nntp_load_description =
2104 cs_subset_bool(NeoMutt->sub, "nntp_load_description");
2105 if (c_nntp_load_description)
2106 {
2107 unsigned int count = 0;
2108 struct Progress *progress = progress_new(
2109 _("Loading descriptions..."), MUTT_PROGRESS_READ, adata->groups_num - i);
2110
2111 for (i = groups_num; i < adata->groups_num; i++)
2112 {
2113 struct NntpMboxData *mdata = adata->groups_list[i];
2114
2115 if (get_description(mdata, NULL, NULL) < 0)
2116 {
2117 progress_free(&progress);
2118 return -1;
2119 }
2120 progress_update(progress, ++count, -1);
2121 }
2122 progress_free(&progress);
2123 }
2124 update_active = true;
2125 rc = 1;
2126 }
2127 if (update_active)
2128 nntp_active_save_cache(adata);
2129 mutt_clear_error();
2130 return rc;
2131 }
2132
2133 /**
2134 * nntp_check_msgid - Fetch article by Message-ID
2135 * @param m Mailbox
2136 * @param msgid Message ID
2137 * @retval 0 Success
2138 * @retval 1 No such article
2139 * @retval -1 Error
2140 */
nntp_check_msgid(struct Mailbox * m,const char * msgid)2141 int nntp_check_msgid(struct Mailbox *m, const char *msgid)
2142 {
2143 if (!m)
2144 return -1;
2145
2146 struct NntpMboxData *mdata = m->mdata;
2147 char buf[1024];
2148
2149 FILE *fp = mutt_file_mkstemp();
2150 if (!fp)
2151 {
2152 mutt_perror(_("Can't create temporary file"));
2153 return -1;
2154 }
2155
2156 snprintf(buf, sizeof(buf), "HEAD %s\r\n", msgid);
2157 int rc = nntp_fetch_lines(mdata, buf, sizeof(buf), NULL, fetch_tempfile, fp);
2158 if (rc)
2159 {
2160 mutt_file_fclose(&fp);
2161 if (rc < 0)
2162 return -1;
2163 if (mutt_str_startswith(buf, "430"))
2164 return 1;
2165 mutt_error("HEAD: %s", buf);
2166 return -1;
2167 }
2168
2169 /* parse header */
2170 if (m->msg_count == m->email_max)
2171 mx_alloc_memory(m);
2172 m->emails[m->msg_count] = email_new();
2173 struct Email *e = m->emails[m->msg_count];
2174 e->edata = nntp_edata_new();
2175 e->edata_free = nntp_edata_free;
2176 e->env = mutt_rfc822_read_header(fp, e, false, false);
2177 mutt_file_fclose(&fp);
2178
2179 /* get article number */
2180 if (e->env->xref)
2181 nntp_parse_xref(m, e);
2182 else
2183 {
2184 snprintf(buf, sizeof(buf), "STAT %s\r\n", msgid);
2185 if (nntp_query(mdata, buf, sizeof(buf)) < 0)
2186 {
2187 email_free(&e);
2188 return -1;
2189 }
2190 sscanf(buf + 4, ANUM, &nntp_edata_get(e)->article_num);
2191 }
2192
2193 /* reset flags */
2194 e->read = false;
2195 e->old = false;
2196 e->deleted = false;
2197 e->changed = true;
2198 e->received = e->date_sent;
2199 e->index = m->msg_count++;
2200 mailbox_changed(m, NT_MAILBOX_INVALID);
2201 return 0;
2202 }
2203
2204 /**
2205 * nntp_check_children - Fetch children of article with the Message-ID
2206 * @param m Mailbox
2207 * @param msgid Message ID to find
2208 * @retval 0 Success
2209 * @retval -1 Failure
2210 */
nntp_check_children(struct Mailbox * m,const char * msgid)2211 int nntp_check_children(struct Mailbox *m, const char *msgid)
2212 {
2213 if (!m)
2214 return -1;
2215
2216 struct NntpMboxData *mdata = m->mdata;
2217 struct ChildCtx cc;
2218 char buf[256];
2219 int rc;
2220 void *hc = NULL;
2221
2222 if (!mdata || !mdata->adata)
2223 return -1;
2224 if (mdata->first_message > mdata->last_loaded)
2225 return 0;
2226
2227 /* init context */
2228 cc.mailbox = m;
2229 cc.num = 0;
2230 cc.max = 10;
2231 cc.child = mutt_mem_malloc(sizeof(anum_t) * cc.max);
2232
2233 /* fetch numbers of child messages */
2234 snprintf(buf, sizeof(buf), "XPAT References %u-%u *%s*\r\n",
2235 mdata->first_message, mdata->last_loaded, msgid);
2236 rc = nntp_fetch_lines(mdata, buf, sizeof(buf), NULL, fetch_children, &cc);
2237 if (rc)
2238 {
2239 FREE(&cc.child);
2240 if (rc > 0)
2241 {
2242 if (!mutt_str_startswith(buf, "500"))
2243 mutt_error("XPAT: %s", buf);
2244 else
2245 {
2246 mutt_error(_("Unable to find child articles because server does not "
2247 "support XPAT command"));
2248 }
2249 }
2250 return -1;
2251 }
2252
2253 /* fetch all found messages */
2254 bool verbose = m->verbose;
2255 m->verbose = false;
2256 #ifdef USE_HCACHE
2257 hc = nntp_hcache_open(mdata);
2258 #endif
2259 int old_msg_count = m->msg_count;
2260 for (int i = 0; i < cc.num; i++)
2261 {
2262 rc = nntp_fetch_headers(m, hc, cc.child[i], cc.child[i], true);
2263 if (rc < 0)
2264 break;
2265 }
2266 if (m->msg_count > old_msg_count)
2267 mailbox_changed(m, NT_MAILBOX_INVALID);
2268
2269 #ifdef USE_HCACHE
2270 mutt_hcache_close(hc);
2271 #endif
2272 m->verbose = verbose;
2273 FREE(&cc.child);
2274 return (rc < 0) ? -1 : 0;
2275 }
2276
2277 /**
2278 * nntp_compare_order - Sort to mailbox order - Implements ::sort_mail_t - @ingroup sort_mail_api
2279 */
nntp_compare_order(const struct Email * a,const struct Email * b,bool reverse)2280 int nntp_compare_order(const struct Email *a, const struct Email *b, bool reverse)
2281 {
2282 anum_t na = nntp_edata_get((struct Email *) a)->article_num;
2283 anum_t nb = nntp_edata_get((struct Email *) b)->article_num;
2284 int result = (na == nb) ? 0 : (na > nb) ? 1 : -1;
2285 return reverse ? -result : result;
2286 }
2287
2288 /**
2289 * nntp_ac_owns_path - Check whether an Account owns a Mailbox path - Implements MxOps::ac_owns_path() - @ingroup mx_ac_owns_path
2290 */
nntp_ac_owns_path(struct Account * a,const char * path)2291 static bool nntp_ac_owns_path(struct Account *a, const char *path)
2292 {
2293 return true;
2294 }
2295
2296 /**
2297 * nntp_ac_add - Add a Mailbox to an Account - Implements MxOps::ac_add() - @ingroup mx_ac_add
2298 */
nntp_ac_add(struct Account * a,struct Mailbox * m)2299 static bool nntp_ac_add(struct Account *a, struct Mailbox *m)
2300 {
2301 return true;
2302 }
2303
2304 /**
2305 * nntp_mbox_open - Open a Mailbox - Implements MxOps::mbox_open() - @ingroup mx_mbox_open
2306 */
nntp_mbox_open(struct Mailbox * m)2307 static enum MxOpenReturns nntp_mbox_open(struct Mailbox *m)
2308 {
2309 if (!m->account)
2310 return MX_OPEN_ERROR;
2311
2312 char buf[8192];
2313 char server[1024];
2314 char *group = NULL;
2315 int rc;
2316 void *hc = NULL;
2317 anum_t first, last, count = 0;
2318
2319 struct Url *url = url_parse(mailbox_path(m));
2320 if (!url || !url->host || !url->path ||
2321 !((url->scheme == U_NNTP) || (url->scheme == U_NNTPS)))
2322 {
2323 url_free(&url);
2324 mutt_error(_("%s is an invalid newsgroup specification"), mailbox_path(m));
2325 return MX_OPEN_ERROR;
2326 }
2327
2328 group = url->path;
2329 if (group[0] == '/') /* Skip a leading '/' */
2330 group++;
2331
2332 url->path = strchr(url->path, '\0');
2333 url_tostring(url, server, sizeof(server), U_NO_FLAGS);
2334
2335 mutt_account_hook(m->realpath);
2336 struct NntpAccountData *adata = m->account->adata;
2337 if (!adata)
2338 {
2339 adata = nntp_select_server(m, server, true);
2340 m->account->adata = adata;
2341 m->account->adata_free = nntp_adata_free;
2342 }
2343
2344 if (!adata)
2345 {
2346 url_free(&url);
2347 return MX_OPEN_ERROR;
2348 }
2349 CurrentNewsSrv = adata;
2350
2351 m->msg_count = 0;
2352 m->msg_unread = 0;
2353 m->vcount = 0;
2354
2355 if (group[0] == '/')
2356 group++;
2357
2358 /* find news group data structure */
2359 struct NntpMboxData *mdata = mutt_hash_find(adata->groups_hash, group);
2360 if (!mdata)
2361 {
2362 nntp_newsrc_close(adata);
2363 mutt_error(_("Newsgroup %s not found on the server"), group);
2364 url_free(&url);
2365 return MX_OPEN_ERROR;
2366 }
2367
2368 m->rights &= ~MUTT_ACL_INSERT; // Clear the flag
2369 const bool c_save_unsubscribed =
2370 cs_subset_bool(NeoMutt->sub, "save_unsubscribed");
2371 if (!mdata->newsrc_ent && !mdata->subscribed && !c_save_unsubscribed)
2372 m->readonly = true;
2373
2374 /* select newsgroup */
2375 mutt_message(_("Selecting %s..."), group);
2376 url_free(&url);
2377 buf[0] = '\0';
2378 if (nntp_query(mdata, buf, sizeof(buf)) < 0)
2379 {
2380 nntp_newsrc_close(adata);
2381 return MX_OPEN_ERROR;
2382 }
2383
2384 /* newsgroup not found, remove it */
2385 if (mutt_str_startswith(buf, "411"))
2386 {
2387 mutt_error(_("Newsgroup %s has been removed from the server"), mdata->group);
2388 if (!mdata->deleted)
2389 {
2390 mdata->deleted = true;
2391 nntp_active_save_cache(adata);
2392 }
2393 if (mdata->newsrc_ent && !mdata->subscribed && !c_save_unsubscribed)
2394 {
2395 FREE(&mdata->newsrc_ent);
2396 mdata->newsrc_len = 0;
2397 nntp_delete_group_cache(mdata);
2398 nntp_newsrc_update(adata);
2399 }
2400 }
2401
2402 /* parse newsgroup info */
2403 else
2404 {
2405 if (sscanf(buf, "211 " ANUM " " ANUM " " ANUM, &count, &first, &last) != 3)
2406 {
2407 nntp_newsrc_close(adata);
2408 mutt_error("GROUP: %s", buf);
2409 return MX_OPEN_ERROR;
2410 }
2411 mdata->first_message = first;
2412 mdata->last_message = last;
2413 mdata->deleted = false;
2414
2415 /* get description if empty */
2416 const bool c_nntp_load_description =
2417 cs_subset_bool(NeoMutt->sub, "nntp_load_description");
2418 if (c_nntp_load_description && !mdata->desc)
2419 {
2420 if (get_description(mdata, NULL, NULL) < 0)
2421 {
2422 nntp_newsrc_close(adata);
2423 return MX_OPEN_ERROR;
2424 }
2425 if (mdata->desc)
2426 nntp_active_save_cache(adata);
2427 }
2428 }
2429
2430 adata->check_time = mutt_date_epoch();
2431 m->mdata = mdata;
2432 // Every known newsgroup has an mdata which is stored in adata->groups_list.
2433 // Currently we don't let the Mailbox free the mdata.
2434 // m->mdata_free = nntp_mdata_free;
2435 if (!mdata->bcache && (mdata->newsrc_ent || mdata->subscribed || c_save_unsubscribed))
2436 mdata->bcache = mutt_bcache_open(&adata->conn->account, mdata->group);
2437
2438 /* strip off extra articles if adding context is greater than $nntp_context */
2439 first = mdata->first_message;
2440 const short c_nntp_context = cs_subset_number(NeoMutt->sub, "nntp_context");
2441 if (c_nntp_context && (mdata->last_message - first + 1 > c_nntp_context))
2442 first = mdata->last_message - c_nntp_context + 1;
2443 mdata->last_loaded = first ? first - 1 : 0;
2444 count = mdata->first_message;
2445 mdata->first_message = first;
2446 nntp_bcache_update(mdata);
2447 mdata->first_message = count;
2448 #ifdef USE_HCACHE
2449 hc = nntp_hcache_open(mdata);
2450 nntp_hcache_update(mdata, hc);
2451 #endif
2452 if (!hc)
2453 m->rights &= ~(MUTT_ACL_WRITE | MUTT_ACL_DELETE); // Clear the flags
2454
2455 nntp_newsrc_close(adata);
2456 rc = nntp_fetch_headers(m, hc, first, mdata->last_message, false);
2457 #ifdef USE_HCACHE
2458 mutt_hcache_close(hc);
2459 #endif
2460 if (rc < 0)
2461 return MX_OPEN_ERROR;
2462 mdata->last_loaded = mdata->last_message;
2463 adata->newsrc_modified = false;
2464 return MX_OPEN_OK;
2465 }
2466
2467 /**
2468 * nntp_mbox_check - Check for new mail - Implements MxOps::mbox_check() - @ingroup mx_mbox_check
2469 * @param m Mailbox
2470 * @retval enum #MxStatus
2471 */
nntp_mbox_check(struct Mailbox * m)2472 static enum MxStatus nntp_mbox_check(struct Mailbox *m)
2473 {
2474 enum MxStatus rc = check_mailbox(m);
2475 if (rc == MX_STATUS_OK)
2476 {
2477 struct NntpMboxData *mdata = m->mdata;
2478 struct NntpAccountData *adata = mdata->adata;
2479 nntp_newsrc_close(adata);
2480 }
2481 return rc;
2482 }
2483
2484 /**
2485 * nntp_mbox_sync - Save changes to the Mailbox - Implements MxOps::mbox_sync() - @ingroup mx_mbox_sync
2486 *
2487 * @note May also return values from check_mailbox()
2488 */
nntp_mbox_sync(struct Mailbox * m)2489 static enum MxStatus nntp_mbox_sync(struct Mailbox *m)
2490 {
2491 struct NntpMboxData *mdata = m->mdata;
2492
2493 /* check for new articles */
2494 mdata->adata->check_time = 0;
2495 enum MxStatus check = check_mailbox(m);
2496 if (check != MX_STATUS_OK)
2497 return check;
2498
2499 #ifdef USE_HCACHE
2500 mdata->last_cached = 0;
2501 struct HeaderCache *hc = nntp_hcache_open(mdata);
2502 #endif
2503
2504 for (int i = 0; i < m->msg_count; i++)
2505 {
2506 struct Email *e = m->emails[i];
2507 if (!e)
2508 break;
2509
2510 char buf[16];
2511
2512 snprintf(buf, sizeof(buf), ANUM, nntp_edata_get(e)->article_num);
2513 if (mdata->bcache && e->deleted)
2514 {
2515 mutt_debug(LL_DEBUG2, "mutt_bcache_del %s\n", buf);
2516 mutt_bcache_del(mdata->bcache, buf);
2517 }
2518
2519 #ifdef USE_HCACHE
2520 if (hc && (e->changed || e->deleted))
2521 {
2522 if (e->deleted && !e->read)
2523 mdata->unread--;
2524 mutt_debug(LL_DEBUG2, "mutt_hcache_store %s\n", buf);
2525 mutt_hcache_store(hc, buf, strlen(buf), e, 0);
2526 }
2527 #endif
2528 }
2529
2530 #ifdef USE_HCACHE
2531 if (hc)
2532 {
2533 mutt_hcache_close(hc);
2534 mdata->last_cached = mdata->last_loaded;
2535 }
2536 #endif
2537
2538 /* save .newsrc entries */
2539 nntp_newsrc_gen_entries(m);
2540 nntp_newsrc_update(mdata->adata);
2541 nntp_newsrc_close(mdata->adata);
2542 return MX_STATUS_OK;
2543 }
2544
2545 /**
2546 * nntp_mbox_close - Close a Mailbox - Implements MxOps::mbox_close() - @ingroup mx_mbox_close
2547 * @retval 0 Always
2548 */
nntp_mbox_close(struct Mailbox * m)2549 static enum MxStatus nntp_mbox_close(struct Mailbox *m)
2550 {
2551 struct NntpMboxData *mdata = m->mdata;
2552 struct NntpMboxData *tmp_mdata = NULL;
2553 if (!mdata)
2554 return MX_STATUS_OK;
2555
2556 mdata->unread = m->msg_unread;
2557
2558 nntp_acache_free(mdata);
2559 if (!mdata->adata || !mdata->adata->groups_hash || !mdata->group)
2560 return MX_STATUS_OK;
2561
2562 tmp_mdata = mutt_hash_find(mdata->adata->groups_hash, mdata->group);
2563 if (!tmp_mdata || (tmp_mdata != mdata))
2564 nntp_mdata_free((void **) &mdata);
2565 return MX_STATUS_OK;
2566 }
2567
2568 /**
2569 * nntp_msg_open - Open an email message in a Mailbox - Implements MxOps::msg_open() - @ingroup mx_msg_open
2570 */
nntp_msg_open(struct Mailbox * m,struct Message * msg,int msgno)2571 static bool nntp_msg_open(struct Mailbox *m, struct Message *msg, int msgno)
2572 {
2573 struct NntpMboxData *mdata = m->mdata;
2574 struct Email *e = m->emails[msgno];
2575 if (!e)
2576 return false;
2577
2578 char article[16];
2579
2580 /* try to get article from cache */
2581 struct NntpAcache *acache = &mdata->acache[e->index % NNTP_ACACHE_LEN];
2582 if (acache->path)
2583 {
2584 if (acache->index == e->index)
2585 {
2586 msg->fp = mutt_file_fopen(acache->path, "r");
2587 if (msg->fp)
2588 return true;
2589 }
2590 /* clear previous entry */
2591 else
2592 {
2593 unlink(acache->path);
2594 FREE(&acache->path);
2595 }
2596 }
2597 snprintf(article, sizeof(article), ANUM, nntp_edata_get(e)->article_num);
2598 msg->fp = mutt_bcache_get(mdata->bcache, article);
2599 if (msg->fp)
2600 {
2601 if (nntp_edata_get(e)->parsed)
2602 return true;
2603 }
2604 else
2605 {
2606 char buf[PATH_MAX];
2607 /* don't try to fetch article from removed newsgroup */
2608 if (mdata->deleted)
2609 return false;
2610
2611 /* create new cache file */
2612 const char *fetch_msg = _("Fetching message...");
2613 mutt_message(fetch_msg);
2614 msg->fp = mutt_bcache_put(mdata->bcache, article);
2615 if (!msg->fp)
2616 {
2617 mutt_mktemp(buf, sizeof(buf));
2618 acache->path = mutt_str_dup(buf);
2619 acache->index = e->index;
2620 msg->fp = mutt_file_fopen(acache->path, "w+");
2621 if (!msg->fp)
2622 {
2623 mutt_perror(acache->path);
2624 unlink(acache->path);
2625 FREE(&acache->path);
2626 return false;
2627 }
2628 }
2629
2630 /* fetch message to cache file */
2631 snprintf(buf, sizeof(buf), "ARTICLE %s\r\n",
2632 nntp_edata_get(e)->article_num ? article : e->env->message_id);
2633 const int rc =
2634 nntp_fetch_lines(mdata, buf, sizeof(buf), fetch_msg, fetch_tempfile, msg->fp);
2635 if (rc)
2636 {
2637 mutt_file_fclose(&msg->fp);
2638 if (acache->path)
2639 {
2640 unlink(acache->path);
2641 FREE(&acache->path);
2642 }
2643 if (rc > 0)
2644 {
2645 if (mutt_str_startswith(buf, nntp_edata_get(e)->article_num ? "423" : "430"))
2646 {
2647 mutt_error(_("Article %s not found on the server"),
2648 nntp_edata_get(e)->article_num ? article : e->env->message_id);
2649 }
2650 else
2651 mutt_error("ARTICLE: %s", buf);
2652 }
2653 return false;
2654 }
2655
2656 if (!acache->path)
2657 mutt_bcache_commit(mdata->bcache, article);
2658 }
2659
2660 /* replace envelope with new one
2661 * hash elements must be updated because pointers will be changed */
2662 if (m->id_hash && e->env->message_id)
2663 mutt_hash_delete(m->id_hash, e->env->message_id, e);
2664 if (m->subj_hash && e->env->real_subj)
2665 mutt_hash_delete(m->subj_hash, e->env->real_subj, e);
2666
2667 mutt_env_free(&e->env);
2668 e->env = mutt_rfc822_read_header(msg->fp, e, false, false);
2669
2670 if (m->id_hash && e->env->message_id)
2671 mutt_hash_insert(m->id_hash, e->env->message_id, e);
2672 if (m->subj_hash && e->env->real_subj)
2673 mutt_hash_insert(m->subj_hash, e->env->real_subj, e);
2674
2675 /* fix content length */
2676 fseek(msg->fp, 0, SEEK_END);
2677 e->body->length = ftell(msg->fp) - e->body->offset;
2678
2679 /* this is called in neomutt before the open which fetches the message,
2680 * which is probably wrong, but we just call it again here to handle
2681 * the problem instead of fixing it */
2682 nntp_edata_get(e)->parsed = true;
2683 mutt_parse_mime_message(e, msg->fp);
2684
2685 /* these would normally be updated in ctx_update(), but the
2686 * full headers aren't parsed with overview, so the information wasn't
2687 * available then */
2688 if (WithCrypto)
2689 e->security = crypt_query(e->body);
2690
2691 rewind(msg->fp);
2692 mutt_clear_error();
2693 return true;
2694 }
2695
2696 /**
2697 * nntp_msg_close - Close an email - Implements MxOps::msg_close() - @ingroup mx_msg_close
2698 *
2699 * @note May also return EOF Failure, see errno
2700 */
nntp_msg_close(struct Mailbox * m,struct Message * msg)2701 static int nntp_msg_close(struct Mailbox *m, struct Message *msg)
2702 {
2703 return mutt_file_fclose(&msg->fp);
2704 }
2705
2706 /**
2707 * nntp_path_probe - Is this an NNTP Mailbox? - Implements MxOps::path_probe() - @ingroup mx_path_probe
2708 */
nntp_path_probe(const char * path,const struct stat * st)2709 enum MailboxType nntp_path_probe(const char *path, const struct stat *st)
2710 {
2711 if (mutt_istr_startswith(path, "news://"))
2712 return MUTT_NNTP;
2713
2714 if (mutt_istr_startswith(path, "snews://"))
2715 return MUTT_NNTP;
2716
2717 return MUTT_UNKNOWN;
2718 }
2719
2720 /**
2721 * nntp_path_canon - Canonicalise a Mailbox path - Implements MxOps::path_canon() - @ingroup mx_path_canon
2722 */
nntp_path_canon(char * buf,size_t buflen)2723 static int nntp_path_canon(char *buf, size_t buflen)
2724 {
2725 return 0;
2726 }
2727
2728 /**
2729 * nntp_path_pretty - Abbreviate a Mailbox path - Implements MxOps::path_pretty() - @ingroup mx_path_pretty
2730 */
nntp_path_pretty(char * buf,size_t buflen,const char * folder)2731 static int nntp_path_pretty(char *buf, size_t buflen, const char *folder)
2732 {
2733 /* Succeed, but don't do anything, for now */
2734 return 0;
2735 }
2736
2737 /**
2738 * nntp_path_parent - Find the parent of a Mailbox path - Implements MxOps::path_parent() - @ingroup mx_path_parent
2739 */
nntp_path_parent(char * buf,size_t buflen)2740 static int nntp_path_parent(char *buf, size_t buflen)
2741 {
2742 /* Succeed, but don't do anything, for now */
2743 return 0;
2744 }
2745
2746 /**
2747 * MxNntpOps - NNTP Mailbox - Implements ::MxOps - @ingroup mx_api
2748 */
2749 struct MxOps MxNntpOps = {
2750 // clang-format off
2751 .type = MUTT_NNTP,
2752 .name = "nntp",
2753 .is_local = false,
2754 .ac_owns_path = nntp_ac_owns_path,
2755 .ac_add = nntp_ac_add,
2756 .mbox_open = nntp_mbox_open,
2757 .mbox_open_append = NULL,
2758 .mbox_check = nntp_mbox_check,
2759 .mbox_check_stats = NULL,
2760 .mbox_sync = nntp_mbox_sync,
2761 .mbox_close = nntp_mbox_close,
2762 .msg_open = nntp_msg_open,
2763 .msg_open_new = NULL,
2764 .msg_commit = NULL,
2765 .msg_close = nntp_msg_close,
2766 .msg_padding_size = NULL,
2767 .msg_save_hcache = NULL,
2768 .tags_edit = NULL,
2769 .tags_commit = NULL,
2770 .path_probe = nntp_path_probe,
2771 .path_canon = nntp_path_canon,
2772 .path_pretty = nntp_path_pretty,
2773 .path_parent = nntp_path_parent,
2774 // clang-format on
2775 };
2776