1 /* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
2
3 #include "lib.h"
4 #include "str.h"
5 #include "ioloop.h"
6 #include "istream.h"
7 #include "istream-concat.h"
8 #include "istream-header-filter.h"
9 #include "message-header-parser.h"
10 #include "imap-arg.h"
11 #include "imap-util.h"
12 #include "imap-date.h"
13 #include "imap-quote.h"
14 #include "imap-bodystructure.h"
15 #include "imap-resp-code.h"
16 #include "imapc-mail.h"
17 #include "imapc-storage.h"
18
imapc_mail_set_failure(struct imapc_mail * mail,const struct imapc_command_reply * reply)19 static void imapc_mail_set_failure(struct imapc_mail *mail,
20 const struct imapc_command_reply *reply)
21 {
22 struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box);
23
24 mail->last_fetch_reply = p_strdup(mail->imail.mail.pool, reply->text_full);
25
26 switch (reply->state) {
27 case IMAPC_COMMAND_STATE_OK:
28 break;
29 case IMAPC_COMMAND_STATE_NO:
30 if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS)) {
31 /* fetch-fix-broken-mails feature disabled -
32 fail any mails with missing replies */
33 break;
34 }
35 if (reply->resp_text_key != NULL &&
36 (strcasecmp(reply->resp_text_key, IMAP_RESP_CODE_SERVERBUG) == 0 ||
37 strcasecmp(reply->resp_text_key, IMAP_RESP_CODE_LIMIT) == 0)) {
38 /* this is a temporary error, retrying should work.
39 Yahoo sends * BYE +
40 NO [LIMIT] UID FETCH Rate limit hit. */
41 } else {
42 /* hopefully this is a permanent failure */
43 mail->fetch_ignore_if_missing = TRUE;
44 }
45 break;
46 case IMAPC_COMMAND_STATE_BAD:
47 case IMAPC_COMMAND_STATE_DISCONNECTED:
48 case IMAPC_COMMAND_STATE_AUTH_FAILED:
49 mail->fetch_failed = TRUE;
50 break;
51 }
52 }
53
54 static void
imapc_mail_fetch_callback(const struct imapc_command_reply * reply,void * context)55 imapc_mail_fetch_callback(const struct imapc_command_reply *reply,
56 void *context)
57 {
58 struct imapc_fetch_request *request = context;
59 struct imapc_fetch_request *const *requests;
60 struct imapc_mail *mail;
61 struct imapc_mailbox *mbox = NULL;
62 unsigned int i, count;
63
64 array_foreach_elem(&request->mails, mail) {
65 i_assert(mail->fetch_count > 0);
66 imapc_mail_set_failure(mail, reply);
67 if (--mail->fetch_count == 0)
68 mail->fetching_fields = 0;
69 mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box);
70 }
71 i_assert(mbox != NULL);
72
73 requests = array_get(&mbox->fetch_requests, &count);
74 for (i = 0; i < count; i++) {
75 if (requests[i] == request) {
76 array_delete(&mbox->fetch_requests, i, 1);
77 break;
78 }
79 }
80 i_assert(i < count);
81
82 array_free(&request->mails);
83 i_free(request);
84
85 if (reply->state == IMAPC_COMMAND_STATE_OK)
86 ;
87 else if (reply->state == IMAPC_COMMAND_STATE_NO) {
88 imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS,
89 reply);
90 } else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) {
91 /* The disconnection message was already logged */
92 mail_storage_set_internal_error(&mbox->storage->storage);
93 } else {
94 mailbox_set_critical(&mbox->box,
95 "imapc: Mail FETCH failed: %s", reply->text_full);
96 }
97 imapc_client_stop(mbox->storage->client->client);
98 }
99
100 static bool
headers_have_subset(const char * const * superset,const char * const * subset)101 headers_have_subset(const char *const *superset, const char *const *subset)
102 {
103 unsigned int i;
104
105 if (superset == NULL)
106 return FALSE;
107 if (subset != NULL) {
108 for (i = 0; subset[i] != NULL; i++) {
109 if (!str_array_icase_find(superset, subset[i]))
110 return FALSE;
111 }
112 }
113 return TRUE;
114 }
115
116 static const char *const *
headers_merge(pool_t pool,const char * const * h1,const char * const * h2)117 headers_merge(pool_t pool, const char *const *h1, const char *const *h2)
118 {
119 ARRAY_TYPE(const_string) headers;
120 const char *value;
121 unsigned int i;
122
123 p_array_init(&headers, pool, 16);
124 if (h1 != NULL) {
125 for (i = 0; h1[i] != NULL; i++) {
126 value = p_strdup(pool, h1[i]);
127 array_push_back(&headers, &value);
128 }
129 }
130 if (h2 != NULL) {
131 for (i = 0; h2[i] != NULL; i++) {
132 if (h1 == NULL || !str_array_icase_find(h1, h2[i])) {
133 value = p_strdup(pool, h2[i]);
134 array_push_back(&headers, &value);
135 }
136 }
137 }
138 array_append_zero(&headers);
139 return array_front(&headers);
140 }
141
142 static bool
imapc_mail_try_merge_fetch(struct imapc_mailbox * mbox,string_t * str)143 imapc_mail_try_merge_fetch(struct imapc_mailbox *mbox, string_t *str)
144 {
145 const char *s1 = str_c(str);
146 const char *s2 = str_c(mbox->pending_fetch_cmd);
147 const char *p1, *p2;
148
149 i_assert(str_begins(s1, "UID FETCH "));
150 i_assert(str_begins(s2, "UID FETCH "));
151
152 /* skip over UID range */
153 p1 = strchr(s1+10, ' ');
154 p2 = strchr(s2+10, ' ');
155
156 if (null_strcmp(p1, p2) != 0)
157 return FALSE;
158 /* append the new UID to the pending FETCH UID range */
159 str_truncate(str, p1-s1);
160 str_insert(mbox->pending_fetch_cmd, p2-s2, ",");
161 str_insert(mbox->pending_fetch_cmd, p2-s2+1, str_c(str) + 10);
162 return TRUE;
163 }
164
165 static void
imapc_mail_delayed_send_or_merge(struct imapc_mail * mail,string_t * str)166 imapc_mail_delayed_send_or_merge(struct imapc_mail *mail, string_t *str)
167 {
168 struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box);
169
170 if (mbox->pending_fetch_request != NULL &&
171 !imapc_mail_try_merge_fetch(mbox, str)) {
172 /* send the previous FETCH and create a new one */
173 imapc_mail_fetch_flush(mbox);
174 }
175 if (mbox->pending_fetch_request == NULL) {
176 mbox->pending_fetch_request =
177 i_new(struct imapc_fetch_request, 1);
178 i_array_init(&mbox->pending_fetch_request->mails, 4);
179 i_assert(mbox->pending_fetch_cmd->used == 0);
180 str_append_str(mbox->pending_fetch_cmd, str);
181 }
182 array_push_back(&mbox->pending_fetch_request->mails, &mail);
183
184 if (mbox->to_pending_fetch_send == NULL &&
185 array_count(&mbox->pending_fetch_request->mails) >
186 mbox->box.storage->set->mail_prefetch_count) {
187 /* we're now prefetching the maximum number of mails. this
188 most likely means that we need to flush out the command now
189 before sending anything else. delay it a little bit though
190 in case the sending code doesn't actually use
191 mail_prefetch_count and wants to fetch more.
192
193 note that we don't want to add this timeout too early,
194 because we want to optimize the maximum number of messages
195 placed into a single FETCH. even without timeout the command
196 gets flushed by imapc_mail_fetch() call. */
197 mbox->to_pending_fetch_send =
198 timeout_add_short(0, imapc_mail_fetch_flush, mbox);
199 }
200 }
201
202 static int
imapc_mail_send_fetch(struct mail * _mail,enum mail_fetch_field fields,const char * const * headers)203 imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields,
204 const char *const *headers)
205 {
206 struct imapc_mail *mail = IMAPC_MAIL(_mail);
207 struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
208 struct mail_index_view *view;
209 string_t *str;
210 uint32_t seq;
211 unsigned int i;
212
213 i_assert(headers == NULL ||
214 IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS));
215
216 if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
217 mail_set_aborted(_mail);
218 return -1;
219 }
220 _mail->mail_stream_opened = TRUE;
221
222 /* drop any fields that we may already be fetching currently */
223 fields &= ENUM_NEGATE(mail->fetching_fields);
224 if (headers_have_subset(mail->fetching_headers, headers))
225 headers = NULL;
226 if (fields == 0 && headers == NULL)
227 return mail->fetch_sent ? 0 : 1;
228
229 if (!_mail->saving) {
230 /* if we already know that the mail is expunged,
231 don't try to FETCH it */
232 view = mbox->delayed_sync_view != NULL ?
233 mbox->delayed_sync_view : mbox->box.view;
234 if (!mail_index_lookup_seq(view, _mail->uid, &seq) ||
235 mail_index_is_expunged(view, seq)) {
236 mail_set_expunged(_mail);
237 return -1;
238 }
239 } else if (mbox->client_box == NULL) {
240 /* opened as save-only. we'll need to fetch the mail,
241 so actually SELECT/EXAMINE the mailbox */
242 i_assert(mbox->box.opened);
243
244 if (imapc_mailbox_select(mbox) < 0)
245 return -1;
246 }
247
248 if ((fields & MAIL_FETCH_STREAM_BODY) != 0)
249 fields |= MAIL_FETCH_STREAM_HEADER;
250
251 str = t_str_new(64);
252 str_printfa(str, "UID FETCH %u (", _mail->uid);
253 if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0)
254 str_append(str, "INTERNALDATE ");
255 if ((fields & MAIL_FETCH_SAVE_DATE) != 0) {
256 i_assert(HAS_ALL_BITS(mbox->capabilities,
257 IMAPC_CAPABILITY_SAVEDATE));
258 str_append(str, "SAVEDATE ");
259 }
260 if ((fields & (MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE)) != 0)
261 str_append(str, "RFC822.SIZE ");
262 if ((fields & MAIL_FETCH_GUID) != 0) {
263 str_append(str, mbox->guid_fetch_field_name);
264 str_append_c(str, ' ');
265 }
266 if ((fields & MAIL_FETCH_IMAP_BODY) != 0)
267 str_append(str, "BODY ");
268 if ((fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0)
269 str_append(str, "BODYSTRUCTURE ");
270
271 if ((fields & MAIL_FETCH_STREAM_BODY) != 0) {
272 if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_ZIMBRA_WORKAROUNDS))
273 str_append(str, "BODY.PEEK[] ");
274 else {
275 /* BODY.PEEK[] can return different headers than
276 BODY.PEEK[HEADER] (e.g. invalid 8bit chars replaced
277 with '?' in HEADER) - this violates IMAP protocol
278 and messes up dsync since it sometimes fetches the
279 full body and sometimes only the headers. */
280 str_append(str, "BODY.PEEK[HEADER] BODY.PEEK[TEXT] ");
281 }
282 fields |= MAIL_FETCH_STREAM_HEADER;
283 } else if ((fields & MAIL_FETCH_STREAM_HEADER) != 0)
284 str_append(str, "BODY.PEEK[HEADER] ");
285 else if (headers != NULL) {
286 mail->fetching_headers =
287 headers_merge(mail->imail.mail.data_pool, headers,
288 mail->fetching_headers);
289 str_append(str, "BODY.PEEK[HEADER.FIELDS (");
290 for (i = 0; mail->fetching_headers[i] != NULL; i++) {
291 if (i > 0)
292 str_append_c(str, ' ');
293 imap_append_astring(str, mail->fetching_headers[i]);
294 }
295 str_append(str, ")] ");
296 mail->header_list_fetched = FALSE;
297 }
298 str_truncate(str, str_len(str)-1);
299 str_append_c(str, ')');
300
301 mail->fetching_fields |= fields;
302 mail->fetch_count++;
303 mail->fetch_sent = FALSE;
304 mail->fetch_failed = FALSE;
305
306 imapc_mail_delayed_send_or_merge(mail, str);
307 return 1;
308 }
309
imapc_mail_cache_get(struct imapc_mail * mail,struct imapc_mail_cache * cache)310 static void imapc_mail_cache_get(struct imapc_mail *mail,
311 struct imapc_mail_cache *cache)
312 {
313 if (mail->body_fetched)
314 return;
315
316 if (cache->fd != -1) {
317 mail->fd = cache->fd;
318 mail->imail.data.stream = i_stream_create_fd(mail->fd, 0);
319 cache->fd = -1;
320 } else if (cache->buf != NULL) {
321 mail->body = cache->buf;
322 mail->imail.data.stream =
323 i_stream_create_from_data(mail->body->data,
324 mail->body->used);
325 cache->buf = NULL;
326 } else {
327 return;
328 }
329 mail->header_fetched = TRUE;
330 mail->body_fetched = TRUE;
331 imapc_mail_init_stream(mail);
332 }
333
334 static enum mail_fetch_field
imapc_mail_get_wanted_fetch_fields(struct imapc_mail * mail)335 imapc_mail_get_wanted_fetch_fields(struct imapc_mail *mail)
336 {
337 struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box);
338 struct index_mail_data *data = &mail->imail.data;
339 enum mail_fetch_field fields = 0;
340
341 if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0 &&
342 data->received_date == (time_t)-1)
343 fields |= MAIL_FETCH_RECEIVED_DATE;
344 if ((data->wanted_fields & MAIL_FETCH_SAVE_DATE) != 0 &&
345 data->save_date == (time_t)-1) {
346 if (HAS_ALL_BITS(mbox->capabilities, IMAPC_CAPABILITY_SAVEDATE))
347 fields |= MAIL_FETCH_SAVE_DATE;
348 else
349 fields |= MAIL_FETCH_RECEIVED_DATE;
350 }
351 if ((data->wanted_fields & (MAIL_FETCH_PHYSICAL_SIZE |
352 MAIL_FETCH_VIRTUAL_SIZE)) != 0 &&
353 data->physical_size == UOFF_T_MAX &&
354 IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE))
355 fields |= MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE;
356 if ((data->wanted_fields & MAIL_FETCH_IMAP_BODY) != 0 &&
357 data->body == NULL &&
358 IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE))
359 fields |= MAIL_FETCH_IMAP_BODY;
360 if ((data->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0 &&
361 data->bodystructure == NULL &&
362 IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE))
363 fields |= MAIL_FETCH_IMAP_BODYSTRUCTURE;
364 if ((data->wanted_fields & MAIL_FETCH_GUID) != 0 &&
365 data->guid == NULL && mbox->guid_fetch_field_name != NULL)
366 fields |= MAIL_FETCH_GUID;
367
368 if (data->stream == NULL && data->access_part != 0) {
369 if ((data->access_part & (READ_BODY | PARSE_BODY)) != 0)
370 fields |= MAIL_FETCH_STREAM_BODY;
371 fields |= MAIL_FETCH_STREAM_HEADER;
372 }
373 return fields;
374 }
375
imapc_mail_try_init_stream_from_cache(struct imapc_mail * mail)376 void imapc_mail_try_init_stream_from_cache(struct imapc_mail *mail)
377 {
378 struct mail *_mail = &mail->imail.mail.mail;
379 struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
380
381 if (mbox->prev_mail_cache.uid == _mail->uid)
382 imapc_mail_cache_get(mail, &mbox->prev_mail_cache);
383 }
384
imapc_mail_prefetch(struct mail * _mail)385 bool imapc_mail_prefetch(struct mail *_mail)
386 {
387 struct imapc_mail *mail = IMAPC_MAIL(_mail);
388 struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
389 struct index_mail_data *data = &mail->imail.data;
390 enum mail_fetch_field fields;
391 const char *const *headers = NULL;
392
393 /* try to get as much from cache as possible */
394 imapc_mail_update_access_parts(&mail->imail);
395 /* If mail is already cached we can avoid re-FETCHing the mail.
396 However, don't initialize the stream if we don't actually want to
397 access the mail. */
398 if (mail->imail.data.access_part != 0)
399 imapc_mail_try_init_stream_from_cache(mail);
400
401 fields = imapc_mail_get_wanted_fetch_fields(mail);
402 if (data->wanted_headers != NULL && data->stream == NULL &&
403 (fields & MAIL_FETCH_STREAM_HEADER) == 0 &&
404 !imapc_mail_has_headers_in_cache(&mail->imail, data->wanted_headers)) {
405 /* fetch specific headers */
406 if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS))
407 headers = data->wanted_headers->name;
408 else
409 fields |= MAIL_FETCH_STREAM_HEADER;
410 }
411 if (fields != 0 || headers != NULL) T_BEGIN {
412 if (imapc_mail_send_fetch(_mail, fields, headers) > 0)
413 mail->imail.data.prefetch_sent = TRUE;
414 } T_END;
415 return !mail->imail.data.prefetch_sent;
416 }
417
418 static bool
imapc_mail_have_fields(struct imapc_mail * imail,enum mail_fetch_field fields)419 imapc_mail_have_fields(struct imapc_mail *imail, enum mail_fetch_field fields)
420 {
421 struct imapc_mailbox *mbox = IMAPC_MAILBOX(imail->imail.mail.mail.box);
422
423 if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0) {
424 if (imail->imail.data.received_date == (time_t)-1)
425 return FALSE;
426 fields &= ENUM_NEGATE(MAIL_FETCH_RECEIVED_DATE);
427 }
428 if ((fields & MAIL_FETCH_SAVE_DATE) != 0) {
429 i_assert(HAS_ALL_BITS(mbox->capabilities,
430 IMAPC_CAPABILITY_SAVEDATE));
431 if (imail->imail.data.save_date == (time_t)-1)
432 return FALSE;
433 fields &= ENUM_NEGATE(MAIL_FETCH_SAVE_DATE);
434 }
435 if ((fields & (MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE)) != 0) {
436 if (imail->imail.data.physical_size == UOFF_T_MAX)
437 return FALSE;
438 fields &= ENUM_NEGATE(MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE);
439 }
440 if ((fields & MAIL_FETCH_GUID) != 0) {
441 if (imail->imail.data.guid == NULL)
442 return FALSE;
443 fields &= ENUM_NEGATE(MAIL_FETCH_GUID);
444 }
445 if ((fields & MAIL_FETCH_IMAP_BODY) != 0) {
446 if (imail->imail.data.body == NULL)
447 return FALSE;
448 fields &= ENUM_NEGATE(MAIL_FETCH_IMAP_BODY);
449 }
450 if ((fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0) {
451 if (imail->imail.data.bodystructure == NULL)
452 return FALSE;
453 fields &= ENUM_NEGATE(MAIL_FETCH_IMAP_BODYSTRUCTURE);
454 }
455 if ((fields & (MAIL_FETCH_STREAM_HEADER |
456 MAIL_FETCH_STREAM_BODY)) != 0) {
457 if (imail->imail.data.stream == NULL)
458 return FALSE;
459 fields &= ENUM_NEGATE(MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY);
460 }
461 i_assert(fields == 0);
462 return TRUE;
463 }
464
imapc_mail_fetch(struct mail * _mail,enum mail_fetch_field fields,const char * const * headers)465 int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields,
466 const char *const *headers)
467 {
468 struct imapc_mail *imail = IMAPC_MAIL(_mail);
469 struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
470 int ret;
471
472 if ((fields & MAIL_FETCH_GUID) != 0 &&
473 mbox->guid_fetch_field_name == NULL) {
474 mail_storage_set_error(_mail->box->storage,
475 MAIL_ERROR_NOTPOSSIBLE,
476 "Message GUID not available in this server");
477 return -1;
478 }
479 if (_mail->saving) {
480 mail_storage_set_error(_mail->box->storage,
481 MAIL_ERROR_NOTPOSSIBLE,
482 "Attempting to issue FETCH for a mail not yet committed");
483 return -1;
484 }
485
486 fields |= imapc_mail_get_wanted_fetch_fields(imail);
487 T_BEGIN {
488 ret = imapc_mail_send_fetch(_mail, fields, headers);
489 } T_END;
490 if (ret < 0)
491 return -1;
492
493 /* we'll continue waiting until we've got all the fields we wanted,
494 or until all FETCH replies have been received (i.e. some FETCHes
495 failed) */
496 if (ret > 0)
497 imapc_mail_fetch_flush(mbox);
498 while (imail->fetch_count > 0 &&
499 (!imapc_mail_have_fields(imail, fields) ||
500 !imail->header_list_fetched)) {
501 imapc_mailbox_run_nofetch(mbox);
502 }
503 if (imail->fetch_failed) {
504 mail_storage_set_internal_error(&mbox->storage->storage);
505 return -1;
506 }
507 return 0;
508 }
509
imapc_mail_fetch_flush(struct imapc_mailbox * mbox)510 void imapc_mail_fetch_flush(struct imapc_mailbox *mbox)
511 {
512 struct imapc_command *cmd;
513 struct imapc_mail *mail;
514
515 if (mbox->pending_fetch_request == NULL) {
516 i_assert(mbox->to_pending_fetch_send == NULL);
517 return;
518 }
519
520 array_foreach_elem(&mbox->pending_fetch_request->mails, mail)
521 mail->fetch_sent = TRUE;
522
523 cmd = imapc_client_mailbox_cmd(mbox->client_box,
524 imapc_mail_fetch_callback,
525 mbox->pending_fetch_request);
526 imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
527 array_push_back(&mbox->fetch_requests, &mbox->pending_fetch_request);
528
529 imapc_command_send(cmd, str_c(mbox->pending_fetch_cmd));
530
531 mbox->pending_fetch_request = NULL;
532 timeout_remove(&mbox->to_pending_fetch_send);
533 str_truncate(mbox->pending_fetch_cmd, 0);
534 }
535
imapc_find_lfile_arg(const struct imapc_untagged_reply * reply,const struct imap_arg * arg,int * fd_r)536 static bool imapc_find_lfile_arg(const struct imapc_untagged_reply *reply,
537 const struct imap_arg *arg, int *fd_r)
538 {
539 const struct imap_arg *list;
540 unsigned int i, count;
541
542 for (i = 0; i < reply->file_args_count; i++) {
543 const struct imapc_arg_file *farg = &reply->file_args[i];
544
545 if (farg->parent_arg == arg->parent &&
546 imap_arg_get_list_full(arg->parent, &list, &count) &&
547 farg->list_idx < count && &list[farg->list_idx] == arg) {
548 *fd_r = farg->fd;
549 return TRUE;
550 }
551 }
552 return FALSE;
553 }
554
imapc_stream_filter(struct istream ** input)555 static void imapc_stream_filter(struct istream **input)
556 {
557 static const char *imapc_hide_headers[] = {
558 /* Added by MS Exchange 2010 when \Flagged flag is set.
559 This violates IMAP guarantee of messages being immutable. */
560 "X-Message-Flag"
561 };
562 struct istream *filter_input;
563
564 filter_input = i_stream_create_header_filter(*input,
565 HEADER_FILTER_EXCLUDE,
566 imapc_hide_headers, N_ELEMENTS(imapc_hide_headers),
567 *null_header_filter_callback, NULL);
568 i_stream_unref(input);
569 *input = filter_input;
570 }
571
imapc_mail_init_stream(struct imapc_mail * mail)572 void imapc_mail_init_stream(struct imapc_mail *mail)
573 {
574 struct index_mail *imail = &mail->imail;
575 struct mail *_mail = &imail->mail.mail;
576 struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
577 struct istream *input;
578 uoff_t size;
579 int ret;
580
581 i_stream_set_name(imail->data.stream,
582 t_strdup_printf("imapc mail uid=%u", _mail->uid));
583 index_mail_set_read_buffer_size(_mail, imail->data.stream);
584
585 if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
586 /* enable filtering only when we're not passing through
587 RFC822.SIZE. otherwise we'll get size mismatches. */
588 imapc_stream_filter(&imail->data.stream);
589 }
590 if (imail->mail.v.istream_opened != NULL) {
591 if (imail->mail.v.istream_opened(_mail,
592 &imail->data.stream) < 0) {
593 index_mail_close_streams(imail);
594 return;
595 }
596 }
597 ret = i_stream_get_size(imail->data.stream, TRUE, &size);
598 if (ret < 0) {
599 index_mail_close_streams(imail);
600 return;
601 }
602 i_assert(ret != 0);
603 /* Once message body is fetched, we can be sure of what its size is.
604 If we had already received RFC822.SIZE, overwrite it here in case
605 it's wrong. Also in more special cases the RFC822.SIZE may be
606 smaller than the fetched message header. In this case change the
607 size as well, otherwise reading via istream-mail will fail. */
608 if (mail->body_fetched || imail->data.physical_size < size) {
609 if (mail->body_fetched) {
610 imail->data.inexact_total_sizes = FALSE;
611 /* Don't trust any existing virtual_size. Also don't
612 set it to size, because there's no guarantees about
613 the content having proper CRLF newlines, especially
614 not if istream_opened() has changed the stream. */
615 imail->data.virtual_size = UOFF_T_MAX;
616 }
617 imail->data.physical_size = size;
618 }
619
620 imail->data.stream_has_only_header = !mail->body_fetched;
621 if (index_mail_init_stream(imail, NULL, NULL, &input) < 0)
622 index_mail_close_streams(imail);
623 }
624
625 static void
imapc_fetch_stream(struct imapc_mail * mail,const struct imapc_untagged_reply * reply,const struct imap_arg * arg,bool have_header,bool have_body)626 imapc_fetch_stream(struct imapc_mail *mail,
627 const struct imapc_untagged_reply *reply,
628 const struct imap_arg *arg,
629 bool have_header, bool have_body)
630 {
631 struct index_mail *imail = &mail->imail;
632 struct imapc_mailbox *mbox = IMAPC_MAILBOX(imail->mail.mail.box);
633 struct istream *hdr_stream = NULL;
634 const char *value;
635 int fd;
636
637 if (imail->data.stream != NULL) {
638 i_assert(mail->header_fetched);
639 if (mail->body_fetched || !have_body)
640 return;
641 if (have_header) {
642 /* replace the existing stream */
643 } else if (mail->fd == -1) {
644 /* append this body stream to the existing
645 header stream */
646 hdr_stream = imail->data.stream;
647 i_stream_ref(hdr_stream);
648 } else {
649 /* append this body stream to the existing
650 header stream. we'll need to recreate the stream
651 with autoclosed fd. */
652 if (lseek(mail->fd, 0, SEEK_SET) < 0)
653 i_error("lseek(imapc) failed: %m");
654 hdr_stream = i_stream_create_fd_autoclose(&mail->fd, 0);
655 }
656 index_mail_close_streams(imail);
657 i_close_fd(&mail->fd);
658 } else {
659 if (!have_header) {
660 /* BODY.PEEK[TEXT] received - we can't currently handle
661 this before receiving BODY.PEEK[HEADER] reply */
662 return;
663 }
664 }
665
666 if (arg->type == IMAP_ARG_LITERAL_SIZE) {
667 if (!imapc_find_lfile_arg(reply, arg, &fd)) {
668 i_stream_unref(&hdr_stream);
669 return;
670 }
671 if ((fd = dup(fd)) == -1) {
672 i_error("dup() failed: %m");
673 i_stream_unref(&hdr_stream);
674 return;
675 }
676 mail->fd = fd;
677 imail->data.stream = i_stream_create_fd(fd, 0);
678 } else {
679 if (!imap_arg_get_nstring(arg, &value))
680 value = NULL;
681 if (value == NULL ||
682 (value[0] == '\0' &&
683 IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_EMPTY_IS_EXPUNGED))) {
684 mail_set_expunged(&imail->mail.mail);
685 i_stream_unref(&hdr_stream);
686 return;
687 }
688 if (mail->body == NULL) {
689 mail->body = buffer_create_dynamic(default_pool,
690 arg->str_len + 1);
691 } else if (!have_header && hdr_stream != NULL) {
692 /* header is already in the buffer - add body now
693 without destroying the existing header data */
694 i_stream_unref(&hdr_stream);
695 } else {
696 buffer_set_used_size(mail->body, 0);
697 }
698 buffer_append(mail->body, value, arg->str_len);
699 imail->data.stream = i_stream_create_from_data(mail->body->data,
700 mail->body->used);
701 }
702 if (have_header)
703 mail->header_fetched = TRUE;
704 mail->body_fetched = have_body;
705
706 if (hdr_stream != NULL) {
707 struct istream *inputs[3];
708
709 inputs[0] = hdr_stream;
710 inputs[1] = imail->data.stream;
711 inputs[2] = NULL;
712 imail->data.stream = i_stream_create_concat(inputs);
713 i_stream_unref(&inputs[0]);
714 i_stream_unref(&inputs[1]);
715 }
716
717 imapc_mail_init_stream(mail);
718 }
719
720 static void
imapc_fetch_header_stream(struct imapc_mail * mail,const struct imapc_untagged_reply * reply,const struct imap_arg * args)721 imapc_fetch_header_stream(struct imapc_mail *mail,
722 const struct imapc_untagged_reply *reply,
723 const struct imap_arg *args)
724 {
725 const enum message_header_parser_flags hdr_parser_flags =
726 MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP |
727 MESSAGE_HEADER_PARSER_FLAG_DROP_CR;
728 const struct imap_arg *hdr_list;
729 struct mailbox_header_lookup_ctx *headers_ctx;
730 struct message_header_parser_ctx *parser;
731 struct message_header_line *hdr;
732 struct istream *input;
733 ARRAY_TYPE(const_string) hdr_arr;
734 const char *value;
735 int ret, fd;
736
737 if (!imap_arg_get_list(args, &hdr_list))
738 return;
739 if (!imap_arg_atom_equals(args+1, "]"))
740 return;
741 args += 2;
742
743 /* see if this is reply to the latest headers list request
744 (parse it even if it's not) */
745 t_array_init(&hdr_arr, 16);
746 while (imap_arg_get_astring(hdr_list, &value)) {
747 array_push_back(&hdr_arr, &value);
748 hdr_list++;
749 }
750 if (hdr_list->type != IMAP_ARG_EOL)
751 return;
752 array_append_zero(&hdr_arr);
753
754 if (headers_have_subset(array_front(&hdr_arr), mail->fetching_headers))
755 mail->header_list_fetched = TRUE;
756
757 if (args->type == IMAP_ARG_LITERAL_SIZE) {
758 if (!imapc_find_lfile_arg(reply, args, &fd))
759 return;
760 input = i_stream_create_fd(fd, 0);
761 } else {
762 if (!imap_arg_get_nstring(args, &value))
763 return;
764 if (value == NULL) {
765 mail_set_expunged(&mail->imail.mail.mail);
766 return;
767 }
768 input = i_stream_create_from_data(value, args->str_len);
769 }
770
771 headers_ctx = mailbox_header_lookup_init(mail->imail.mail.mail.box,
772 array_front(&hdr_arr));
773 index_mail_parse_header_init(&mail->imail, headers_ctx);
774
775 parser = message_parse_header_init(input, NULL, hdr_parser_flags);
776 while ((ret = message_parse_header_next(parser, &hdr)) > 0)
777 index_mail_parse_header(NULL, hdr, &mail->imail);
778 i_assert(ret != 0);
779 index_mail_parse_header(NULL, NULL, &mail->imail);
780 message_parse_header_deinit(&parser);
781
782 mailbox_header_lookup_unref(&headers_ctx);
783 i_stream_destroy(&input);
784 }
785
786 static const char *
imapc_args_to_bodystructure(struct imapc_mail * mail,const struct imap_arg * list_arg,bool extended)787 imapc_args_to_bodystructure(struct imapc_mail *mail,
788 const struct imap_arg *list_arg, bool extended)
789 {
790 const struct imap_arg *args;
791 struct message_part *parts = NULL;
792 const char *ret, *error;
793 pool_t pool;
794
795 if (!imap_arg_get_list(list_arg, &args)) {
796 mail_set_critical(&mail->imail.mail.mail,
797 "imapc: Server sent invalid BODYSTRUCTURE parameters");
798 return NULL;
799 }
800
801 pool = pool_alloconly_create("imap bodystructure", 1024);
802 if (imap_bodystructure_parse_args(args, pool, &parts, &error) < 0) {
803 mail_set_critical(&mail->imail.mail.mail,
804 "imapc: Server sent invalid BODYSTRUCTURE: %s", error);
805 ret = NULL;
806 } else {
807 string_t *str = t_str_new(128);
808 if (imap_bodystructure_write(parts, str, extended, &error) < 0) {
809 /* All the input to imap_bodystructure_write() came
810 from imap_bodystructure_parse_args(). We should never
811 get here. Instead, if something is wrong the
812 parsing should have returned an error already. */
813 str_truncate(str, 0);
814 imap_write_args(str, args);
815 i_panic("Failed to write parsed BODYSTRUCTURE: %s "
816 "(original string: '%s')", error, str_c(str));
817 }
818 ret = p_strdup(mail->imail.mail.data_pool, str_c(str));
819 }
820 pool_unref(&pool);
821 return ret;
822 }
823
imapc_mail_fetch_update(struct imapc_mail * mail,const struct imapc_untagged_reply * reply,const struct imap_arg * args)824 void imapc_mail_fetch_update(struct imapc_mail *mail,
825 const struct imapc_untagged_reply *reply,
826 const struct imap_arg *args)
827 {
828 struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box);
829 const char *key, *value;
830 unsigned int i;
831 uoff_t size;
832 time_t t;
833 int tz;
834 bool match = FALSE;
835
836 for (i = 0; args[i].type != IMAP_ARG_EOL; i += 2) {
837 if (!imap_arg_get_atom(&args[i], &key) ||
838 args[i+1].type == IMAP_ARG_EOL)
839 break;
840
841 if (strcasecmp(key, "BODY[]") == 0) {
842 imapc_fetch_stream(mail, reply, &args[i+1], TRUE, TRUE);
843 match = TRUE;
844 } else if (strcasecmp(key, "BODY[HEADER]") == 0) {
845 imapc_fetch_stream(mail, reply, &args[i+1], TRUE, FALSE);
846 match = TRUE;
847 } else if (strcasecmp(key, "BODY[TEXT]") == 0) {
848 imapc_fetch_stream(mail, reply, &args[i+1], FALSE, TRUE);
849 match = TRUE;
850 } else if (strcasecmp(key, "BODY[HEADER.FIELDS") == 0) {
851 imapc_fetch_header_stream(mail, reply, &args[i+1]);
852 match = TRUE;
853 } else if (strcasecmp(key, "INTERNALDATE") == 0) {
854 if (imap_arg_get_astring(&args[i+1], &value) &&
855 imap_parse_datetime(value, &t, &tz)) {
856 mail->imail.data.received_date = t;
857 if (HAS_NO_BITS(mbox->capabilities,
858 IMAPC_CAPABILITY_SAVEDATE))
859 mail->imail.data.save_date = t;
860 }
861 match = TRUE;
862 } else if (strcasecmp(key, "SAVEDATE") == 0) {
863 if (imap_arg_get_astring(&args[i+1], &value)) {
864 if (strcasecmp(value, "NIL") == 0)
865 mail->imail.data.save_date = 0;
866 else if (imap_parse_datetime(value, &t, &tz))
867 mail->imail.data.save_date = t;
868 }
869 match = TRUE;
870 } else if (strcasecmp(key, "BODY") == 0) {
871 if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) {
872 mail->imail.data.body =
873 imapc_args_to_bodystructure(mail, &args[i+1], FALSE);
874 }
875 match = TRUE;
876 } else if (strcasecmp(key, "BODYSTRUCTURE") == 0) {
877 if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) {
878 mail->imail.data.bodystructure =
879 imapc_args_to_bodystructure(mail, &args[i+1], TRUE);
880 }
881 match = TRUE;
882 } else if (strcasecmp(key, "RFC822.SIZE") == 0) {
883 if (imap_arg_get_atom(&args[i+1], &value) &&
884 str_to_uoff(value, &size) == 0 &&
885 IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
886 mail->imail.data.physical_size = size;
887 mail->imail.data.virtual_size = size;
888 mail->imail.data.inexact_total_sizes = TRUE;
889 }
890 match = TRUE;
891 } else if (strcasecmp(key, "X-GM-MSGID") == 0 ||
892 strcasecmp(key, "X-GUID") == 0) {
893 if (imap_arg_get_astring(&args[i+1], &value)) {
894 mail->imail.data.guid =
895 p_strdup(mail->imail.mail.pool, value);
896 }
897 match = TRUE;
898 }
899 }
900 if (!match) {
901 /* this is only a FETCH FLAGS update for the wanted mail */
902 } else {
903 imapc_client_stop(mbox->storage->client->client);
904 }
905 }
906