1 /* jmap_mail_query.c -- Helper routines for JMAP Email/query
2  *
3  * Copyright (c) 1994-2018 Carnegie Mellon University.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  *
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  *
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in
14  *    the documentation and/or other materials provided with the
15  *    distribution.
16  *
17  * 3. The name "Carnegie Mellon University" must not be used to
18  *    endorse or promote products derived from this software without
19  *    prior written permission. For permission or any legal
20  *    details, please contact
21  *      Carnegie Mellon University
22  *      Center for Technology Transfer and Enterprise Creation
23  *      4615 Forbes Avenue
24  *      Suite 302
25  *      Pittsburgh, PA  15213
26  *      (412) 268-7393, fax: (412) 268-7395
27  *      innovation@andrew.cmu.edu
28  *
29  * 4. Redistributions of any form whatsoever must retain the following
30  *    acknowledgment:
31  *    "This product includes software developed by Computing Services
32  *     at Carnegie Mellon University (http://www.cmu.edu/computing/)."
33  *
34  * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
35  * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
36  * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
37  * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
38  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
39  * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
40  * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
41  *
42  */
43 
44 #include <config.h>
45 
46 #include <string.h>
47 #include <syslog.h>
48 #include <errno.h>
49 
50 #include "libconfig.h"
51 
52 #include "jmap_mail_query.h"
53 #include "jmap_util.h"
54 #include "json_support.h"
55 #include "search_engines.h"
56 
57 #include "imap/imap_err.h"
58 
59 #ifndef JMAP_URN_MAIL
60 #define JMAP_URN_MAIL                "urn:ietf:params:jmap:mail"
61 #endif
62 #ifndef JMAP_MAIL_EXTENSION
63 #define JMAP_MAIL_EXTENSION          "https://cyrusimap.org/ns/jmap/mail"
64 #endif
65 
_email_threadkeyword_is_valid(const char * keyword)66 static int _email_threadkeyword_is_valid(const char *keyword)
67 {
68     /* \Seen is always supported */
69     if (!strcasecmp(keyword, "$Seen"))
70         return 1;
71 
72     const char *counted_flags = config_getstring(IMAPOPT_CONVERSATIONS_COUNTED_FLAGS);
73     if (!counted_flags)
74         return 0;
75 
76     /* We really shouldn't do all this string mangling for each keyword */
77     strarray_t *flags = strarray_split(counted_flags, " ", STRARRAY_TRIM);
78     int i, is_supported = 0;
79     for (i = 0; i < flags->count; i++) {
80         const char *flag = strarray_nth(flags, i);
81         const char *kw = keyword;
82         if (*flag == '\\') { // special case \ => $
83             flag++;
84             if (*kw != '$') continue;
85             kw++;
86         }
87         if (!strcasecmp(flag, kw)) {
88             is_supported = 1;
89             break;
90         }
91     }
92     strarray_free(flags);
93 
94     return is_supported;
95 }
96 
97 
98 #ifdef WITH_DAV
99 
100 #include "annotate.h"
101 #include "carddav_db.h"
102 #include "global.h"
103 #include "hash.h"
104 #include "index.h"
105 #include "search_query.h"
106 #include "times.h"
107 
jmap_email_contactfilter_init(const char * accountid,const char * addressbookid,struct email_contactfilter * cfilter)108 HIDDEN void jmap_email_contactfilter_init(const char *accountid,
109                                           const char *addressbookid,
110                                           struct email_contactfilter *cfilter)
111 {
112     memset(cfilter, 0, sizeof(struct email_contactfilter));
113     cfilter->accountid = accountid;
114     if (addressbookid) {
115         cfilter->addrbook = carddav_mboxname(accountid, addressbookid);
116     }
117 }
118 
jmap_email_contactfilter_fini(struct email_contactfilter * cfilter)119 HIDDEN void jmap_email_contactfilter_fini(struct email_contactfilter *cfilter)
120 {
121     if (cfilter->carddavdb) {
122         carddav_close(cfilter->carddavdb);
123     }
124     free(cfilter->addrbook);
125     free_hash_table(&cfilter->contactgroups, (void(*)(void*))strarray_free);
126 }
127 
128 
_get_sharedaddressbook_cb(const mbentry_t * mbentry,void * rock)129 static int _get_sharedaddressbook_cb(const mbentry_t *mbentry, void *rock)
130 {
131     mbname_t **mbnamep = rock;
132     if (!mbentry) return 0;
133     if (!(mbentry->mbtype & MBTYPE_ADDRESSBOOK)) return 0;
134     mbname_t *mbname = mbname_from_intname(mbentry->name);
135     if (!strcmpsafe(strarray_nth(mbname_boxes(mbname), -1), "Shared")) {
136         *mbnamep = mbname;
137         return CYRUSDB_DONE;
138     }
139     mbname_free(&mbname);
140     return 0;
141 }
142 
143 
_get_sharedaddressbookuser(const char * userid)144 static mbname_t *_get_sharedaddressbookuser(const char *userid)
145 {
146     mbname_t *res = NULL;
147     int flags = MBOXTREE_PLUS_RACL|MBOXTREE_SKIP_ROOT|MBOXTREE_SKIP_CHILDREN;
148     // XXX - do we need to pass req->authstate right through??
149     int r = mboxlist_usermboxtree(userid, NULL, _get_sharedaddressbook_cb, &res, flags);
150     if (r == CYRUSDB_DONE)
151         return res;
152     mbname_free(&res);
153     return NULL;
154 }
155 
156 
157 static const struct contactfilters_t {
158     const char *field;
159     int isany;
160 } contactfilters[] = {
161   { "fromContactGroupId", 0 },
162   { "toContactGroupId", 0 },
163   { "ccContactGroupId", 0 },
164   { "bccContactGroupId", 0 },
165   { "fromAnyContact", 1 },
166   { "toAnyContact", 1 },
167   { "ccAnyContact", 1 },
168   { "bccAnyContact", 1 },
169   { NULL, 0 }
170 };
171 
jmap_email_contactfilter_from_filtercondition(struct jmap_parser * parser,json_t * filter,struct email_contactfilter * cfilter)172 HIDDEN int jmap_email_contactfilter_from_filtercondition(struct jmap_parser *parser,
173                                                          json_t *filter,
174                                                          struct email_contactfilter *cfilter)
175 {
176     int havefield = 0;
177     const struct contactfilters_t *c;
178     mbname_t *othermb = NULL;
179     int r = 0;
180 
181     /* prefilter to see if there are any fields that we will need to look up */
182     for (c = contactfilters; c->field; c++) {
183         json_t *arg = json_object_get(filter, c->field);
184         if (!arg) continue;
185         const char *groupid = c->isany ? (json_is_true(arg) ? "" : NULL) : json_string_value(arg);
186         if (!groupid) continue; // avoid looking up if invalid!
187         havefield = 1;
188         break;
189     }
190     if (!havefield) goto done;
191 
192     /* ensure we have preconditions for lookups */
193     if (!cfilter->contactgroups.size) {
194         /* Initialize groups lookup table */
195         construct_hash_table(&cfilter->contactgroups, 32, 0);
196     }
197 
198     if (!cfilter->carddavdb) {
199         /* Open CardDAV db first time we need it */
200         cfilter->carddavdb = carddav_open_userid(cfilter->accountid);
201         if (!cfilter->carddavdb) {
202             syslog(LOG_ERR, "jmap: carddav_open_userid(%s) failed",
203                    cfilter->accountid);
204             r = CYRUSDB_INTERNAL;
205             goto done;
206         }
207     }
208 
209     othermb = _get_sharedaddressbookuser(cfilter->accountid);
210     if (othermb) {
211         int r2 = carddav_set_otheruser(cfilter->carddavdb, mbname_userid(othermb));
212         if (r2) syslog(LOG_NOTICE, "DBNOTICE: failed to open otheruser %s contacts for %s",
213                  mbname_userid(othermb), cfilter->accountid);
214     }
215 
216     /* fetch members for each filter referenced */
217 
218     for (c = contactfilters; c->field; c++) {
219         json_t *arg = json_object_get(filter, c->field);
220         if (!arg) continue;
221         const char *groupid = c->isany ? (json_is_true(arg) ? "" : NULL) : json_string_value(arg);
222         if (!groupid) continue;
223         if (hash_lookup(groupid, &cfilter->contactgroups)) continue;
224 
225         /* Lookup group member email addresses */
226         strarray_t *members = carddav_getgroup(cfilter->carddavdb, cfilter->addrbook, groupid, othermb);
227         if (!members) {
228             jmap_parser_invalid(parser, c->field);
229         }
230         else {
231             hash_insert(groupid, members, &cfilter->contactgroups);
232         }
233     }
234 
235 done:
236     mbname_free(&othermb);
237     return r;
238 }
239 
jmap_emailbodies_fini(struct emailbodies * bodies)240 HIDDEN void jmap_emailbodies_fini(struct emailbodies *bodies)
241 {
242     ptrarray_fini(&bodies->attslist);
243     ptrarray_fini(&bodies->textlist);
244     ptrarray_fini(&bodies->htmllist);
245 }
246 
_email_extract_bodies_internal(const struct body * parts,int nparts,const char * multipart_type,int in_alternative,ptrarray_t * textlist,ptrarray_t * htmllist,ptrarray_t * attslist)247 static int _email_extract_bodies_internal(const struct body *parts,
248                                           int nparts,
249                                           const char *multipart_type,
250                                           int in_alternative,
251                                           ptrarray_t *textlist,
252                                           ptrarray_t *htmllist,
253                                           ptrarray_t *attslist)
254 {
255     int i;
256 
257     enum parttype { OTHER, PLAIN, HTML, MULTIPART, INLINE_MEDIA, MESSAGE };
258 
259     int textlist_count = textlist ? textlist->count : -1;
260     int htmllist_count = htmllist ? htmllist->count : -1;
261 
262     for (i = 0; i < nparts; i++) {
263         const struct body *part = parts + i;
264 
265         /* Determine part type */
266         enum parttype parttype = OTHER;
267         if (!strcmp(part->type, "TEXT") && !strcmp(part->subtype, "PLAIN"))
268             parttype = PLAIN;
269         else if (!strcmp(part->type, "TEXT") && !strcmp(part->subtype, "RICHTEXT"))
270             parttype = PLAIN; // RFC 1341
271         else if (!strcmp(part->type, "TEXT") && !strcmp(part->subtype, "ENRICHED"))
272             parttype = PLAIN; // RFC 1563
273         else if (!strcmp(part->type, "TEXT") && !strcmp(part->subtype, "HTML"))
274             parttype = HTML;
275         else if (!strcmp(part->type, "MULTIPART"))
276             parttype = MULTIPART;
277         else if (!strcmp(part->type, "IMAGE") || !strcmp(part->type, "AUDIO") || !strcmp(part->type, "VIDEO"))
278             parttype = INLINE_MEDIA;
279 
280         /* Determine disposition name, if any. */
281         const char *dispname = NULL;
282         struct param *param;
283         for (param = part->disposition_params; param; param = param->next) {
284             if (!strncasecmp(param->attribute, "filename", 8)) {
285                 dispname = param->value;
286                 break;
287             }
288         }
289         if (!dispname) {
290             for (param = part->params; param; param = param->next) {
291                 if (!strncasecmp(param->attribute, "name", 4)) {
292                     dispname = param->value;
293                     break;
294                 }
295             }
296         }
297         /* Determine if it's an inlined part */
298         int is_inline =
299             (!part->disposition || strcmp(part->disposition, "ATTACHMENT")) &&
300             /* Must be one of the allowed body types */
301             (parttype == PLAIN || parttype == HTML || parttype == INLINE_MEDIA) &&
302              /* If multipart/related, only the first part can be inline
303               * If a text part with a filename, and not the first item in the
304               * multipart, assume it is an attachment */
305              (i == 0 || (strcmp(multipart_type, "RELATED") &&
306                          (parttype == INLINE_MEDIA || !dispname)));
307         /* Handle by part type */
308         if (parttype == MULTIPART) {
309             _email_extract_bodies_internal(part->subpart, part->numparts,
310                     part->subtype,
311                     in_alternative || !strcmp(part->subtype, "ALTERNATIVE"),
312                     textlist, htmllist, attslist);
313         }
314         else if (is_inline) {
315             if (!strcmp(multipart_type, "ALTERNATIVE")) {
316                 if (parttype == PLAIN && textlist) {
317                     ptrarray_append(textlist, (void*) part);
318                 }
319                 else if (parttype == HTML && htmllist) {
320                     ptrarray_append(htmllist, (void*) part);
321                 }
322                 else {
323                     ptrarray_append(attslist, (void*) part);
324                 }
325                 continue;
326             }
327             else if (in_alternative) {
328                 if (parttype == PLAIN)
329                     htmllist = NULL;
330                 if (parttype == HTML)
331                     textlist = NULL;
332             }
333             if (textlist)
334                 ptrarray_append(textlist, (void*) part);
335             if (htmllist)
336                 ptrarray_append(htmllist, (void*) part);
337             if ((!textlist || !htmllist) && parttype == INLINE_MEDIA)
338                 ptrarray_append(attslist, (void*) part);
339         }
340         else {
341             ptrarray_append(attslist, (void*) part);
342         }
343     }
344 
345     if (!strcmp(multipart_type, "ALTERNATIVE")) {
346         int j;
347         /* Found HTML part only */
348         if (textlist && htmllist && textlist_count == textlist->count) {
349             for (j = htmllist_count; j < htmllist->count; j++)
350                 ptrarray_append(textlist, ptrarray_nth(htmllist, j));
351         }
352         /* Found TEXT part only */
353         if (htmllist && textlist && htmllist_count == htmllist->count) {
354             for (j = textlist_count; j < textlist->count; j++)
355                 ptrarray_append(htmllist, ptrarray_nth(textlist, j));
356         }
357     }
358 
359     return 0;
360 }
361 
jmap_emailbodies_extract(const struct body * root,struct emailbodies * bodies)362 HIDDEN int jmap_emailbodies_extract(const struct body *root,
363                                      struct emailbodies *bodies)
364 {
365     return _email_extract_bodies_internal(root, 1, "MIXED", 0,
366             &bodies->textlist, &bodies->htmllist,
367             &bodies->attslist);
368 }
369 
370 struct matchmime_receiver {
371     struct search_text_receiver super;
372     xapian_dbw_t *dbw;
373     struct buf buf;
374 };
375 
_matchmime_tr_begin_mailbox(search_text_receiver_t * rx,struct mailbox * mailbox,int incremental)376 static int _matchmime_tr_begin_mailbox(search_text_receiver_t *rx __attribute__((unused)),
377                                        struct mailbox *mailbox __attribute__((unused)),
378                                        int incremental __attribute__((unused)))
379 {
380     return 0;
381 }
382 
_matchmime_tr_first_unindexed_uid(search_text_receiver_t * rx)383 static uint32_t _matchmime_tr_first_unindexed_uid(search_text_receiver_t *rx __attribute__((unused)))
384 {
385     return 1;
386 }
387 
_matchmime_tr_is_indexed(search_text_receiver_t * rx,message_t * msg)388 static uint8_t _matchmime_tr_is_indexed(search_text_receiver_t *rx __attribute__((unused)),
389                                         message_t *msg __attribute__((unused)))
390 {
391     return 0;
392 }
393 
_matchmime_tr_begin_message(search_text_receiver_t * rx,message_t * msg)394 static int _matchmime_tr_begin_message(search_text_receiver_t *rx, message_t *msg)
395 {
396     const struct message_guid *guid;
397     int r = message_get_guid(msg, &guid);
398     if (r) return r;
399 
400     struct matchmime_receiver *tr = (struct matchmime_receiver *) rx;
401     return xapian_dbw_begin_doc(tr->dbw, guid, 'G');
402 }
403 
_matchmime_tr_begin_part(search_text_receiver_t * rx,int part,const struct message_guid * content_guid)404 static void _matchmime_tr_begin_part(search_text_receiver_t *rx __attribute__((unused)),
405                                      int part __attribute__((unused)),
406                                      const struct message_guid *content_guid __attribute__((unused)))
407 {
408 }
409 
_matchmime_tr_append_text(search_text_receiver_t * rx,const struct buf * text)410 static void _matchmime_tr_append_text(search_text_receiver_t *rx,
411                                       const struct buf *text)
412 {
413     struct matchmime_receiver *tr = (struct matchmime_receiver *) rx;
414 
415     if (buf_len(&tr->buf) >= SEARCH_MAX_PARTS_SIZE) return;
416 
417     size_t n = SEARCH_MAX_PARTS_SIZE - buf_len(&tr->buf);
418     if (n > buf_len(text)) {
419         n = buf_len(text);
420     }
421     buf_appendmap(&tr->buf, buf_base(text), n);
422 }
423 
_matchmime_tr_end_part(search_text_receiver_t * rx,int part)424 static void _matchmime_tr_end_part(search_text_receiver_t *rx, int part)
425 {
426     struct matchmime_receiver *tr = (struct matchmime_receiver *) rx;
427     xapian_dbw_doc_part(tr->dbw, &tr->buf, part);
428     buf_reset(&tr->buf);
429 }
430 
_matchmime_tr_end_message(search_text_receiver_t * rx,uint8_t indexlevel)431 static int _matchmime_tr_end_message(search_text_receiver_t *rx, uint8_t indexlevel)
432 {
433     struct matchmime_receiver *tr = (struct matchmime_receiver *) rx;
434     return xapian_dbw_end_doc(tr->dbw, indexlevel);
435 }
436 
_matchmime_tr_end_mailbox(search_text_receiver_t * rx,struct mailbox * mailbox)437 static int _matchmime_tr_end_mailbox(search_text_receiver_t *rx __attribute__((unused)),
438                                      struct mailbox *mailbox __attribute__((unused)))
439 {
440     return 0;
441 }
442 
_matchmime_tr_flush(search_text_receiver_t * rx)443 static int _matchmime_tr_flush(search_text_receiver_t *rx __attribute__((unused)))
444 {
445     return 0;
446 }
447 
_matchmime_tr_audit_mailbox(search_text_receiver_t * rx,bitvector_t * unindexed)448 static int _matchmime_tr_audit_mailbox(search_text_receiver_t *rx __attribute__((unused)),
449                                        bitvector_t *unindexed __attribute__((unused)))
450 {
451     return 0;
452 }
453 
_matchmime_tr_index_charset_flags(int base_flags)454 static int _matchmime_tr_index_charset_flags(int base_flags)
455 {
456     return base_flags | CHARSET_KEEPCASE;
457 }
458 
_matchmime_tr_index_message_format(int format,int is_snippet)459 static int _matchmime_tr_index_message_format(int format __attribute__((unused)),
460                                               int is_snippet __attribute__((unused)))
461 {
462     return MESSAGE_SNIPPET;
463 }
464 
_email_matchmime_evaluate_xcb(void * data,size_t n,void * rock)465 static int _email_matchmime_evaluate_xcb(void *data __attribute__((unused)),
466                                          size_t n, void *rock)
467 {
468     int *matches = rock;
469     /* There's just a single message in the in-memory database,
470      * so no need to check the message guid in the search result. */
471     *matches = n > 0;
472     return 0;
473 }
474 
_email_matchmime_contactgroup(const char * groupid,int part,xapian_db_t * db,struct email_contactfilter * cfilter)475 static xapian_query_t *_email_matchmime_contactgroup(const char *groupid,
476                                                      int part,
477                                                      xapian_db_t *db,
478                                                      struct email_contactfilter *cfilter)
479 {
480     xapian_query_t *xq = NULL;
481 
482     if (cfilter->contactgroups.size) {
483         strarray_t *members = hash_lookup(groupid, &cfilter->contactgroups);
484         if (members && strarray_size(members)) {
485             ptrarray_t xsubqs = PTRARRAY_INITIALIZER;
486             int i;
487             for (i = 0; i < strarray_size(members); i++) {
488                 const char *member = strarray_nth(members, i);
489                 if (!strchr(member, '@')) continue;
490                 xapian_query_t *xsubq = xapian_query_new_match(db, part, member);
491                 if (xsubq) ptrarray_append(&xsubqs, xsubq);
492             }
493             if (ptrarray_size(&xsubqs)) {
494                 xq = xapian_query_new_compound(db, /*is_or*/1,
495                         (xapian_query_t **) xsubqs.data, xsubqs.count);
496             }
497         }
498     }
499     if (!xq) {
500         xq = xapian_query_new_not(db, xapian_query_new_matchall(db));
501     }
502 
503     return xq;
504 }
505 
build_type_query(xapian_db_t * db,const char * type)506 static xapian_query_t *build_type_query(xapian_db_t *db, const char *type)
507 {
508     strarray_t types = STRARRAY_INITIALIZER;
509     ptrarray_t xqs = PTRARRAY_INITIALIZER;
510 
511     /* Handle type wildcards */
512     if (!strcasecmp(type, "image")) {
513         strarray_append(&types, "image/gif");
514         strarray_append(&types, "image/jpeg");
515         strarray_append(&types, "image/pjpeg");
516         strarray_append(&types, "image/jpg");
517         strarray_append(&types, "image/png");
518         strarray_append(&types, "image/bmp");
519         strarray_append(&types, "image/tiff");
520     }
521     else if (!strcasecmp(type, "document")) {
522         strarray_append(&types, "application/msword");
523         strarray_append(&types, "application/vnd.openxmlformats-officedocument.wordprocessingml.document");
524         strarray_append(&types, "application/vnd.openxmlformats-officedocument.wordprocessingml.template");
525         strarray_append(&types, "application/vnd.sun.xml.writer");
526         strarray_append(&types, "application/vnd.sun.xml.writer.template");
527         strarray_append(&types, "application/vnd.oasis.opendocument.text");
528         strarray_append(&types, "application/vnd.oasis.opendocument.text-template");
529         strarray_append(&types, "application/x-iwork-pages-sffpages");
530         strarray_append(&types, "application/vnd.apple.pages");
531     }
532     else if (!strcasecmp(type, "spreadsheet")) {
533         strarray_append(&types, "application/vnd.ms-excel");
534         strarray_append(&types, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
535         strarray_append(&types, "application/vnd.openxmlformats-officedocument.spreadsheetml.template");
536         strarray_append(&types, "application/vnd.sun.xml.calc");
537         strarray_append(&types, "application/vnd.sun.xml.calc.template");
538         strarray_append(&types, "application/vnd.oasis.opendocument.spreadsheet");
539         strarray_append(&types, "application/vnd.oasis.opendocument.spreadsheet-template");
540         strarray_append(&types, "application/x-iwork-numbers-sffnumbers");
541         strarray_append(&types, "application/vnd.apple.numbers");
542     }
543     else if (!strcasecmp(type, "presentation")) {
544         strarray_append(&types, "application/vnd.ms-powerpoint");
545         strarray_append(&types, "application/vnd.openxmlformats-officedocument.presentationml.presentation");
546         strarray_append(&types, "application/vnd.openxmlformats-officedocument.presentationml.template");
547         strarray_append(&types, "application/vnd.openxmlformats-officedocument.presentationml.slideshow");
548         strarray_append(&types, "application/vnd.sun.xml.impress");
549         strarray_append(&types, "application/vnd.sun.xml.impress.template");
550         strarray_append(&types, "application/vnd.oasis.opendocument.presentation");
551         strarray_append(&types, "application/vnd.oasis.opendocument.presentation-template");
552         strarray_append(&types, "application/x-iwork-keynote-sffkey");
553         strarray_append(&types, "application/vnd.apple.keynote");
554     }
555     else if (!strcasecmp(type, "email")) {
556         strarray_append(&types, "message/rfc822");
557     }
558     else if (!strcasecmp(type, "pdf")) {
559         strarray_append(&types, "application/pdf");
560     }
561     else {
562         strarray_append(&types, type);
563     }
564 
565     /* Build expression */
566     int i;
567     for (i = 0; i < strarray_size(&types); i++) {
568         const char *t = strarray_nth(&types, i);
569         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_TYPE, t);
570         if (xq) ptrarray_append(&xqs, xq);
571     }
572     xapian_query_t *xq = xapian_query_new_compound(db, /*is_or*/1,
573                           (xapian_query_t **) xqs.data, xqs.count);
574 
575     ptrarray_fini(&xqs);
576     strarray_fini(&types);
577     return xq;
578 }
579 
_email_matchmime_evaluate(json_t * filter,message_t * m,xapian_db_t * db,struct email_contactfilter * cfilter,time_t internaldate)580 static int _email_matchmime_evaluate(json_t *filter,
581                                      message_t *m,
582                                      xapian_db_t *db,
583                                      struct email_contactfilter *cfilter,
584                                      time_t internaldate)
585 {
586     if (!json_object_size(filter)) {
587         /* Match all */
588         return 1;
589     }
590 
591     json_t *conditions = json_object_get(filter, "conditions");
592     if (json_is_array(conditions)) {
593 
594         /* Evaluate FilterOperator */
595 
596         const char *strop = json_string_value(json_object_get(filter, "operator"));
597         enum search_op op = SEOP_UNKNOWN;
598         int matches;
599 
600         if (!strcasecmpsafe(strop, "AND")) {
601             op = SEOP_AND;
602             matches = 1;
603         }
604         else if (!strcasecmpsafe(strop, "OR")) {
605             op = SEOP_OR;
606             matches = json_array_size(conditions) == 0;
607         }
608         else if (!strcasecmpsafe(strop, "NOT")) {
609             op = SEOP_NOT;
610             matches = json_array_size(conditions) != 0;
611         }
612         else return 0;
613 
614         json_t *condition;
615         size_t i;
616         json_array_foreach(conditions, i, condition) {
617             int cond_matches = _email_matchmime_evaluate(condition, m, db, cfilter, internaldate);
618             if (op == SEOP_AND && !cond_matches) {
619                 return 0;
620             }
621             if (op == SEOP_OR && cond_matches) {
622                 return 1;
623             }
624             if (op == SEOP_NOT && cond_matches) {
625                 return 0;
626             }
627         }
628 
629         return matches;
630     }
631 
632     /* Evaluate FilterCondition */
633 
634     int need_matches = json_object_size(filter);
635     int have_matches = 0;
636     json_t *jval;
637 
638 #define MATCHMIME_XQ_OR_MATCHALL(_xq) \
639     ((_xq) ? _xq : xapian_query_new_matchall(db))
640 
641     /* Xapian-backed criteria */
642     ptrarray_t xqs = PTRARRAY_INITIALIZER;
643     const char *match;
644     if ((match = json_string_value(json_object_get(filter, "text")))) {
645         ptrarray_t childqueries = PTRARRAY_INITIALIZER;
646         int i;
647         for (i = 0 ; i < SEARCH_NUM_PARTS ; i++) {
648             switch (i) {
649                 case SEARCH_PART_LISTID:
650                 case SEARCH_PART_TYPE:
651                 case SEARCH_PART_LANGUAGE:
652                 case SEARCH_PART_PRIORITY:
653                 case SEARCH_PART_ATTACHMENTBODY:
654                     continue;
655             }
656             void *xq = xapian_query_new_match(db, i, match);
657             if (xq) ptrarray_push(&childqueries, xq);
658         }
659         xapian_query_t *xq = xapian_query_new_compound(db, /*is_or*/1,
660                                        (xapian_query_t **)childqueries.data,
661                                        childqueries.count);
662         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
663         ptrarray_fini(&childqueries);
664     }
665     if ((match = json_string_value(json_object_get(filter, "from")))) {
666         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_FROM, match);
667         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
668     }
669     if ((match = json_string_value(json_object_get(filter, "to")))) {
670         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_TO, match);
671         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
672     }
673     if ((match = json_string_value(json_object_get(filter, "cc")))) {
674         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_CC, match);
675         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
676     }
677     if ((match = json_string_value(json_object_get(filter, "bcc")))) {
678         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_BCC, match);
679         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
680     }
681     if ((match = json_string_value(json_object_get(filter, "deliveredTo")))) {
682         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_DELIVEREDTO, match);
683         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
684     }
685     if ((match = json_string_value(json_object_get(filter, "subject")))) {
686         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_SUBJECT, match);
687         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
688     }
689     if ((match = json_string_value(json_object_get(filter, "body")))) {
690         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_BODY, match);
691         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
692     }
693     if ((match = json_string_value(json_object_get(filter, "fromContactGroupId")))) {
694         xapian_query_t *xq = _email_matchmime_contactgroup(match, SEARCH_PART_FROM, db, cfilter);
695         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
696     }
697     if ((match = json_string_value(json_object_get(filter, "toContactGroupId")))) {
698         xapian_query_t *xq = _email_matchmime_contactgroup(match, SEARCH_PART_TO, db, cfilter);
699         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
700     }
701     if ((match = json_string_value(json_object_get(filter, "ccContactGroupId")))) {
702         xapian_query_t *xq = _email_matchmime_contactgroup(match, SEARCH_PART_CC, db, cfilter);
703         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
704     }
705     if ((match = json_string_value(json_object_get(filter, "bccContactGroupId")))) {
706         xapian_query_t *xq = _email_matchmime_contactgroup(match, SEARCH_PART_BCC, db, cfilter);
707         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
708     }
709     if ((json_is_true(json_object_get(filter, "fromAnyContact")))) {
710         xapian_query_t *xq = _email_matchmime_contactgroup("", SEARCH_PART_FROM, db, cfilter);
711         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
712     }
713     if ((json_is_true(json_object_get(filter, "toAnyContact")))) {
714         xapian_query_t *xq = _email_matchmime_contactgroup("", SEARCH_PART_TO, db, cfilter);
715         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
716     }
717     if ((json_is_true(json_object_get(filter, "ccAnyContact")))) {
718         xapian_query_t *xq = _email_matchmime_contactgroup("", SEARCH_PART_CC, db, cfilter);
719         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
720     }
721     if ((json_is_true(json_object_get(filter, "bccAnyContact")))) {
722         xapian_query_t *xq = _email_matchmime_contactgroup("", SEARCH_PART_BCC, db, cfilter);
723         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
724     }
725     if ((match = json_string_value(json_object_get(filter, "attachmentName")))) {
726         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_ATTACHMENTNAME, match);
727         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
728     }
729     if ((match = json_string_value(json_object_get(filter, "attachmentType")))) {
730         xapian_query_t *xq = build_type_query(db, match);
731         ptrarray_append(&xqs, MATCHMIME_XQ_OR_MATCHALL(xq));
732     }
733     if ((match = json_string_value(json_object_get(filter, "listId")))) {
734         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_LISTID, match);
735         if (xq) ptrarray_append(&xqs, xq);
736     }
737     if (JNOTNULL(jval = json_object_get(filter, "isHighPriority"))) {
738         xapian_query_t *xq = xapian_query_new_match(db, SEARCH_PART_PRIORITY, "1");
739         if (xq && !json_boolean_value(jval)) {
740             xq = xapian_query_new_not(db, xq);
741         }
742         if (xq) ptrarray_append(&xqs, xq);
743     }
744     // ignore attachmentBody
745 
746 #undef MATCHMIME_XQ_OR_MATCHALL
747 
748     if (xqs.count) {
749         int matches = 0;
750         xapian_query_t *xq = xapian_query_new_compound(db, /*is_or*/0,
751                 (xapian_query_t **) xqs.data, xqs.count);
752         xapian_query_run(db, xq, 0, _email_matchmime_evaluate_xcb, &matches);
753         xapian_query_free(xq);
754         size_t xqs_count = xqs.count;
755         ptrarray_fini(&xqs);
756         if (matches) {
757             have_matches += xqs_count; // assumes one xapian query per criteria
758         }
759         else return 0;
760     }
761 
762     /* size */
763     if (json_object_get(filter, "minSize") || json_object_get(filter, "maxSize")) {
764         uint32_t size;
765         if (message_get_size(m, &size) == 0) {
766             json_int_t jint;
767             if ((jint = json_integer_value(json_object_get(filter, "minSize"))) > 0) {
768                 if (size >= jint) {
769                     have_matches++;
770                 }
771                 else return 0;
772             }
773             if ((jint = json_integer_value(json_object_get(filter, "maxSize"))) > 0) {
774                 if (size < jint) {
775                     have_matches++;
776                 }
777                 else return 0;
778             }
779         }
780     }
781 
782     /* hasAttachment */
783     if (JNOTNULL(jval = json_object_get(filter, "hasAttachment"))) {
784         const struct body *body;
785         if (message_get_cachebody(m, &body) == 0) {
786             struct emailbodies bodies = EMAILBODIES_INITIALIZER;
787             if (jmap_emailbodies_extract(body, &bodies) == 0) {
788                 int have = ptrarray_size(&bodies.attslist) > 0;
789                 int want = jval == json_true();
790                 jmap_emailbodies_fini(&bodies);
791                 if (have == want) {
792                     have_matches++;
793                 }
794                 else return 0;
795             }
796         }
797     }
798 
799     /* header */
800     if (JNOTNULL((jval = json_object_get(filter, "header")))) {
801         const char *hdr, *val;
802 
803         if (json_array_size(jval) == 2) {
804             hdr = json_string_value(json_array_get(jval, 0));
805             val = json_string_value(json_array_get(jval, 1));
806         } else {
807             hdr = json_string_value(json_array_get(jval, 0));
808             val = NULL; // match any value
809         }
810 
811         int matches = 0;
812 
813         /* Replicate match_header logic in search_expr.c */
814         char *lhdr = lcase(xstrdup(hdr));
815         struct buf buf = BUF_INITIALIZER;
816         int r = message_get_field(m, lhdr,
817                 MESSAGE_DECODED|MESSAGE_APPEND|MESSAGE_MULTIPLE, &buf);
818         if (!r) {
819             if (val) {
820                 charset_t utf8 = charset_lookupname("utf-8");
821                 char *v = NULL;
822                 if ((v = charset_convert(val, utf8, charset_flags))) {
823                     comp_pat *pat = charset_compilepat(v);
824                     if (pat) {
825                         matches = charset_searchstring(v, pat, buf.s, buf.len, charset_flags);
826                     }
827                     charset_freepat(pat);
828                 }
829                 free(v);
830                 charset_free(&utf8);
831             }
832             else {
833                 matches = buf_len(&buf) > 0;
834             }
835         }
836         buf_free(&buf);
837         free(lhdr);
838         if (matches) {
839             have_matches++;
840         } else return 0;
841     }
842 
843     /* before */
844     if (JNOTNULL(jval = json_object_get(filter, "before"))) {
845         time_t t;
846         time_from_iso8601(json_string_value(jval), &t);
847         if (internaldate < t) {
848             have_matches++;
849         } else return 0;
850     }
851     /* after */
852     if (JNOTNULL(jval = json_object_get(filter, "after"))) {
853         time_t t;
854         time_from_iso8601(json_string_value(jval), &t);
855         if (internaldate >= t) {
856             have_matches++;
857         } else return 0;
858     }
859 
860     return need_matches == have_matches;
861 }
862 
jmap_filter_parser_invalid(const char * field,void * rock)863 HIDDEN void jmap_filter_parser_invalid(const char *field, void *rock)
864 {
865     struct jmap_email_filter_parser_rock *frock =
866         (struct jmap_email_filter_parser_rock *) rock;
867 
868     jmap_parser_invalid(frock->parser, field);
869 }
870 
jmap_filter_parser_push_index(const char * field,size_t index,const char * name,void * rock)871 HIDDEN void jmap_filter_parser_push_index(const char *field, size_t index,
872                                           const char *name, void *rock)
873 {
874     struct jmap_email_filter_parser_rock *frock =
875         (struct jmap_email_filter_parser_rock *) rock;
876 
877     jmap_parser_push_index(frock->parser, field, index, name);
878 }
879 
jmap_filter_parser_pop(void * rock)880 HIDDEN void jmap_filter_parser_pop(void *rock)
881 {
882     struct jmap_email_filter_parser_rock *frock =
883         (struct jmap_email_filter_parser_rock *) rock;
884 
885     jmap_parser_pop(frock->parser);
886 }
887 
jmap_email_filtercondition_validate(const char * field,json_t * arg,void * rock)888 HIDDEN void jmap_email_filtercondition_validate(const char *field, json_t *arg,
889                                                 void *rock)
890 {
891     struct jmap_email_filter_parser_rock *frock =
892         (struct jmap_email_filter_parser_rock *) rock;
893 
894     if (!strcmp(field, "inMailbox")) {
895         if (!json_is_string(arg)) {
896             jmap_parser_invalid(frock->parser, field);
897         }
898     }
899     else if (!strcmp(field, "inMailboxOtherThan")) {
900         if (!json_is_array(arg)) {
901             jmap_parser_invalid(frock->parser, field);
902         }
903     }
904     else if (!strcmp(field, "allInThreadHaveKeyword") ||
905              !strcmp(field, "someInThreadHaveKeyword") ||
906              !strcmp(field, "noneInThreadHaveKeyword")) {
907         const char *s;
908 
909         if (!json_is_string(arg) ||
910             !(s = json_string_value(arg)) ||
911             !jmap_email_keyword_is_valid(s)) {
912             jmap_parser_invalid(frock->parser, field);
913         }
914         else if (!_email_threadkeyword_is_valid(s)) {
915             json_array_append_new(frock->unsupported,
916                                   json_pack("{s:s}", field, s));
917         }
918     }
919     else if (!strcmp(field, "hasKeyword") ||
920              !strcmp(field, "notKeyword")) {
921         if (!json_is_string(arg) ||
922             !jmap_email_keyword_is_valid(json_string_value(arg))) {
923             jmap_parser_invalid(frock->parser, field);
924         }
925     }
926     else {
927         jmap_parser_invalid(frock->parser, field);
928     }
929 }
930 
jmap_email_matchmime_init(const struct buf * mime,json_t ** err)931 HIDDEN matchmime_t *jmap_email_matchmime_init(const struct buf *mime, json_t **err)
932 {
933     matchmime_t *matchmime = xzmalloc(sizeof(matchmime_t));
934     int r = 0;
935 
936     matchmime->mime = mime;
937 
938     /* Parse message into memory */
939     matchmime->m = message_new_from_data(buf_base(mime), buf_len(mime));
940     if (!matchmime->m) {
941         syslog(LOG_ERR, "jmap_matchmime: can't create Cyrus message");
942         *err = jmap_server_error(r);
943         jmap_email_matchmime_free(&matchmime);
944         return NULL;
945     }
946 
947     /* Open temporary database */
948     matchmime->dbpath = create_tempdir(config_getstring(IMAPOPT_TEMP_PATH), "matchmime");
949     if (!matchmime->dbpath) {
950         syslog(LOG_ERR, "jmap_matchmime: can't create tempdir: %s", strerror(errno));
951         *err = jmap_server_error(IMAP_INTERNAL);
952         jmap_email_matchmime_free(&matchmime);
953         return NULL;
954     }
955 
956     /* Open search index in temp directory */
957     const char *paths[2];
958     paths[0] = matchmime->dbpath;
959     paths[1] = NULL;
960     r = xapian_dbw_open(paths, &matchmime->dbw, /*mode*/0, /*nosync*/1);
961     if (r) {
962         syslog(LOG_ERR, "jmap_matchmime: can't open search backend: %s",
963                 error_message(r));
964         *err = jmap_server_error(r);
965         jmap_email_matchmime_free(&matchmime);
966         return NULL;
967     }
968 
969     /* Index message bodies in-memory */
970     struct matchmime_receiver tr = {
971         {
972             _matchmime_tr_begin_mailbox,
973             _matchmime_tr_first_unindexed_uid,
974             _matchmime_tr_is_indexed,
975             _matchmime_tr_begin_message,
976             _matchmime_tr_begin_part,
977             _matchmime_tr_append_text,
978             _matchmime_tr_end_part,
979             _matchmime_tr_end_message,
980             _matchmime_tr_end_mailbox,
981             _matchmime_tr_flush,
982             _matchmime_tr_audit_mailbox,
983             _matchmime_tr_index_charset_flags,
984             _matchmime_tr_index_message_format
985         },
986         matchmime->dbw, BUF_INITIALIZER
987     };
988     r = index_getsearchtext(matchmime->m, NULL, (struct search_text_receiver*) &tr, 0);
989     buf_free(&tr.buf);
990     if (r) {
991         syslog(LOG_ERR, "jmap_matchmime: can't index MIME message: %s",
992                 error_message(r));
993         *err = jmap_server_error(r);
994         jmap_email_matchmime_free(&matchmime);
995         return NULL;
996     }
997 
998     return matchmime;
999 }
1000 
jmap_email_matchmime_free(matchmime_t ** matchmimep)1001 HIDDEN void jmap_email_matchmime_free(matchmime_t **matchmimep)
1002 {
1003     matchmime_t *matchmime = *matchmimep;
1004     if (!matchmime) return;
1005 
1006     if (matchmime->m) message_unref(&matchmime->m);
1007     if (matchmime->dbw) xapian_dbw_close(matchmime->dbw);
1008     if (matchmime->dbpath) removedir(matchmime->dbpath);
1009     free(matchmime->dbpath);
1010 
1011     free(matchmime);
1012     *matchmimep = NULL;
1013 }
1014 
jmap_email_matchmime(matchmime_t * matchmime,json_t * jfilter,const char * accountid,time_t internaldate,json_t ** err)1015 HIDDEN int jmap_email_matchmime(matchmime_t *matchmime,
1016                                 json_t *jfilter,
1017                                 const char *accountid,
1018                                 time_t internaldate,
1019                                 json_t **err)
1020 {
1021     int r = 0;
1022     int matches = 0;
1023 
1024     struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
1025     strarray_t capabilities = STRARRAY_INITIALIZER;
1026     struct email_contactfilter cfilter;
1027     json_t *unsupported = json_array();
1028     struct jmap_email_filter_parser_rock frock = { &parser, unsupported } ;
1029     jmap_email_filter_parse_ctx_t parse_ctx = {
1030         &jmap_email_filtercondition_validate,
1031         &jmap_filter_parser_invalid,
1032         &jmap_filter_parser_push_index,
1033         &jmap_filter_parser_pop,
1034         &capabilities,
1035         &frock
1036     };
1037 
1038     /* Parse filter */
1039     strarray_append(&capabilities, JMAP_URN_MAIL);
1040     strarray_append(&capabilities, JMAP_MAIL_EXTENSION);
1041     jmap_email_filter_parse(jfilter, &parse_ctx);
1042 
1043     /* Gather contactgroup ids */
1044     jmap_email_contactfilter_init(accountid, /*addressbookid*/NULL, &cfilter);
1045     ptrarray_t work = PTRARRAY_INITIALIZER;
1046     ptrarray_push(&work, jfilter);
1047     json_t *jf;
1048     while ((jf = ptrarray_pop(&work))) {
1049         size_t i;
1050         json_t *jval;
1051         json_array_foreach(json_object_get(jf, "conditions"), i, jval) {
1052             ptrarray_push(&work, jval);
1053         }
1054         r = jmap_email_contactfilter_from_filtercondition(&parser, jf, &cfilter);
1055         if (r) break;
1056 
1057     }
1058     ptrarray_fini(&work);
1059     if (r) {
1060         syslog(LOG_ERR, "jmap_matchmime: can't load contactgroups from filter: %s",
1061                 error_message(r));
1062         *err = jmap_server_error(r);
1063         goto done;
1064     }
1065     else if (json_array_size(parser.invalid)) {
1066         *err = json_pack("{s:s s:O}", "type", "invalidArguments",
1067                 "arguments", parser.invalid);
1068         goto done;
1069     }
1070     else if (json_array_size(unsupported)) {
1071         *err = json_pack("{s:s s:O}", "type", "unsupportedFilter",
1072                          "filters", unsupported);
1073         goto done;
1074     }
1075 
1076     /* Evaluate filter */
1077     xapian_db_t *db = NULL;
1078     r = xapian_db_opendbw(matchmime->dbw, &db);
1079     if (r) {
1080         syslog(LOG_ERR, "jmap_matchmime: can't open query backend: %s",
1081                 error_message(r));
1082         *err = jmap_server_error(r);
1083         goto done;
1084     }
1085     matches = _email_matchmime_evaluate(jfilter, matchmime->m, db, &cfilter, internaldate);
1086     xapian_db_close(db);
1087 
1088 done:
1089     jmap_email_contactfilter_fini(&cfilter);
1090     jmap_parser_fini(&parser);
1091     strarray_fini(&capabilities);
1092     json_decref(unsupported);
1093     return matches;
1094 }
1095 
1096 #endif /* WITH_DAV */
1097 
1098