1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
4
5 /* Copyright (c) University of Cambridge 1995 - 2015 */
6 /* Copyright (c) The Exim Maintainers 2020 */
7 /* See the file NOTICE for conditions of use and distribution. */
8
9 /* A set of functions to search databases in various formats. An open
10 database is represented by a void * value which is returned from a lookup-
11 specific "open" function. These are now all held in individual modules in the
12 lookups subdirectory and the functions here form a generic interface.
13
14 Caching is used to improve performance. Open files are cached until a tidyup
15 function is called, and for each file the result of the last lookup is cached.
16 However, if too many files are opened, some of those that are not in use have
17 to be closed. Those open items that use real files are kept on a LRU chain to
18 help with this.
19
20 All the data is held in permanent store so as to be independent of the stacking
21 pool that is reset from time to time. In fact, we use malloc'd store so that it
22 can be freed when the caches are tidied up. It isn't actually clear whether
23 this is a benefit or not, to be honest. */
24
25 #include "exim.h"
26
27
28 /* Tree in which to cache open files until tidyup called. */
29
30 static tree_node *search_tree = NULL;
31
32 /* Two-way chain of open databases that use real files. This is maintained in
33 recently-used order for the purposes of closing the least recently used when
34 too many files are open. */
35
36 static tree_node *open_top = NULL;
37 static tree_node *open_bot = NULL;
38
39 /* Count of open databases that use real files */
40
41 static int open_filecount = 0;
42
43 /* Allow us to reset store used for lookups and lookup caching */
44
45 static rmark search_reset_point = NULL;
46
47
48
49 /*************************************************
50 * Validate a plain lookup type name *
51 *************************************************/
52
53 /* Only those names that are recognized and whose code is included in the
54 binary give an OK response. Use a binary chop search now that the list has got
55 so long.
56
57 Arguments:
58 name lookup type name - not necessarily zero terminated (e.g. dbm*)
59 len length of the name
60
61 Returns: +ve => valid lookup name; value is offset in lookup_list
62 -ve => invalid name; message in search_error_message.
63 */
64
65 int
search_findtype(const uschar * name,int len)66 search_findtype(const uschar *name, int len)
67 {
68 int bot = 0;
69 int top = lookup_list_count;
70 while (top > bot)
71 {
72 int mid = (top + bot)/2;
73 int c = Ustrncmp(name, lookup_list[mid]->name, len);
74
75 /* If c == 0 we have matched the incoming name with the start of the search
76 type name. However, some search types are substrings of others (e.g. nis and
77 nisplus) so we need to check that the lengths are the same. The length of the
78 type name cannot be shorter (else c would not be 0); if it is not equal it
79 must be longer, and in that case, the incoming name comes before the name we
80 are testing. By leaving c == 0 when the lengths are different, and doing a
81 > 0 test below, this all falls out correctly. */
82
83 if (c == 0 && Ustrlen(lookup_list[mid]->name) == len)
84 {
85 if (lookup_list[mid]->find != NULL) return mid;
86 search_error_message = string_sprintf("lookup type \"%.*s\" is not "
87 "available (not in the binary - check buildtime LOOKUP configuration)",
88 len, name);
89 return -1;
90 }
91
92 if (c > 0) bot = mid + 1; else top = mid;
93 }
94
95 search_error_message = string_sprintf("unknown lookup type \"%.*s\"",len,name);
96 return -1;
97 }
98
99
100
101 /*************************************************
102 * Validate a full lookup type name *
103 *************************************************/
104
105 /* This function recognizes the "partial-" prefix and also terminating * and *@
106 suffixes.
107
108 Arguments:
109 name the full lookup type name
110 ptypeptr where to put the partial type
111 after subtraction of 1024 or 2048:
112 negative => no partial matching
113 non-negative => minimum number of non-wild components
114 ptypeaff where to put a pointer to the affix
115 the affix is within name if supplied therein
116 otherwise it's a literal string
117 afflen the length of the affix
118 starflags where to put the SEARCH_STAR and SEARCH_STARAT flags
119 opts where to put the options
120
121 Returns: +ve => valid lookup name; value is offset in lookup_list
122 -ve => invalid name; message in search_error_message.
123 */
124
125 int
search_findtype_partial(const uschar * name,int * ptypeptr,const uschar ** ptypeaff,int * afflen,int * starflags,const uschar ** opts)126 search_findtype_partial(const uschar *name, int *ptypeptr, const uschar **ptypeaff,
127 int *afflen, int *starflags, const uschar ** opts)
128 {
129 int len, stype;
130 int pv = -1;
131 const uschar *ss = name;
132 const uschar * t;
133
134 *starflags = 0;
135 *ptypeaff = NULL;
136
137 /* Check for a partial matching type. It must start with "partial", optionally
138 followed by a sequence of digits. If this is followed by "-", the affix is the
139 default "*." string. Otherwise we expect an affix in parentheses. Affixes are a
140 limited number of characters, not including parens. */
141
142 if (Ustrncmp(name, "partial", 7) == 0)
143 {
144 ss += 7;
145 if (isdigit (*ss))
146 {
147 pv = 0;
148 while (isdigit(*ss)) pv = pv*10 + *ss++ - '0';
149 }
150 else pv = 2; /* Default number of wild components */
151
152 if (*ss == '(')
153 {
154 *ptypeaff = ++ss;
155 while (ispunct(*ss) && *ss != ')') ss++;
156 if (*ss != ')') goto BAD_TYPE;
157 *afflen = ss++ - *ptypeaff;
158 }
159 else if (*ss++ == '-')
160 {
161 *ptypeaff = US "*.";
162 *afflen = 2;
163 }
164 else
165 {
166 BAD_TYPE:
167 search_error_message = string_sprintf("format error in lookup type \"%s\"",
168 name);
169 return -1;
170 }
171 }
172
173 /* Now we are left with a lookup name, possibly followed by * or *@,
174 and then by options starting with a "," */
175
176 len = Ustrlen(ss);
177 if ((t = Ustrchr(ss, '*')))
178 {
179 len = t - ss;
180 *starflags |= (t[1] == '@' ? SEARCH_STARAT : SEARCH_STAR);
181 }
182 else
183 t = ss;
184
185 if ((t = Ustrchr(t, ',')))
186 {
187 int l = t - ss;
188 if (l < len) len = l;
189 *opts = string_copy(t+1);
190 }
191 else
192 *opts = NULL;
193
194 /* Check for the individual search type. Only those that are actually in the
195 binary are valid. For query-style types, "partial" and default types are
196 erroneous. */
197
198 stype = search_findtype(ss, len);
199 if (stype >= 0 && mac_islookup(stype, lookup_querystyle))
200 {
201 if (pv >= 0)
202 {
203 search_error_message = string_sprintf("\"partial\" is not permitted "
204 "for lookup type \"%s\"", ss);
205 return -1;
206 }
207 if ((*starflags & (SEARCH_STAR|SEARCH_STARAT)) != 0)
208 {
209 search_error_message = string_sprintf("defaults using \"*\" or \"*@\" are "
210 "not permitted for lookup type \"%s\"", ss);
211 return -1;
212 }
213 }
214
215 *ptypeptr = pv;
216 return stype;
217 }
218
219
220 /* Set the parameters for the three different kinds of lookup.
221 Arguments:
222 search_type the search-type code
223 search the search-type string
224 query argument for the search; filename or query
225 fnamep pointer to return filename
226 opts options
227
228 Return: keyquery the search-type (for single-key) or query (for query-type)
229 */
230 uschar *
search_args(int search_type,uschar * search,uschar * query,uschar ** fnamep,const uschar * opts)231 search_args(int search_type, uschar * search, uschar * query, uschar ** fnamep,
232 const uschar * opts)
233 {
234 Uskip_whitespace(&query);
235 if (mac_islookup(search_type, lookup_absfilequery))
236 { /* query-style but with file (sqlite) */
237 int sep = ',';
238
239 /* Check options first for new-style file spec */
240 if (opts) for (uschar * s; s = string_nextinlist(&opts, &sep, NULL, 0); )
241 if (Ustrncmp(s, "file=", 5) == 0)
242 {
243 *fnamep = s+5;
244 return query;
245 }
246
247 /* If no filename from options, use old-tyle space-sep prefix on query */
248 if (*query == '/')
249 {
250 uschar * s = query;
251 while (*query && !isspace(*query)) query++;
252 *fnamep = string_copyn(s, query - s);
253 Uskip_whitespace(&query);
254 }
255 else
256 *fnamep = NULL;
257 return query; /* remainder after file skipped */
258 }
259 if (!mac_islookup(search_type, lookup_querystyle))
260 { /* single-key */
261 *fnamep = query;
262 return search; /* modifiers important so use "keyquery" for them */
263 }
264 *fnamep = NULL; /* else query-style */
265 return query;
266 }
267
268
269
270 /*************************************************
271 * Release cached resources *
272 *************************************************/
273
274 /* When search_open is called it caches the "file" that it opens in
275 search_tree. The name of the tree node is a concatenation of the search type
276 with the file name. For query-style lookups, the file name is empty. Real files
277 are normally closed only when this tidyup routine is called, typically at the
278 end of sections of code where a number of lookups might occur. However, if too
279 many files are open simultaneously, some get closed beforehand. They can't be
280 removed from the tree. There is also a general tidyup function which is called
281 for the lookup driver, if it exists.
282
283 First, there is an internal, recursive subroutine.
284
285 Argument: a pointer to a search_openfile tree node
286 Returns: nothing
287 */
288
289 static void
tidyup_subtree(tree_node * t)290 tidyup_subtree(tree_node *t)
291 {
292 search_cache * c = (search_cache *)(t->data.ptr);
293 if (t->left) tidyup_subtree(t->left);
294 if (t->right) tidyup_subtree(t->right);
295 if (c && c->handle && lookup_list[c->search_type]->close)
296 lookup_list[c->search_type]->close(c->handle);
297 }
298
299
300 /* The external entry point
301
302 Argument: none
303 Returns: nothing
304 */
305
306 void
search_tidyup(void)307 search_tidyup(void)
308 {
309 int old_pool = store_pool;
310
311 DEBUG(D_lookup) debug_printf_indent("search_tidyup called\n");
312
313 /* Close individually each cached open file. */
314
315 store_pool = POOL_SEARCH;
316 if (search_tree)
317 {
318 tidyup_subtree(search_tree);
319 search_tree = NULL;
320 }
321 open_top = open_bot = NULL;
322 open_filecount = 0;
323
324 /* Call the general tidyup entry for any drivers that have one. */
325
326 for (int i = 0; i < lookup_list_count; i++) if (lookup_list[i]->tidy)
327 (lookup_list[i]->tidy)();
328
329 if (search_reset_point) search_reset_point = store_reset(search_reset_point);
330 store_pool = old_pool;
331 }
332
333
334
335
336 /*************************************************
337 * Open search database *
338 *************************************************/
339
340 /* A mode, and lists of owners and groups, are passed over for checking in
341 the cases where the database is one or more files. Return NULL, with a message
342 pointed to by message, in cases of error.
343
344 For search types that use a file or files, check up on the mode after
345 opening. It is tempting to do a stat before opening the file, and use it as
346 an existence check. However, doing that opens a small security loophole in
347 that the status could be changed before the file is opened. Can't quite see
348 what problems this might lead to, but you can't be too careful where security
349 is concerned. Fstat() on an open file can normally be expected to succeed,
350 but there are some NFS states where it does not.
351
352 There are two styles of query: (1) in the "single-key+file" style, a single
353 key string and a file name are given, for example, for linear searches, DBM
354 files, or for NIS. (2) In the "query" style, no "filename" is given; instead
355 just a single query string is passed. This applies to multiple-key lookup
356 types such as NIS+.
357
358 Before opening, scan the tree of cached files to see if this file is already
359 open for the correct search type. If so, return the saved handle. If not, put
360 the handle in the tree for possible subsequent use. See search_tidyup above for
361 closing all the cached files.
362
363 A count of open databases which use real files is maintained, and if this
364 gets too large, we have to close a cached file. Its entry remains in the tree,
365 but is marked closed.
366
367 Arguments:
368 filename the name of the file for single-key+file style lookups,
369 NULL for query-style lookups
370 search_type the type of search required
371 modemask if a real single file is used, this specifies mode bits that
372 must not be set; otherwise it is ignored
373 owners if a real single file is used, this specifies the possible
374 owners of the file; otherwise it is ignored
375 owngroups if a real single file is used, this specifies the possible
376 group owners of the file; otherwise it is ignored
377
378 Returns: an identifying handle for the open database;
379 this is the pointer to the tree block in the
380 cache of open files; return NULL on open failure, with
381 a message in search_error_message
382 */
383
384 void *
search_open(const uschar * filename,int search_type,int modemask,uid_t * owners,gid_t * owngroups)385 search_open(const uschar * filename, int search_type, int modemask,
386 uid_t * owners, gid_t * owngroups)
387 {
388 void *handle;
389 tree_node *t;
390 search_cache *c;
391 lookup_info *lk = lookup_list[search_type];
392 uschar keybuffer[256];
393 int old_pool = store_pool;
394
395 if (filename && is_tainted2(filename, LOG_MAIN|LOG_PANIC, "Tainted filename for search: '%s'", filename))
396 return NULL;
397
398 /* Change to the search store pool and remember our reset point */
399
400 store_pool = POOL_SEARCH;
401 if (!search_reset_point) search_reset_point = store_mark();
402
403 DEBUG(D_lookup) debug_printf_indent("search_open: %s \"%s\"\n", lk->name,
404 filename ? filename : US"NULL");
405
406 /* See if we already have this open for this type of search, and if so,
407 pass back the tree block as the handle. The key for the tree node is the search
408 type plus '0' concatenated with the file name. There may be entries in the tree
409 with closed files if a lot of files have been opened. */
410
411 sprintf(CS keybuffer, "%c%.254s", search_type + '0',
412 filename ? filename : US"");
413
414 if ((t = tree_search(search_tree, keybuffer)))
415 {
416 if ((c = (search_cache *)t->data.ptr)->handle)
417 {
418 DEBUG(D_lookup) debug_printf_indent(" cached open\n");
419 store_pool = old_pool;
420 return t;
421 }
422 DEBUG(D_lookup) debug_printf_indent(" cached closed\n");
423 }
424
425 /* Otherwise, we need to open the file or database - each search type has its
426 own code, which is now split off into separately compiled modules. Before doing
427 this, if the search type is one that uses real files, check on the number that
428 we are holding open in the cache. If the limit is reached, close the least
429 recently used one. */
430
431 if (lk->type == lookup_absfile && open_filecount >= lookup_open_max)
432 if (!open_bot)
433 log_write(0, LOG_MAIN|LOG_PANIC, "too many lookups open, but can't find "
434 "one to close");
435 else
436 {
437 search_cache *c = (search_cache *)(open_bot->data.ptr);
438 DEBUG(D_lookup) debug_printf_indent("Too many lookup files open\n closing %s\n",
439 open_bot->name);
440 if ((open_bot = c->up))
441 ((search_cache *)(open_bot->data.ptr))->down = NULL;
442 else
443 open_top = NULL;
444 ((lookup_list[c->search_type])->close)(c->handle);
445 c->handle = NULL;
446 open_filecount--;
447 }
448
449 /* If opening is successful, call the file-checking function if there is one,
450 and if all is still well, enter the open database into the tree. */
451
452 if (!(handle = (lk->open)(filename, &search_error_message)))
453 {
454 store_pool = old_pool;
455 return NULL;
456 }
457
458 if ( lk->check
459 && !lk->check(handle, filename, modemask, owners, owngroups,
460 &search_error_message))
461 {
462 lk->close(handle);
463 store_pool = old_pool;
464 return NULL;
465 }
466
467 /* If this is a search type that uses real files, keep count. */
468
469 if (lk->type == lookup_absfile) open_filecount++;
470
471 /* If we found a previously opened entry in the tree, re-use it; otherwise
472 insert a new entry. On re-use, leave any cached lookup data and the lookup
473 count alone. */
474
475 if (!t)
476 {
477 t = store_get(sizeof(tree_node) + Ustrlen(keybuffer), FALSE);
478 t->data.ptr = c = store_get(sizeof(search_cache), FALSE);
479 c->item_cache = NULL;
480 Ustrcpy(t->name, keybuffer);
481 tree_insertnode(&search_tree, t);
482 }
483 else c = t->data.ptr;
484
485 c->handle = handle;
486 c->search_type = search_type;
487 c->up = c->down = NULL;
488
489 store_pool = old_pool;
490 return t;
491 }
492
493
494
495
496
497 /*************************************************
498 * Internal function: Find one item in database *
499 *************************************************/
500
501 /* The answer is always put into dynamic store. The last lookup for each handle
502 is cached.
503
504 Arguments:
505 handle the handle from search_open; points to tree node
506 filename the filename that was handed to search_open, or
507 NULL for query-style searches
508 keystring the keystring for single-key+file lookups, or
509 the querystring for query-style lookups
510 cache_rd FALSE to avoid lookup in cache layer
511 opts type-specific options
512
513 Returns: a pointer to a dynamic string containing the answer,
514 or NULL if the query failed or was deferred; in the
515 latter case, search_find_defer is set TRUE; after an unusual
516 failure, there may be a message in search_error_message.
517 */
518
519 static uschar *
internal_search_find(void * handle,const uschar * filename,uschar * keystring,BOOL cache_rd,const uschar * opts)520 internal_search_find(void * handle, const uschar * filename, uschar * keystring,
521 BOOL cache_rd, const uschar * opts)
522 {
523 tree_node * t = (tree_node *)handle;
524 search_cache * c = (search_cache *)(t->data.ptr);
525 expiring_data * e = NULL; /* compiler quietening */
526 uschar * data = NULL;
527 int search_type = t->name[0] - '0';
528 int old_pool = store_pool;
529
530 /* Lookups that return DEFER may not always set an error message. So that
531 the callers don't have to test for NULL, set an empty string. */
532
533 search_error_message = US"";
534 f.search_find_defer = FALSE;
535
536 DEBUG(D_lookup) debug_printf_indent("internal_search_find: file=\"%s\"\n "
537 "type=%s key=\"%s\" opts=%s%s%s\n", filename,
538 lookup_list[search_type]->name, keystring,
539 opts ? "\"" : "", opts, opts ? "\"" : "");
540
541 /* Insurance. If the keystring is empty, just fail. */
542
543 if (keystring[0] == 0) return NULL;
544
545 /* Use the special store pool for search data */
546
547 store_pool = POOL_SEARCH;
548
549 /* Look up the data for the key, unless it is already in the cache for this
550 file. No need to check c->item_cache for NULL, tree_search will do so. Check
551 whether we want to use the cache entry last so that we can always replace it. */
552
553 if ( (t = tree_search(c->item_cache, keystring))
554 && (!(e = t->data.ptr)->expiry || e->expiry > time(NULL))
555 && (!opts && !e->opts || opts && e->opts && Ustrcmp(opts, e->opts) == 0)
556 && cache_rd
557 )
558 { /* Data was in the cache already; set the pointer from the tree node */
559 data = e->data.ptr;
560 DEBUG(D_lookup) debug_printf_indent("cached data used for lookup of %s%s%s\n",
561 keystring,
562 filename ? US"\n in " : US"", filename ? filename : US"");
563 }
564 else
565 {
566 uint do_cache = UINT_MAX;
567 int keylength = Ustrlen(keystring);
568
569 DEBUG(D_lookup)
570 {
571 if (t)
572 debug_printf_indent("cached data found but %s; ",
573 e->expiry && e->expiry <= time(NULL) ? "out-of-date"
574 : cache_rd ? "wrong opts" : "no_rd option set");
575 debug_printf_indent("%s lookup required for %s%s%s\n",
576 filename ? US"file" : US"database",
577 keystring,
578 filename ? US"\n in " : US"", filename ? filename : US"");
579 }
580
581 /* Call the code for the different kinds of search. DEFER is handled
582 like FAIL, except that search_find_defer is set so the caller can
583 distinguish if necessary. */
584
585 if (lookup_list[search_type]->find(c->handle, filename, keystring, keylength,
586 &data, &search_error_message, &do_cache, opts) == DEFER)
587 f.search_find_defer = TRUE;
588
589 /* A record that has been found is now in data, which is either NULL
590 or points to a bit of dynamic store. Cache the result of the lookup if
591 caching is permitted. Lookups can disable caching, when they did something
592 that changes their data. The mysql and pgsql lookups do this when an
593 UPDATE/INSERT query was executed. Lookups can also set a TTL for the
594 cache entry; the dnsdb lookup does.
595 Finally, the caller can request no caching by setting an option. */
596
597 else if (do_cache)
598 {
599 DEBUG(D_lookup) debug_printf_indent("%s cache entry\n",
600 t ? "replacing old" : "creating new");
601 if (!t) /* No existing entry. Create new one. */
602 {
603 int len = keylength + 1;
604 e = store_get(sizeof(expiring_data) + sizeof(tree_node) + len,
605 is_tainted(keystring));
606 t = (tree_node *)(e+1);
607 memcpy(t->name, keystring, len);
608 t->data.ptr = e;
609 tree_insertnode(&c->item_cache, t);
610 }
611 /* Else previous, out-of-date cache entry. Update with the */
612 /* new result and forget the old one */
613 e->expiry = do_cache == UINT_MAX ? 0 : time(NULL)+do_cache;
614 e->opts = opts ? string_copy(opts) : NULL;
615 e->data.ptr = data;
616 }
617
618 /* If caching was disabled, empty the cache tree. We just set the cache
619 pointer to NULL here, because we cannot release the store at this stage. */
620
621 else
622 {
623 DEBUG(D_lookup) debug_printf_indent("lookup forced cache cleanup\n");
624 c->item_cache = NULL; /* forget all lookups on this connection */
625 }
626 }
627
628 DEBUG(D_lookup)
629 {
630 if (data)
631 debug_printf_indent("lookup yielded: %s\n", data);
632 else if (f.search_find_defer)
633 debug_printf_indent("lookup deferred: %s\n", search_error_message);
634 else debug_printf_indent("lookup failed\n");
635 }
636
637 /* Return it in new dynamic store in the regular pool */
638
639 store_pool = old_pool;
640 return data ? string_copy(data) : NULL;
641 }
642
643
644
645
646 /*************************************************
647 * Find one item in database, possibly wildcarded *
648 *************************************************/
649
650 /* This function calls the internal function above; once only if there
651 is no partial matching, but repeatedly when partial matching is requested.
652
653 Arguments:
654 handle the handle from search_open
655 filename the filename that was handed to search_open, or
656 NULL for query-style searches
657 keystring the keystring for single-key+file lookups, or
658 the querystring for query-style lookups
659 partial -1 means no partial matching;
660 otherwise it's the minimum number of components;
661 affix the affix string for partial matching
662 affixlen the length of the affix string
663 starflags SEARCH_STAR and SEARCH_STARAT flags
664 expand_setup pointer to offset for setting up expansion strings;
665 don't do any if < 0
666 opts type-specific options
667
668 Returns: a pointer to a dynamic string containing the answer,
669 or NULL if the query failed or was deferred; in the
670 latter case, search_find_defer is set TRUE
671 */
672
673 uschar *
search_find(void * handle,const uschar * filename,uschar * keystring,int partial,const uschar * affix,int affixlen,int starflags,int * expand_setup,const uschar * opts)674 search_find(void * handle, const uschar * filename, uschar * keystring,
675 int partial, const uschar * affix, int affixlen, int starflags,
676 int * expand_setup, const uschar * opts)
677 {
678 tree_node * t = (tree_node *)handle;
679 BOOL set_null_wild = FALSE, cache_rd = TRUE, ret_key = FALSE;
680 uschar * yield;
681
682 DEBUG(D_lookup)
683 {
684 if (partial < 0) affixlen = 99; /* So that "NULL" prints */
685 debug_printf_indent("search_find: file=\"%s\"\n key=\"%s\" "
686 "partial=%d affix=%.*s starflags=%x opts=%s%s%s\n",
687 filename ? filename : US"NULL",
688 keystring, partial, affixlen, affix, starflags,
689 opts ? "\"" : "", opts, opts ? "\"" : "");
690
691 }
692
693 /* Parse global lookup options. Also, create a new options list with
694 the global options dropped so that the cache-modifiers are not
695 used in the cache key. */
696
697 if (opts)
698 {
699 int sep = ',';
700 gstring * g = NULL;
701
702 for (uschar * ele; ele = string_nextinlist(&opts, &sep, NULL, 0); )
703 if (Ustrcmp(ele, "ret=key") == 0) ret_key = TRUE;
704 else if (Ustrcmp(ele, "cache=no_rd") == 0) cache_rd = FALSE;
705 else g = string_append_listele(g, ',', ele);
706
707 opts = string_from_gstring(g);
708 }
709
710 /* Arrange to put this database at the top of the LRU chain if it is a type
711 that opens real files. */
712
713 if ( open_top != (tree_node *)handle
714 && lookup_list[t->name[0]-'0']->type == lookup_absfile)
715 {
716 search_cache *c = (search_cache *)(t->data.ptr);
717 tree_node *up = c->up;
718 tree_node *down = c->down;
719
720 /* Cut it out of the list. A newly opened file will a NULL up pointer.
721 Otherwise there will be a non-NULL up pointer, since we checked above that
722 this block isn't already at the top of the list. */
723
724 if (up)
725 {
726 ((search_cache *)(up->data.ptr))->down = down;
727 if (down)
728 ((search_cache *)(down->data.ptr))->up = up;
729 else
730 open_bot = up;
731 }
732
733 /* Now put it at the head of the list. */
734
735 c->up = NULL;
736 c->down = open_top;
737 if (!open_top) open_bot = t;
738 else ((search_cache *)(open_top->data.ptr))->up = t;
739 open_top = t;
740 }
741
742 DEBUG(D_lookup)
743 {
744 debug_printf_indent("LRU list:\n");
745 for (tree_node *t = open_top; t; )
746 {
747 search_cache *c = (search_cache *)(t->data.ptr);
748 debug_printf_indent(" %s\n", t->name);
749 if (t == open_bot) debug_printf_indent(" End\n");
750 t = c->down;
751 }
752 }
753
754 /* First of all, try to match the key string verbatim. If matched a complete
755 entry but could have been partial, flag to set up variables. */
756
757 yield = internal_search_find(handle, filename, keystring, cache_rd, opts);
758 if (f.search_find_defer) return NULL;
759
760 if (yield) { if (partial >= 0) set_null_wild = TRUE; }
761
762 /* Not matched a complete entry; handle partial lookups, but only if the full
763 search didn't defer. Don't use string_sprintf() to construct the initial key,
764 just in case the original key is too long for the string_sprintf() buffer (it
765 *has* happened!). The case of a zero-length affix has to be treated specially.
766 */
767
768 else if (partial >= 0)
769 {
770 int len = Ustrlen(keystring);
771 uschar *keystring2;
772
773 /* Try with the affix on the front, except for a zero-length affix */
774
775 if (affixlen == 0) keystring2 = keystring; else
776 {
777 keystring2 = store_get(len + affixlen + 1,
778 is_tainted(keystring) || is_tainted(affix));
779 Ustrncpy(keystring2, affix, affixlen);
780 Ustrcpy(keystring2 + affixlen, keystring);
781 DEBUG(D_lookup) debug_printf_indent("trying partial match %s\n", keystring2);
782 yield = internal_search_find(handle, filename, keystring2, cache_rd, opts);
783 if (f.search_find_defer) return NULL;
784 }
785
786 /* The key in its entirety did not match a wild entry; try chopping off
787 leading components. */
788
789 if (!yield)
790 {
791 int dotcount = 0;
792 uschar *keystring3 = keystring2 + affixlen;
793 uschar *s = keystring3;
794 while (*s != 0) if (*s++ == '.') dotcount++;
795
796 while (dotcount-- >= partial)
797 {
798 while (*keystring3 != 0 && *keystring3 != '.') keystring3++;
799
800 /* If we get right to the end of the string (which will be the last time
801 through this loop), we've failed if the affix is null. Otherwise do one
802 last lookup for the affix itself, but if it is longer than 1 character,
803 remove the last character if it is ".". */
804
805 if (*keystring3 == 0)
806 {
807 if (affixlen < 1) break;
808 if (affixlen > 1 && affix[affixlen-1] == '.') affixlen--;
809 Ustrncpy(keystring2, affix, affixlen);
810 keystring2[affixlen] = 0;
811 keystring3 = keystring2;
812 }
813 else
814 {
815 keystring3 -= affixlen - 1;
816 if (affixlen > 0) Ustrncpy(keystring3, affix, affixlen);
817 }
818
819 DEBUG(D_lookup) debug_printf_indent("trying partial match %s\n", keystring3);
820 yield = internal_search_find(handle, filename, keystring3,
821 cache_rd, opts);
822 if (f.search_find_defer) return NULL;
823 if (yield)
824 {
825 /* First variable is the wild part; second is the fixed part. Take care
826 to get it right when keystring3 is just "*". */
827
828 if (expand_setup && *expand_setup >= 0)
829 {
830 int fixedlength = Ustrlen(keystring3) - affixlen;
831 int wildlength = Ustrlen(keystring) - fixedlength - 1;
832 *expand_setup += 1;
833 expand_nstring[*expand_setup] = keystring;
834 expand_nlength[*expand_setup] = wildlength;
835 *expand_setup += 1;
836 expand_nstring[*expand_setup] = keystring + wildlength + 1;
837 expand_nlength[*expand_setup] = (fixedlength < 0)? 0 : fixedlength;
838 }
839 break;
840 }
841 keystring3 += affixlen;
842 }
843 }
844
845 else set_null_wild = TRUE; /* Matched a wild entry without any wild part */
846 }
847
848 /* If nothing has been matched, but the option to look for "*@" is set, try
849 replacing everything to the left of @ by *. After a match, the wild part
850 is set to the string to the left of the @. */
851
852 if (!yield && starflags & SEARCH_STARAT)
853 {
854 uschar *atat = Ustrrchr(keystring, '@');
855 if (atat != NULL && atat > keystring)
856 {
857 int savechar;
858 savechar = *(--atat);
859 *atat = '*';
860
861 DEBUG(D_lookup) debug_printf_indent("trying default match %s\n", atat);
862 yield = internal_search_find(handle, filename, atat, cache_rd, opts);
863 *atat = savechar;
864 if (f.search_find_defer) return NULL;
865
866 if (yield && expand_setup && *expand_setup >= 0)
867 {
868 *expand_setup += 1;
869 expand_nstring[*expand_setup] = keystring;
870 expand_nlength[*expand_setup] = atat - keystring + 1;
871 *expand_setup += 1;
872 expand_nstring[*expand_setup] = keystring;
873 expand_nlength[*expand_setup] = 0;
874 }
875 }
876 }
877
878 /* If we still haven't matched anything, and the option to look for "*" is set,
879 try that. If we do match, the first variable (the wild part) is the whole key,
880 and the second is empty. */
881
882 if (!yield && starflags & (SEARCH_STAR|SEARCH_STARAT))
883 {
884 DEBUG(D_lookup) debug_printf_indent("trying to match *\n");
885 yield = internal_search_find(handle, filename, US"*", cache_rd, opts);
886 if (yield && expand_setup && *expand_setup >= 0)
887 {
888 *expand_setup += 1;
889 expand_nstring[*expand_setup] = keystring;
890 expand_nlength[*expand_setup] = Ustrlen(keystring);
891 *expand_setup += 1;
892 expand_nstring[*expand_setup] = keystring;
893 expand_nlength[*expand_setup] = 0;
894 }
895 }
896
897 /* If this was a potentially partial lookup, and we matched either a
898 complete non-wild domain entry, or we matched a wild-carded entry without
899 chopping off any of the domain components, set up the expansion variables
900 (if required) so that the first one is empty, and the second one is the
901 fixed part of the domain. The set_null_wild flag is set only when yield is not
902 NULL. */
903
904 if (set_null_wild && expand_setup && *expand_setup >= 0)
905 {
906 *expand_setup += 1;
907 expand_nstring[*expand_setup] = keystring;
908 expand_nlength[*expand_setup] = 0;
909 *expand_setup += 1;
910 expand_nstring[*expand_setup] = keystring;
911 expand_nlength[*expand_setup] = Ustrlen(keystring);
912 }
913
914 /* If we have a result, check the options to see if the key was wanted rather
915 than the result. Return a de-tainted version of the key on the grounds that
916 it have been validated by the lookup. */
917
918 if (yield && ret_key)
919 yield = string_copy_taint(keystring, FALSE);
920
921 return yield;
922 }
923
924 /* End of search.c */
925