1 /* $OpenBSD: queue_fs.c,v 1.19 2019/06/28 13:32:51 deraadt Exp $ */ 2 3 /* 4 * Copyright (c) 2011 Gilles Chehade <gilles@poolp.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/types.h> 20 #include <sys/mount.h> 21 #include <sys/queue.h> 22 #include <sys/tree.h> 23 #include <sys/socket.h> 24 #include <sys/stat.h> 25 26 #include <ctype.h> 27 #include <dirent.h> 28 #include <err.h> 29 #include <errno.h> 30 #include <event.h> 31 #include <fcntl.h> 32 #include <fts.h> 33 #include <imsg.h> 34 #include <inttypes.h> 35 #include <pwd.h> 36 #include <stdio.h> 37 #include <stdlib.h> 38 #include <string.h> 39 #include <time.h> 40 #include <unistd.h> 41 42 #include "smtpd.h" 43 #include "log.h" 44 45 #define PATH_QUEUE "/queue" 46 #define PATH_INCOMING "/incoming" 47 #define PATH_EVPTMP PATH_INCOMING "/envelope.tmp" 48 #define PATH_MESSAGE "/message" 49 50 /* percentage of remaining space / inodes required to accept new messages */ 51 #define MINSPACE 5 52 #define MININODES 5 53 54 struct qwalk { 55 FTS *fts; 56 int depth; 57 }; 58 59 static int fsqueue_check_space(void); 60 static void fsqueue_envelope_path(uint64_t, char *, size_t); 61 static void fsqueue_envelope_incoming_path(uint64_t, char *, size_t); 62 static int fsqueue_envelope_dump(char *, const char *, size_t, int, int); 63 static void fsqueue_message_path(uint32_t, char *, size_t); 64 static void fsqueue_message_incoming_path(uint32_t, char *, size_t); 65 static void *fsqueue_qwalk_new(void); 66 static int fsqueue_qwalk(void *, uint64_t *); 67 static void fsqueue_qwalk_close(void *); 68 69 struct tree evpcount; 70 static struct timespec startup; 71 72 #define REF (int*)0xf00 73 74 static int 75 queue_fs_message_create(uint32_t *msgid) 76 { 77 char rootdir[PATH_MAX]; 78 struct stat sb; 79 80 if (!fsqueue_check_space()) 81 return 0; 82 83 again: 84 *msgid = queue_generate_msgid(); 85 86 /* prevent possible collision later when moving to Q_QUEUE */ 87 fsqueue_message_path(*msgid, rootdir, sizeof(rootdir)); 88 if (stat(rootdir, &sb) != -1) 89 goto again; 90 91 /* we hit an unexpected error, temporarily fail */ 92 if (errno != ENOENT) { 93 *msgid = 0; 94 return 0; 95 } 96 97 fsqueue_message_incoming_path(*msgid, rootdir, sizeof(rootdir)); 98 if (mkdir(rootdir, 0700) == -1) { 99 if (errno == EEXIST) 100 goto again; 101 102 if (errno == ENOSPC) { 103 *msgid = 0; 104 return 0; 105 } 106 107 log_warn("warn: queue-fs: mkdir"); 108 *msgid = 0; 109 return 0; 110 } 111 112 return (1); 113 } 114 115 static int 116 queue_fs_message_commit(uint32_t msgid, const char *path) 117 { 118 char incomingdir[PATH_MAX]; 119 char queuedir[PATH_MAX]; 120 char msgdir[PATH_MAX]; 121 char msgpath[PATH_MAX]; 122 123 /* before-first, move the message content in the incoming directory */ 124 fsqueue_message_incoming_path(msgid, msgpath, sizeof(msgpath)); 125 if (strlcat(msgpath, PATH_MESSAGE, sizeof(msgpath)) 126 >= sizeof(msgpath)) 127 return (0); 128 if (rename(path, msgpath) == -1) 129 return (0); 130 131 fsqueue_message_incoming_path(msgid, incomingdir, sizeof(incomingdir)); 132 fsqueue_message_path(msgid, msgdir, sizeof(msgdir)); 133 if (strlcpy(queuedir, msgdir, sizeof(queuedir)) 134 >= sizeof(queuedir)) 135 return (0); 136 137 /* first attempt to rename */ 138 if (rename(incomingdir, msgdir) == 0) 139 return 1; 140 if (errno == ENOSPC) 141 return 0; 142 if (errno != ENOENT) { 143 log_warn("warn: queue-fs: rename"); 144 return 0; 145 } 146 147 /* create the bucket */ 148 *strrchr(queuedir, '/') = '\0'; 149 if (mkdir(queuedir, 0700) == -1) { 150 if (errno == ENOSPC) 151 return 0; 152 if (errno != EEXIST) { 153 log_warn("warn: queue-fs: mkdir"); 154 return 0; 155 } 156 } 157 158 /* rename */ 159 if (rename(incomingdir, msgdir) == -1) { 160 if (errno == ENOSPC) 161 return 0; 162 log_warn("warn: queue-fs: rename"); 163 return 0; 164 } 165 166 return 1; 167 } 168 169 static int 170 queue_fs_message_fd_r(uint32_t msgid) 171 { 172 int fd; 173 char path[PATH_MAX]; 174 175 fsqueue_message_path(msgid, path, sizeof(path)); 176 if (strlcat(path, PATH_MESSAGE, sizeof(path)) 177 >= sizeof(path)) 178 return -1; 179 180 if ((fd = open(path, O_RDONLY)) == -1) { 181 log_warn("warn: queue-fs: open"); 182 return -1; 183 } 184 185 return fd; 186 } 187 188 static int 189 queue_fs_message_delete(uint32_t msgid) 190 { 191 char path[PATH_MAX]; 192 struct stat sb; 193 194 fsqueue_message_incoming_path(msgid, path, sizeof(path)); 195 if (stat(path, &sb) == -1) 196 fsqueue_message_path(msgid, path, sizeof(path)); 197 198 if (rmtree(path, 0) == -1) 199 log_warn("warn: queue-fs: rmtree"); 200 201 tree_pop(&evpcount, msgid); 202 203 return 1; 204 } 205 206 static int 207 queue_fs_envelope_create(uint32_t msgid, const char *buf, size_t len, 208 uint64_t *evpid) 209 { 210 char path[PATH_MAX]; 211 int queued = 0, i, r = 0, *n; 212 struct stat sb; 213 214 if (msgid == 0) { 215 log_warnx("warn: queue-fs: msgid=0, evpid=%016"PRIx64, *evpid); 216 goto done; 217 } 218 219 fsqueue_message_incoming_path(msgid, path, sizeof(path)); 220 if (stat(path, &sb) == -1) 221 queued = 1; 222 223 for (i = 0; i < 20; i ++) { 224 *evpid = queue_generate_evpid(msgid); 225 if (queued) 226 fsqueue_envelope_path(*evpid, path, sizeof(path)); 227 else 228 fsqueue_envelope_incoming_path(*evpid, path, 229 sizeof(path)); 230 231 r = fsqueue_envelope_dump(path, buf, len, 0, 0); 232 if (r >= 0) 233 goto done; 234 } 235 r = 0; 236 log_warnx("warn: queue-fs: could not allocate evpid"); 237 238 done: 239 if (r) { 240 n = tree_pop(&evpcount, msgid); 241 if (n == NULL) 242 n = REF; 243 n += 1; 244 tree_xset(&evpcount, msgid, n); 245 } 246 return (r); 247 } 248 249 static int 250 queue_fs_envelope_load(uint64_t evpid, char *buf, size_t len) 251 { 252 char pathname[PATH_MAX]; 253 FILE *fp; 254 size_t r; 255 256 fsqueue_envelope_path(evpid, pathname, sizeof(pathname)); 257 258 fp = fopen(pathname, "r"); 259 if (fp == NULL) { 260 if (errno != ENOENT && errno != ENFILE) 261 log_warn("warn: queue-fs: fopen"); 262 return 0; 263 } 264 265 r = fread(buf, 1, len, fp); 266 if (r) { 267 if (r == len) { 268 log_warn("warn: queue-fs: too large"); 269 r = 0; 270 } 271 else 272 buf[r] = '\0'; 273 } 274 fclose(fp); 275 276 return (r); 277 } 278 279 static int 280 queue_fs_envelope_update(uint64_t evpid, const char *buf, size_t len) 281 { 282 char dest[PATH_MAX]; 283 284 fsqueue_envelope_path(evpid, dest, sizeof(dest)); 285 286 return (fsqueue_envelope_dump(dest, buf, len, 1, 1)); 287 } 288 289 static int 290 queue_fs_envelope_delete(uint64_t evpid) 291 { 292 char pathname[PATH_MAX]; 293 uint32_t msgid; 294 int *n; 295 296 fsqueue_envelope_path(evpid, pathname, sizeof(pathname)); 297 if (unlink(pathname) == -1) 298 if (errno != ENOENT) 299 return 0; 300 301 msgid = evpid_to_msgid(evpid); 302 n = tree_pop(&evpcount, msgid); 303 n -= 1; 304 305 if (n - REF == 0) 306 queue_fs_message_delete(msgid); 307 else 308 tree_xset(&evpcount, msgid, n); 309 310 return (1); 311 } 312 313 static int 314 queue_fs_message_walk(uint64_t *evpid, char *buf, size_t len, 315 uint32_t msgid, int *done, void **data) 316 { 317 struct dirent *dp; 318 DIR *dir = *data; 319 char path[PATH_MAX]; 320 char msgid_str[9]; 321 char *tmp; 322 int r, *n; 323 324 if (*done) 325 return (-1); 326 327 if (!bsnprintf(path, sizeof path, "%s/%02x/%08x", 328 PATH_QUEUE, (msgid & 0xff000000) >> 24, msgid)) 329 fatalx("queue_fs_message_walk: path does not fit buffer"); 330 331 if (dir == NULL) { 332 if ((dir = opendir(path)) == NULL) { 333 log_warn("warn: queue_fs: opendir: %s", path); 334 *done = 1; 335 return (-1); 336 } 337 338 *data = dir; 339 } 340 341 (void)snprintf(msgid_str, sizeof msgid_str, "%08" PRIx32, msgid); 342 while ((dp = readdir(dir)) != NULL) { 343 if (dp->d_type != DT_REG) 344 continue; 345 346 /* ignore files other than envelopes */ 347 if (strlen(dp->d_name) != 16 || 348 strncmp(dp->d_name, msgid_str, 8)) 349 continue; 350 351 tmp = NULL; 352 *evpid = strtoull(dp->d_name, &tmp, 16); 353 if (tmp && *tmp != '\0') { 354 log_debug("debug: fsqueue: bogus file %s", dp->d_name); 355 continue; 356 } 357 358 memset(buf, 0, len); 359 r = queue_fs_envelope_load(*evpid, buf, len); 360 if (r) { 361 n = tree_pop(&evpcount, msgid); 362 if (n == NULL) 363 n = REF; 364 365 n += 1; 366 tree_xset(&evpcount, msgid, n); 367 } 368 369 return (r); 370 } 371 372 (void)closedir(dir); 373 *done = 1; 374 return (-1); 375 } 376 377 static int 378 queue_fs_envelope_walk(uint64_t *evpid, char *buf, size_t len) 379 { 380 static int done = 0; 381 static void *hdl = NULL; 382 int r, *n; 383 uint32_t msgid; 384 385 if (done) 386 return (-1); 387 388 if (hdl == NULL) 389 hdl = fsqueue_qwalk_new(); 390 391 if (fsqueue_qwalk(hdl, evpid)) { 392 memset(buf, 0, len); 393 r = queue_fs_envelope_load(*evpid, buf, len); 394 if (r) { 395 msgid = evpid_to_msgid(*evpid); 396 n = tree_pop(&evpcount, msgid); 397 if (n == NULL) 398 n = REF; 399 n += 1; 400 tree_xset(&evpcount, msgid, n); 401 } 402 return (r); 403 } 404 405 fsqueue_qwalk_close(hdl); 406 done = 1; 407 return (-1); 408 } 409 410 static int 411 fsqueue_check_space(void) 412 { 413 struct statfs buf; 414 uint64_t used; 415 uint64_t total; 416 417 if (statfs(PATH_QUEUE, &buf) == -1) { 418 log_warn("warn: queue-fs: statfs"); 419 return 0; 420 } 421 422 /* 423 * f_bfree and f_ffree is not set on all filesystems. 424 * They could be signed or unsigned integers. 425 * Some systems will set them to 0, others will set them to -1. 426 */ 427 if (buf.f_bfree == 0 || buf.f_ffree == 0 || 428 (int64_t)buf.f_bfree == -1 || (int64_t)buf.f_ffree == -1) 429 return 1; 430 431 used = buf.f_blocks - buf.f_bfree; 432 total = buf.f_bavail + used; 433 if (total != 0) 434 used = (float)used / (float)total * 100; 435 else 436 used = 100; 437 if (100 - used < MINSPACE) { 438 log_warnx("warn: not enough disk space: %llu%% left", 439 (unsigned long long) 100 - used); 440 log_warnx("warn: temporarily rejecting messages"); 441 return 0; 442 } 443 444 used = buf.f_files - buf.f_ffree; 445 total = buf.f_favail + used; 446 if (total != 0) 447 used = (float)used / (float)total * 100; 448 else 449 used = 100; 450 if (100 - used < MININODES) { 451 log_warnx("warn: not enough inodes: %llu%% left", 452 (unsigned long long) 100 - used); 453 log_warnx("warn: temporarily rejecting messages"); 454 return 0; 455 } 456 457 return 1; 458 } 459 460 static void 461 fsqueue_envelope_path(uint64_t evpid, char *buf, size_t len) 462 { 463 if (!bsnprintf(buf, len, "%s/%02x/%08x/%016" PRIx64, 464 PATH_QUEUE, 465 (evpid_to_msgid(evpid) & 0xff000000) >> 24, 466 evpid_to_msgid(evpid), 467 evpid)) 468 fatalx("fsqueue_envelope_path: path does not fit buffer"); 469 } 470 471 static void 472 fsqueue_envelope_incoming_path(uint64_t evpid, char *buf, size_t len) 473 { 474 if (!bsnprintf(buf, len, "%s/%08x/%016" PRIx64, 475 PATH_INCOMING, 476 evpid_to_msgid(evpid), 477 evpid)) 478 fatalx("fsqueue_envelope_incoming_path: path does not fit buffer"); 479 } 480 481 static int 482 fsqueue_envelope_dump(char *dest, const char *evpbuf, size_t evplen, 483 int do_atomic, int do_sync) 484 { 485 const char *path = do_atomic ? PATH_EVPTMP : dest; 486 FILE *fp = NULL; 487 int fd; 488 size_t w; 489 490 if ((fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0600)) == -1) { 491 log_warn("warn: queue-fs: open"); 492 goto tempfail; 493 } 494 495 if ((fp = fdopen(fd, "w")) == NULL) { 496 log_warn("warn: queue-fs: fdopen"); 497 goto tempfail; 498 } 499 500 w = fwrite(evpbuf, 1, evplen, fp); 501 if (w < evplen) { 502 log_warn("warn: queue-fs: short write"); 503 goto tempfail; 504 } 505 if (fflush(fp)) { 506 log_warn("warn: queue-fs: fflush"); 507 goto tempfail; 508 } 509 if (do_sync && fsync(fileno(fp))) { 510 log_warn("warn: queue-fs: fsync"); 511 goto tempfail; 512 } 513 if (fclose(fp) != 0) { 514 log_warn("warn: queue-fs: fclose"); 515 fp = NULL; 516 goto tempfail; 517 } 518 fp = NULL; 519 fd = -1; 520 521 if (do_atomic && rename(path, dest) == -1) { 522 log_warn("warn: queue-fs: rename"); 523 goto tempfail; 524 } 525 return (1); 526 527 tempfail: 528 if (fp) 529 fclose(fp); 530 else if (fd != -1) 531 close(fd); 532 if (unlink(path) == -1) 533 log_warn("warn: queue-fs: unlink"); 534 return (0); 535 } 536 537 static void 538 fsqueue_message_path(uint32_t msgid, char *buf, size_t len) 539 { 540 if (!bsnprintf(buf, len, "%s/%02x/%08x", 541 PATH_QUEUE, 542 (msgid & 0xff000000) >> 24, 543 msgid)) 544 fatalx("fsqueue_message_path: path does not fit buffer"); 545 } 546 547 static void 548 fsqueue_message_incoming_path(uint32_t msgid, char *buf, size_t len) 549 { 550 if (!bsnprintf(buf, len, "%s/%08x", 551 PATH_INCOMING, 552 msgid)) 553 fatalx("fsqueue_message_incoming_path: path does not fit buffer"); 554 } 555 556 static void * 557 fsqueue_qwalk_new(void) 558 { 559 char path[PATH_MAX]; 560 char * const path_argv[] = { path, NULL }; 561 struct qwalk *q; 562 563 q = xcalloc(1, sizeof(*q)); 564 (void)strlcpy(path, PATH_QUEUE, sizeof(path)); 565 q->fts = fts_open(path_argv, 566 FTS_PHYSICAL | FTS_NOCHDIR, NULL); 567 568 if (q->fts == NULL) 569 err(1, "fsqueue_qwalk_new: fts_open: %s", path); 570 571 return (q); 572 } 573 574 static void 575 fsqueue_qwalk_close(void *hdl) 576 { 577 struct qwalk *q = hdl; 578 579 fts_close(q->fts); 580 581 free(q); 582 } 583 584 static int 585 fsqueue_qwalk(void *hdl, uint64_t *evpid) 586 { 587 struct qwalk *q = hdl; 588 FTSENT *e; 589 char *tmp; 590 591 while ((e = fts_read(q->fts)) != NULL) { 592 switch (e->fts_info) { 593 case FTS_D: 594 q->depth += 1; 595 if (q->depth == 2 && e->fts_namelen != 2) { 596 log_debug("debug: fsqueue: bogus directory %s", 597 e->fts_path); 598 fts_set(q->fts, e, FTS_SKIP); 599 break; 600 } 601 if (q->depth == 3 && e->fts_namelen != 8) { 602 log_debug("debug: fsqueue: bogus directory %s", 603 e->fts_path); 604 fts_set(q->fts, e, FTS_SKIP); 605 break; 606 } 607 break; 608 609 case FTS_DP: 610 case FTS_DNR: 611 q->depth -= 1; 612 break; 613 614 case FTS_F: 615 if (q->depth != 3) 616 break; 617 if (e->fts_namelen != 16) 618 break; 619 if (timespeccmp(&e->fts_statp->st_mtim, &startup, >)) 620 break; 621 tmp = NULL; 622 *evpid = strtoull(e->fts_name, &tmp, 16); 623 if (tmp && *tmp != '\0') { 624 log_debug("debug: fsqueue: bogus file %s", 625 e->fts_path); 626 break; 627 } 628 return (1); 629 default: 630 break; 631 } 632 } 633 634 return (0); 635 } 636 637 static int 638 queue_fs_init(struct passwd *pw, int server, const char *conf) 639 { 640 unsigned int n; 641 char *paths[] = { PATH_QUEUE, PATH_INCOMING }; 642 char path[PATH_MAX]; 643 int ret; 644 645 /* remove incoming/ if it exists */ 646 if (server) 647 mvpurge(PATH_SPOOL PATH_INCOMING, PATH_SPOOL PATH_PURGE); 648 649 fsqueue_envelope_path(0, path, sizeof(path)); 650 651 ret = 1; 652 for (n = 0; n < nitems(paths); n++) { 653 (void)strlcpy(path, PATH_SPOOL, sizeof(path)); 654 if (strlcat(path, paths[n], sizeof(path)) >= sizeof(path)) 655 errx(1, "path too long %s%s", PATH_SPOOL, paths[n]); 656 if (ckdir(path, 0700, pw->pw_uid, 0, server) == 0) 657 ret = 0; 658 } 659 660 if (clock_gettime(CLOCK_REALTIME, &startup)) 661 err(1, "clock_gettime"); 662 663 tree_init(&evpcount); 664 665 queue_api_on_message_create(queue_fs_message_create); 666 queue_api_on_message_commit(queue_fs_message_commit); 667 queue_api_on_message_delete(queue_fs_message_delete); 668 queue_api_on_message_fd_r(queue_fs_message_fd_r); 669 queue_api_on_envelope_create(queue_fs_envelope_create); 670 queue_api_on_envelope_delete(queue_fs_envelope_delete); 671 queue_api_on_envelope_update(queue_fs_envelope_update); 672 queue_api_on_envelope_load(queue_fs_envelope_load); 673 queue_api_on_envelope_walk(queue_fs_envelope_walk); 674 queue_api_on_message_walk(queue_fs_message_walk); 675 676 return (ret); 677 } 678 679 struct queue_backend queue_backend_fs = { 680 queue_fs_init, 681 }; 682