1 /* $OpenBSD: queue_fs.c,v 1.22 2021/06/14 17:58:16 eric Exp $ */ 2 3 /* 4 * Copyright (c) 2011 Gilles Chehade <gilles@poolp.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/types.h> 20 #include <sys/mount.h> 21 #include <sys/stat.h> 22 23 #include <dirent.h> 24 #include <errno.h> 25 #include <fcntl.h> 26 #include <fts.h> 27 #include <inttypes.h> 28 #include <pwd.h> 29 #include <stdlib.h> 30 #include <string.h> 31 #include <unistd.h> 32 33 #include "smtpd.h" 34 #include "log.h" 35 36 #define PATH_QUEUE "/queue" 37 #define PATH_INCOMING "/incoming" 38 #define PATH_EVPTMP PATH_INCOMING "/envelope.tmp" 39 #define PATH_MESSAGE "/message" 40 41 /* percentage of remaining space / inodes required to accept new messages */ 42 #define MINSPACE 5 43 #define MININODES 5 44 45 struct qwalk { 46 FTS *fts; 47 int depth; 48 }; 49 50 static int fsqueue_check_space(void); 51 static void fsqueue_envelope_path(uint64_t, char *, size_t); 52 static void fsqueue_envelope_incoming_path(uint64_t, char *, size_t); 53 static int fsqueue_envelope_dump(char *, const char *, size_t, int, int); 54 static void fsqueue_message_path(uint32_t, char *, size_t); 55 static void fsqueue_message_incoming_path(uint32_t, char *, size_t); 56 static void *fsqueue_qwalk_new(void); 57 static int fsqueue_qwalk(void *, uint64_t *); 58 static void fsqueue_qwalk_close(void *); 59 60 struct tree evpcount; 61 static struct timespec startup; 62 63 #define REF (int*)0xf00 64 65 static int 66 queue_fs_message_create(uint32_t *msgid) 67 { 68 char rootdir[PATH_MAX]; 69 struct stat sb; 70 71 if (!fsqueue_check_space()) 72 return 0; 73 74 again: 75 *msgid = queue_generate_msgid(); 76 77 /* prevent possible collision later when moving to Q_QUEUE */ 78 fsqueue_message_path(*msgid, rootdir, sizeof(rootdir)); 79 if (stat(rootdir, &sb) != -1) 80 goto again; 81 82 /* we hit an unexpected error, temporarily fail */ 83 if (errno != ENOENT) { 84 *msgid = 0; 85 return 0; 86 } 87 88 fsqueue_message_incoming_path(*msgid, rootdir, sizeof(rootdir)); 89 if (mkdir(rootdir, 0700) == -1) { 90 if (errno == EEXIST) 91 goto again; 92 93 if (errno == ENOSPC) { 94 *msgid = 0; 95 return 0; 96 } 97 98 log_warn("warn: queue-fs: mkdir"); 99 *msgid = 0; 100 return 0; 101 } 102 103 return (1); 104 } 105 106 static int 107 queue_fs_message_commit(uint32_t msgid, const char *path) 108 { 109 char incomingdir[PATH_MAX]; 110 char queuedir[PATH_MAX]; 111 char msgdir[PATH_MAX]; 112 char msgpath[PATH_MAX]; 113 114 /* before-first, move the message content in the incoming directory */ 115 fsqueue_message_incoming_path(msgid, msgpath, sizeof(msgpath)); 116 if (strlcat(msgpath, PATH_MESSAGE, sizeof(msgpath)) 117 >= sizeof(msgpath)) 118 return (0); 119 if (rename(path, msgpath) == -1) 120 return (0); 121 122 fsqueue_message_incoming_path(msgid, incomingdir, sizeof(incomingdir)); 123 fsqueue_message_path(msgid, msgdir, sizeof(msgdir)); 124 if (strlcpy(queuedir, msgdir, sizeof(queuedir)) 125 >= sizeof(queuedir)) 126 return (0); 127 128 /* first attempt to rename */ 129 if (rename(incomingdir, msgdir) == 0) 130 return 1; 131 if (errno == ENOSPC) 132 return 0; 133 if (errno != ENOENT) { 134 log_warn("warn: queue-fs: rename"); 135 return 0; 136 } 137 138 /* create the bucket */ 139 *strrchr(queuedir, '/') = '\0'; 140 if (mkdir(queuedir, 0700) == -1) { 141 if (errno == ENOSPC) 142 return 0; 143 if (errno != EEXIST) { 144 log_warn("warn: queue-fs: mkdir"); 145 return 0; 146 } 147 } 148 149 /* rename */ 150 if (rename(incomingdir, msgdir) == -1) { 151 if (errno == ENOSPC) 152 return 0; 153 log_warn("warn: queue-fs: rename"); 154 return 0; 155 } 156 157 return 1; 158 } 159 160 static int 161 queue_fs_message_fd_r(uint32_t msgid) 162 { 163 int fd; 164 char path[PATH_MAX]; 165 166 fsqueue_message_path(msgid, path, sizeof(path)); 167 if (strlcat(path, PATH_MESSAGE, sizeof(path)) 168 >= sizeof(path)) 169 return -1; 170 171 if ((fd = open(path, O_RDONLY)) == -1) { 172 log_warn("warn: queue-fs: open"); 173 return -1; 174 } 175 176 return fd; 177 } 178 179 static int 180 queue_fs_message_delete(uint32_t msgid) 181 { 182 char path[PATH_MAX]; 183 struct stat sb; 184 185 fsqueue_message_incoming_path(msgid, path, sizeof(path)); 186 if (stat(path, &sb) == -1) 187 fsqueue_message_path(msgid, path, sizeof(path)); 188 189 if (rmtree(path, 0) == -1) 190 log_warn("warn: queue-fs: rmtree"); 191 192 tree_pop(&evpcount, msgid); 193 194 return 1; 195 } 196 197 static int 198 queue_fs_envelope_create(uint32_t msgid, const char *buf, size_t len, 199 uint64_t *evpid) 200 { 201 char path[PATH_MAX]; 202 int queued = 0, i, r = 0, *n; 203 struct stat sb; 204 205 if (msgid == 0) { 206 log_warnx("warn: queue-fs: msgid=0, evpid=%016"PRIx64, *evpid); 207 goto done; 208 } 209 210 fsqueue_message_incoming_path(msgid, path, sizeof(path)); 211 if (stat(path, &sb) == -1) 212 queued = 1; 213 214 for (i = 0; i < 20; i ++) { 215 *evpid = queue_generate_evpid(msgid); 216 if (queued) 217 fsqueue_envelope_path(*evpid, path, sizeof(path)); 218 else 219 fsqueue_envelope_incoming_path(*evpid, path, 220 sizeof(path)); 221 222 if ((r = fsqueue_envelope_dump(path, buf, len, 0, 0)) != 0) 223 goto done; 224 } 225 r = 0; 226 log_warnx("warn: queue-fs: could not allocate evpid"); 227 228 done: 229 if (r) { 230 n = tree_pop(&evpcount, msgid); 231 if (n == NULL) 232 n = REF; 233 n += 1; 234 tree_xset(&evpcount, msgid, n); 235 } 236 return (r); 237 } 238 239 static int 240 queue_fs_envelope_load(uint64_t evpid, char *buf, size_t len) 241 { 242 char pathname[PATH_MAX]; 243 FILE *fp; 244 size_t r; 245 246 fsqueue_envelope_path(evpid, pathname, sizeof(pathname)); 247 248 fp = fopen(pathname, "r"); 249 if (fp == NULL) { 250 if (errno != ENOENT && errno != ENFILE) 251 log_warn("warn: queue-fs: fopen"); 252 return 0; 253 } 254 255 r = fread(buf, 1, len, fp); 256 if (r) { 257 if (r == len) { 258 log_warn("warn: queue-fs: too large"); 259 r = 0; 260 } 261 else 262 buf[r] = '\0'; 263 } 264 fclose(fp); 265 266 return (r); 267 } 268 269 static int 270 queue_fs_envelope_update(uint64_t evpid, const char *buf, size_t len) 271 { 272 char dest[PATH_MAX]; 273 274 fsqueue_envelope_path(evpid, dest, sizeof(dest)); 275 276 return (fsqueue_envelope_dump(dest, buf, len, 1, 1)); 277 } 278 279 static int 280 queue_fs_envelope_delete(uint64_t evpid) 281 { 282 char pathname[PATH_MAX]; 283 uint32_t msgid; 284 int *n; 285 286 fsqueue_envelope_path(evpid, pathname, sizeof(pathname)); 287 if (unlink(pathname) == -1) 288 if (errno != ENOENT) 289 return 0; 290 291 msgid = evpid_to_msgid(evpid); 292 n = tree_pop(&evpcount, msgid); 293 n -= 1; 294 295 if (n - REF == 0) 296 queue_fs_message_delete(msgid); 297 else 298 tree_xset(&evpcount, msgid, n); 299 300 return (1); 301 } 302 303 static int 304 queue_fs_message_walk(uint64_t *evpid, char *buf, size_t len, 305 uint32_t msgid, int *done, void **data) 306 { 307 struct dirent *dp; 308 DIR *dir = *data; 309 char path[PATH_MAX]; 310 char msgid_str[9]; 311 char *tmp; 312 int r, *n; 313 314 if (*done) 315 return (-1); 316 317 if (!bsnprintf(path, sizeof path, "%s/%02x/%08x", 318 PATH_QUEUE, (msgid & 0xff000000) >> 24, msgid)) 319 fatalx("queue_fs_message_walk: path does not fit buffer"); 320 321 if (dir == NULL) { 322 if ((dir = opendir(path)) == NULL) { 323 log_warn("warn: queue_fs: opendir: %s", path); 324 *done = 1; 325 return (-1); 326 } 327 328 *data = dir; 329 } 330 331 (void)snprintf(msgid_str, sizeof msgid_str, "%08" PRIx32, msgid); 332 while ((dp = readdir(dir)) != NULL) { 333 if (dp->d_type != DT_REG) 334 continue; 335 336 /* ignore files other than envelopes */ 337 if (strlen(dp->d_name) != 16 || 338 strncmp(dp->d_name, msgid_str, 8)) 339 continue; 340 341 tmp = NULL; 342 *evpid = strtoull(dp->d_name, &tmp, 16); 343 if (tmp && *tmp != '\0') { 344 log_debug("debug: fsqueue: bogus file %s", dp->d_name); 345 continue; 346 } 347 348 memset(buf, 0, len); 349 r = queue_fs_envelope_load(*evpid, buf, len); 350 if (r) { 351 n = tree_pop(&evpcount, msgid); 352 if (n == NULL) 353 n = REF; 354 355 n += 1; 356 tree_xset(&evpcount, msgid, n); 357 } 358 359 return (r); 360 } 361 362 (void)closedir(dir); 363 *done = 1; 364 return (-1); 365 } 366 367 static int 368 queue_fs_envelope_walk(uint64_t *evpid, char *buf, size_t len) 369 { 370 static int done = 0; 371 static void *hdl = NULL; 372 int r, *n; 373 uint32_t msgid; 374 375 if (done) 376 return (-1); 377 378 if (hdl == NULL) 379 hdl = fsqueue_qwalk_new(); 380 381 if (fsqueue_qwalk(hdl, evpid)) { 382 memset(buf, 0, len); 383 r = queue_fs_envelope_load(*evpid, buf, len); 384 if (r) { 385 msgid = evpid_to_msgid(*evpid); 386 n = tree_pop(&evpcount, msgid); 387 if (n == NULL) 388 n = REF; 389 n += 1; 390 tree_xset(&evpcount, msgid, n); 391 } 392 return (r); 393 } 394 395 fsqueue_qwalk_close(hdl); 396 done = 1; 397 return (-1); 398 } 399 400 static int 401 fsqueue_check_space(void) 402 { 403 struct statfs buf; 404 uint64_t used; 405 uint64_t total; 406 407 if (statfs(PATH_QUEUE, &buf) == -1) { 408 log_warn("warn: queue-fs: statfs"); 409 return 0; 410 } 411 412 /* 413 * f_bfree and f_ffree is not set on all filesystems. 414 * They could be signed or unsigned integers. 415 * Some systems will set them to 0, others will set them to -1. 416 */ 417 if (buf.f_bfree == 0 || buf.f_ffree == 0 || 418 (int64_t)buf.f_bfree == -1 || (int64_t)buf.f_ffree == -1) 419 return 1; 420 421 used = buf.f_blocks - buf.f_bfree; 422 total = buf.f_bavail + used; 423 if (total != 0) 424 used = (float)used / (float)total * 100; 425 else 426 used = 100; 427 if (100 - used < MINSPACE) { 428 log_warnx("warn: not enough disk space: %llu%% left", 429 (unsigned long long) 100 - used); 430 log_warnx("warn: temporarily rejecting messages"); 431 return 0; 432 } 433 434 used = buf.f_files - buf.f_ffree; 435 total = buf.f_favail + used; 436 if (total != 0) 437 used = (float)used / (float)total * 100; 438 else 439 used = 100; 440 if (100 - used < MININODES) { 441 log_warnx("warn: not enough inodes: %llu%% left", 442 (unsigned long long) 100 - used); 443 log_warnx("warn: temporarily rejecting messages"); 444 return 0; 445 } 446 447 return 1; 448 } 449 450 static void 451 fsqueue_envelope_path(uint64_t evpid, char *buf, size_t len) 452 { 453 if (!bsnprintf(buf, len, "%s/%02x/%08x/%016" PRIx64, 454 PATH_QUEUE, 455 (evpid_to_msgid(evpid) & 0xff000000) >> 24, 456 evpid_to_msgid(evpid), 457 evpid)) 458 fatalx("fsqueue_envelope_path: path does not fit buffer"); 459 } 460 461 static void 462 fsqueue_envelope_incoming_path(uint64_t evpid, char *buf, size_t len) 463 { 464 if (!bsnprintf(buf, len, "%s/%08x/%016" PRIx64, 465 PATH_INCOMING, 466 evpid_to_msgid(evpid), 467 evpid)) 468 fatalx("fsqueue_envelope_incoming_path: path does not fit buffer"); 469 } 470 471 static int 472 fsqueue_envelope_dump(char *dest, const char *evpbuf, size_t evplen, 473 int do_atomic, int do_sync) 474 { 475 const char *path = do_atomic ? PATH_EVPTMP : dest; 476 FILE *fp = NULL; 477 int fd; 478 size_t w; 479 480 if ((fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0600)) == -1) { 481 log_warn("warn: queue-fs: open"); 482 goto tempfail; 483 } 484 485 if ((fp = fdopen(fd, "w")) == NULL) { 486 log_warn("warn: queue-fs: fdopen"); 487 goto tempfail; 488 } 489 490 w = fwrite(evpbuf, 1, evplen, fp); 491 if (w < evplen) { 492 log_warn("warn: queue-fs: short write"); 493 goto tempfail; 494 } 495 if (fflush(fp)) { 496 log_warn("warn: queue-fs: fflush"); 497 goto tempfail; 498 } 499 if (do_sync && fsync(fileno(fp))) { 500 log_warn("warn: queue-fs: fsync"); 501 goto tempfail; 502 } 503 if (fclose(fp) != 0) { 504 log_warn("warn: queue-fs: fclose"); 505 fp = NULL; 506 goto tempfail; 507 } 508 fp = NULL; 509 fd = -1; 510 511 if (do_atomic && rename(path, dest) == -1) { 512 log_warn("warn: queue-fs: rename"); 513 goto tempfail; 514 } 515 return (1); 516 517 tempfail: 518 if (fp) 519 fclose(fp); 520 else if (fd != -1) 521 close(fd); 522 if (unlink(path) == -1) 523 log_warn("warn: queue-fs: unlink"); 524 return (0); 525 } 526 527 static void 528 fsqueue_message_path(uint32_t msgid, char *buf, size_t len) 529 { 530 if (!bsnprintf(buf, len, "%s/%02x/%08x", 531 PATH_QUEUE, 532 (msgid & 0xff000000) >> 24, 533 msgid)) 534 fatalx("fsqueue_message_path: path does not fit buffer"); 535 } 536 537 static void 538 fsqueue_message_incoming_path(uint32_t msgid, char *buf, size_t len) 539 { 540 if (!bsnprintf(buf, len, "%s/%08x", 541 PATH_INCOMING, 542 msgid)) 543 fatalx("fsqueue_message_incoming_path: path does not fit buffer"); 544 } 545 546 static void * 547 fsqueue_qwalk_new(void) 548 { 549 char path[PATH_MAX]; 550 char * const path_argv[] = { path, NULL }; 551 struct qwalk *q; 552 553 q = xcalloc(1, sizeof(*q)); 554 (void)strlcpy(path, PATH_QUEUE, sizeof(path)); 555 q->fts = fts_open(path_argv, 556 FTS_PHYSICAL | FTS_NOCHDIR, NULL); 557 558 if (q->fts == NULL) 559 fatal("fsqueue_qwalk_new: fts_open: %s", path); 560 561 return (q); 562 } 563 564 static void 565 fsqueue_qwalk_close(void *hdl) 566 { 567 struct qwalk *q = hdl; 568 569 fts_close(q->fts); 570 571 free(q); 572 } 573 574 static int 575 fsqueue_qwalk(void *hdl, uint64_t *evpid) 576 { 577 struct qwalk *q = hdl; 578 FTSENT *e; 579 char *tmp; 580 581 while ((e = fts_read(q->fts)) != NULL) { 582 switch (e->fts_info) { 583 case FTS_D: 584 q->depth += 1; 585 if (q->depth == 2 && e->fts_namelen != 2) { 586 log_debug("debug: fsqueue: bogus directory %s", 587 e->fts_path); 588 fts_set(q->fts, e, FTS_SKIP); 589 break; 590 } 591 if (q->depth == 3 && e->fts_namelen != 8) { 592 log_debug("debug: fsqueue: bogus directory %s", 593 e->fts_path); 594 fts_set(q->fts, e, FTS_SKIP); 595 break; 596 } 597 break; 598 599 case FTS_DP: 600 case FTS_DNR: 601 q->depth -= 1; 602 break; 603 604 case FTS_F: 605 if (q->depth != 3) 606 break; 607 if (e->fts_namelen != 16) 608 break; 609 if (timespeccmp(&e->fts_statp->st_mtim, &startup, >)) 610 break; 611 tmp = NULL; 612 *evpid = strtoull(e->fts_name, &tmp, 16); 613 if (tmp && *tmp != '\0') { 614 log_debug("debug: fsqueue: bogus file %s", 615 e->fts_path); 616 break; 617 } 618 return (1); 619 default: 620 break; 621 } 622 } 623 624 return (0); 625 } 626 627 static int 628 queue_fs_init(struct passwd *pw, int server, const char *conf) 629 { 630 unsigned int n; 631 char *paths[] = { PATH_QUEUE, PATH_INCOMING }; 632 char path[PATH_MAX]; 633 int ret; 634 635 /* remove incoming/ if it exists */ 636 if (server) 637 mvpurge(PATH_SPOOL PATH_INCOMING, PATH_SPOOL PATH_PURGE); 638 639 fsqueue_envelope_path(0, path, sizeof(path)); 640 641 ret = 1; 642 for (n = 0; n < nitems(paths); n++) { 643 (void)strlcpy(path, PATH_SPOOL, sizeof(path)); 644 if (strlcat(path, paths[n], sizeof(path)) >= sizeof(path)) 645 fatalx("path too long %s%s", PATH_SPOOL, paths[n]); 646 if (ckdir(path, 0700, pw->pw_uid, 0, server) == 0) 647 ret = 0; 648 } 649 650 if (clock_gettime(CLOCK_REALTIME, &startup)) 651 fatal("clock_gettime"); 652 653 tree_init(&evpcount); 654 655 queue_api_on_message_create(queue_fs_message_create); 656 queue_api_on_message_commit(queue_fs_message_commit); 657 queue_api_on_message_delete(queue_fs_message_delete); 658 queue_api_on_message_fd_r(queue_fs_message_fd_r); 659 queue_api_on_envelope_create(queue_fs_envelope_create); 660 queue_api_on_envelope_delete(queue_fs_envelope_delete); 661 queue_api_on_envelope_update(queue_fs_envelope_update); 662 queue_api_on_envelope_load(queue_fs_envelope_load); 663 queue_api_on_envelope_walk(queue_fs_envelope_walk); 664 queue_api_on_message_walk(queue_fs_message_walk); 665 666 return (ret); 667 } 668 669 struct queue_backend queue_backend_fs = { 670 queue_fs_init, 671 }; 672