1 /*-
2 * Copyright 2016 Vsevolod Stakhov
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "config.h"
17 #include "libserver/logger.h"
18 #include "libutil/sqlite_utils.h"
19 #include "unix-std.h"
20
21
22 static GQuark
rspamd_sqlite3_quark(void)23 rspamd_sqlite3_quark (void)
24 {
25 return g_quark_from_static_string ("rspamd-sqlite3");
26 }
27
28 GArray*
rspamd_sqlite3_init_prstmt(sqlite3 * db,struct rspamd_sqlite3_prstmt * init_stmt,gint max_idx,GError ** err)29 rspamd_sqlite3_init_prstmt (sqlite3 *db,
30 struct rspamd_sqlite3_prstmt *init_stmt,
31 gint max_idx,
32 GError **err)
33 {
34 gint i;
35 GArray *res;
36 struct rspamd_sqlite3_prstmt *nst;
37
38 res = g_array_sized_new (FALSE, TRUE, sizeof (struct rspamd_sqlite3_prstmt),
39 max_idx);
40 g_array_set_size (res, max_idx);
41
42 for (i = 0; i < max_idx; i ++) {
43 nst = &g_array_index (res, struct rspamd_sqlite3_prstmt, i);
44 memcpy (nst, &init_stmt[i], sizeof (*nst));
45
46 if (sqlite3_prepare_v2 (db, init_stmt[i].sql, -1,
47 &nst->stmt, NULL) != SQLITE_OK) {
48 g_set_error (err, rspamd_sqlite3_quark (),
49 -1, "Cannot initialize prepared sql `%s`: %s",
50 nst->sql, sqlite3_errmsg (db));
51 rspamd_sqlite3_close_prstmt (db, res);
52
53 return NULL;
54 }
55 }
56
57 return res;
58 }
59
60 int
rspamd_sqlite3_run_prstmt(rspamd_mempool_t * pool,sqlite3 * db,GArray * stmts,gint idx,...)61 rspamd_sqlite3_run_prstmt (rspamd_mempool_t *pool, sqlite3 *db, GArray *stmts,
62 gint idx, ...)
63 {
64 gint retcode;
65 va_list ap;
66 sqlite3_stmt *stmt;
67 gint i, rowid, nargs, j;
68 gint64 len;
69 gpointer p;
70 struct rspamd_sqlite3_prstmt *nst;
71 const char *argtypes;
72
73 if (idx < 0 || idx >= (gint)stmts->len) {
74
75 return -1;
76 }
77
78 nst = &g_array_index (stmts, struct rspamd_sqlite3_prstmt, idx);
79 stmt = nst->stmt;
80
81 g_assert (nst != NULL);
82
83 msg_debug_pool ("executing `%s`", nst->sql);
84 argtypes = nst->args;
85 sqlite3_clear_bindings (stmt);
86 sqlite3_reset (stmt);
87 va_start (ap, idx);
88 nargs = 1;
89
90 for (i = 0, rowid = 1; argtypes[i] != '\0'; i ++) {
91 switch (argtypes[i]) {
92 case 'T':
93
94 for (j = 0; j < nargs; j ++, rowid ++) {
95 sqlite3_bind_text (stmt, rowid, va_arg (ap, const char*), -1,
96 SQLITE_STATIC);
97 }
98
99 nargs = 1;
100 break;
101 case 'V':
102 case 'B':
103
104 for (j = 0; j < nargs; j ++, rowid ++) {
105 len = va_arg (ap, gint64);
106 sqlite3_bind_text (stmt, rowid, va_arg (ap, const char*), len,
107 SQLITE_STATIC);
108 }
109
110 nargs = 1;
111 break;
112 case 'I':
113
114 for (j = 0; j < nargs; j ++, rowid ++) {
115 sqlite3_bind_int64 (stmt, rowid, va_arg (ap, gint64));
116 }
117
118 nargs = 1;
119 break;
120 case 'S':
121
122 for (j = 0; j < nargs; j ++, rowid ++) {
123 sqlite3_bind_int (stmt, rowid, va_arg (ap, gint));
124 }
125
126 nargs = 1;
127 break;
128 case '*':
129 nargs = va_arg (ap, gint);
130 break;
131 }
132 }
133
134 retcode = sqlite3_step (stmt);
135
136 if (retcode == nst->result) {
137 argtypes = nst->ret;
138
139 for (i = 0; argtypes != NULL && argtypes[i] != '\0'; i ++) {
140 switch (argtypes[i]) {
141 case 'T':
142 *va_arg (ap, char**) = g_strdup (sqlite3_column_text (stmt, i));
143 break;
144 case 'I':
145 *va_arg (ap, gint64*) = sqlite3_column_int64 (stmt, i);
146 break;
147 case 'S':
148 *va_arg (ap, int*) = sqlite3_column_int (stmt, i);
149 break;
150 case 'L':
151 *va_arg (ap, gint64*) = sqlite3_last_insert_rowid (db);
152 break;
153 case 'B':
154 len = sqlite3_column_bytes (stmt, i);
155 g_assert (len >= 0);
156 p = g_malloc (len);
157 memcpy (p, sqlite3_column_blob (stmt, i), len);
158 *va_arg (ap, gint64*) = len;
159 *va_arg (ap, gpointer*) = p;
160 break;
161 }
162 }
163
164 if (!(nst->flags & RSPAMD_SQLITE3_STMT_MULTIPLE)) {
165 sqlite3_clear_bindings (stmt);
166 sqlite3_reset (stmt);
167 }
168
169 va_end (ap);
170
171 return SQLITE_OK;
172 }
173 else if (retcode != SQLITE_DONE && retcode != SQLITE_OK && retcode != SQLITE_ROW) {
174 msg_warn_pool ("failed to execute query %s: %d, %s", nst->sql,
175 retcode, sqlite3_errmsg (db));
176 }
177
178 if (!(nst->flags & RSPAMD_SQLITE3_STMT_MULTIPLE)) {
179 sqlite3_clear_bindings (stmt);
180 sqlite3_reset (stmt);
181 }
182
183 va_end (ap);
184
185 return retcode;
186 }
187
188 void
rspamd_sqlite3_close_prstmt(sqlite3 * db,GArray * stmts)189 rspamd_sqlite3_close_prstmt (sqlite3 *db, GArray *stmts)
190 {
191 guint i;
192 struct rspamd_sqlite3_prstmt *nst;
193
194 for (i = 0; i < stmts->len; i++) {
195 nst = &g_array_index (stmts, struct rspamd_sqlite3_prstmt, i);
196 if (nst->stmt != NULL) {
197 sqlite3_finalize (nst->stmt);
198 }
199 }
200
201 g_array_free (stmts, TRUE);
202
203 return;
204 }
205
206 static gboolean
rspamd_sqlite3_wait(rspamd_mempool_t * pool,const gchar * lock)207 rspamd_sqlite3_wait (rspamd_mempool_t *pool, const gchar *lock)
208 {
209 gint fd;
210 pid_t pid;
211 gssize r;
212 struct timespec sleep_ts = {
213 .tv_sec = 0,
214 .tv_nsec = 1000000
215 };
216
217 while ((fd = open (lock, O_WRONLY|O_CREAT|O_EXCL, 00600)) == -1) {
218 if (errno != EBUSY && errno != EEXIST) {
219 msg_err_pool_check ("cannot open lock file %s: %s", lock,
220 strerror (errno));
221
222 return FALSE;
223 }
224
225 fd = open (lock, O_RDONLY);
226
227 if (fd == -1) {
228 msg_err_pool_check ("cannot open lock file %s: %s", lock,
229 strerror (errno));
230
231 return FALSE;
232 }
233
234 r = read (fd, &pid, sizeof (pid));
235
236 if (r != sizeof (pid)) {
237 msg_warn_pool_check ("stale lock file %s, removing", lock);
238 unlink (lock);
239 close (fd);
240
241 return TRUE;
242 }
243
244 /* Now check for process existence */
245 if (pid == getpid ()) {
246 msg_warn_pool_check ("lock file %s, belongs to me, removing", lock);
247 unlink (lock);
248 close (fd);
249
250 return TRUE;
251 }
252 else if (kill (pid, 0) == -1) {
253 if (errno == ESRCH) {
254 /* Process is already dead */
255 msg_warn_pool_check ("stale lock file %s from pid: %P, removing",
256 lock, pid);
257 unlink (lock);
258 close (fd);
259
260 return TRUE;
261 }
262 }
263
264 close (fd);
265
266 if (nanosleep (&sleep_ts, NULL) == -1 && errno != EINTR) {
267 msg_err_pool_check ("cannot sleep open lock file %s: %s", lock,
268 strerror (errno));
269
270 return FALSE;
271 }
272 }
273
274 unlink (lock);
275 close (fd);
276
277 return TRUE;
278 }
279
280 #define RSPAMD_SQLITE_MMAP_LIMIT 268435456
281 #define RSPAMD_SQLITE_CACHE_SIZE 262144
282
283 sqlite3 *
rspamd_sqlite3_open_or_create(rspamd_mempool_t * pool,const gchar * path,const gchar * create_sql,guint version,GError ** err)284 rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
285 gchar *create_sql, guint version, GError **err)
286 {
287 sqlite3 *sqlite;
288 gint rc, flags, lock_fd;
289 gchar lock_path[PATH_MAX], dbdir[PATH_MAX], *pdir;
290 static const char sqlite_wal[] =
291 "PRAGMA journal_mode=\"wal\";"
292 "PRAGMA wal_autocheckpoint = 16;"
293 "PRAGMA journal_size_limit = 1536;",
294 exclusive_lock_sql[] = "PRAGMA locking_mode=\"exclusive\";",
295
296 fsync_sql[] = "PRAGMA synchronous=\"NORMAL\";",
297
298 foreign_keys[] = "PRAGMA foreign_keys=\"ON\";",
299
300 #if defined(__LP64__) || defined(_LP64)
301 enable_mmap[] = "PRAGMA mmap_size="
302 G_STRINGIFY(RSPAMD_SQLITE_MMAP_LIMIT) ";",
303 #endif
304
305 other_pragmas[] = "PRAGMA read_uncommitted=\"ON\";"
306 "PRAGMA cache_size="
307 G_STRINGIFY(RSPAMD_SQLITE_CACHE_SIZE) ";",
308 db_version[] = "PRAGMA user_version;";
309 gboolean create = FALSE, has_lock = FALSE;
310
311 flags = SQLITE_OPEN_READWRITE;
312 #ifdef SQLITE_OPEN_SHAREDCACHE
313 flags |= SQLITE_OPEN_SHAREDCACHE;
314 #endif
315 #ifdef SQLITE_OPEN_WAL
316 flags |= SQLITE_OPEN_WAL;
317 #endif
318
319 rspamd_strlcpy (dbdir, path, sizeof (dbdir));
320 pdir = dirname (dbdir);
321
322 if (access (pdir, W_OK) == -1) {
323 g_set_error (err, rspamd_sqlite3_quark (),
324 errno, "cannot open sqlite directory %s: %s",
325 pdir, strerror (errno));
326
327 return NULL;
328 }
329
330 rspamd_snprintf (lock_path, sizeof (lock_path), "%s.lock", path);
331
332 if (access (path, R_OK) == -1) {
333 flags |= SQLITE_OPEN_CREATE;
334 create = TRUE;
335 }
336
337
338 rspamd_snprintf (lock_path, sizeof (lock_path), "%s.lock", path);
339 lock_fd = open (lock_path, O_WRONLY|O_CREAT|O_EXCL, 00600);
340
341 if (lock_fd == -1) {
342 if (errno == EEXIST || errno == EBUSY) {
343 msg_debug_pool_check ("checking %s to wait for db being initialized", lock_path);
344
345 if (!rspamd_sqlite3_wait(pool, lock_path)) {
346 g_set_error(err, rspamd_sqlite3_quark(),
347 errno, "cannot create sqlite file %s: %s",
348 path, strerror(errno));
349
350 return NULL;
351 }
352
353
354 /* At this point we have database created */
355 create = FALSE;
356 has_lock = FALSE;
357 }
358 else {
359 g_set_error(err, rspamd_sqlite3_quark(),
360 errno, "cannot lock sqlite file %s: %s",
361 path, strerror(errno));
362 }
363 }
364 else {
365 pid_t myself = getpid ();
366 msg_debug_pool_check ("locking %s to block other processes", lock_path);
367 (void)write (lock_fd, &myself, sizeof (myself));
368
369 g_assert (rspamd_file_lock (lock_fd, FALSE));
370 has_lock = TRUE;
371 }
372
373 if ((rc = sqlite3_open_v2 (path, &sqlite,
374 flags, NULL)) != SQLITE_OK) {
375 #if SQLITE_VERSION_NUMBER >= 3008000
376 g_set_error (err, rspamd_sqlite3_quark (),
377 rc, "cannot open sqlite db %s: %s",
378 path, sqlite3_errstr (rc));
379 #else
380 g_set_error (err, rspamd_sqlite3_quark (),
381 rc, "cannot open sqlite db %s: %d",
382 path, rc);
383 #endif
384
385 if (has_lock && lock_fd != -1) {
386 msg_debug_pool_check ("removing lock from %s", lock_path);
387 rspamd_file_unlock (lock_fd, FALSE);
388 unlink (lock_path);
389 close (lock_fd);
390 }
391
392 return NULL;
393 }
394
395 if (create && has_lock) {
396 while ((rc = sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL)) != SQLITE_OK) {
397 if (rc == SQLITE_BUSY) {
398 struct timespec sleep_ts = {
399 .tv_sec = 0,
400 .tv_nsec = 1000000
401 };
402
403 nanosleep (&sleep_ts, NULL);
404
405 continue;
406 }
407
408 msg_warn_pool_check ("WAL mode is not supported (%s), locking issues might occur",
409 sqlite3_errmsg (sqlite));
410 break;
411 }
412
413 if (sqlite3_exec (sqlite, exclusive_lock_sql, NULL, NULL, NULL) != SQLITE_OK) {
414 msg_warn_pool_check ("cannot exclusively lock database to create schema: %s",
415 sqlite3_errmsg (sqlite));
416 }
417
418 if (create_sql) {
419 while ((rc = sqlite3_exec (sqlite, create_sql, NULL, NULL, NULL)) != SQLITE_OK) {
420 if (rc == SQLITE_BUSY) {
421 struct timespec sleep_ts = {
422 .tv_sec = 0,
423 .tv_nsec = 1000000
424 };
425
426 nanosleep (&sleep_ts, NULL);
427
428 continue;
429 }
430
431 g_set_error (err, rspamd_sqlite3_quark (),
432 -1, "cannot execute create sql `%s`: %s",
433 create_sql, sqlite3_errmsg (sqlite));
434 sqlite3_close (sqlite);
435 rspamd_file_unlock (lock_fd, FALSE);
436 unlink (lock_path);
437 if (lock_fd != -1) {
438 close (lock_fd);
439 }
440
441 return NULL;
442 }
443 }
444
445 sqlite3_close (sqlite);
446
447 /* Reopen in normal mode */
448 msg_debug_pool_check ("reopening %s in normal mode", path);
449 flags &= ~SQLITE_OPEN_CREATE;
450
451 if ((rc = sqlite3_open_v2 (path, &sqlite,
452 flags, NULL)) != SQLITE_OK) {
453 #if SQLITE_VERSION_NUMBER >= 3008000
454 g_set_error (err, rspamd_sqlite3_quark (),
455 rc, "cannot open sqlite db after creation %s: %s",
456 path, sqlite3_errstr (rc));
457 #else
458 g_set_error (err, rspamd_sqlite3_quark (),
459 rc, "cannot open sqlite db after creation %s: %d",
460 path, rc);
461 #endif
462 rspamd_file_unlock (lock_fd, FALSE);
463 unlink (lock_path);
464
465 if (lock_fd != -1) {
466 close (lock_fd);
467 }
468
469 return NULL;
470 }
471 }
472 else if (has_lock && version > 0) {
473 /* Check user version */
474 sqlite3_stmt *stmt = NULL;
475 guint32 db_ver;
476 GString *new_ver_sql;
477
478 if (sqlite3_prepare (sqlite, db_version, -1, &stmt, NULL) != SQLITE_OK) {
479 msg_warn_pool_check ("Cannot get user version pragma: %s",
480 sqlite3_errmsg (sqlite));
481 }
482 else {
483 if (sqlite3_step (stmt) != SQLITE_ROW) {
484 msg_warn_pool_check ("Cannot get user version pragma, step failed: %s",
485 sqlite3_errmsg (sqlite));
486 sqlite3_finalize (stmt);
487 }
488 else {
489 db_ver = sqlite3_column_int (stmt, 0);
490 sqlite3_reset (stmt);
491 sqlite3_finalize (stmt);
492
493 if (version > db_ver) {
494 msg_warn_pool_check ("Database version %ud is less than "
495 "desired version %ud, run create script", db_ver,
496 version);
497
498 if (create_sql) {
499 if (sqlite3_exec (sqlite, create_sql, NULL, NULL, NULL) != SQLITE_OK) {
500 g_set_error (err, rspamd_sqlite3_quark (),
501 -1, "cannot execute create sql `%s`: %s",
502 create_sql, sqlite3_errmsg (sqlite));
503 sqlite3_close (sqlite);
504 rspamd_file_unlock (lock_fd, FALSE);
505 unlink (lock_path);
506 if (lock_fd != -1) {
507 close (lock_fd);
508 }
509
510 return NULL;
511 }
512 }
513
514 new_ver_sql = g_string_new ("PRAGMA user_version=");
515 rspamd_printf_gstring (new_ver_sql, "%ud", version);
516
517 if (sqlite3_exec (sqlite, new_ver_sql->str, NULL, NULL, NULL)
518 != SQLITE_OK) {
519 g_set_error (err, rspamd_sqlite3_quark (),
520 -1, "cannot execute update version sql `%s`: %s",
521 new_ver_sql->str, sqlite3_errmsg (sqlite));
522 sqlite3_close (sqlite);
523 rspamd_file_unlock (lock_fd, FALSE);
524 unlink (lock_path);
525 if (lock_fd != -1) {
526 close (lock_fd);
527 }
528
529 g_string_free (new_ver_sql, TRUE);
530
531 return NULL;
532 }
533
534 g_string_free (new_ver_sql, TRUE);
535 }
536 else if (db_ver > version) {
537 msg_warn_pool_check ("Database version %ud is more than "
538 "desired version %ud, this could cause"
539 " unexpected behaviour", db_ver,
540 version);
541 }
542 }
543 }
544 }
545
546 while ((rc = sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL)) != SQLITE_OK) {
547 if (rc == SQLITE_BUSY) {
548 struct timespec sleep_ts = {
549 .tv_sec = 0,
550 .tv_nsec = 1000000
551 };
552
553 nanosleep (&sleep_ts, NULL);
554
555 continue;
556 }
557
558 msg_warn_pool_check ("WAL mode is not supported (%s), locking issues might occur",
559 sqlite3_errmsg (sqlite));
560 break;
561 }
562
563 if (sqlite3_exec (sqlite, fsync_sql, NULL, NULL, NULL) != SQLITE_OK) {
564 msg_warn_pool_check ("cannot set synchronous: %s",
565 sqlite3_errmsg (sqlite));
566 }
567
568 if ((rc = sqlite3_exec (sqlite, foreign_keys, NULL, NULL, NULL)) !=
569 SQLITE_OK) {
570 msg_warn_pool_check ("cannot enable foreign keys: %s",
571 sqlite3_errmsg (sqlite));
572 }
573
574 #if defined(__LP64__) || defined(_LP64)
575 if ((rc = sqlite3_exec (sqlite, enable_mmap, NULL, NULL, NULL)) != SQLITE_OK) {
576 msg_warn_pool_check ("cannot enable mmap: %s",
577 sqlite3_errmsg (sqlite));
578 }
579 #endif
580
581 if ((rc = sqlite3_exec (sqlite, other_pragmas, NULL, NULL, NULL)) !=
582 SQLITE_OK) {
583 msg_warn_pool_check ("cannot execute tuning pragmas: %s",
584 sqlite3_errmsg (sqlite));
585 }
586
587 if (has_lock && lock_fd != -1) {
588 msg_debug_pool_check ("removing lock from %s", lock_path);
589 rspamd_file_unlock (lock_fd, FALSE);
590 unlink (lock_path);
591 close (lock_fd);
592 }
593
594 return sqlite;
595 }
596
597 gboolean
rspamd_sqlite3_sync(sqlite3 * db,gint * wal_frames,gint * wal_checkpoints)598 rspamd_sqlite3_sync (sqlite3 *db, gint *wal_frames, gint *wal_checkpoints)
599 {
600 gint wf = 0, wc = 0, mode;
601
602 #ifdef SQLITE_OPEN_WAL
603 #ifdef SQLITE_CHECKPOINT_TRUNCATE
604 mode = SQLITE_CHECKPOINT_TRUNCATE;
605 #elif defined(SQLITE_CHECKPOINT_RESTART)
606 mode = SQLITE_CHECKPOINT_RESTART;
607 #elif defined(SQLITE_CHECKPOINT_FULL)
608 mode = SQLITE_CHECKPOINT_FULL;
609 #endif
610 /* Perform wal checkpoint (might be long) */
611 if (sqlite3_wal_checkpoint_v2 (db,
612 NULL,
613 mode,
614 &wf,
615 &wc) != SQLITE_OK) {
616 return FALSE;
617
618 }
619 #endif
620
621 if (wal_frames) {
622 *wal_frames = wf;
623 }
624 if (wal_checkpoints) {
625 *wal_checkpoints = wc;
626 }
627
628 return TRUE;
629 }
630