1 /*-------------------------------------------------------------------------
2 *
3 * pg_stat_statements.c
4 * Track statement execution times across a whole database cluster.
5 *
6 * Execution costs are totalled for each distinct source query, and kept in
7 * a shared hashtable. (We track only as many distinct queries as will fit
8 * in the designated amount of shared memory.)
9 *
10 * As of Postgres 9.2, this module normalizes query entries. Normalization
11 * is a process whereby similar queries, typically differing only in their
12 * constants (though the exact rules are somewhat more subtle than that) are
13 * recognized as equivalent, and are tracked as a single entry. This is
14 * particularly useful for non-prepared queries.
15 *
16 * Normalization is implemented by fingerprinting queries, selectively
17 * serializing those fields of each query tree's nodes that are judged to be
18 * essential to the query. This is referred to as a query jumble. This is
19 * distinct from a regular serialization in that various extraneous
20 * information is ignored as irrelevant or not essential to the query, such
21 * as the collations of Vars and, most notably, the values of constants.
22 *
23 * This jumble is acquired at the end of parse analysis of each query, and
24 * a 32-bit hash of it is stored into the query's Query.queryId field.
25 * The server then copies this value around, making it available in plan
26 * tree(s) generated from the query. The executor can then use this value
27 * to blame query costs on the proper queryId.
28 *
29 * To facilitate presenting entries to users, we create "representative" query
30 * strings in which constants are replaced with '?' characters, to make it
31 * clearer what a normalized entry can represent. To save on shared memory,
32 * and to avoid having to truncate oversized query strings, we store these
33 * strings in a temporary external query-texts file. Offsets into this
34 * file are kept in shared memory.
35 *
36 * Note about locking issues: to create or delete an entry in the shared
37 * hashtable, one must hold pgss->lock exclusively. Modifying any field
38 * in an entry except the counters requires the same. To look up an entry,
39 * one must hold the lock shared. To read or update the counters within
40 * an entry, one must hold the lock shared or exclusive (so the entry doesn't
41 * disappear!) and also take the entry's mutex spinlock.
42 * The shared state variable pgss->extent (the next free spot in the external
43 * query-text file) should be accessed only while holding either the
44 * pgss->mutex spinlock, or exclusive lock on pgss->lock. We use the mutex to
45 * allow reserving file space while holding only shared lock on pgss->lock.
46 * Rewriting the entire external query-text file, eg for garbage collection,
47 * requires holding pgss->lock exclusively; this allows individual entries
48 * in the file to be read or written while holding only shared lock.
49 *
50 *
51 * Copyright (c) 2008-2016, PostgreSQL Global Development Group
52 *
53 * IDENTIFICATION
54 * contrib/pg_stat_statements/pg_stat_statements.c
55 *
56 *-------------------------------------------------------------------------
57 */
58 #include "postgres.h"
59
60 #include <math.h>
61 #include <sys/stat.h>
62 #include <unistd.h>
63
64 #include "access/hash.h"
65 #include "executor/instrument.h"
66 #include "funcapi.h"
67 #include "mb/pg_wchar.h"
68 #include "miscadmin.h"
69 #include "parser/analyze.h"
70 #include "parser/parsetree.h"
71 #include "parser/scanner.h"
72 #include "pgstat.h"
73 #include "storage/fd.h"
74 #include "storage/ipc.h"
75 #include "storage/spin.h"
76 #include "tcop/utility.h"
77 #include "utils/builtins.h"
78 #include "utils/memutils.h"
79
80 PG_MODULE_MAGIC;
81
82 /* Location of permanent stats file (valid when database is shut down) */
83 #define PGSS_DUMP_FILE PGSTAT_STAT_PERMANENT_DIRECTORY "/pg_stat_statements.stat"
84
85 /*
86 * Location of external query text file. We don't keep it in the core
87 * system's stats_temp_directory. The core system can safely use that GUC
88 * setting, because the statistics collector temp file paths are set only once
89 * as part of changing the GUC, but pg_stat_statements has no way of avoiding
90 * race conditions. Besides, we only expect modest, infrequent I/O for query
91 * strings, so placing the file on a faster filesystem is not compelling.
92 */
93 #define PGSS_TEXT_FILE PG_STAT_TMP_DIR "/pgss_query_texts.stat"
94
95 /* Magic number identifying the stats file format */
96 static const uint32 PGSS_FILE_HEADER = 0x20140125;
97
98 /* PostgreSQL major version number, changes in which invalidate all entries */
99 static const uint32 PGSS_PG_MAJOR_VERSION = PG_VERSION_NUM / 100;
100
101 /* XXX: Should USAGE_EXEC reflect execution time and/or buffer usage? */
102 #define USAGE_EXEC(duration) (1.0)
103 #define USAGE_INIT (1.0) /* including initial planning */
104 #define ASSUMED_MEDIAN_INIT (10.0) /* initial assumed median usage */
105 #define ASSUMED_LENGTH_INIT 1024 /* initial assumed mean query length */
106 #define USAGE_DECREASE_FACTOR (0.99) /* decreased every entry_dealloc */
107 #define STICKY_DECREASE_FACTOR (0.50) /* factor for sticky entries */
108 #define USAGE_DEALLOC_PERCENT 5 /* free this % of entries at once */
109
110 #define JUMBLE_SIZE 1024 /* query serialization buffer size */
111
112 /*
113 * Extension version number, for supporting older extension versions' objects
114 */
115 typedef enum pgssVersion
116 {
117 PGSS_V1_0 = 0,
118 PGSS_V1_1,
119 PGSS_V1_2,
120 PGSS_V1_3
121 } pgssVersion;
122
123 /*
124 * Hashtable key that defines the identity of a hashtable entry. We separate
125 * queries by user and by database even if they are otherwise identical.
126 */
127 typedef struct pgssHashKey
128 {
129 Oid userid; /* user OID */
130 Oid dbid; /* database OID */
131 uint32 queryid; /* query identifier */
132 } pgssHashKey;
133
134 /*
135 * The actual stats counters kept within pgssEntry.
136 */
137 typedef struct Counters
138 {
139 int64 calls; /* # of times executed */
140 double total_time; /* total execution time, in msec */
141 double min_time; /* minimum execution time in msec */
142 double max_time; /* maximum execution time in msec */
143 double mean_time; /* mean execution time in msec */
144 double sum_var_time; /* sum of variances in execution time in msec */
145 int64 rows; /* total # of retrieved or affected rows */
146 int64 shared_blks_hit; /* # of shared buffer hits */
147 int64 shared_blks_read; /* # of shared disk blocks read */
148 int64 shared_blks_dirtied; /* # of shared disk blocks dirtied */
149 int64 shared_blks_written; /* # of shared disk blocks written */
150 int64 local_blks_hit; /* # of local buffer hits */
151 int64 local_blks_read; /* # of local disk blocks read */
152 int64 local_blks_dirtied; /* # of local disk blocks dirtied */
153 int64 local_blks_written; /* # of local disk blocks written */
154 int64 temp_blks_read; /* # of temp blocks read */
155 int64 temp_blks_written; /* # of temp blocks written */
156 double blk_read_time; /* time spent reading, in msec */
157 double blk_write_time; /* time spent writing, in msec */
158 double usage; /* usage factor */
159 } Counters;
160
161 /*
162 * Statistics per statement
163 *
164 * Note: in event of a failure in garbage collection of the query text file,
165 * we reset query_offset to zero and query_len to -1. This will be seen as
166 * an invalid state by qtext_fetch().
167 */
168 typedef struct pgssEntry
169 {
170 pgssHashKey key; /* hash key of entry - MUST BE FIRST */
171 Counters counters; /* the statistics for this query */
172 Size query_offset; /* query text offset in external file */
173 int query_len; /* # of valid bytes in query string, or -1 */
174 int encoding; /* query text encoding */
175 slock_t mutex; /* protects the counters only */
176 } pgssEntry;
177
178 /*
179 * Global shared state
180 */
181 typedef struct pgssSharedState
182 {
183 LWLock *lock; /* protects hashtable search/modification */
184 double cur_median_usage; /* current median usage in hashtable */
185 Size mean_query_len; /* current mean entry text length */
186 slock_t mutex; /* protects following fields only: */
187 Size extent; /* current extent of query file */
188 int n_writers; /* number of active writers to query file */
189 int gc_count; /* query file garbage collection cycle count */
190 } pgssSharedState;
191
192 /*
193 * Struct for tracking locations/lengths of constants during normalization
194 */
195 typedef struct pgssLocationLen
196 {
197 int location; /* start offset in query text */
198 int length; /* length in bytes, or -1 to ignore */
199 } pgssLocationLen;
200
201 /*
202 * Working state for computing a query jumble and producing a normalized
203 * query string
204 */
205 typedef struct pgssJumbleState
206 {
207 /* Jumble of current query tree */
208 unsigned char *jumble;
209
210 /* Number of bytes used in jumble[] */
211 Size jumble_len;
212
213 /* Array of locations of constants that should be removed */
214 pgssLocationLen *clocations;
215
216 /* Allocated length of clocations array */
217 int clocations_buf_size;
218
219 /* Current number of valid entries in clocations array */
220 int clocations_count;
221 } pgssJumbleState;
222
223 /*---- Local variables ----*/
224
225 /* Current nesting depth of ExecutorRun+ProcessUtility calls */
226 static int nested_level = 0;
227
228 /* Saved hook values in case of unload */
229 static shmem_startup_hook_type prev_shmem_startup_hook = NULL;
230 static post_parse_analyze_hook_type prev_post_parse_analyze_hook = NULL;
231 static ExecutorStart_hook_type prev_ExecutorStart = NULL;
232 static ExecutorRun_hook_type prev_ExecutorRun = NULL;
233 static ExecutorFinish_hook_type prev_ExecutorFinish = NULL;
234 static ExecutorEnd_hook_type prev_ExecutorEnd = NULL;
235 static ProcessUtility_hook_type prev_ProcessUtility = NULL;
236
237 /* Links to shared memory state */
238 static pgssSharedState *pgss = NULL;
239 static HTAB *pgss_hash = NULL;
240
241 /*---- GUC variables ----*/
242
243 typedef enum
244 {
245 PGSS_TRACK_NONE, /* track no statements */
246 PGSS_TRACK_TOP, /* only top level statements */
247 PGSS_TRACK_ALL /* all statements, including nested ones */
248 } PGSSTrackLevel;
249
250 static const struct config_enum_entry track_options[] =
251 {
252 {"none", PGSS_TRACK_NONE, false},
253 {"top", PGSS_TRACK_TOP, false},
254 {"all", PGSS_TRACK_ALL, false},
255 {NULL, 0, false}
256 };
257
258 static int pgss_max; /* max # statements to track */
259 static int pgss_track; /* tracking level */
260 static bool pgss_track_utility; /* whether to track utility commands */
261 static bool pgss_save; /* whether to save stats across shutdown */
262
263
264 #define pgss_enabled() \
265 (pgss_track == PGSS_TRACK_ALL || \
266 (pgss_track == PGSS_TRACK_TOP && nested_level == 0))
267
268 #define record_gc_qtexts() \
269 do { \
270 volatile pgssSharedState *s = (volatile pgssSharedState *) pgss; \
271 SpinLockAcquire(&s->mutex); \
272 s->gc_count++; \
273 SpinLockRelease(&s->mutex); \
274 } while(0)
275
276 /*---- Function declarations ----*/
277
278 void _PG_init(void);
279 void _PG_fini(void);
280
281 PG_FUNCTION_INFO_V1(pg_stat_statements_reset);
282 PG_FUNCTION_INFO_V1(pg_stat_statements_1_2);
283 PG_FUNCTION_INFO_V1(pg_stat_statements_1_3);
284 PG_FUNCTION_INFO_V1(pg_stat_statements);
285
286 static void pgss_shmem_startup(void);
287 static void pgss_shmem_shutdown(int code, Datum arg);
288 static void pgss_post_parse_analyze(ParseState *pstate, Query *query);
289 static void pgss_ExecutorStart(QueryDesc *queryDesc, int eflags);
290 static void pgss_ExecutorRun(QueryDesc *queryDesc,
291 ScanDirection direction,
292 uint64 count);
293 static void pgss_ExecutorFinish(QueryDesc *queryDesc);
294 static void pgss_ExecutorEnd(QueryDesc *queryDesc);
295 static void pgss_ProcessUtility(Node *parsetree, const char *queryString,
296 ProcessUtilityContext context, ParamListInfo params,
297 DestReceiver *dest, char *completionTag);
298 static uint32 pgss_hash_fn(const void *key, Size keysize);
299 static int pgss_match_fn(const void *key1, const void *key2, Size keysize);
300 static uint32 pgss_hash_string(const char *str);
301 static void pgss_store(const char *query, uint32 queryId,
302 double total_time, uint64 rows,
303 const BufferUsage *bufusage,
304 pgssJumbleState *jstate);
305 static void pg_stat_statements_internal(FunctionCallInfo fcinfo,
306 pgssVersion api_version,
307 bool showtext);
308 static Size pgss_memsize(void);
309 static pgssEntry *entry_alloc(pgssHashKey *key, Size query_offset, int query_len,
310 int encoding, bool sticky);
311 static void entry_dealloc(void);
312 static bool qtext_store(const char *query, int query_len,
313 Size *query_offset, int *gc_count);
314 static char *qtext_load_file(Size *buffer_size);
315 static char *qtext_fetch(Size query_offset, int query_len,
316 char *buffer, Size buffer_size);
317 static bool need_gc_qtexts(void);
318 static void gc_qtexts(void);
319 static void entry_reset(void);
320 static void AppendJumble(pgssJumbleState *jstate,
321 const unsigned char *item, Size size);
322 static void JumbleQuery(pgssJumbleState *jstate, Query *query);
323 static void JumbleRangeTable(pgssJumbleState *jstate, List *rtable);
324 static void JumbleExpr(pgssJumbleState *jstate, Node *node);
325 static void RecordConstLocation(pgssJumbleState *jstate, int location);
326 static char *generate_normalized_query(pgssJumbleState *jstate, const char *query,
327 int *query_len_p, int encoding);
328 static void fill_in_constant_lengths(pgssJumbleState *jstate, const char *query);
329 static int comp_location(const void *a, const void *b);
330
331
332 /*
333 * Module load callback
334 */
335 void
_PG_init(void)336 _PG_init(void)
337 {
338 /*
339 * In order to create our shared memory area, we have to be loaded via
340 * shared_preload_libraries. If not, fall out without hooking into any of
341 * the main system. (We don't throw error here because it seems useful to
342 * allow the pg_stat_statements functions to be created even when the
343 * module isn't active. The functions must protect themselves against
344 * being called then, however.)
345 */
346 if (!process_shared_preload_libraries_in_progress)
347 return;
348
349 /*
350 * Define (or redefine) custom GUC variables.
351 */
352 DefineCustomIntVariable("pg_stat_statements.max",
353 "Sets the maximum number of statements tracked by pg_stat_statements.",
354 NULL,
355 &pgss_max,
356 5000,
357 100,
358 INT_MAX,
359 PGC_POSTMASTER,
360 0,
361 NULL,
362 NULL,
363 NULL);
364
365 DefineCustomEnumVariable("pg_stat_statements.track",
366 "Selects which statements are tracked by pg_stat_statements.",
367 NULL,
368 &pgss_track,
369 PGSS_TRACK_TOP,
370 track_options,
371 PGC_SUSET,
372 0,
373 NULL,
374 NULL,
375 NULL);
376
377 DefineCustomBoolVariable("pg_stat_statements.track_utility",
378 "Selects whether utility commands are tracked by pg_stat_statements.",
379 NULL,
380 &pgss_track_utility,
381 true,
382 PGC_SUSET,
383 0,
384 NULL,
385 NULL,
386 NULL);
387
388 DefineCustomBoolVariable("pg_stat_statements.save",
389 "Save pg_stat_statements statistics across server shutdowns.",
390 NULL,
391 &pgss_save,
392 true,
393 PGC_SIGHUP,
394 0,
395 NULL,
396 NULL,
397 NULL);
398
399 EmitWarningsOnPlaceholders("pg_stat_statements");
400
401 /*
402 * Request additional shared resources. (These are no-ops if we're not in
403 * the postmaster process.) We'll allocate or attach to the shared
404 * resources in pgss_shmem_startup().
405 */
406 RequestAddinShmemSpace(pgss_memsize());
407 RequestNamedLWLockTranche("pg_stat_statements", 1);
408
409 /*
410 * Install hooks.
411 */
412 prev_shmem_startup_hook = shmem_startup_hook;
413 shmem_startup_hook = pgss_shmem_startup;
414 prev_post_parse_analyze_hook = post_parse_analyze_hook;
415 post_parse_analyze_hook = pgss_post_parse_analyze;
416 prev_ExecutorStart = ExecutorStart_hook;
417 ExecutorStart_hook = pgss_ExecutorStart;
418 prev_ExecutorRun = ExecutorRun_hook;
419 ExecutorRun_hook = pgss_ExecutorRun;
420 prev_ExecutorFinish = ExecutorFinish_hook;
421 ExecutorFinish_hook = pgss_ExecutorFinish;
422 prev_ExecutorEnd = ExecutorEnd_hook;
423 ExecutorEnd_hook = pgss_ExecutorEnd;
424 prev_ProcessUtility = ProcessUtility_hook;
425 ProcessUtility_hook = pgss_ProcessUtility;
426 }
427
428 /*
429 * Module unload callback
430 */
431 void
_PG_fini(void)432 _PG_fini(void)
433 {
434 /* Uninstall hooks. */
435 shmem_startup_hook = prev_shmem_startup_hook;
436 post_parse_analyze_hook = prev_post_parse_analyze_hook;
437 ExecutorStart_hook = prev_ExecutorStart;
438 ExecutorRun_hook = prev_ExecutorRun;
439 ExecutorFinish_hook = prev_ExecutorFinish;
440 ExecutorEnd_hook = prev_ExecutorEnd;
441 ProcessUtility_hook = prev_ProcessUtility;
442 }
443
444 /*
445 * shmem_startup hook: allocate or attach to shared memory,
446 * then load any pre-existing statistics from file.
447 * Also create and load the query-texts file, which is expected to exist
448 * (even if empty) while the module is enabled.
449 */
450 static void
pgss_shmem_startup(void)451 pgss_shmem_startup(void)
452 {
453 bool found;
454 HASHCTL info;
455 FILE *file = NULL;
456 FILE *qfile = NULL;
457 uint32 header;
458 int32 num;
459 int32 pgver;
460 int32 i;
461 int buffer_size;
462 char *buffer = NULL;
463
464 if (prev_shmem_startup_hook)
465 prev_shmem_startup_hook();
466
467 /* reset in case this is a restart within the postmaster */
468 pgss = NULL;
469 pgss_hash = NULL;
470
471 /*
472 * Create or attach to the shared memory state, including hash table
473 */
474 LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
475
476 pgss = ShmemInitStruct("pg_stat_statements",
477 sizeof(pgssSharedState),
478 &found);
479
480 if (!found)
481 {
482 /* First time through ... */
483 pgss->lock = &(GetNamedLWLockTranche("pg_stat_statements"))->lock;
484 pgss->cur_median_usage = ASSUMED_MEDIAN_INIT;
485 pgss->mean_query_len = ASSUMED_LENGTH_INIT;
486 SpinLockInit(&pgss->mutex);
487 pgss->extent = 0;
488 pgss->n_writers = 0;
489 pgss->gc_count = 0;
490 }
491
492 memset(&info, 0, sizeof(info));
493 info.keysize = sizeof(pgssHashKey);
494 info.entrysize = sizeof(pgssEntry);
495 info.hash = pgss_hash_fn;
496 info.match = pgss_match_fn;
497 pgss_hash = ShmemInitHash("pg_stat_statements hash",
498 pgss_max, pgss_max,
499 &info,
500 HASH_ELEM | HASH_FUNCTION | HASH_COMPARE);
501
502 LWLockRelease(AddinShmemInitLock);
503
504 /*
505 * If we're in the postmaster (or a standalone backend...), set up a shmem
506 * exit hook to dump the statistics to disk.
507 */
508 if (!IsUnderPostmaster)
509 on_shmem_exit(pgss_shmem_shutdown, (Datum) 0);
510
511 /*
512 * Done if some other process already completed our initialization.
513 */
514 if (found)
515 return;
516
517 /*
518 * Note: we don't bother with locks here, because there should be no other
519 * processes running when this code is reached.
520 */
521
522 /* Unlink query text file possibly left over from crash */
523 unlink(PGSS_TEXT_FILE);
524
525 /* Allocate new query text temp file */
526 qfile = AllocateFile(PGSS_TEXT_FILE, PG_BINARY_W);
527 if (qfile == NULL)
528 goto write_error;
529
530 /*
531 * If we were told not to load old statistics, we're done. (Note we do
532 * not try to unlink any old dump file in this case. This seems a bit
533 * questionable but it's the historical behavior.)
534 */
535 if (!pgss_save)
536 {
537 FreeFile(qfile);
538 return;
539 }
540
541 /*
542 * Attempt to load old statistics from the dump file.
543 */
544 file = AllocateFile(PGSS_DUMP_FILE, PG_BINARY_R);
545 if (file == NULL)
546 {
547 if (errno != ENOENT)
548 goto read_error;
549 /* No existing persisted stats file, so we're done */
550 FreeFile(qfile);
551 return;
552 }
553
554 buffer_size = 2048;
555 buffer = (char *) palloc(buffer_size);
556
557 if (fread(&header, sizeof(uint32), 1, file) != 1 ||
558 fread(&pgver, sizeof(uint32), 1, file) != 1 ||
559 fread(&num, sizeof(int32), 1, file) != 1)
560 goto read_error;
561
562 if (header != PGSS_FILE_HEADER ||
563 pgver != PGSS_PG_MAJOR_VERSION)
564 goto data_error;
565
566 for (i = 0; i < num; i++)
567 {
568 pgssEntry temp;
569 pgssEntry *entry;
570 Size query_offset;
571
572 if (fread(&temp, sizeof(pgssEntry), 1, file) != 1)
573 goto read_error;
574
575 /* Encoding is the only field we can easily sanity-check */
576 if (!PG_VALID_BE_ENCODING(temp.encoding))
577 goto data_error;
578
579 /* Resize buffer as needed */
580 if (temp.query_len >= buffer_size)
581 {
582 buffer_size = Max(buffer_size * 2, temp.query_len + 1);
583 buffer = repalloc(buffer, buffer_size);
584 }
585
586 if (fread(buffer, 1, temp.query_len + 1, file) != temp.query_len + 1)
587 goto read_error;
588
589 /* Should have a trailing null, but let's make sure */
590 buffer[temp.query_len] = '\0';
591
592 /* Skip loading "sticky" entries */
593 if (temp.counters.calls == 0)
594 continue;
595
596 /* Store the query text */
597 query_offset = pgss->extent;
598 if (fwrite(buffer, 1, temp.query_len + 1, qfile) != temp.query_len + 1)
599 goto write_error;
600 pgss->extent += temp.query_len + 1;
601
602 /* make the hashtable entry (discards old entries if too many) */
603 entry = entry_alloc(&temp.key, query_offset, temp.query_len,
604 temp.encoding,
605 false);
606
607 /* copy in the actual stats */
608 entry->counters = temp.counters;
609 }
610
611 pfree(buffer);
612 FreeFile(file);
613 FreeFile(qfile);
614
615 /*
616 * Remove the persisted stats file so it's not included in
617 * backups/replication slaves, etc. A new file will be written on next
618 * shutdown.
619 *
620 * Note: it's okay if the PGSS_TEXT_FILE is included in a basebackup,
621 * because we remove that file on startup; it acts inversely to
622 * PGSS_DUMP_FILE, in that it is only supposed to be around when the
623 * server is running, whereas PGSS_DUMP_FILE is only supposed to be around
624 * when the server is not running. Leaving the file creates no danger of
625 * a newly restored database having a spurious record of execution costs,
626 * which is what we're really concerned about here.
627 */
628 unlink(PGSS_DUMP_FILE);
629
630 return;
631
632 read_error:
633 ereport(LOG,
634 (errcode_for_file_access(),
635 errmsg("could not read pg_stat_statement file \"%s\": %m",
636 PGSS_DUMP_FILE)));
637 goto fail;
638 data_error:
639 ereport(LOG,
640 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
641 errmsg("ignoring invalid data in pg_stat_statement file \"%s\"",
642 PGSS_DUMP_FILE)));
643 goto fail;
644 write_error:
645 ereport(LOG,
646 (errcode_for_file_access(),
647 errmsg("could not write pg_stat_statement file \"%s\": %m",
648 PGSS_TEXT_FILE)));
649 fail:
650 if (buffer)
651 pfree(buffer);
652 if (file)
653 FreeFile(file);
654 if (qfile)
655 FreeFile(qfile);
656 /* If possible, throw away the bogus file; ignore any error */
657 unlink(PGSS_DUMP_FILE);
658
659 /*
660 * Don't unlink PGSS_TEXT_FILE here; it should always be around while the
661 * server is running with pg_stat_statements enabled
662 */
663 }
664
665 /*
666 * shmem_shutdown hook: Dump statistics into file.
667 *
668 * Note: we don't bother with acquiring lock, because there should be no
669 * other processes running when this is called.
670 */
671 static void
pgss_shmem_shutdown(int code,Datum arg)672 pgss_shmem_shutdown(int code, Datum arg)
673 {
674 FILE *file;
675 char *qbuffer = NULL;
676 Size qbuffer_size = 0;
677 HASH_SEQ_STATUS hash_seq;
678 int32 num_entries;
679 pgssEntry *entry;
680
681 /* Don't try to dump during a crash. */
682 if (code)
683 return;
684
685 /* Safety check ... shouldn't get here unless shmem is set up. */
686 if (!pgss || !pgss_hash)
687 return;
688
689 /* Don't dump if told not to. */
690 if (!pgss_save)
691 return;
692
693 file = AllocateFile(PGSS_DUMP_FILE ".tmp", PG_BINARY_W);
694 if (file == NULL)
695 goto error;
696
697 if (fwrite(&PGSS_FILE_HEADER, sizeof(uint32), 1, file) != 1)
698 goto error;
699 if (fwrite(&PGSS_PG_MAJOR_VERSION, sizeof(uint32), 1, file) != 1)
700 goto error;
701 num_entries = hash_get_num_entries(pgss_hash);
702 if (fwrite(&num_entries, sizeof(int32), 1, file) != 1)
703 goto error;
704
705 qbuffer = qtext_load_file(&qbuffer_size);
706 if (qbuffer == NULL)
707 goto error;
708
709 /*
710 * When serializing to disk, we store query texts immediately after their
711 * entry data. Any orphaned query texts are thereby excluded.
712 */
713 hash_seq_init(&hash_seq, pgss_hash);
714 while ((entry = hash_seq_search(&hash_seq)) != NULL)
715 {
716 int len = entry->query_len;
717 char *qstr = qtext_fetch(entry->query_offset, len,
718 qbuffer, qbuffer_size);
719
720 if (qstr == NULL)
721 continue; /* Ignore any entries with bogus texts */
722
723 if (fwrite(entry, sizeof(pgssEntry), 1, file) != 1 ||
724 fwrite(qstr, 1, len + 1, file) != len + 1)
725 {
726 /* note: we assume hash_seq_term won't change errno */
727 hash_seq_term(&hash_seq);
728 goto error;
729 }
730 }
731
732 free(qbuffer);
733 qbuffer = NULL;
734
735 if (FreeFile(file))
736 {
737 file = NULL;
738 goto error;
739 }
740
741 /*
742 * Rename file into place, so we atomically replace any old one.
743 */
744 (void) durable_rename(PGSS_DUMP_FILE ".tmp", PGSS_DUMP_FILE, LOG);
745
746 /* Unlink query-texts file; it's not needed while shutdown */
747 unlink(PGSS_TEXT_FILE);
748
749 return;
750
751 error:
752 ereport(LOG,
753 (errcode_for_file_access(),
754 errmsg("could not write pg_stat_statement file \"%s\": %m",
755 PGSS_DUMP_FILE ".tmp")));
756 if (qbuffer)
757 free(qbuffer);
758 if (file)
759 FreeFile(file);
760 unlink(PGSS_DUMP_FILE ".tmp");
761 unlink(PGSS_TEXT_FILE);
762 }
763
764 /*
765 * Post-parse-analysis hook: mark query with a queryId
766 */
767 static void
pgss_post_parse_analyze(ParseState * pstate,Query * query)768 pgss_post_parse_analyze(ParseState *pstate, Query *query)
769 {
770 pgssJumbleState jstate;
771
772 if (prev_post_parse_analyze_hook)
773 prev_post_parse_analyze_hook(pstate, query);
774
775 /* Assert we didn't do this already */
776 Assert(query->queryId == 0);
777
778 /* Safety check... */
779 if (!pgss || !pgss_hash)
780 return;
781
782 /*
783 * Utility statements get queryId zero. We do this even in cases where
784 * the statement contains an optimizable statement for which a queryId
785 * could be derived (such as EXPLAIN or DECLARE CURSOR). For such cases,
786 * runtime control will first go through ProcessUtility and then the
787 * executor, and we don't want the executor hooks to do anything, since we
788 * are already measuring the statement's costs at the utility level.
789 */
790 if (query->utilityStmt)
791 {
792 query->queryId = 0;
793 return;
794 }
795
796 /* Set up workspace for query jumbling */
797 jstate.jumble = (unsigned char *) palloc(JUMBLE_SIZE);
798 jstate.jumble_len = 0;
799 jstate.clocations_buf_size = 32;
800 jstate.clocations = (pgssLocationLen *)
801 palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen));
802 jstate.clocations_count = 0;
803
804 /* Compute query ID and mark the Query node with it */
805 JumbleQuery(&jstate, query);
806 query->queryId = hash_any(jstate.jumble, jstate.jumble_len);
807
808 /*
809 * If we are unlucky enough to get a hash of zero, use 1 instead, to
810 * prevent confusion with the utility-statement case.
811 */
812 if (query->queryId == 0)
813 query->queryId = 1;
814
815 /*
816 * If we were able to identify any ignorable constants, we immediately
817 * create a hash table entry for the query, so that we can record the
818 * normalized form of the query string. If there were no such constants,
819 * the normalized string would be the same as the query text anyway, so
820 * there's no need for an early entry.
821 */
822 if (jstate.clocations_count > 0)
823 pgss_store(pstate->p_sourcetext,
824 query->queryId,
825 0,
826 0,
827 NULL,
828 &jstate);
829 }
830
831 /*
832 * ExecutorStart hook: start up tracking if needed
833 */
834 static void
pgss_ExecutorStart(QueryDesc * queryDesc,int eflags)835 pgss_ExecutorStart(QueryDesc *queryDesc, int eflags)
836 {
837 if (prev_ExecutorStart)
838 prev_ExecutorStart(queryDesc, eflags);
839 else
840 standard_ExecutorStart(queryDesc, eflags);
841
842 /*
843 * If query has queryId zero, don't track it. This prevents double
844 * counting of optimizable statements that are directly contained in
845 * utility statements.
846 */
847 if (pgss_enabled() && queryDesc->plannedstmt->queryId != 0)
848 {
849 /*
850 * Set up to track total elapsed time in ExecutorRun. Make sure the
851 * space is allocated in the per-query context so it will go away at
852 * ExecutorEnd.
853 */
854 if (queryDesc->totaltime == NULL)
855 {
856 MemoryContext oldcxt;
857
858 oldcxt = MemoryContextSwitchTo(queryDesc->estate->es_query_cxt);
859 queryDesc->totaltime = InstrAlloc(1, INSTRUMENT_ALL);
860 MemoryContextSwitchTo(oldcxt);
861 }
862 }
863 }
864
865 /*
866 * ExecutorRun hook: all we need do is track nesting depth
867 */
868 static void
pgss_ExecutorRun(QueryDesc * queryDesc,ScanDirection direction,uint64 count)869 pgss_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count)
870 {
871 nested_level++;
872 PG_TRY();
873 {
874 if (prev_ExecutorRun)
875 prev_ExecutorRun(queryDesc, direction, count);
876 else
877 standard_ExecutorRun(queryDesc, direction, count);
878 nested_level--;
879 }
880 PG_CATCH();
881 {
882 nested_level--;
883 PG_RE_THROW();
884 }
885 PG_END_TRY();
886 }
887
888 /*
889 * ExecutorFinish hook: all we need do is track nesting depth
890 */
891 static void
pgss_ExecutorFinish(QueryDesc * queryDesc)892 pgss_ExecutorFinish(QueryDesc *queryDesc)
893 {
894 nested_level++;
895 PG_TRY();
896 {
897 if (prev_ExecutorFinish)
898 prev_ExecutorFinish(queryDesc);
899 else
900 standard_ExecutorFinish(queryDesc);
901 nested_level--;
902 }
903 PG_CATCH();
904 {
905 nested_level--;
906 PG_RE_THROW();
907 }
908 PG_END_TRY();
909 }
910
911 /*
912 * ExecutorEnd hook: store results if needed
913 */
914 static void
pgss_ExecutorEnd(QueryDesc * queryDesc)915 pgss_ExecutorEnd(QueryDesc *queryDesc)
916 {
917 uint32 queryId = queryDesc->plannedstmt->queryId;
918
919 if (queryId != 0 && queryDesc->totaltime && pgss_enabled())
920 {
921 /*
922 * Make sure stats accumulation is done. (Note: it's okay if several
923 * levels of hook all do this.)
924 */
925 InstrEndLoop(queryDesc->totaltime);
926
927 pgss_store(queryDesc->sourceText,
928 queryId,
929 queryDesc->totaltime->total * 1000.0, /* convert to msec */
930 queryDesc->estate->es_processed,
931 &queryDesc->totaltime->bufusage,
932 NULL);
933 }
934
935 if (prev_ExecutorEnd)
936 prev_ExecutorEnd(queryDesc);
937 else
938 standard_ExecutorEnd(queryDesc);
939 }
940
941 /*
942 * ProcessUtility hook
943 */
944 static void
pgss_ProcessUtility(Node * parsetree,const char * queryString,ProcessUtilityContext context,ParamListInfo params,DestReceiver * dest,char * completionTag)945 pgss_ProcessUtility(Node *parsetree, const char *queryString,
946 ProcessUtilityContext context, ParamListInfo params,
947 DestReceiver *dest, char *completionTag)
948 {
949 /*
950 * If it's an EXECUTE statement, we don't track it and don't increment the
951 * nesting level. This allows the cycles to be charged to the underlying
952 * PREPARE instead (by the Executor hooks), which is much more useful.
953 *
954 * We also don't track execution of PREPARE. If we did, we would get one
955 * hash table entry for the PREPARE (with hash calculated from the query
956 * string), and then a different one with the same query string (but hash
957 * calculated from the query tree) would be used to accumulate costs of
958 * ensuing EXECUTEs. This would be confusing, and inconsistent with other
959 * cases where planning time is not included at all.
960 *
961 * Likewise, we don't track execution of DEALLOCATE.
962 */
963 if (pgss_track_utility && pgss_enabled() &&
964 !IsA(parsetree, ExecuteStmt) &&
965 !IsA(parsetree, PrepareStmt) &&
966 !IsA(parsetree, DeallocateStmt))
967 {
968 instr_time start;
969 instr_time duration;
970 uint64 rows;
971 BufferUsage bufusage_start,
972 bufusage;
973 uint32 queryId;
974
975 bufusage_start = pgBufferUsage;
976 INSTR_TIME_SET_CURRENT(start);
977
978 nested_level++;
979 PG_TRY();
980 {
981 if (prev_ProcessUtility)
982 prev_ProcessUtility(parsetree, queryString,
983 context, params,
984 dest, completionTag);
985 else
986 standard_ProcessUtility(parsetree, queryString,
987 context, params,
988 dest, completionTag);
989 nested_level--;
990 }
991 PG_CATCH();
992 {
993 nested_level--;
994 PG_RE_THROW();
995 }
996 PG_END_TRY();
997
998 INSTR_TIME_SET_CURRENT(duration);
999 INSTR_TIME_SUBTRACT(duration, start);
1000
1001 /* parse command tag to retrieve the number of affected rows. */
1002 if (completionTag &&
1003 strncmp(completionTag, "COPY ", 5) == 0)
1004 rows = pg_strtouint64(completionTag + 5, NULL, 10);
1005 else
1006 rows = 0;
1007
1008 /* calc differences of buffer counters. */
1009 bufusage.shared_blks_hit =
1010 pgBufferUsage.shared_blks_hit - bufusage_start.shared_blks_hit;
1011 bufusage.shared_blks_read =
1012 pgBufferUsage.shared_blks_read - bufusage_start.shared_blks_read;
1013 bufusage.shared_blks_dirtied =
1014 pgBufferUsage.shared_blks_dirtied - bufusage_start.shared_blks_dirtied;
1015 bufusage.shared_blks_written =
1016 pgBufferUsage.shared_blks_written - bufusage_start.shared_blks_written;
1017 bufusage.local_blks_hit =
1018 pgBufferUsage.local_blks_hit - bufusage_start.local_blks_hit;
1019 bufusage.local_blks_read =
1020 pgBufferUsage.local_blks_read - bufusage_start.local_blks_read;
1021 bufusage.local_blks_dirtied =
1022 pgBufferUsage.local_blks_dirtied - bufusage_start.local_blks_dirtied;
1023 bufusage.local_blks_written =
1024 pgBufferUsage.local_blks_written - bufusage_start.local_blks_written;
1025 bufusage.temp_blks_read =
1026 pgBufferUsage.temp_blks_read - bufusage_start.temp_blks_read;
1027 bufusage.temp_blks_written =
1028 pgBufferUsage.temp_blks_written - bufusage_start.temp_blks_written;
1029 bufusage.blk_read_time = pgBufferUsage.blk_read_time;
1030 INSTR_TIME_SUBTRACT(bufusage.blk_read_time, bufusage_start.blk_read_time);
1031 bufusage.blk_write_time = pgBufferUsage.blk_write_time;
1032 INSTR_TIME_SUBTRACT(bufusage.blk_write_time, bufusage_start.blk_write_time);
1033
1034 /* For utility statements, we just hash the query string directly */
1035 queryId = pgss_hash_string(queryString);
1036
1037 pgss_store(queryString,
1038 queryId,
1039 INSTR_TIME_GET_MILLISEC(duration),
1040 rows,
1041 &bufusage,
1042 NULL);
1043 }
1044 else
1045 {
1046 if (prev_ProcessUtility)
1047 prev_ProcessUtility(parsetree, queryString,
1048 context, params,
1049 dest, completionTag);
1050 else
1051 standard_ProcessUtility(parsetree, queryString,
1052 context, params,
1053 dest, completionTag);
1054 }
1055 }
1056
1057 /*
1058 * Calculate hash value for a key
1059 */
1060 static uint32
pgss_hash_fn(const void * key,Size keysize)1061 pgss_hash_fn(const void *key, Size keysize)
1062 {
1063 const pgssHashKey *k = (const pgssHashKey *) key;
1064
1065 return hash_uint32((uint32) k->userid) ^
1066 hash_uint32((uint32) k->dbid) ^
1067 hash_uint32((uint32) k->queryid);
1068 }
1069
1070 /*
1071 * Compare two keys - zero means match
1072 */
1073 static int
pgss_match_fn(const void * key1,const void * key2,Size keysize)1074 pgss_match_fn(const void *key1, const void *key2, Size keysize)
1075 {
1076 const pgssHashKey *k1 = (const pgssHashKey *) key1;
1077 const pgssHashKey *k2 = (const pgssHashKey *) key2;
1078
1079 if (k1->userid == k2->userid &&
1080 k1->dbid == k2->dbid &&
1081 k1->queryid == k2->queryid)
1082 return 0;
1083 else
1084 return 1;
1085 }
1086
1087 /*
1088 * Given an arbitrarily long query string, produce a hash for the purposes of
1089 * identifying the query, without normalizing constants. Used when hashing
1090 * utility statements.
1091 */
1092 static uint32
pgss_hash_string(const char * str)1093 pgss_hash_string(const char *str)
1094 {
1095 return hash_any((const unsigned char *) str, strlen(str));
1096 }
1097
1098 /*
1099 * Store some statistics for a statement.
1100 *
1101 * If jstate is not NULL then we're trying to create an entry for which
1102 * we have no statistics as yet; we just want to record the normalized
1103 * query string. total_time, rows, bufusage are ignored in this case.
1104 */
1105 static void
pgss_store(const char * query,uint32 queryId,double total_time,uint64 rows,const BufferUsage * bufusage,pgssJumbleState * jstate)1106 pgss_store(const char *query, uint32 queryId,
1107 double total_time, uint64 rows,
1108 const BufferUsage *bufusage,
1109 pgssJumbleState *jstate)
1110 {
1111 pgssHashKey key;
1112 pgssEntry *entry;
1113 char *norm_query = NULL;
1114 int encoding = GetDatabaseEncoding();
1115 int query_len;
1116
1117 Assert(query != NULL);
1118
1119 /* Safety check... */
1120 if (!pgss || !pgss_hash)
1121 return;
1122
1123 query_len = strlen(query);
1124
1125 /* Set up key for hashtable search */
1126 key.userid = GetUserId();
1127 key.dbid = MyDatabaseId;
1128 key.queryid = queryId;
1129
1130 /* Lookup the hash table entry with shared lock. */
1131 LWLockAcquire(pgss->lock, LW_SHARED);
1132
1133 entry = (pgssEntry *) hash_search(pgss_hash, &key, HASH_FIND, NULL);
1134
1135 /* Create new entry, if not present */
1136 if (!entry)
1137 {
1138 Size query_offset;
1139 int gc_count;
1140 bool stored;
1141 bool do_gc;
1142
1143 /*
1144 * Create a new, normalized query string if caller asked. We don't
1145 * need to hold the lock while doing this work. (Note: in any case,
1146 * it's possible that someone else creates a duplicate hashtable entry
1147 * in the interval where we don't hold the lock below. That case is
1148 * handled by entry_alloc.)
1149 */
1150 if (jstate)
1151 {
1152 LWLockRelease(pgss->lock);
1153 norm_query = generate_normalized_query(jstate, query,
1154 &query_len,
1155 encoding);
1156 LWLockAcquire(pgss->lock, LW_SHARED);
1157 }
1158
1159 /* Append new query text to file with only shared lock held */
1160 stored = qtext_store(norm_query ? norm_query : query, query_len,
1161 &query_offset, &gc_count);
1162
1163 /*
1164 * Determine whether we need to garbage collect external query texts
1165 * while the shared lock is still held. This micro-optimization
1166 * avoids taking the time to decide this while holding exclusive lock.
1167 */
1168 do_gc = need_gc_qtexts();
1169
1170 /* Need exclusive lock to make a new hashtable entry - promote */
1171 LWLockRelease(pgss->lock);
1172 LWLockAcquire(pgss->lock, LW_EXCLUSIVE);
1173
1174 /*
1175 * A garbage collection may have occurred while we weren't holding the
1176 * lock. In the unlikely event that this happens, the query text we
1177 * stored above will have been garbage collected, so write it again.
1178 * This should be infrequent enough that doing it while holding
1179 * exclusive lock isn't a performance problem.
1180 */
1181 if (!stored || pgss->gc_count != gc_count)
1182 stored = qtext_store(norm_query ? norm_query : query, query_len,
1183 &query_offset, NULL);
1184
1185 /* If we failed to write to the text file, give up */
1186 if (!stored)
1187 goto done;
1188
1189 /* OK to create a new hashtable entry */
1190 entry = entry_alloc(&key, query_offset, query_len, encoding,
1191 jstate != NULL);
1192
1193 /* If needed, perform garbage collection while exclusive lock held */
1194 if (do_gc)
1195 gc_qtexts();
1196 }
1197
1198 /* Increment the counts, except when jstate is not NULL */
1199 if (!jstate)
1200 {
1201 /*
1202 * Grab the spinlock while updating the counters (see comment about
1203 * locking rules at the head of the file)
1204 */
1205 volatile pgssEntry *e = (volatile pgssEntry *) entry;
1206
1207 SpinLockAcquire(&e->mutex);
1208
1209 /* "Unstick" entry if it was previously sticky */
1210 if (e->counters.calls == 0)
1211 e->counters.usage = USAGE_INIT;
1212
1213 e->counters.calls += 1;
1214 e->counters.total_time += total_time;
1215 if (e->counters.calls == 1)
1216 {
1217 e->counters.min_time = total_time;
1218 e->counters.max_time = total_time;
1219 e->counters.mean_time = total_time;
1220 }
1221 else
1222 {
1223 /*
1224 * Welford's method for accurately computing variance. See
1225 * <http://www.johndcook.com/blog/standard_deviation/>
1226 */
1227 double old_mean = e->counters.mean_time;
1228
1229 e->counters.mean_time +=
1230 (total_time - old_mean) / e->counters.calls;
1231 e->counters.sum_var_time +=
1232 (total_time - old_mean) * (total_time - e->counters.mean_time);
1233
1234 /* calculate min and max time */
1235 if (e->counters.min_time > total_time)
1236 e->counters.min_time = total_time;
1237 if (e->counters.max_time < total_time)
1238 e->counters.max_time = total_time;
1239 }
1240 e->counters.rows += rows;
1241 e->counters.shared_blks_hit += bufusage->shared_blks_hit;
1242 e->counters.shared_blks_read += bufusage->shared_blks_read;
1243 e->counters.shared_blks_dirtied += bufusage->shared_blks_dirtied;
1244 e->counters.shared_blks_written += bufusage->shared_blks_written;
1245 e->counters.local_blks_hit += bufusage->local_blks_hit;
1246 e->counters.local_blks_read += bufusage->local_blks_read;
1247 e->counters.local_blks_dirtied += bufusage->local_blks_dirtied;
1248 e->counters.local_blks_written += bufusage->local_blks_written;
1249 e->counters.temp_blks_read += bufusage->temp_blks_read;
1250 e->counters.temp_blks_written += bufusage->temp_blks_written;
1251 e->counters.blk_read_time += INSTR_TIME_GET_MILLISEC(bufusage->blk_read_time);
1252 e->counters.blk_write_time += INSTR_TIME_GET_MILLISEC(bufusage->blk_write_time);
1253 e->counters.usage += USAGE_EXEC(total_time);
1254
1255 SpinLockRelease(&e->mutex);
1256 }
1257
1258 done:
1259 LWLockRelease(pgss->lock);
1260
1261 /* We postpone this clean-up until we're out of the lock */
1262 if (norm_query)
1263 pfree(norm_query);
1264 }
1265
1266 /*
1267 * Reset all statement statistics.
1268 */
1269 Datum
pg_stat_statements_reset(PG_FUNCTION_ARGS)1270 pg_stat_statements_reset(PG_FUNCTION_ARGS)
1271 {
1272 if (!pgss || !pgss_hash)
1273 ereport(ERROR,
1274 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1275 errmsg("pg_stat_statements must be loaded via shared_preload_libraries")));
1276 entry_reset();
1277 PG_RETURN_VOID();
1278 }
1279
1280 /* Number of output arguments (columns) for various API versions */
1281 #define PG_STAT_STATEMENTS_COLS_V1_0 14
1282 #define PG_STAT_STATEMENTS_COLS_V1_1 18
1283 #define PG_STAT_STATEMENTS_COLS_V1_2 19
1284 #define PG_STAT_STATEMENTS_COLS_V1_3 23
1285 #define PG_STAT_STATEMENTS_COLS 23 /* maximum of above */
1286
1287 /*
1288 * Retrieve statement statistics.
1289 *
1290 * The SQL API of this function has changed multiple times, and will likely
1291 * do so again in future. To support the case where a newer version of this
1292 * loadable module is being used with an old SQL declaration of the function,
1293 * we continue to support the older API versions. For 1.2 and later, the
1294 * expected API version is identified by embedding it in the C name of the
1295 * function. Unfortunately we weren't bright enough to do that for 1.1.
1296 */
1297 Datum
pg_stat_statements_1_3(PG_FUNCTION_ARGS)1298 pg_stat_statements_1_3(PG_FUNCTION_ARGS)
1299 {
1300 bool showtext = PG_GETARG_BOOL(0);
1301
1302 pg_stat_statements_internal(fcinfo, PGSS_V1_3, showtext);
1303
1304 return (Datum) 0;
1305 }
1306
1307 Datum
pg_stat_statements_1_2(PG_FUNCTION_ARGS)1308 pg_stat_statements_1_2(PG_FUNCTION_ARGS)
1309 {
1310 bool showtext = PG_GETARG_BOOL(0);
1311
1312 pg_stat_statements_internal(fcinfo, PGSS_V1_2, showtext);
1313
1314 return (Datum) 0;
1315 }
1316
1317 /*
1318 * Legacy entry point for pg_stat_statements() API versions 1.0 and 1.1.
1319 * This can be removed someday, perhaps.
1320 */
1321 Datum
pg_stat_statements(PG_FUNCTION_ARGS)1322 pg_stat_statements(PG_FUNCTION_ARGS)
1323 {
1324 /* If it's really API 1.1, we'll figure that out below */
1325 pg_stat_statements_internal(fcinfo, PGSS_V1_0, true);
1326
1327 return (Datum) 0;
1328 }
1329
1330 /* Common code for all versions of pg_stat_statements() */
1331 static void
pg_stat_statements_internal(FunctionCallInfo fcinfo,pgssVersion api_version,bool showtext)1332 pg_stat_statements_internal(FunctionCallInfo fcinfo,
1333 pgssVersion api_version,
1334 bool showtext)
1335 {
1336 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
1337 TupleDesc tupdesc;
1338 Tuplestorestate *tupstore;
1339 MemoryContext per_query_ctx;
1340 MemoryContext oldcontext;
1341 Oid userid = GetUserId();
1342 bool is_superuser = superuser();
1343 char *qbuffer = NULL;
1344 Size qbuffer_size = 0;
1345 Size extent = 0;
1346 int gc_count = 0;
1347 HASH_SEQ_STATUS hash_seq;
1348 pgssEntry *entry;
1349
1350 /* hash table must exist already */
1351 if (!pgss || !pgss_hash)
1352 ereport(ERROR,
1353 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1354 errmsg("pg_stat_statements must be loaded via shared_preload_libraries")));
1355
1356 /* check to see if caller supports us returning a tuplestore */
1357 if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
1358 ereport(ERROR,
1359 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1360 errmsg("set-valued function called in context that cannot accept a set")));
1361 if (!(rsinfo->allowedModes & SFRM_Materialize))
1362 ereport(ERROR,
1363 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1364 errmsg("materialize mode required, but it is not " \
1365 "allowed in this context")));
1366
1367 /* Switch into long-lived context to construct returned data structures */
1368 per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
1369 oldcontext = MemoryContextSwitchTo(per_query_ctx);
1370
1371 /* Build a tuple descriptor for our result type */
1372 if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
1373 elog(ERROR, "return type must be a row type");
1374
1375 /*
1376 * Check we have the expected number of output arguments. Aside from
1377 * being a good safety check, we need a kluge here to detect API version
1378 * 1.1, which was wedged into the code in an ill-considered way.
1379 */
1380 switch (tupdesc->natts)
1381 {
1382 case PG_STAT_STATEMENTS_COLS_V1_0:
1383 if (api_version != PGSS_V1_0)
1384 elog(ERROR, "incorrect number of output arguments");
1385 break;
1386 case PG_STAT_STATEMENTS_COLS_V1_1:
1387 /* pg_stat_statements() should have told us 1.0 */
1388 if (api_version != PGSS_V1_0)
1389 elog(ERROR, "incorrect number of output arguments");
1390 api_version = PGSS_V1_1;
1391 break;
1392 case PG_STAT_STATEMENTS_COLS_V1_2:
1393 if (api_version != PGSS_V1_2)
1394 elog(ERROR, "incorrect number of output arguments");
1395 break;
1396 case PG_STAT_STATEMENTS_COLS_V1_3:
1397 if (api_version != PGSS_V1_3)
1398 elog(ERROR, "incorrect number of output arguments");
1399 break;
1400 default:
1401 elog(ERROR, "incorrect number of output arguments");
1402 }
1403
1404 tupstore = tuplestore_begin_heap(true, false, work_mem);
1405 rsinfo->returnMode = SFRM_Materialize;
1406 rsinfo->setResult = tupstore;
1407 rsinfo->setDesc = tupdesc;
1408
1409 MemoryContextSwitchTo(oldcontext);
1410
1411 /*
1412 * We'd like to load the query text file (if needed) while not holding any
1413 * lock on pgss->lock. In the worst case we'll have to do this again
1414 * after we have the lock, but it's unlikely enough to make this a win
1415 * despite occasional duplicated work. We need to reload if anybody
1416 * writes to the file (either a retail qtext_store(), or a garbage
1417 * collection) between this point and where we've gotten shared lock. If
1418 * a qtext_store is actually in progress when we look, we might as well
1419 * skip the speculative load entirely.
1420 */
1421 if (showtext)
1422 {
1423 int n_writers;
1424
1425 /* Take the mutex so we can examine variables */
1426 {
1427 volatile pgssSharedState *s = (volatile pgssSharedState *) pgss;
1428
1429 SpinLockAcquire(&s->mutex);
1430 extent = s->extent;
1431 n_writers = s->n_writers;
1432 gc_count = s->gc_count;
1433 SpinLockRelease(&s->mutex);
1434 }
1435
1436 /* No point in loading file now if there are active writers */
1437 if (n_writers == 0)
1438 qbuffer = qtext_load_file(&qbuffer_size);
1439 }
1440
1441 /*
1442 * Get shared lock, load or reload the query text file if we must, and
1443 * iterate over the hashtable entries.
1444 *
1445 * With a large hash table, we might be holding the lock rather longer
1446 * than one could wish. However, this only blocks creation of new hash
1447 * table entries, and the larger the hash table the less likely that is to
1448 * be needed. So we can hope this is okay. Perhaps someday we'll decide
1449 * we need to partition the hash table to limit the time spent holding any
1450 * one lock.
1451 */
1452 LWLockAcquire(pgss->lock, LW_SHARED);
1453
1454 if (showtext)
1455 {
1456 /*
1457 * Here it is safe to examine extent and gc_count without taking the
1458 * mutex. Note that although other processes might change
1459 * pgss->extent just after we look at it, the strings they then write
1460 * into the file cannot yet be referenced in the hashtable, so we
1461 * don't care whether we see them or not.
1462 *
1463 * If qtext_load_file fails, we just press on; we'll return NULL for
1464 * every query text.
1465 */
1466 if (qbuffer == NULL ||
1467 pgss->extent != extent ||
1468 pgss->gc_count != gc_count)
1469 {
1470 if (qbuffer)
1471 free(qbuffer);
1472 qbuffer = qtext_load_file(&qbuffer_size);
1473 }
1474 }
1475
1476 hash_seq_init(&hash_seq, pgss_hash);
1477 while ((entry = hash_seq_search(&hash_seq)) != NULL)
1478 {
1479 Datum values[PG_STAT_STATEMENTS_COLS];
1480 bool nulls[PG_STAT_STATEMENTS_COLS];
1481 int i = 0;
1482 Counters tmp;
1483 double stddev;
1484 int64 queryid = entry->key.queryid;
1485
1486 memset(values, 0, sizeof(values));
1487 memset(nulls, 0, sizeof(nulls));
1488
1489 values[i++] = ObjectIdGetDatum(entry->key.userid);
1490 values[i++] = ObjectIdGetDatum(entry->key.dbid);
1491
1492 if (is_superuser || entry->key.userid == userid)
1493 {
1494 if (api_version >= PGSS_V1_2)
1495 values[i++] = Int64GetDatumFast(queryid);
1496
1497 if (showtext)
1498 {
1499 char *qstr = qtext_fetch(entry->query_offset,
1500 entry->query_len,
1501 qbuffer,
1502 qbuffer_size);
1503
1504 if (qstr)
1505 {
1506 char *enc;
1507
1508 enc = pg_any_to_server(qstr,
1509 entry->query_len,
1510 entry->encoding);
1511
1512 values[i++] = CStringGetTextDatum(enc);
1513
1514 if (enc != qstr)
1515 pfree(enc);
1516 }
1517 else
1518 {
1519 /* Just return a null if we fail to find the text */
1520 nulls[i++] = true;
1521 }
1522 }
1523 else
1524 {
1525 /* Query text not requested */
1526 nulls[i++] = true;
1527 }
1528 }
1529 else
1530 {
1531 /* Don't show queryid */
1532 if (api_version >= PGSS_V1_2)
1533 nulls[i++] = true;
1534
1535 /*
1536 * Don't show query text, but hint as to the reason for not doing
1537 * so if it was requested
1538 */
1539 if (showtext)
1540 values[i++] = CStringGetTextDatum("<insufficient privilege>");
1541 else
1542 nulls[i++] = true;
1543 }
1544
1545 /* copy counters to a local variable to keep locking time short */
1546 {
1547 volatile pgssEntry *e = (volatile pgssEntry *) entry;
1548
1549 SpinLockAcquire(&e->mutex);
1550 tmp = e->counters;
1551 SpinLockRelease(&e->mutex);
1552 }
1553
1554 /* Skip entry if unexecuted (ie, it's a pending "sticky" entry) */
1555 if (tmp.calls == 0)
1556 continue;
1557
1558 values[i++] = Int64GetDatumFast(tmp.calls);
1559 values[i++] = Float8GetDatumFast(tmp.total_time);
1560 if (api_version >= PGSS_V1_3)
1561 {
1562 values[i++] = Float8GetDatumFast(tmp.min_time);
1563 values[i++] = Float8GetDatumFast(tmp.max_time);
1564 values[i++] = Float8GetDatumFast(tmp.mean_time);
1565
1566 /*
1567 * Note we are calculating the population variance here, not the
1568 * sample variance, as we have data for the whole population, so
1569 * Bessel's correction is not used, and we don't divide by
1570 * tmp.calls - 1.
1571 */
1572 if (tmp.calls > 1)
1573 stddev = sqrt(tmp.sum_var_time / tmp.calls);
1574 else
1575 stddev = 0.0;
1576 values[i++] = Float8GetDatumFast(stddev);
1577 }
1578 values[i++] = Int64GetDatumFast(tmp.rows);
1579 values[i++] = Int64GetDatumFast(tmp.shared_blks_hit);
1580 values[i++] = Int64GetDatumFast(tmp.shared_blks_read);
1581 if (api_version >= PGSS_V1_1)
1582 values[i++] = Int64GetDatumFast(tmp.shared_blks_dirtied);
1583 values[i++] = Int64GetDatumFast(tmp.shared_blks_written);
1584 values[i++] = Int64GetDatumFast(tmp.local_blks_hit);
1585 values[i++] = Int64GetDatumFast(tmp.local_blks_read);
1586 if (api_version >= PGSS_V1_1)
1587 values[i++] = Int64GetDatumFast(tmp.local_blks_dirtied);
1588 values[i++] = Int64GetDatumFast(tmp.local_blks_written);
1589 values[i++] = Int64GetDatumFast(tmp.temp_blks_read);
1590 values[i++] = Int64GetDatumFast(tmp.temp_blks_written);
1591 if (api_version >= PGSS_V1_1)
1592 {
1593 values[i++] = Float8GetDatumFast(tmp.blk_read_time);
1594 values[i++] = Float8GetDatumFast(tmp.blk_write_time);
1595 }
1596
1597 Assert(i == (api_version == PGSS_V1_0 ? PG_STAT_STATEMENTS_COLS_V1_0 :
1598 api_version == PGSS_V1_1 ? PG_STAT_STATEMENTS_COLS_V1_1 :
1599 api_version == PGSS_V1_2 ? PG_STAT_STATEMENTS_COLS_V1_2 :
1600 api_version == PGSS_V1_3 ? PG_STAT_STATEMENTS_COLS_V1_3 :
1601 -1 /* fail if you forget to update this assert */ ));
1602
1603 tuplestore_putvalues(tupstore, tupdesc, values, nulls);
1604 }
1605
1606 /* clean up and return the tuplestore */
1607 LWLockRelease(pgss->lock);
1608
1609 if (qbuffer)
1610 free(qbuffer);
1611
1612 tuplestore_donestoring(tupstore);
1613 }
1614
1615 /*
1616 * Estimate shared memory space needed.
1617 */
1618 static Size
pgss_memsize(void)1619 pgss_memsize(void)
1620 {
1621 Size size;
1622
1623 size = MAXALIGN(sizeof(pgssSharedState));
1624 size = add_size(size, hash_estimate_size(pgss_max, sizeof(pgssEntry)));
1625
1626 return size;
1627 }
1628
1629 /*
1630 * Allocate a new hashtable entry.
1631 * caller must hold an exclusive lock on pgss->lock
1632 *
1633 * "query" need not be null-terminated; we rely on query_len instead
1634 *
1635 * If "sticky" is true, make the new entry artificially sticky so that it will
1636 * probably still be there when the query finishes execution. We do this by
1637 * giving it a median usage value rather than the normal value. (Strictly
1638 * speaking, query strings are normalized on a best effort basis, though it
1639 * would be difficult to demonstrate this even under artificial conditions.)
1640 *
1641 * Note: despite needing exclusive lock, it's not an error for the target
1642 * entry to already exist. This is because pgss_store releases and
1643 * reacquires lock after failing to find a match; so someone else could
1644 * have made the entry while we waited to get exclusive lock.
1645 */
1646 static pgssEntry *
entry_alloc(pgssHashKey * key,Size query_offset,int query_len,int encoding,bool sticky)1647 entry_alloc(pgssHashKey *key, Size query_offset, int query_len, int encoding,
1648 bool sticky)
1649 {
1650 pgssEntry *entry;
1651 bool found;
1652
1653 /* Make space if needed */
1654 while (hash_get_num_entries(pgss_hash) >= pgss_max)
1655 entry_dealloc();
1656
1657 /* Find or create an entry with desired hash code */
1658 entry = (pgssEntry *) hash_search(pgss_hash, key, HASH_ENTER, &found);
1659
1660 if (!found)
1661 {
1662 /* New entry, initialize it */
1663
1664 /* reset the statistics */
1665 memset(&entry->counters, 0, sizeof(Counters));
1666 /* set the appropriate initial usage count */
1667 entry->counters.usage = sticky ? pgss->cur_median_usage : USAGE_INIT;
1668 /* re-initialize the mutex each time ... we assume no one using it */
1669 SpinLockInit(&entry->mutex);
1670 /* ... and don't forget the query text metadata */
1671 Assert(query_len >= 0);
1672 entry->query_offset = query_offset;
1673 entry->query_len = query_len;
1674 entry->encoding = encoding;
1675 }
1676
1677 return entry;
1678 }
1679
1680 /*
1681 * qsort comparator for sorting into increasing usage order
1682 */
1683 static int
entry_cmp(const void * lhs,const void * rhs)1684 entry_cmp(const void *lhs, const void *rhs)
1685 {
1686 double l_usage = (*(pgssEntry *const *) lhs)->counters.usage;
1687 double r_usage = (*(pgssEntry *const *) rhs)->counters.usage;
1688
1689 if (l_usage < r_usage)
1690 return -1;
1691 else if (l_usage > r_usage)
1692 return +1;
1693 else
1694 return 0;
1695 }
1696
1697 /*
1698 * Deallocate least-used entries.
1699 *
1700 * Caller must hold an exclusive lock on pgss->lock.
1701 */
1702 static void
entry_dealloc(void)1703 entry_dealloc(void)
1704 {
1705 HASH_SEQ_STATUS hash_seq;
1706 pgssEntry **entries;
1707 pgssEntry *entry;
1708 int nvictims;
1709 int i;
1710 Size tottextlen;
1711 int nvalidtexts;
1712
1713 /*
1714 * Sort entries by usage and deallocate USAGE_DEALLOC_PERCENT of them.
1715 * While we're scanning the table, apply the decay factor to the usage
1716 * values, and update the mean query length.
1717 *
1718 * Note that the mean query length is almost immediately obsolete, since
1719 * we compute it before not after discarding the least-used entries.
1720 * Hopefully, that doesn't affect the mean too much; it doesn't seem worth
1721 * making two passes to get a more current result. Likewise, the new
1722 * cur_median_usage includes the entries we're about to zap.
1723 */
1724
1725 entries = palloc(hash_get_num_entries(pgss_hash) * sizeof(pgssEntry *));
1726
1727 i = 0;
1728 tottextlen = 0;
1729 nvalidtexts = 0;
1730
1731 hash_seq_init(&hash_seq, pgss_hash);
1732 while ((entry = hash_seq_search(&hash_seq)) != NULL)
1733 {
1734 entries[i++] = entry;
1735 /* "Sticky" entries get a different usage decay rate. */
1736 if (entry->counters.calls == 0)
1737 entry->counters.usage *= STICKY_DECREASE_FACTOR;
1738 else
1739 entry->counters.usage *= USAGE_DECREASE_FACTOR;
1740 /* In the mean length computation, ignore dropped texts. */
1741 if (entry->query_len >= 0)
1742 {
1743 tottextlen += entry->query_len + 1;
1744 nvalidtexts++;
1745 }
1746 }
1747
1748 /* Sort into increasing order by usage */
1749 qsort(entries, i, sizeof(pgssEntry *), entry_cmp);
1750
1751 /* Record the (approximate) median usage */
1752 if (i > 0)
1753 pgss->cur_median_usage = entries[i / 2]->counters.usage;
1754 /* Record the mean query length */
1755 if (nvalidtexts > 0)
1756 pgss->mean_query_len = tottextlen / nvalidtexts;
1757 else
1758 pgss->mean_query_len = ASSUMED_LENGTH_INIT;
1759
1760 /* Now zap an appropriate fraction of lowest-usage entries */
1761 nvictims = Max(10, i * USAGE_DEALLOC_PERCENT / 100);
1762 nvictims = Min(nvictims, i);
1763
1764 for (i = 0; i < nvictims; i++)
1765 {
1766 hash_search(pgss_hash, &entries[i]->key, HASH_REMOVE, NULL);
1767 }
1768
1769 pfree(entries);
1770 }
1771
1772 /*
1773 * Given a null-terminated string, allocate a new entry in the external query
1774 * text file and store the string there.
1775 *
1776 * Although we could compute the string length via strlen(), callers already
1777 * have it handy, so we require them to pass it too.
1778 *
1779 * If successful, returns true, and stores the new entry's offset in the file
1780 * into *query_offset. Also, if gc_count isn't NULL, *gc_count is set to the
1781 * number of garbage collections that have occurred so far.
1782 *
1783 * On failure, returns false.
1784 *
1785 * At least a shared lock on pgss->lock must be held by the caller, so as
1786 * to prevent a concurrent garbage collection. Share-lock-holding callers
1787 * should pass a gc_count pointer to obtain the number of garbage collections,
1788 * so that they can recheck the count after obtaining exclusive lock to
1789 * detect whether a garbage collection occurred (and removed this entry).
1790 */
1791 static bool
qtext_store(const char * query,int query_len,Size * query_offset,int * gc_count)1792 qtext_store(const char *query, int query_len,
1793 Size *query_offset, int *gc_count)
1794 {
1795 Size off;
1796 int fd;
1797
1798 /*
1799 * We use a spinlock to protect extent/n_writers/gc_count, so that
1800 * multiple processes may execute this function concurrently.
1801 */
1802 {
1803 volatile pgssSharedState *s = (volatile pgssSharedState *) pgss;
1804
1805 SpinLockAcquire(&s->mutex);
1806 off = s->extent;
1807 s->extent += query_len + 1;
1808 s->n_writers++;
1809 if (gc_count)
1810 *gc_count = s->gc_count;
1811 SpinLockRelease(&s->mutex);
1812 }
1813
1814 *query_offset = off;
1815
1816 /* Now write the data into the successfully-reserved part of the file */
1817 fd = OpenTransientFile(PGSS_TEXT_FILE, O_RDWR | O_CREAT | PG_BINARY,
1818 S_IRUSR | S_IWUSR);
1819 if (fd < 0)
1820 goto error;
1821
1822 if (lseek(fd, off, SEEK_SET) != off)
1823 goto error;
1824
1825 if (write(fd, query, query_len + 1) != query_len + 1)
1826 goto error;
1827
1828 CloseTransientFile(fd);
1829
1830 /* Mark our write complete */
1831 {
1832 volatile pgssSharedState *s = (volatile pgssSharedState *) pgss;
1833
1834 SpinLockAcquire(&s->mutex);
1835 s->n_writers--;
1836 SpinLockRelease(&s->mutex);
1837 }
1838
1839 return true;
1840
1841 error:
1842 ereport(LOG,
1843 (errcode_for_file_access(),
1844 errmsg("could not write pg_stat_statement file \"%s\": %m",
1845 PGSS_TEXT_FILE)));
1846
1847 if (fd >= 0)
1848 CloseTransientFile(fd);
1849
1850 /* Mark our write complete */
1851 {
1852 volatile pgssSharedState *s = (volatile pgssSharedState *) pgss;
1853
1854 SpinLockAcquire(&s->mutex);
1855 s->n_writers--;
1856 SpinLockRelease(&s->mutex);
1857 }
1858
1859 return false;
1860 }
1861
1862 /*
1863 * Read the external query text file into a malloc'd buffer.
1864 *
1865 * Returns NULL (without throwing an error) if unable to read, eg
1866 * file not there or insufficient memory.
1867 *
1868 * On success, the buffer size is also returned into *buffer_size.
1869 *
1870 * This can be called without any lock on pgss->lock, but in that case
1871 * the caller is responsible for verifying that the result is sane.
1872 */
1873 static char *
qtext_load_file(Size * buffer_size)1874 qtext_load_file(Size *buffer_size)
1875 {
1876 char *buf;
1877 int fd;
1878 struct stat stat;
1879 Size nread;
1880
1881 fd = OpenTransientFile(PGSS_TEXT_FILE, O_RDONLY | PG_BINARY, 0);
1882 if (fd < 0)
1883 {
1884 if (errno != ENOENT)
1885 ereport(LOG,
1886 (errcode_for_file_access(),
1887 errmsg("could not read pg_stat_statement file \"%s\": %m",
1888 PGSS_TEXT_FILE)));
1889 return NULL;
1890 }
1891
1892 /* Get file length */
1893 if (fstat(fd, &stat))
1894 {
1895 ereport(LOG,
1896 (errcode_for_file_access(),
1897 errmsg("could not stat pg_stat_statement file \"%s\": %m",
1898 PGSS_TEXT_FILE)));
1899 CloseTransientFile(fd);
1900 return NULL;
1901 }
1902
1903 /* Allocate buffer; beware that off_t might be wider than size_t */
1904 if (stat.st_size <= MaxAllocHugeSize)
1905 buf = (char *) malloc(stat.st_size);
1906 else
1907 buf = NULL;
1908 if (buf == NULL)
1909 {
1910 ereport(LOG,
1911 (errcode(ERRCODE_OUT_OF_MEMORY),
1912 errmsg("out of memory"),
1913 errdetail("Could not allocate enough memory to read pg_stat_statement file \"%s\".",
1914 PGSS_TEXT_FILE)));
1915 CloseTransientFile(fd);
1916 return NULL;
1917 }
1918
1919 /*
1920 * OK, slurp in the file. Windows fails if we try to read more than
1921 * INT_MAX bytes at once, and other platforms might not like that either,
1922 * so read a very large file in 1GB segments.
1923 */
1924 nread = 0;
1925 while (nread < stat.st_size)
1926 {
1927 int toread = Min(1024 * 1024 * 1024, stat.st_size - nread);
1928
1929 /*
1930 * If we get a short read and errno doesn't get set, the reason is
1931 * probably that garbage collection truncated the file since we did
1932 * the fstat(), so we don't log a complaint --- but we don't return
1933 * the data, either, since it's most likely corrupt due to concurrent
1934 * writes from garbage collection.
1935 */
1936 errno = 0;
1937 if (read(fd, buf + nread, toread) != toread)
1938 {
1939 if (errno)
1940 ereport(LOG,
1941 (errcode_for_file_access(),
1942 errmsg("could not read pg_stat_statement file \"%s\": %m",
1943 PGSS_TEXT_FILE)));
1944 free(buf);
1945 CloseTransientFile(fd);
1946 return NULL;
1947 }
1948 nread += toread;
1949 }
1950
1951 CloseTransientFile(fd);
1952
1953 *buffer_size = nread;
1954 return buf;
1955 }
1956
1957 /*
1958 * Locate a query text in the file image previously read by qtext_load_file().
1959 *
1960 * We validate the given offset/length, and return NULL if bogus. Otherwise,
1961 * the result points to a null-terminated string within the buffer.
1962 */
1963 static char *
qtext_fetch(Size query_offset,int query_len,char * buffer,Size buffer_size)1964 qtext_fetch(Size query_offset, int query_len,
1965 char *buffer, Size buffer_size)
1966 {
1967 /* File read failed? */
1968 if (buffer == NULL)
1969 return NULL;
1970 /* Bogus offset/length? */
1971 if (query_len < 0 ||
1972 query_offset + query_len >= buffer_size)
1973 return NULL;
1974 /* As a further sanity check, make sure there's a trailing null */
1975 if (buffer[query_offset + query_len] != '\0')
1976 return NULL;
1977 /* Looks OK */
1978 return buffer + query_offset;
1979 }
1980
1981 /*
1982 * Do we need to garbage-collect the external query text file?
1983 *
1984 * Caller should hold at least a shared lock on pgss->lock.
1985 */
1986 static bool
need_gc_qtexts(void)1987 need_gc_qtexts(void)
1988 {
1989 Size extent;
1990
1991 /* Read shared extent pointer */
1992 {
1993 volatile pgssSharedState *s = (volatile pgssSharedState *) pgss;
1994
1995 SpinLockAcquire(&s->mutex);
1996 extent = s->extent;
1997 SpinLockRelease(&s->mutex);
1998 }
1999
2000 /* Don't proceed if file does not exceed 512 bytes per possible entry */
2001 if (extent < 512 * pgss_max)
2002 return false;
2003
2004 /*
2005 * Don't proceed if file is less than about 50% bloat. Nothing can or
2006 * should be done in the event of unusually large query texts accounting
2007 * for file's large size. We go to the trouble of maintaining the mean
2008 * query length in order to prevent garbage collection from thrashing
2009 * uselessly.
2010 */
2011 if (extent < pgss->mean_query_len * pgss_max * 2)
2012 return false;
2013
2014 return true;
2015 }
2016
2017 /*
2018 * Garbage-collect orphaned query texts in external file.
2019 *
2020 * This won't be called often in the typical case, since it's likely that
2021 * there won't be too much churn, and besides, a similar compaction process
2022 * occurs when serializing to disk at shutdown or as part of resetting.
2023 * Despite this, it seems prudent to plan for the edge case where the file
2024 * becomes unreasonably large, with no other method of compaction likely to
2025 * occur in the foreseeable future.
2026 *
2027 * The caller must hold an exclusive lock on pgss->lock.
2028 *
2029 * At the first sign of trouble we unlink the query text file to get a clean
2030 * slate (although existing statistics are retained), rather than risk
2031 * thrashing by allowing the same problem case to recur indefinitely.
2032 */
2033 static void
gc_qtexts(void)2034 gc_qtexts(void)
2035 {
2036 char *qbuffer;
2037 Size qbuffer_size;
2038 FILE *qfile = NULL;
2039 HASH_SEQ_STATUS hash_seq;
2040 pgssEntry *entry;
2041 Size extent;
2042 int nentries;
2043
2044 /*
2045 * When called from pgss_store, some other session might have proceeded
2046 * with garbage collection in the no-lock-held interim of lock strength
2047 * escalation. Check once more that this is actually necessary.
2048 */
2049 if (!need_gc_qtexts())
2050 return;
2051
2052 /*
2053 * Load the old texts file. If we fail (out of memory, for instance),
2054 * invalidate query texts. Hopefully this is rare. It might seem better
2055 * to leave things alone on an OOM failure, but the problem is that the
2056 * file is only going to get bigger; hoping for a future non-OOM result is
2057 * risky and can easily lead to complete denial of service.
2058 */
2059 qbuffer = qtext_load_file(&qbuffer_size);
2060 if (qbuffer == NULL)
2061 goto gc_fail;
2062
2063 /*
2064 * We overwrite the query texts file in place, so as to reduce the risk of
2065 * an out-of-disk-space failure. Since the file is guaranteed not to get
2066 * larger, this should always work on traditional filesystems; though we
2067 * could still lose on copy-on-write filesystems.
2068 */
2069 qfile = AllocateFile(PGSS_TEXT_FILE, PG_BINARY_W);
2070 if (qfile == NULL)
2071 {
2072 ereport(LOG,
2073 (errcode_for_file_access(),
2074 errmsg("could not write pg_stat_statement file \"%s\": %m",
2075 PGSS_TEXT_FILE)));
2076 goto gc_fail;
2077 }
2078
2079 extent = 0;
2080 nentries = 0;
2081
2082 hash_seq_init(&hash_seq, pgss_hash);
2083 while ((entry = hash_seq_search(&hash_seq)) != NULL)
2084 {
2085 int query_len = entry->query_len;
2086 char *qry = qtext_fetch(entry->query_offset,
2087 query_len,
2088 qbuffer,
2089 qbuffer_size);
2090
2091 if (qry == NULL)
2092 {
2093 /* Trouble ... drop the text */
2094 entry->query_offset = 0;
2095 entry->query_len = -1;
2096 /* entry will not be counted in mean query length computation */
2097 continue;
2098 }
2099
2100 if (fwrite(qry, 1, query_len + 1, qfile) != query_len + 1)
2101 {
2102 ereport(LOG,
2103 (errcode_for_file_access(),
2104 errmsg("could not write pg_stat_statement file \"%s\": %m",
2105 PGSS_TEXT_FILE)));
2106 hash_seq_term(&hash_seq);
2107 goto gc_fail;
2108 }
2109
2110 entry->query_offset = extent;
2111 extent += query_len + 1;
2112 nentries++;
2113 }
2114
2115 /*
2116 * Truncate away any now-unused space. If this fails for some odd reason,
2117 * we log it, but there's no need to fail.
2118 */
2119 if (ftruncate(fileno(qfile), extent) != 0)
2120 ereport(LOG,
2121 (errcode_for_file_access(),
2122 errmsg("could not truncate pg_stat_statement file \"%s\": %m",
2123 PGSS_TEXT_FILE)));
2124
2125 if (FreeFile(qfile))
2126 {
2127 ereport(LOG,
2128 (errcode_for_file_access(),
2129 errmsg("could not write pg_stat_statement file \"%s\": %m",
2130 PGSS_TEXT_FILE)));
2131 qfile = NULL;
2132 goto gc_fail;
2133 }
2134
2135 elog(DEBUG1, "pgss gc of queries file shrunk size from %zu to %zu",
2136 pgss->extent, extent);
2137
2138 /* Reset the shared extent pointer */
2139 pgss->extent = extent;
2140
2141 /*
2142 * Also update the mean query length, to be sure that need_gc_qtexts()
2143 * won't still think we have a problem.
2144 */
2145 if (nentries > 0)
2146 pgss->mean_query_len = extent / nentries;
2147 else
2148 pgss->mean_query_len = ASSUMED_LENGTH_INIT;
2149
2150 free(qbuffer);
2151
2152 /*
2153 * OK, count a garbage collection cycle. (Note: even though we have
2154 * exclusive lock on pgss->lock, we must take pgss->mutex for this, since
2155 * other processes may examine gc_count while holding only the mutex.
2156 * Also, we have to advance the count *after* we've rewritten the file,
2157 * else other processes might not realize they read a stale file.)
2158 */
2159 record_gc_qtexts();
2160
2161 return;
2162
2163 gc_fail:
2164 /* clean up resources */
2165 if (qfile)
2166 FreeFile(qfile);
2167 if (qbuffer)
2168 free(qbuffer);
2169
2170 /*
2171 * Since the contents of the external file are now uncertain, mark all
2172 * hashtable entries as having invalid texts.
2173 */
2174 hash_seq_init(&hash_seq, pgss_hash);
2175 while ((entry = hash_seq_search(&hash_seq)) != NULL)
2176 {
2177 entry->query_offset = 0;
2178 entry->query_len = -1;
2179 }
2180
2181 /*
2182 * Destroy the query text file and create a new, empty one
2183 */
2184 (void) unlink(PGSS_TEXT_FILE);
2185 qfile = AllocateFile(PGSS_TEXT_FILE, PG_BINARY_W);
2186 if (qfile == NULL)
2187 ereport(LOG,
2188 (errcode_for_file_access(),
2189 errmsg("could not write new pg_stat_statement file \"%s\": %m",
2190 PGSS_TEXT_FILE)));
2191 else
2192 FreeFile(qfile);
2193
2194 /* Reset the shared extent pointer */
2195 pgss->extent = 0;
2196
2197 /* Reset mean_query_len to match the new state */
2198 pgss->mean_query_len = ASSUMED_LENGTH_INIT;
2199
2200 /*
2201 * Bump the GC count even though we failed.
2202 *
2203 * This is needed to make concurrent readers of file without any lock on
2204 * pgss->lock notice existence of new version of file. Once readers
2205 * subsequently observe a change in GC count with pgss->lock held, that
2206 * forces a safe reopen of file. Writers also require that we bump here,
2207 * of course. (As required by locking protocol, readers and writers don't
2208 * trust earlier file contents until gc_count is found unchanged after
2209 * pgss->lock acquired in shared or exclusive mode respectively.)
2210 */
2211 record_gc_qtexts();
2212 }
2213
2214 /*
2215 * Release all entries.
2216 */
2217 static void
entry_reset(void)2218 entry_reset(void)
2219 {
2220 HASH_SEQ_STATUS hash_seq;
2221 pgssEntry *entry;
2222 FILE *qfile;
2223
2224 LWLockAcquire(pgss->lock, LW_EXCLUSIVE);
2225
2226 hash_seq_init(&hash_seq, pgss_hash);
2227 while ((entry = hash_seq_search(&hash_seq)) != NULL)
2228 {
2229 hash_search(pgss_hash, &entry->key, HASH_REMOVE, NULL);
2230 }
2231
2232 /*
2233 * Write new empty query file, perhaps even creating a new one to recover
2234 * if the file was missing.
2235 */
2236 qfile = AllocateFile(PGSS_TEXT_FILE, PG_BINARY_W);
2237 if (qfile == NULL)
2238 {
2239 ereport(LOG,
2240 (errcode_for_file_access(),
2241 errmsg("could not create pg_stat_statement file \"%s\": %m",
2242 PGSS_TEXT_FILE)));
2243 goto done;
2244 }
2245
2246 /* If ftruncate fails, log it, but it's not a fatal problem */
2247 if (ftruncate(fileno(qfile), 0) != 0)
2248 ereport(LOG,
2249 (errcode_for_file_access(),
2250 errmsg("could not truncate pg_stat_statement file \"%s\": %m",
2251 PGSS_TEXT_FILE)));
2252
2253 FreeFile(qfile);
2254
2255 done:
2256 pgss->extent = 0;
2257 /* This counts as a query text garbage collection for our purposes */
2258 record_gc_qtexts();
2259
2260 LWLockRelease(pgss->lock);
2261 }
2262
2263 /*
2264 * AppendJumble: Append a value that is substantive in a given query to
2265 * the current jumble.
2266 */
2267 static void
AppendJumble(pgssJumbleState * jstate,const unsigned char * item,Size size)2268 AppendJumble(pgssJumbleState *jstate, const unsigned char *item, Size size)
2269 {
2270 unsigned char *jumble = jstate->jumble;
2271 Size jumble_len = jstate->jumble_len;
2272
2273 /*
2274 * Whenever the jumble buffer is full, we hash the current contents and
2275 * reset the buffer to contain just that hash value, thus relying on the
2276 * hash to summarize everything so far.
2277 */
2278 while (size > 0)
2279 {
2280 Size part_size;
2281
2282 if (jumble_len >= JUMBLE_SIZE)
2283 {
2284 uint32 start_hash = hash_any(jumble, JUMBLE_SIZE);
2285
2286 memcpy(jumble, &start_hash, sizeof(start_hash));
2287 jumble_len = sizeof(start_hash);
2288 }
2289 part_size = Min(size, JUMBLE_SIZE - jumble_len);
2290 memcpy(jumble + jumble_len, item, part_size);
2291 jumble_len += part_size;
2292 item += part_size;
2293 size -= part_size;
2294 }
2295 jstate->jumble_len = jumble_len;
2296 }
2297
2298 /*
2299 * Wrappers around AppendJumble to encapsulate details of serialization
2300 * of individual local variable elements.
2301 */
2302 #define APP_JUMB(item) \
2303 AppendJumble(jstate, (const unsigned char *) &(item), sizeof(item))
2304 #define APP_JUMB_STRING(str) \
2305 AppendJumble(jstate, (const unsigned char *) (str), strlen(str) + 1)
2306
2307 /*
2308 * JumbleQuery: Selectively serialize the query tree, appending significant
2309 * data to the "query jumble" while ignoring nonsignificant data.
2310 *
2311 * Rule of thumb for what to include is that we should ignore anything not
2312 * semantically significant (such as alias names) as well as anything that can
2313 * be deduced from child nodes (else we'd just be double-hashing that piece
2314 * of information).
2315 */
2316 static void
JumbleQuery(pgssJumbleState * jstate,Query * query)2317 JumbleQuery(pgssJumbleState *jstate, Query *query)
2318 {
2319 Assert(IsA(query, Query));
2320 Assert(query->utilityStmt == NULL);
2321
2322 APP_JUMB(query->commandType);
2323 /* resultRelation is usually predictable from commandType */
2324 JumbleExpr(jstate, (Node *) query->cteList);
2325 JumbleRangeTable(jstate, query->rtable);
2326 JumbleExpr(jstate, (Node *) query->jointree);
2327 JumbleExpr(jstate, (Node *) query->targetList);
2328 JumbleExpr(jstate, (Node *) query->onConflict);
2329 JumbleExpr(jstate, (Node *) query->returningList);
2330 JumbleExpr(jstate, (Node *) query->groupClause);
2331 JumbleExpr(jstate, (Node *) query->groupingSets);
2332 JumbleExpr(jstate, query->havingQual);
2333 JumbleExpr(jstate, (Node *) query->windowClause);
2334 JumbleExpr(jstate, (Node *) query->distinctClause);
2335 JumbleExpr(jstate, (Node *) query->sortClause);
2336 JumbleExpr(jstate, query->limitOffset);
2337 JumbleExpr(jstate, query->limitCount);
2338 /* we ignore rowMarks */
2339 JumbleExpr(jstate, query->setOperations);
2340 }
2341
2342 /*
2343 * Jumble a range table
2344 */
2345 static void
JumbleRangeTable(pgssJumbleState * jstate,List * rtable)2346 JumbleRangeTable(pgssJumbleState *jstate, List *rtable)
2347 {
2348 ListCell *lc;
2349
2350 foreach(lc, rtable)
2351 {
2352 RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc);
2353
2354 Assert(IsA(rte, RangeTblEntry));
2355 APP_JUMB(rte->rtekind);
2356 switch (rte->rtekind)
2357 {
2358 case RTE_RELATION:
2359 APP_JUMB(rte->relid);
2360 JumbleExpr(jstate, (Node *) rte->tablesample);
2361 break;
2362 case RTE_SUBQUERY:
2363 JumbleQuery(jstate, rte->subquery);
2364 break;
2365 case RTE_JOIN:
2366 APP_JUMB(rte->jointype);
2367 break;
2368 case RTE_FUNCTION:
2369 JumbleExpr(jstate, (Node *) rte->functions);
2370 break;
2371 case RTE_VALUES:
2372 JumbleExpr(jstate, (Node *) rte->values_lists);
2373 break;
2374 case RTE_CTE:
2375
2376 /*
2377 * Depending on the CTE name here isn't ideal, but it's the
2378 * only info we have to identify the referenced WITH item.
2379 */
2380 APP_JUMB_STRING(rte->ctename);
2381 APP_JUMB(rte->ctelevelsup);
2382 break;
2383 default:
2384 elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind);
2385 break;
2386 }
2387 }
2388 }
2389
2390 /*
2391 * Jumble an expression tree
2392 *
2393 * In general this function should handle all the same node types that
2394 * expression_tree_walker() does, and therefore it's coded to be as parallel
2395 * to that function as possible. However, since we are only invoked on
2396 * queries immediately post-parse-analysis, we need not handle node types
2397 * that only appear in planning.
2398 *
2399 * Note: the reason we don't simply use expression_tree_walker() is that the
2400 * point of that function is to support tree walkers that don't care about
2401 * most tree node types, but here we care about all types. We should complain
2402 * about any unrecognized node type.
2403 */
2404 static void
JumbleExpr(pgssJumbleState * jstate,Node * node)2405 JumbleExpr(pgssJumbleState *jstate, Node *node)
2406 {
2407 ListCell *temp;
2408
2409 if (node == NULL)
2410 return;
2411
2412 /* Guard against stack overflow due to overly complex expressions */
2413 check_stack_depth();
2414
2415 /*
2416 * We always emit the node's NodeTag, then any additional fields that are
2417 * considered significant, and then we recurse to any child nodes.
2418 */
2419 APP_JUMB(node->type);
2420
2421 switch (nodeTag(node))
2422 {
2423 case T_Var:
2424 {
2425 Var *var = (Var *) node;
2426
2427 APP_JUMB(var->varno);
2428 APP_JUMB(var->varattno);
2429 APP_JUMB(var->varlevelsup);
2430 }
2431 break;
2432 case T_Const:
2433 {
2434 Const *c = (Const *) node;
2435
2436 /* We jumble only the constant's type, not its value */
2437 APP_JUMB(c->consttype);
2438 /* Also, record its parse location for query normalization */
2439 RecordConstLocation(jstate, c->location);
2440 }
2441 break;
2442 case T_Param:
2443 {
2444 Param *p = (Param *) node;
2445
2446 APP_JUMB(p->paramkind);
2447 APP_JUMB(p->paramid);
2448 APP_JUMB(p->paramtype);
2449 }
2450 break;
2451 case T_Aggref:
2452 {
2453 Aggref *expr = (Aggref *) node;
2454
2455 APP_JUMB(expr->aggfnoid);
2456 JumbleExpr(jstate, (Node *) expr->aggdirectargs);
2457 JumbleExpr(jstate, (Node *) expr->args);
2458 JumbleExpr(jstate, (Node *) expr->aggorder);
2459 JumbleExpr(jstate, (Node *) expr->aggdistinct);
2460 JumbleExpr(jstate, (Node *) expr->aggfilter);
2461 }
2462 break;
2463 case T_GroupingFunc:
2464 {
2465 GroupingFunc *grpnode = (GroupingFunc *) node;
2466
2467 JumbleExpr(jstate, (Node *) grpnode->refs);
2468 }
2469 break;
2470 case T_WindowFunc:
2471 {
2472 WindowFunc *expr = (WindowFunc *) node;
2473
2474 APP_JUMB(expr->winfnoid);
2475 APP_JUMB(expr->winref);
2476 JumbleExpr(jstate, (Node *) expr->args);
2477 JumbleExpr(jstate, (Node *) expr->aggfilter);
2478 }
2479 break;
2480 case T_ArrayRef:
2481 {
2482 ArrayRef *aref = (ArrayRef *) node;
2483
2484 JumbleExpr(jstate, (Node *) aref->refupperindexpr);
2485 JumbleExpr(jstate, (Node *) aref->reflowerindexpr);
2486 JumbleExpr(jstate, (Node *) aref->refexpr);
2487 JumbleExpr(jstate, (Node *) aref->refassgnexpr);
2488 }
2489 break;
2490 case T_FuncExpr:
2491 {
2492 FuncExpr *expr = (FuncExpr *) node;
2493
2494 APP_JUMB(expr->funcid);
2495 JumbleExpr(jstate, (Node *) expr->args);
2496 }
2497 break;
2498 case T_NamedArgExpr:
2499 {
2500 NamedArgExpr *nae = (NamedArgExpr *) node;
2501
2502 APP_JUMB(nae->argnumber);
2503 JumbleExpr(jstate, (Node *) nae->arg);
2504 }
2505 break;
2506 case T_OpExpr:
2507 case T_DistinctExpr: /* struct-equivalent to OpExpr */
2508 case T_NullIfExpr: /* struct-equivalent to OpExpr */
2509 {
2510 OpExpr *expr = (OpExpr *) node;
2511
2512 APP_JUMB(expr->opno);
2513 JumbleExpr(jstate, (Node *) expr->args);
2514 }
2515 break;
2516 case T_ScalarArrayOpExpr:
2517 {
2518 ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node;
2519
2520 APP_JUMB(expr->opno);
2521 APP_JUMB(expr->useOr);
2522 JumbleExpr(jstate, (Node *) expr->args);
2523 }
2524 break;
2525 case T_BoolExpr:
2526 {
2527 BoolExpr *expr = (BoolExpr *) node;
2528
2529 APP_JUMB(expr->boolop);
2530 JumbleExpr(jstate, (Node *) expr->args);
2531 }
2532 break;
2533 case T_SubLink:
2534 {
2535 SubLink *sublink = (SubLink *) node;
2536
2537 APP_JUMB(sublink->subLinkType);
2538 APP_JUMB(sublink->subLinkId);
2539 JumbleExpr(jstate, (Node *) sublink->testexpr);
2540 JumbleQuery(jstate, (Query *) sublink->subselect);
2541 }
2542 break;
2543 case T_FieldSelect:
2544 {
2545 FieldSelect *fs = (FieldSelect *) node;
2546
2547 APP_JUMB(fs->fieldnum);
2548 JumbleExpr(jstate, (Node *) fs->arg);
2549 }
2550 break;
2551 case T_FieldStore:
2552 {
2553 FieldStore *fstore = (FieldStore *) node;
2554
2555 JumbleExpr(jstate, (Node *) fstore->arg);
2556 JumbleExpr(jstate, (Node *) fstore->newvals);
2557 }
2558 break;
2559 case T_RelabelType:
2560 {
2561 RelabelType *rt = (RelabelType *) node;
2562
2563 APP_JUMB(rt->resulttype);
2564 JumbleExpr(jstate, (Node *) rt->arg);
2565 }
2566 break;
2567 case T_CoerceViaIO:
2568 {
2569 CoerceViaIO *cio = (CoerceViaIO *) node;
2570
2571 APP_JUMB(cio->resulttype);
2572 JumbleExpr(jstate, (Node *) cio->arg);
2573 }
2574 break;
2575 case T_ArrayCoerceExpr:
2576 {
2577 ArrayCoerceExpr *acexpr = (ArrayCoerceExpr *) node;
2578
2579 APP_JUMB(acexpr->resulttype);
2580 JumbleExpr(jstate, (Node *) acexpr->arg);
2581 }
2582 break;
2583 case T_ConvertRowtypeExpr:
2584 {
2585 ConvertRowtypeExpr *crexpr = (ConvertRowtypeExpr *) node;
2586
2587 APP_JUMB(crexpr->resulttype);
2588 JumbleExpr(jstate, (Node *) crexpr->arg);
2589 }
2590 break;
2591 case T_CollateExpr:
2592 {
2593 CollateExpr *ce = (CollateExpr *) node;
2594
2595 APP_JUMB(ce->collOid);
2596 JumbleExpr(jstate, (Node *) ce->arg);
2597 }
2598 break;
2599 case T_CaseExpr:
2600 {
2601 CaseExpr *caseexpr = (CaseExpr *) node;
2602
2603 JumbleExpr(jstate, (Node *) caseexpr->arg);
2604 foreach(temp, caseexpr->args)
2605 {
2606 CaseWhen *when = (CaseWhen *) lfirst(temp);
2607
2608 Assert(IsA(when, CaseWhen));
2609 JumbleExpr(jstate, (Node *) when->expr);
2610 JumbleExpr(jstate, (Node *) when->result);
2611 }
2612 JumbleExpr(jstate, (Node *) caseexpr->defresult);
2613 }
2614 break;
2615 case T_CaseTestExpr:
2616 {
2617 CaseTestExpr *ct = (CaseTestExpr *) node;
2618
2619 APP_JUMB(ct->typeId);
2620 }
2621 break;
2622 case T_ArrayExpr:
2623 JumbleExpr(jstate, (Node *) ((ArrayExpr *) node)->elements);
2624 break;
2625 case T_RowExpr:
2626 JumbleExpr(jstate, (Node *) ((RowExpr *) node)->args);
2627 break;
2628 case T_RowCompareExpr:
2629 {
2630 RowCompareExpr *rcexpr = (RowCompareExpr *) node;
2631
2632 APP_JUMB(rcexpr->rctype);
2633 JumbleExpr(jstate, (Node *) rcexpr->largs);
2634 JumbleExpr(jstate, (Node *) rcexpr->rargs);
2635 }
2636 break;
2637 case T_CoalesceExpr:
2638 JumbleExpr(jstate, (Node *) ((CoalesceExpr *) node)->args);
2639 break;
2640 case T_MinMaxExpr:
2641 {
2642 MinMaxExpr *mmexpr = (MinMaxExpr *) node;
2643
2644 APP_JUMB(mmexpr->op);
2645 JumbleExpr(jstate, (Node *) mmexpr->args);
2646 }
2647 break;
2648 case T_XmlExpr:
2649 {
2650 XmlExpr *xexpr = (XmlExpr *) node;
2651
2652 APP_JUMB(xexpr->op);
2653 JumbleExpr(jstate, (Node *) xexpr->named_args);
2654 JumbleExpr(jstate, (Node *) xexpr->args);
2655 }
2656 break;
2657 case T_NullTest:
2658 {
2659 NullTest *nt = (NullTest *) node;
2660
2661 APP_JUMB(nt->nulltesttype);
2662 JumbleExpr(jstate, (Node *) nt->arg);
2663 }
2664 break;
2665 case T_BooleanTest:
2666 {
2667 BooleanTest *bt = (BooleanTest *) node;
2668
2669 APP_JUMB(bt->booltesttype);
2670 JumbleExpr(jstate, (Node *) bt->arg);
2671 }
2672 break;
2673 case T_CoerceToDomain:
2674 {
2675 CoerceToDomain *cd = (CoerceToDomain *) node;
2676
2677 APP_JUMB(cd->resulttype);
2678 JumbleExpr(jstate, (Node *) cd->arg);
2679 }
2680 break;
2681 case T_CoerceToDomainValue:
2682 {
2683 CoerceToDomainValue *cdv = (CoerceToDomainValue *) node;
2684
2685 APP_JUMB(cdv->typeId);
2686 }
2687 break;
2688 case T_SetToDefault:
2689 {
2690 SetToDefault *sd = (SetToDefault *) node;
2691
2692 APP_JUMB(sd->typeId);
2693 }
2694 break;
2695 case T_CurrentOfExpr:
2696 {
2697 CurrentOfExpr *ce = (CurrentOfExpr *) node;
2698
2699 APP_JUMB(ce->cvarno);
2700 if (ce->cursor_name)
2701 APP_JUMB_STRING(ce->cursor_name);
2702 APP_JUMB(ce->cursor_param);
2703 }
2704 break;
2705 case T_InferenceElem:
2706 {
2707 InferenceElem *ie = (InferenceElem *) node;
2708
2709 APP_JUMB(ie->infercollid);
2710 APP_JUMB(ie->inferopclass);
2711 JumbleExpr(jstate, ie->expr);
2712 }
2713 break;
2714 case T_TargetEntry:
2715 {
2716 TargetEntry *tle = (TargetEntry *) node;
2717
2718 APP_JUMB(tle->resno);
2719 APP_JUMB(tle->ressortgroupref);
2720 JumbleExpr(jstate, (Node *) tle->expr);
2721 }
2722 break;
2723 case T_RangeTblRef:
2724 {
2725 RangeTblRef *rtr = (RangeTblRef *) node;
2726
2727 APP_JUMB(rtr->rtindex);
2728 }
2729 break;
2730 case T_JoinExpr:
2731 {
2732 JoinExpr *join = (JoinExpr *) node;
2733
2734 APP_JUMB(join->jointype);
2735 APP_JUMB(join->isNatural);
2736 APP_JUMB(join->rtindex);
2737 JumbleExpr(jstate, join->larg);
2738 JumbleExpr(jstate, join->rarg);
2739 JumbleExpr(jstate, join->quals);
2740 }
2741 break;
2742 case T_FromExpr:
2743 {
2744 FromExpr *from = (FromExpr *) node;
2745
2746 JumbleExpr(jstate, (Node *) from->fromlist);
2747 JumbleExpr(jstate, from->quals);
2748 }
2749 break;
2750 case T_OnConflictExpr:
2751 {
2752 OnConflictExpr *conf = (OnConflictExpr *) node;
2753
2754 APP_JUMB(conf->action);
2755 JumbleExpr(jstate, (Node *) conf->arbiterElems);
2756 JumbleExpr(jstate, conf->arbiterWhere);
2757 JumbleExpr(jstate, (Node *) conf->onConflictSet);
2758 JumbleExpr(jstate, conf->onConflictWhere);
2759 APP_JUMB(conf->constraint);
2760 APP_JUMB(conf->exclRelIndex);
2761 JumbleExpr(jstate, (Node *) conf->exclRelTlist);
2762 }
2763 break;
2764 case T_List:
2765 foreach(temp, (List *) node)
2766 {
2767 JumbleExpr(jstate, (Node *) lfirst(temp));
2768 }
2769 break;
2770 case T_IntList:
2771 foreach(temp, (List *) node)
2772 {
2773 APP_JUMB(lfirst_int(temp));
2774 }
2775 break;
2776 case T_SortGroupClause:
2777 {
2778 SortGroupClause *sgc = (SortGroupClause *) node;
2779
2780 APP_JUMB(sgc->tleSortGroupRef);
2781 APP_JUMB(sgc->eqop);
2782 APP_JUMB(sgc->sortop);
2783 APP_JUMB(sgc->nulls_first);
2784 }
2785 break;
2786 case T_GroupingSet:
2787 {
2788 GroupingSet *gsnode = (GroupingSet *) node;
2789
2790 JumbleExpr(jstate, (Node *) gsnode->content);
2791 }
2792 break;
2793 case T_WindowClause:
2794 {
2795 WindowClause *wc = (WindowClause *) node;
2796
2797 APP_JUMB(wc->winref);
2798 APP_JUMB(wc->frameOptions);
2799 JumbleExpr(jstate, (Node *) wc->partitionClause);
2800 JumbleExpr(jstate, (Node *) wc->orderClause);
2801 JumbleExpr(jstate, wc->startOffset);
2802 JumbleExpr(jstate, wc->endOffset);
2803 }
2804 break;
2805 case T_CommonTableExpr:
2806 {
2807 CommonTableExpr *cte = (CommonTableExpr *) node;
2808
2809 /* we store the string name because RTE_CTE RTEs need it */
2810 APP_JUMB_STRING(cte->ctename);
2811 JumbleQuery(jstate, (Query *) cte->ctequery);
2812 }
2813 break;
2814 case T_SetOperationStmt:
2815 {
2816 SetOperationStmt *setop = (SetOperationStmt *) node;
2817
2818 APP_JUMB(setop->op);
2819 APP_JUMB(setop->all);
2820 JumbleExpr(jstate, setop->larg);
2821 JumbleExpr(jstate, setop->rarg);
2822 }
2823 break;
2824 case T_RangeTblFunction:
2825 {
2826 RangeTblFunction *rtfunc = (RangeTblFunction *) node;
2827
2828 JumbleExpr(jstate, rtfunc->funcexpr);
2829 }
2830 break;
2831 case T_TableSampleClause:
2832 {
2833 TableSampleClause *tsc = (TableSampleClause *) node;
2834
2835 APP_JUMB(tsc->tsmhandler);
2836 JumbleExpr(jstate, (Node *) tsc->args);
2837 JumbleExpr(jstate, (Node *) tsc->repeatable);
2838 }
2839 break;
2840 default:
2841 /* Only a warning, since we can stumble along anyway */
2842 elog(WARNING, "unrecognized node type: %d",
2843 (int) nodeTag(node));
2844 break;
2845 }
2846 }
2847
2848 /*
2849 * Record location of constant within query string of query tree
2850 * that is currently being walked.
2851 */
2852 static void
RecordConstLocation(pgssJumbleState * jstate,int location)2853 RecordConstLocation(pgssJumbleState *jstate, int location)
2854 {
2855 /* -1 indicates unknown or undefined location */
2856 if (location >= 0)
2857 {
2858 /* enlarge array if needed */
2859 if (jstate->clocations_count >= jstate->clocations_buf_size)
2860 {
2861 jstate->clocations_buf_size *= 2;
2862 jstate->clocations = (pgssLocationLen *)
2863 repalloc(jstate->clocations,
2864 jstate->clocations_buf_size *
2865 sizeof(pgssLocationLen));
2866 }
2867 jstate->clocations[jstate->clocations_count].location = location;
2868 /* initialize lengths to -1 to simplify fill_in_constant_lengths */
2869 jstate->clocations[jstate->clocations_count].length = -1;
2870 jstate->clocations_count++;
2871 }
2872 }
2873
2874 /*
2875 * Generate a normalized version of the query string that will be used to
2876 * represent all similar queries.
2877 *
2878 * Note that the normalized representation may well vary depending on
2879 * just which "equivalent" query is used to create the hashtable entry.
2880 * We assume this is OK.
2881 *
2882 * *query_len_p contains the input string length, and is updated with
2883 * the result string length (which cannot be longer) on exit.
2884 *
2885 * Returns a palloc'd string.
2886 */
2887 static char *
generate_normalized_query(pgssJumbleState * jstate,const char * query,int * query_len_p,int encoding)2888 generate_normalized_query(pgssJumbleState *jstate, const char *query,
2889 int *query_len_p, int encoding)
2890 {
2891 char *norm_query;
2892 int query_len = *query_len_p;
2893 int i,
2894 len_to_wrt, /* Length (in bytes) to write */
2895 quer_loc = 0, /* Source query byte location */
2896 n_quer_loc = 0, /* Normalized query byte location */
2897 last_off = 0, /* Offset from start for previous tok */
2898 last_tok_len = 0; /* Length (in bytes) of that tok */
2899
2900 /*
2901 * Get constants' lengths (core system only gives us locations). Note
2902 * this also ensures the items are sorted by location.
2903 */
2904 fill_in_constant_lengths(jstate, query);
2905
2906 /* Allocate result buffer */
2907 norm_query = palloc(query_len + 1);
2908
2909 for (i = 0; i < jstate->clocations_count; i++)
2910 {
2911 int off, /* Offset from start for cur tok */
2912 tok_len; /* Length (in bytes) of that tok */
2913
2914 off = jstate->clocations[i].location;
2915 tok_len = jstate->clocations[i].length;
2916
2917 if (tok_len < 0)
2918 continue; /* ignore any duplicates */
2919
2920 /* Copy next chunk (what precedes the next constant) */
2921 len_to_wrt = off - last_off;
2922 len_to_wrt -= last_tok_len;
2923
2924 Assert(len_to_wrt >= 0);
2925 memcpy(norm_query + n_quer_loc, query + quer_loc, len_to_wrt);
2926 n_quer_loc += len_to_wrt;
2927
2928 /* And insert a '?' in place of the constant token */
2929 norm_query[n_quer_loc++] = '?';
2930
2931 quer_loc = off + tok_len;
2932 last_off = off;
2933 last_tok_len = tok_len;
2934 }
2935
2936 /*
2937 * We've copied up until the last ignorable constant. Copy over the
2938 * remaining bytes of the original query string.
2939 */
2940 len_to_wrt = query_len - quer_loc;
2941
2942 Assert(len_to_wrt >= 0);
2943 memcpy(norm_query + n_quer_loc, query + quer_loc, len_to_wrt);
2944 n_quer_loc += len_to_wrt;
2945
2946 Assert(n_quer_loc <= query_len);
2947 norm_query[n_quer_loc] = '\0';
2948
2949 *query_len_p = n_quer_loc;
2950 return norm_query;
2951 }
2952
2953 /*
2954 * Given a valid SQL string and an array of constant-location records,
2955 * fill in the textual lengths of those constants.
2956 *
2957 * The constants may use any allowed constant syntax, such as float literals,
2958 * bit-strings, single-quoted strings and dollar-quoted strings. This is
2959 * accomplished by using the public API for the core scanner.
2960 *
2961 * It is the caller's job to ensure that the string is a valid SQL statement
2962 * with constants at the indicated locations. Since in practice the string
2963 * has already been parsed, and the locations that the caller provides will
2964 * have originated from within the authoritative parser, this should not be
2965 * a problem.
2966 *
2967 * Duplicate constant pointers are possible, and will have their lengths
2968 * marked as '-1', so that they are later ignored. (Actually, we assume the
2969 * lengths were initialized as -1 to start with, and don't change them here.)
2970 *
2971 * N.B. There is an assumption that a '-' character at a Const location begins
2972 * a negative numeric constant. This precludes there ever being another
2973 * reason for a constant to start with a '-'.
2974 */
2975 static void
fill_in_constant_lengths(pgssJumbleState * jstate,const char * query)2976 fill_in_constant_lengths(pgssJumbleState *jstate, const char *query)
2977 {
2978 pgssLocationLen *locs;
2979 core_yyscan_t yyscanner;
2980 core_yy_extra_type yyextra;
2981 core_YYSTYPE yylval;
2982 YYLTYPE yylloc;
2983 int last_loc = -1;
2984 int i;
2985
2986 /*
2987 * Sort the records by location so that we can process them in order while
2988 * scanning the query text.
2989 */
2990 if (jstate->clocations_count > 1)
2991 qsort(jstate->clocations, jstate->clocations_count,
2992 sizeof(pgssLocationLen), comp_location);
2993 locs = jstate->clocations;
2994
2995 /* initialize the flex scanner --- should match raw_parser() */
2996 yyscanner = scanner_init(query,
2997 &yyextra,
2998 ScanKeywords,
2999 NumScanKeywords);
3000
3001 /* we don't want to re-emit any escape string warnings */
3002 yyextra.escape_string_warning = false;
3003
3004 /* Search for each constant, in sequence */
3005 for (i = 0; i < jstate->clocations_count; i++)
3006 {
3007 int loc = locs[i].location;
3008 int tok;
3009
3010 Assert(loc >= 0);
3011
3012 if (loc <= last_loc)
3013 continue; /* Duplicate constant, ignore */
3014
3015 /* Lex tokens until we find the desired constant */
3016 for (;;)
3017 {
3018 tok = core_yylex(&yylval, &yylloc, yyscanner);
3019
3020 /* We should not hit end-of-string, but if we do, behave sanely */
3021 if (tok == 0)
3022 break; /* out of inner for-loop */
3023
3024 /*
3025 * We should find the token position exactly, but if we somehow
3026 * run past it, work with that.
3027 */
3028 if (yylloc >= loc)
3029 {
3030 if (query[loc] == '-')
3031 {
3032 /*
3033 * It's a negative value - this is the one and only case
3034 * where we replace more than a single token.
3035 *
3036 * Do not compensate for the core system's special-case
3037 * adjustment of location to that of the leading '-'
3038 * operator in the event of a negative constant. It is
3039 * also useful for our purposes to start from the minus
3040 * symbol. In this way, queries like "select * from foo
3041 * where bar = 1" and "select * from foo where bar = -2"
3042 * will have identical normalized query strings.
3043 */
3044 tok = core_yylex(&yylval, &yylloc, yyscanner);
3045 if (tok == 0)
3046 break; /* out of inner for-loop */
3047 }
3048
3049 /*
3050 * We now rely on the assumption that flex has placed a zero
3051 * byte after the text of the current token in scanbuf.
3052 */
3053 locs[i].length = strlen(yyextra.scanbuf + loc);
3054 break; /* out of inner for-loop */
3055 }
3056 }
3057
3058 /* If we hit end-of-string, give up, leaving remaining lengths -1 */
3059 if (tok == 0)
3060 break;
3061
3062 last_loc = loc;
3063 }
3064
3065 scanner_finish(yyscanner);
3066 }
3067
3068 /*
3069 * comp_location: comparator for qsorting pgssLocationLen structs by location
3070 */
3071 static int
comp_location(const void * a,const void * b)3072 comp_location(const void *a, const void *b)
3073 {
3074 int l = ((const pgssLocationLen *) a)->location;
3075 int r = ((const pgssLocationLen *) b)->location;
3076
3077 if (l < r)
3078 return -1;
3079 else if (l > r)
3080 return +1;
3081 else
3082 return 0;
3083 }
3084