1 /*-------------------------------------------------------------------------
2 *
3 * llvmjit.c
4 * Core part of the LLVM JIT provider.
5 *
6 * Copyright (c) 2016-2021, PostgreSQL Global Development Group
7 *
8 * IDENTIFICATION
9 * src/backend/jit/llvm/llvmjit.c
10 *
11 *-------------------------------------------------------------------------
12 */
13
14 #include "postgres.h"
15
16 #include <llvm-c/Analysis.h>
17 #include <llvm-c/BitReader.h>
18 #include <llvm-c/BitWriter.h>
19 #include <llvm-c/Core.h>
20 #include <llvm-c/ExecutionEngine.h>
21 #if LLVM_VERSION_MAJOR > 11
22 #include <llvm-c/Orc.h>
23 #include <llvm-c/OrcEE.h>
24 #include <llvm-c/LLJIT.h>
25 #else
26 #include <llvm-c/OrcBindings.h>
27 #endif
28 #include <llvm-c/Support.h>
29 #include <llvm-c/Target.h>
30 #include <llvm-c/Transforms/IPO.h>
31 #include <llvm-c/Transforms/PassManagerBuilder.h>
32 #include <llvm-c/Transforms/Scalar.h>
33 #if LLVM_VERSION_MAJOR > 6
34 #include <llvm-c/Transforms/Utils.h>
35 #endif
36
37 #include "jit/llvmjit.h"
38 #include "jit/llvmjit_emit.h"
39 #include "miscadmin.h"
40 #include "portability/instr_time.h"
41 #include "storage/ipc.h"
42 #include "utils/memutils.h"
43 #include "utils/resowner_private.h"
44
45 /* Handle of a module emitted via ORC JIT */
46 typedef struct LLVMJitHandle
47 {
48 #if LLVM_VERSION_MAJOR > 11
49 LLVMOrcLLJITRef lljit;
50 LLVMOrcResourceTrackerRef resource_tracker;
51 #else
52 LLVMOrcJITStackRef stack;
53 LLVMOrcModuleHandle orc_handle;
54 #endif
55 } LLVMJitHandle;
56
57
58 /* types & functions commonly needed for JITing */
59 LLVMTypeRef TypeSizeT;
60 LLVMTypeRef TypeParamBool;
61 LLVMTypeRef TypeStorageBool;
62 LLVMTypeRef TypePGFunction;
63 LLVMTypeRef StructNullableDatum;
64 LLVMTypeRef StructHeapTupleFieldsField3;
65 LLVMTypeRef StructHeapTupleFields;
66 LLVMTypeRef StructHeapTupleHeaderData;
67 LLVMTypeRef StructHeapTupleDataChoice;
68 LLVMTypeRef StructHeapTupleData;
69 LLVMTypeRef StructMinimalTupleData;
70 LLVMTypeRef StructItemPointerData;
71 LLVMTypeRef StructBlockId;
72 LLVMTypeRef StructFormPgAttribute;
73 LLVMTypeRef StructTupleConstr;
74 LLVMTypeRef StructTupleDescData;
75 LLVMTypeRef StructTupleTableSlot;
76 LLVMTypeRef StructHeapTupleTableSlot;
77 LLVMTypeRef StructMinimalTupleTableSlot;
78 LLVMTypeRef StructMemoryContextData;
79 LLVMTypeRef StructPGFinfoRecord;
80 LLVMTypeRef StructFmgrInfo;
81 LLVMTypeRef StructFunctionCallInfoData;
82 LLVMTypeRef StructExprContext;
83 LLVMTypeRef StructExprEvalStep;
84 LLVMTypeRef StructExprState;
85 LLVMTypeRef StructAggState;
86 LLVMTypeRef StructAggStatePerGroupData;
87 LLVMTypeRef StructAggStatePerTransData;
88
89 LLVMValueRef AttributeTemplate;
90
91 LLVMModuleRef llvm_types_module = NULL;
92
93 static bool llvm_session_initialized = false;
94 static size_t llvm_generation = 0;
95 static const char *llvm_triple = NULL;
96 static const char *llvm_layout = NULL;
97
98
99 static LLVMTargetRef llvm_targetref;
100 #if LLVM_VERSION_MAJOR > 11
101 static LLVMOrcThreadSafeContextRef llvm_ts_context;
102 static LLVMOrcLLJITRef llvm_opt0_orc;
103 static LLVMOrcLLJITRef llvm_opt3_orc;
104 #else /* LLVM_VERSION_MAJOR > 11 */
105 static LLVMOrcJITStackRef llvm_opt0_orc;
106 static LLVMOrcJITStackRef llvm_opt3_orc;
107 #endif /* LLVM_VERSION_MAJOR > 11 */
108
109
110 static void llvm_release_context(JitContext *context);
111 static void llvm_session_initialize(void);
112 static void llvm_shutdown(int code, Datum arg);
113 static void llvm_compile_module(LLVMJitContext *context);
114 static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
115
116 static void llvm_create_types(void);
117 static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
118
119 #if LLVM_VERSION_MAJOR > 11
120 static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
121 static char *llvm_error_message(LLVMErrorRef error);
122 #endif /* LLVM_VERSION_MAJOR > 11 */
123
124 PG_MODULE_MAGIC;
125
126
127 /*
128 * Initialize LLVM JIT provider.
129 */
130 void
_PG_jit_provider_init(JitProviderCallbacks * cb)131 _PG_jit_provider_init(JitProviderCallbacks *cb)
132 {
133 cb->reset_after_error = llvm_reset_after_error;
134 cb->release_context = llvm_release_context;
135 cb->compile_expr = llvm_compile_expr;
136 }
137
138 /*
139 * Create a context for JITing work.
140 *
141 * The context, including subsidiary resources, will be cleaned up either when
142 * the context is explicitly released, or when the lifetime of
143 * CurrentResourceOwner ends (usually the end of the current [sub]xact).
144 */
145 LLVMJitContext *
llvm_create_context(int jitFlags)146 llvm_create_context(int jitFlags)
147 {
148 LLVMJitContext *context;
149
150 llvm_assert_in_fatal_section();
151
152 llvm_session_initialize();
153
154 ResourceOwnerEnlargeJIT(CurrentResourceOwner);
155
156 context = MemoryContextAllocZero(TopMemoryContext,
157 sizeof(LLVMJitContext));
158 context->base.flags = jitFlags;
159
160 /* ensure cleanup */
161 context->base.resowner = CurrentResourceOwner;
162 ResourceOwnerRememberJIT(CurrentResourceOwner, PointerGetDatum(context));
163
164 return context;
165 }
166
167 /*
168 * Release resources required by one llvm context.
169 */
170 static void
llvm_release_context(JitContext * context)171 llvm_release_context(JitContext *context)
172 {
173 LLVMJitContext *llvm_context = (LLVMJitContext *) context;
174 ListCell *lc;
175
176 /*
177 * When this backend is exiting, don't clean up LLVM. As an error might
178 * have occurred from within LLVM, we do not want to risk reentering. All
179 * resource cleanup is going to happen through process exit.
180 */
181 if (proc_exit_inprogress)
182 return;
183
184 llvm_enter_fatal_on_oom();
185
186 if (llvm_context->module)
187 {
188 LLVMDisposeModule(llvm_context->module);
189 llvm_context->module = NULL;
190 }
191
192 foreach(lc, llvm_context->handles)
193 {
194 LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
195
196 #if LLVM_VERSION_MAJOR > 11
197 {
198 LLVMOrcExecutionSessionRef ee;
199 LLVMOrcSymbolStringPoolRef sp;
200
201 LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
202 LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
203
204 /*
205 * Without triggering cleanup of the string pool, we'd leak
206 * memory. It'd be sufficient to do this far less often, but in
207 * experiments the required time was small enough to just always
208 * do it.
209 */
210 ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
211 sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
212 LLVMOrcSymbolStringPoolClearDeadEntries(sp);
213 }
214 #else /* LLVM_VERSION_MAJOR > 11 */
215 {
216 LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
217 }
218 #endif /* LLVM_VERSION_MAJOR > 11 */
219
220 pfree(jit_handle);
221 }
222 list_free(llvm_context->handles);
223 llvm_context->handles = NIL;
224 }
225
226 /*
227 * Return module which may be modified, e.g. by creating new functions.
228 */
229 LLVMModuleRef
llvm_mutable_module(LLVMJitContext * context)230 llvm_mutable_module(LLVMJitContext *context)
231 {
232 llvm_assert_in_fatal_section();
233
234 /*
235 * If there's no in-progress module, create a new one.
236 */
237 if (!context->module)
238 {
239 context->compiled = false;
240 context->module_generation = llvm_generation++;
241 context->module = LLVMModuleCreateWithName("pg");
242 LLVMSetTarget(context->module, llvm_triple);
243 LLVMSetDataLayout(context->module, llvm_layout);
244 }
245
246 return context->module;
247 }
248
249 /*
250 * Expand function name to be non-conflicting. This should be used by code
251 * generating code, when adding new externally visible function definitions to
252 * a Module.
253 */
254 char *
llvm_expand_funcname(struct LLVMJitContext * context,const char * basename)255 llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
256 {
257 Assert(context->module != NULL);
258
259 context->base.instr.created_functions++;
260
261 /*
262 * Previously we used dots to separate, but turns out some tools, e.g.
263 * GDB, don't like that and truncate name.
264 */
265 return psprintf("%s_%zu_%d",
266 basename,
267 context->module_generation,
268 context->counter++);
269 }
270
271 /*
272 * Return pointer to function funcname, which has to exist. If there's pending
273 * code to be optimized and emitted, do so first.
274 */
275 void *
llvm_get_function(LLVMJitContext * context,const char * funcname)276 llvm_get_function(LLVMJitContext *context, const char *funcname)
277 {
278 #if LLVM_VERSION_MAJOR > 11 || \
279 defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
280 ListCell *lc;
281 #endif
282
283 llvm_assert_in_fatal_section();
284
285 /*
286 * If there is a pending / not emitted module, compile and emit now.
287 * Otherwise we might not find the [correct] function.
288 */
289 if (!context->compiled)
290 {
291 llvm_compile_module(context);
292 }
293
294 /*
295 * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
296 * to mangle here.
297 */
298
299 #if LLVM_VERSION_MAJOR > 11
300 foreach(lc, context->handles)
301 {
302 LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
303 instr_time starttime;
304 instr_time endtime;
305 LLVMErrorRef error;
306 LLVMOrcJITTargetAddress addr;
307
308 INSTR_TIME_SET_CURRENT(starttime);
309
310 addr = 0;
311 error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
312 if (error)
313 elog(ERROR, "failed to look up symbol \"%s\": %s",
314 funcname, llvm_error_message(error));
315
316 /*
317 * LLJIT only actually emits code the first time a symbol is
318 * referenced. Thus add lookup time to emission time. That's counting
319 * a bit more than with older LLVM versions, but unlikely to ever
320 * matter.
321 */
322 INSTR_TIME_SET_CURRENT(endtime);
323 INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
324 endtime, starttime);
325
326 if (addr)
327 return (void *) (uintptr_t) addr;
328 }
329 #elif defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
330 foreach(lc, context->handles)
331 {
332 LLVMOrcTargetAddress addr;
333 LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
334
335 addr = 0;
336 if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
337 elog(ERROR, "failed to look up symbol \"%s\"", funcname);
338 if (addr)
339 return (void *) (uintptr_t) addr;
340 }
341 #elif LLVM_VERSION_MAJOR < 5
342 {
343 LLVMOrcTargetAddress addr;
344
345 if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
346 return (void *) (uintptr_t) addr;
347 if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
348 return (void *) (uintptr_t) addr;
349 }
350 #else
351 {
352 LLVMOrcTargetAddress addr;
353
354 if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
355 elog(ERROR, "failed to look up symbol \"%s\"", funcname);
356 if (addr)
357 return (void *) (uintptr_t) addr;
358 if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
359 elog(ERROR, "failed to look up symbol \"%s\"", funcname);
360 if (addr)
361 return (void *) (uintptr_t) addr;
362 }
363 #endif
364
365 elog(ERROR, "failed to JIT: %s", funcname);
366
367 return NULL;
368 }
369
370 /*
371 * Return type of a variable in llvmjit_types.c. This is useful to keep types
372 * in sync between plain C and JIT related code.
373 */
374 LLVMTypeRef
llvm_pg_var_type(const char * varname)375 llvm_pg_var_type(const char *varname)
376 {
377 LLVMValueRef v_srcvar;
378 LLVMTypeRef typ;
379
380 /* this'll return a *pointer* to the global */
381 v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
382 if (!v_srcvar)
383 elog(ERROR, "variable %s not in llvmjit_types.c", varname);
384
385 /* look at the contained type */
386 typ = LLVMTypeOf(v_srcvar);
387 Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMPointerTypeKind);
388 typ = LLVMGetElementType(typ);
389 Assert(typ != NULL);
390
391 return typ;
392 }
393
394 /*
395 * Return function type of a variable in llvmjit_types.c. This is useful to
396 * keep function types in sync between C and JITed code.
397 */
398 LLVMTypeRef
llvm_pg_var_func_type(const char * varname)399 llvm_pg_var_func_type(const char *varname)
400 {
401 LLVMTypeRef typ = llvm_pg_var_type(varname);
402
403 /* look at the contained type */
404 Assert(LLVMGetTypeKind(typ) == LLVMPointerTypeKind);
405 typ = LLVMGetElementType(typ);
406 Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMFunctionTypeKind);
407
408 return typ;
409 }
410
411 /*
412 * Return declaration for a function referenced in llvmjit_types.c, adding it
413 * to the module if necessary.
414 *
415 * This is used to make functions discovered via llvm_create_types() known to
416 * the module that's currently being worked on.
417 */
418 LLVMValueRef
llvm_pg_func(LLVMModuleRef mod,const char * funcname)419 llvm_pg_func(LLVMModuleRef mod, const char *funcname)
420 {
421 LLVMValueRef v_srcfn;
422 LLVMValueRef v_fn;
423
424 /* don't repeatedly add function */
425 v_fn = LLVMGetNamedFunction(mod, funcname);
426 if (v_fn)
427 return v_fn;
428
429 v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
430
431 if (!v_srcfn)
432 elog(ERROR, "function %s not in llvmjit_types.c", funcname);
433
434 v_fn = LLVMAddFunction(mod,
435 funcname,
436 LLVMGetElementType(LLVMTypeOf(v_srcfn)));
437 llvm_copy_attributes(v_srcfn, v_fn);
438
439 return v_fn;
440 }
441
442 /*
443 * Copy attributes from one function to another, for a specific index (an
444 * index can reference return value, function and parameter attributes).
445 */
446 static void
llvm_copy_attributes_at_index(LLVMValueRef v_from,LLVMValueRef v_to,uint32 index)447 llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
448 {
449 int num_attributes;
450 LLVMAttributeRef *attrs;
451
452 num_attributes = LLVMGetAttributeCountAtIndexPG(v_from, index);
453
454 /*
455 * Not just for efficiency: LLVM <= 3.9 crashes when
456 * LLVMGetAttributesAtIndex() is called for an index with 0 attributes.
457 */
458 if (num_attributes == 0)
459 return;
460
461 attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
462 LLVMGetAttributesAtIndex(v_from, index, attrs);
463
464 for (int attno = 0; attno < num_attributes; attno++)
465 LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
466
467 pfree(attrs);
468 }
469
470 /*
471 * Copy all attributes from one function to another. I.e. function, return and
472 * parameters will be copied.
473 */
474 void
llvm_copy_attributes(LLVMValueRef v_from,LLVMValueRef v_to)475 llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
476 {
477 uint32 param_count;
478
479 /* copy function attributes */
480 llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
481
482 /* and the return value attributes */
483 llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
484
485 /* and each function parameter's attribute */
486 param_count = LLVMCountParams(v_from);
487
488 for (int paramidx = 1; paramidx <= param_count; paramidx++)
489 llvm_copy_attributes_at_index(v_from, v_to, paramidx);
490 }
491
492 /*
493 * Return a callable LLVMValueRef for fcinfo.
494 */
495 LLVMValueRef
llvm_function_reference(LLVMJitContext * context,LLVMBuilderRef builder,LLVMModuleRef mod,FunctionCallInfo fcinfo)496 llvm_function_reference(LLVMJitContext *context,
497 LLVMBuilderRef builder,
498 LLVMModuleRef mod,
499 FunctionCallInfo fcinfo)
500 {
501 char *modname;
502 char *basename;
503 char *funcname;
504
505 LLVMValueRef v_fn;
506
507 fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
508
509 if (modname != NULL && basename != NULL)
510 {
511 /* external function in loadable library */
512 funcname = psprintf("pgextern.%s.%s", modname, basename);
513 }
514 else if (basename != NULL)
515 {
516 /* internal function */
517 funcname = psprintf("%s", basename);
518 }
519 else
520 {
521 /*
522 * Function we don't know to handle, return pointer. We do so by
523 * creating a global constant containing a pointer to the function.
524 * Makes IR more readable.
525 */
526 LLVMValueRef v_fn_addr;
527
528 funcname = psprintf("pgoidextern.%u",
529 fcinfo->flinfo->fn_oid);
530 v_fn = LLVMGetNamedGlobal(mod, funcname);
531 if (v_fn != 0)
532 return LLVMBuildLoad(builder, v_fn, "");
533
534 v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
535
536 v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
537 LLVMSetInitializer(v_fn, v_fn_addr);
538 LLVMSetGlobalConstant(v_fn, true);
539 LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
540 LLVMSetUnnamedAddr(v_fn, true);
541
542 return LLVMBuildLoad(builder, v_fn, "");
543 }
544
545 /* check if function already has been added */
546 v_fn = LLVMGetNamedFunction(mod, funcname);
547 if (v_fn != 0)
548 return v_fn;
549
550 v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
551
552 return v_fn;
553 }
554
555 /*
556 * Optimize code in module using the flags set in context.
557 */
558 static void
llvm_optimize_module(LLVMJitContext * context,LLVMModuleRef module)559 llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
560 {
561 LLVMPassManagerBuilderRef llvm_pmb;
562 LLVMPassManagerRef llvm_mpm;
563 LLVMPassManagerRef llvm_fpm;
564 LLVMValueRef func;
565 int compile_optlevel;
566
567 if (context->base.flags & PGJIT_OPT3)
568 compile_optlevel = 3;
569 else
570 compile_optlevel = 0;
571
572 /*
573 * Have to create a new pass manager builder every pass through, as the
574 * inliner has some per-builder state. Otherwise one ends up only inlining
575 * a function the first time though.
576 */
577 llvm_pmb = LLVMPassManagerBuilderCreate();
578 LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
579 llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
580
581 if (context->base.flags & PGJIT_OPT3)
582 {
583 /* TODO: Unscientifically determined threshold */
584 LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
585 }
586 else
587 {
588 /* we rely on mem2reg heavily, so emit even in the O0 case */
589 LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
590 }
591
592 LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
593
594 /*
595 * Do function level optimization. This could be moved to the point where
596 * functions are emitted, to reduce memory usage a bit.
597 */
598 LLVMInitializeFunctionPassManager(llvm_fpm);
599 for (func = LLVMGetFirstFunction(context->module);
600 func != NULL;
601 func = LLVMGetNextFunction(func))
602 LLVMRunFunctionPassManager(llvm_fpm, func);
603 LLVMFinalizeFunctionPassManager(llvm_fpm);
604 LLVMDisposePassManager(llvm_fpm);
605
606 /*
607 * Perform module level optimization. We do so even in the non-optimized
608 * case, so always-inline functions etc get inlined. It's cheap enough.
609 */
610 llvm_mpm = LLVMCreatePassManager();
611 LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
612 llvm_mpm);
613 /* always use always-inliner pass */
614 if (!(context->base.flags & PGJIT_OPT3))
615 LLVMAddAlwaysInlinerPass(llvm_mpm);
616 /* if doing inlining, but no expensive optimization, add inlining pass */
617 if (context->base.flags & PGJIT_INLINE
618 && !(context->base.flags & PGJIT_OPT3))
619 LLVMAddFunctionInliningPass(llvm_mpm);
620 LLVMRunPassManager(llvm_mpm, context->module);
621 LLVMDisposePassManager(llvm_mpm);
622
623 LLVMPassManagerBuilderDispose(llvm_pmb);
624 }
625
626 /*
627 * Emit code for the currently pending module.
628 */
629 static void
llvm_compile_module(LLVMJitContext * context)630 llvm_compile_module(LLVMJitContext *context)
631 {
632 LLVMJitHandle *handle;
633 MemoryContext oldcontext;
634 instr_time starttime;
635 instr_time endtime;
636 #if LLVM_VERSION_MAJOR > 11
637 LLVMOrcLLJITRef compile_orc;
638 #else
639 LLVMOrcJITStackRef compile_orc;
640 #endif
641
642 if (context->base.flags & PGJIT_OPT3)
643 compile_orc = llvm_opt3_orc;
644 else
645 compile_orc = llvm_opt0_orc;
646
647 /* perform inlining */
648 if (context->base.flags & PGJIT_INLINE)
649 {
650 INSTR_TIME_SET_CURRENT(starttime);
651 llvm_inline(context->module);
652 INSTR_TIME_SET_CURRENT(endtime);
653 INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
654 endtime, starttime);
655 }
656
657 if (jit_dump_bitcode)
658 {
659 char *filename;
660
661 filename = psprintf("%u.%zu.bc",
662 MyProcPid,
663 context->module_generation);
664 LLVMWriteBitcodeToFile(context->module, filename);
665 pfree(filename);
666 }
667
668
669 /* optimize according to the chosen optimization settings */
670 INSTR_TIME_SET_CURRENT(starttime);
671 llvm_optimize_module(context, context->module);
672 INSTR_TIME_SET_CURRENT(endtime);
673 INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
674 endtime, starttime);
675
676 if (jit_dump_bitcode)
677 {
678 char *filename;
679
680 filename = psprintf("%u.%zu.optimized.bc",
681 MyProcPid,
682 context->module_generation);
683 LLVMWriteBitcodeToFile(context->module, filename);
684 pfree(filename);
685 }
686
687 handle = (LLVMJitHandle *)
688 MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
689
690 /*
691 * Emit the code. Note that this can, depending on the optimization
692 * settings, take noticeable resources as code emission executes low-level
693 * instruction combining/selection passes etc. Without optimization a
694 * faster instruction selection mechanism is used.
695 */
696 INSTR_TIME_SET_CURRENT(starttime);
697 #if LLVM_VERSION_MAJOR > 11
698 {
699 LLVMOrcThreadSafeModuleRef ts_module;
700 LLVMErrorRef error;
701 LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
702
703 ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
704
705 handle->lljit = compile_orc;
706 handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
707
708 /*
709 * NB: This doesn't actually emit code. That happens lazily the first
710 * time a symbol defined in the module is requested. Due to that
711 * llvm_get_function() also accounts for emission time.
712 */
713
714 context->module = NULL; /* will be owned by LLJIT */
715 error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
716 handle->resource_tracker,
717 ts_module);
718
719 if (error)
720 elog(ERROR, "failed to JIT module: %s",
721 llvm_error_message(error));
722
723 handle->lljit = compile_orc;
724
725 /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
726 }
727 #elif LLVM_VERSION_MAJOR > 6
728 {
729 handle->stack = compile_orc;
730 if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
731 llvm_resolve_symbol, NULL))
732 elog(ERROR, "failed to JIT module");
733
734 /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
735 }
736 #elif LLVM_VERSION_MAJOR > 4
737 {
738 LLVMSharedModuleRef smod;
739
740 smod = LLVMOrcMakeSharedModule(context->module);
741 handle->stack = compile_orc;
742 if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, smod,
743 llvm_resolve_symbol, NULL))
744 elog(ERROR, "failed to JIT module");
745
746 LLVMOrcDisposeSharedModuleRef(smod);
747 }
748 #else /* LLVM 4.0 and 3.9 */
749 {
750 handle->stack = compile_orc;
751 handle->orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
752 llvm_resolve_symbol, NULL);
753
754 LLVMDisposeModule(context->module);
755 }
756 #endif
757
758 INSTR_TIME_SET_CURRENT(endtime);
759 INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
760 endtime, starttime);
761
762 context->module = NULL;
763 context->compiled = true;
764
765 /* remember emitted code for cleanup and lookups */
766 oldcontext = MemoryContextSwitchTo(TopMemoryContext);
767 context->handles = lappend(context->handles, handle);
768 MemoryContextSwitchTo(oldcontext);
769
770 ereport(DEBUG1,
771 (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
772 INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
773 INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
774 INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
775 errhidestmt(true),
776 errhidecontext(true)));
777 }
778
779 /*
780 * Per session initialization.
781 */
782 static void
llvm_session_initialize(void)783 llvm_session_initialize(void)
784 {
785 MemoryContext oldcontext;
786 char *error = NULL;
787 char *cpu = NULL;
788 char *features = NULL;
789 LLVMTargetMachineRef opt0_tm;
790 LLVMTargetMachineRef opt3_tm;
791
792 if (llvm_session_initialized)
793 return;
794
795 oldcontext = MemoryContextSwitchTo(TopMemoryContext);
796
797 LLVMInitializeNativeTarget();
798 LLVMInitializeNativeAsmPrinter();
799 LLVMInitializeNativeAsmParser();
800
801 /*
802 * Synchronize types early, as that also includes inferring the target
803 * triple.
804 */
805 llvm_create_types();
806
807 if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
808 {
809 elog(FATAL, "failed to query triple %s\n", error);
810 }
811
812 /*
813 * We want the generated code to use all available features. Therefore
814 * grab the host CPU string and detect features of the current CPU. The
815 * latter is needed because some CPU architectures default to enabling
816 * features not all CPUs have (weird, huh).
817 */
818 cpu = LLVMGetHostCPUName();
819 features = LLVMGetHostCPUFeatures();
820 elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
821 cpu, features);
822
823 opt0_tm =
824 LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
825 LLVMCodeGenLevelNone,
826 LLVMRelocDefault,
827 LLVMCodeModelJITDefault);
828 opt3_tm =
829 LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
830 LLVMCodeGenLevelAggressive,
831 LLVMRelocDefault,
832 LLVMCodeModelJITDefault);
833
834 LLVMDisposeMessage(cpu);
835 cpu = NULL;
836 LLVMDisposeMessage(features);
837 features = NULL;
838
839 /* force symbols in main binary to be loaded */
840 LLVMLoadLibraryPermanently(NULL);
841
842 #if LLVM_VERSION_MAJOR > 11
843 {
844 llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
845
846 llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
847 opt0_tm = 0;
848
849 llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
850 opt3_tm = 0;
851 }
852 #else /* LLVM_VERSION_MAJOR > 11 */
853 {
854 llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
855 llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
856
857 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
858 if (jit_debugging_support)
859 {
860 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
861
862 LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
863 LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
864 }
865 #endif
866 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
867 if (jit_profiling_support)
868 {
869 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
870
871 LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
872 LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
873 }
874 #endif
875 }
876 #endif /* LLVM_VERSION_MAJOR > 11 */
877
878 on_proc_exit(llvm_shutdown, 0);
879
880 llvm_session_initialized = true;
881
882 MemoryContextSwitchTo(oldcontext);
883 }
884
885 static void
llvm_shutdown(int code,Datum arg)886 llvm_shutdown(int code, Datum arg)
887 {
888 /*
889 * If llvm_shutdown() is reached while in a fatal-on-oom section an error
890 * has occurred in the middle of LLVM code. It is not safe to call back
891 * into LLVM (which is why a FATAL error was thrown).
892 *
893 * We do need to shutdown LLVM in other shutdown cases, otherwise
894 * e.g. profiling data won't be written out.
895 */
896 if (llvm_in_fatal_on_oom())
897 {
898 Assert(proc_exit_inprogress);
899 return;
900 }
901
902 #if LLVM_VERSION_MAJOR > 11
903 {
904 if (llvm_opt3_orc)
905 {
906 LLVMOrcDisposeLLJIT(llvm_opt3_orc);
907 llvm_opt3_orc = NULL;
908 }
909 if (llvm_opt0_orc)
910 {
911 LLVMOrcDisposeLLJIT(llvm_opt0_orc);
912 llvm_opt0_orc = NULL;
913 }
914 if (llvm_ts_context)
915 {
916 LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
917 llvm_ts_context = NULL;
918 }
919 }
920 #else /* LLVM_VERSION_MAJOR > 11 */
921 {
922 /* unregister profiling support, needs to be flushed to be useful */
923
924 if (llvm_opt3_orc)
925 {
926 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
927 if (jit_profiling_support)
928 LLVMOrcUnregisterPerf(llvm_opt3_orc);
929 #endif
930 LLVMOrcDisposeInstance(llvm_opt3_orc);
931 llvm_opt3_orc = NULL;
932 }
933
934 if (llvm_opt0_orc)
935 {
936 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
937 if (jit_profiling_support)
938 LLVMOrcUnregisterPerf(llvm_opt0_orc);
939 #endif
940 LLVMOrcDisposeInstance(llvm_opt0_orc);
941 llvm_opt0_orc = NULL;
942 }
943 }
944 #endif /* LLVM_VERSION_MAJOR > 11 */
945 }
946
947 /* helper for llvm_create_types, returning a function's return type */
948 static LLVMTypeRef
load_return_type(LLVMModuleRef mod,const char * name)949 load_return_type(LLVMModuleRef mod, const char *name)
950 {
951 LLVMValueRef value;
952 LLVMTypeRef typ;
953
954 /* this'll return a *pointer* to the function */
955 value = LLVMGetNamedFunction(mod, name);
956 if (!value)
957 elog(ERROR, "function %s is unknown", name);
958
959 /* get type of function pointer */
960 typ = LLVMTypeOf(value);
961 Assert(typ != NULL);
962 /* dereference pointer */
963 typ = LLVMGetElementType(typ);
964 Assert(typ != NULL);
965 /* and look at return type */
966 typ = LLVMGetReturnType(typ);
967 Assert(typ != NULL);
968
969 return typ;
970 }
971
972 /*
973 * Load required information, types, function signatures from llvmjit_types.c
974 * and make them available in global variables.
975 *
976 * Those global variables are then used while emitting code.
977 */
978 static void
llvm_create_types(void)979 llvm_create_types(void)
980 {
981 char path[MAXPGPATH];
982 LLVMMemoryBufferRef buf;
983 char *msg;
984
985 snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
986
987 /* open file */
988 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
989 {
990 elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
991 path, msg);
992 }
993
994 /* eagerly load contents, going to need it all */
995 if (LLVMParseBitcode2(buf, &llvm_types_module))
996 {
997 elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
998 }
999 LLVMDisposeMemoryBuffer(buf);
1000
1001 /*
1002 * Load triple & layout from clang emitted file so we're guaranteed to be
1003 * compatible.
1004 */
1005 llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
1006 llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
1007
1008 TypeSizeT = llvm_pg_var_type("TypeSizeT");
1009 TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
1010 TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1011 TypePGFunction = llvm_pg_var_type("TypePGFunction");
1012 StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1013 StructExprContext = llvm_pg_var_type("StructExprContext");
1014 StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1015 StructExprState = llvm_pg_var_type("StructExprState");
1016 StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1017 StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1018 StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1019 StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1020 StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1021 StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1022 StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1023 StructAggState = llvm_pg_var_type("StructAggState");
1024 StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1025 StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1026
1027 AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1028 }
1029
1030 /*
1031 * Split a symbol into module / function parts. If the function is in the
1032 * main binary (or an external library) *modname will be NULL.
1033 */
1034 void
llvm_split_symbol_name(const char * name,char ** modname,char ** funcname)1035 llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1036 {
1037 *modname = NULL;
1038 *funcname = NULL;
1039
1040 /*
1041 * Module function names are pgextern.$module.$funcname
1042 */
1043 if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1044 {
1045 /*
1046 * Symbol names cannot contain a ., therefore we can split based on
1047 * first and last occurrence of one.
1048 */
1049 *funcname = rindex(name, '.');
1050 (*funcname)++; /* jump over . */
1051
1052 *modname = pnstrdup(name + strlen("pgextern."),
1053 *funcname - name - strlen("pgextern.") - 1);
1054 Assert(funcname);
1055
1056 *funcname = pstrdup(*funcname);
1057 }
1058 else
1059 {
1060 *modname = NULL;
1061 *funcname = pstrdup(name);
1062 }
1063 }
1064
1065 /*
1066 * Attempt to resolve symbol, so LLVM can emit a reference to it.
1067 */
1068 static uint64_t
llvm_resolve_symbol(const char * symname,void * ctx)1069 llvm_resolve_symbol(const char *symname, void *ctx)
1070 {
1071 uintptr_t addr;
1072 char *funcname;
1073 char *modname;
1074
1075 /*
1076 * macOS prefixes all object level symbols with an underscore. But neither
1077 * dlsym() nor PG's inliner expect that. So undo.
1078 */
1079 #if defined(__darwin__)
1080 if (symname[0] != '_')
1081 elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1082 symname++;
1083 #endif
1084
1085 llvm_split_symbol_name(symname, &modname, &funcname);
1086
1087 /* functions that aren't resolved to names shouldn't ever get here */
1088 Assert(funcname);
1089
1090 if (modname)
1091 addr = (uintptr_t) load_external_function(modname, funcname,
1092 true, NULL);
1093 else
1094 addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1095
1096 pfree(funcname);
1097 if (modname)
1098 pfree(modname);
1099
1100 /* let LLVM will error out - should never happen */
1101 if (!addr)
1102 elog(WARNING, "failed to resolve name %s", symname);
1103
1104 return (uint64_t) addr;
1105 }
1106
1107 #if LLVM_VERSION_MAJOR > 11
1108
1109 static LLVMErrorRef
llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj,void * Ctx,LLVMOrcLookupStateRef * LookupState,LLVMOrcLookupKind Kind,LLVMOrcJITDylibRef JD,LLVMOrcJITDylibLookupFlags JDLookupFlags,LLVMOrcCLookupSet LookupSet,size_t LookupSetSize)1110 llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1111 LLVMOrcLookupStateRef * LookupState, LLVMOrcLookupKind Kind,
1112 LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1113 LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1114 {
1115 LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1116 LLVMErrorRef error;
1117 LLVMOrcMaterializationUnitRef mu;
1118
1119 for (int i = 0; i < LookupSetSize; i++)
1120 {
1121 const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1122
1123 #if LLVM_VERSION_MAJOR > 12
1124 LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1125 #endif
1126 symbols[i].Name = LookupSet[i].Name;
1127 symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1128 symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1129 }
1130
1131 mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1132 error = LLVMOrcJITDylibDefine(JD, mu);
1133 if (error != LLVMErrorSuccess)
1134 LLVMOrcDisposeMaterializationUnit(mu);
1135
1136 pfree(symbols);
1137
1138 return error;
1139 }
1140
1141 /*
1142 * We cannot throw errors through LLVM (without causing a FATAL at least), so
1143 * just use WARNING here. That's OK anyway, as the error is also reported at
1144 * the top level action (with less detail) and there might be multiple
1145 * invocations of errors with details.
1146 *
1147 * This doesn't really happen during normal operation, but in cases like
1148 * symbol resolution breakage. So just using elog(WARNING) is fine.
1149 */
1150 static void
llvm_log_jit_error(void * ctx,LLVMErrorRef error)1151 llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1152 {
1153 elog(WARNING, "error during JITing: %s",
1154 llvm_error_message(error));
1155 }
1156
1157 /*
1158 * Create our own object layer, so we can add event listeners.
1159 */
1160 static LLVMOrcObjectLayerRef
llvm_create_object_layer(void * Ctx,LLVMOrcExecutionSessionRef ES,const char * Triple)1161 llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1162 {
1163 LLVMOrcObjectLayerRef objlayer =
1164 LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1165
1166 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1167 if (jit_debugging_support)
1168 {
1169 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1170
1171 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1172 }
1173 #endif
1174
1175 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1176 if (jit_profiling_support)
1177 {
1178 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1179
1180 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1181 }
1182 #endif
1183
1184 return objlayer;
1185 }
1186
1187 /*
1188 * Create LLJIT instance, using the passed in target machine. Note that the
1189 * target machine afterwards is owned by the LLJIT instance.
1190 */
1191 static LLVMOrcLLJITRef
llvm_create_jit_instance(LLVMTargetMachineRef tm)1192 llvm_create_jit_instance(LLVMTargetMachineRef tm)
1193 {
1194 LLVMOrcLLJITRef lljit;
1195 LLVMOrcJITTargetMachineBuilderRef tm_builder;
1196 LLVMOrcLLJITBuilderRef lljit_builder;
1197 LLVMErrorRef error;
1198 LLVMOrcDefinitionGeneratorRef main_gen;
1199 LLVMOrcDefinitionGeneratorRef ref_gen;
1200
1201 lljit_builder = LLVMOrcCreateLLJITBuilder();
1202 tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1203 LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1204
1205 LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1206 llvm_create_object_layer,
1207 NULL);
1208
1209 error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1210 if (error)
1211 elog(ERROR, "failed to create lljit instance: %s",
1212 llvm_error_message(error));
1213
1214 LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1215 llvm_log_jit_error, NULL);
1216
1217 /*
1218 * Symbol resolution support for symbols in the postgres binary /
1219 * libraries already loaded.
1220 */
1221 error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1222 LLVMOrcLLJITGetGlobalPrefix(lljit),
1223 0, NULL);
1224 if (error)
1225 elog(ERROR, "failed to create generator: %s",
1226 llvm_error_message(error));
1227 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1228
1229 /*
1230 * Symbol resolution support for "special" functions, e.g. a call into an
1231 * SQL callable function.
1232 */
1233 ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1234 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1235
1236 return lljit;
1237 }
1238
1239 static char *
llvm_error_message(LLVMErrorRef error)1240 llvm_error_message(LLVMErrorRef error)
1241 {
1242 char *orig = LLVMGetErrorMessage(error);
1243 char *msg = pstrdup(orig);
1244
1245 LLVMDisposeErrorMessage(orig);
1246
1247 return msg;
1248 }
1249
1250 #endif /* LLVM_VERSION_MAJOR > 11 */
1251