1 /*-------------------------------------------------------------------------
2 *
3 * llvmjit.c
4 * Core part of the LLVM JIT provider.
5 *
6 * Copyright (c) 2016-2018, PostgreSQL Global Development Group
7 *
8 * IDENTIFICATION
9 * src/backend/jit/llvm/llvmjit.c
10 *
11 *-------------------------------------------------------------------------
12 */
13
14 #include "postgres.h"
15
16 #include "jit/llvmjit.h"
17 #include "jit/llvmjit_emit.h"
18
19 #include "miscadmin.h"
20
21 #include "utils/memutils.h"
22 #include "utils/resowner_private.h"
23 #include "portability/instr_time.h"
24 #include "storage/ipc.h"
25
26
27 #include <llvm-c/Analysis.h>
28 #include <llvm-c/BitReader.h>
29 #include <llvm-c/BitWriter.h>
30 #include <llvm-c/Core.h>
31 #include <llvm-c/ExecutionEngine.h>
32 #if LLVM_VERSION_MAJOR > 11
33 #include <llvm-c/Orc.h>
34 #include <llvm-c/OrcEE.h>
35 #include <llvm-c/LLJIT.h>
36 #else
37 #include <llvm-c/OrcBindings.h>
38 #endif
39 #include <llvm-c/Support.h>
40 #include <llvm-c/Target.h>
41 #include <llvm-c/Transforms/IPO.h>
42 #include <llvm-c/Transforms/PassManagerBuilder.h>
43 #include <llvm-c/Transforms/Scalar.h>
44 #if LLVM_VERSION_MAJOR > 6
45 #include <llvm-c/Transforms/Utils.h>
46 #endif
47
48
49 /* Handle of a module emitted via ORC JIT */
50 typedef struct LLVMJitHandle
51 {
52 #if LLVM_VERSION_MAJOR > 11
53 LLVMOrcLLJITRef lljit;
54 LLVMOrcResourceTrackerRef resource_tracker;
55 #else
56 LLVMOrcJITStackRef stack;
57 LLVMOrcModuleHandle orc_handle;
58 #endif
59 } LLVMJitHandle;
60
61
62 /* types & functions commonly needed for JITing */
63 LLVMTypeRef TypeSizeT;
64 LLVMTypeRef TypeParamBool;
65 LLVMTypeRef TypeStorageBool;
66 LLVMTypeRef TypePGFunction;
67 LLVMTypeRef StructHeapTupleFieldsField3;
68 LLVMTypeRef StructHeapTupleFields;
69 LLVMTypeRef StructHeapTupleHeaderData;
70 LLVMTypeRef StructHeapTupleDataChoice;
71 LLVMTypeRef StructHeapTupleData;
72 LLVMTypeRef StructMinimalTupleData;
73 LLVMTypeRef StructItemPointerData;
74 LLVMTypeRef StructBlockId;
75 LLVMTypeRef StructFormPgAttribute;
76 LLVMTypeRef StructTupleConstr;
77 LLVMTypeRef StructtupleDesc;
78 LLVMTypeRef StructTupleTableSlot;
79 LLVMTypeRef StructMemoryContextData;
80 LLVMTypeRef StructPGFinfoRecord;
81 LLVMTypeRef StructFmgrInfo;
82 LLVMTypeRef StructFunctionCallInfoData;
83 LLVMTypeRef StructExprContext;
84 LLVMTypeRef StructExprEvalStep;
85 LLVMTypeRef StructExprState;
86 LLVMTypeRef StructAggState;
87 LLVMTypeRef StructAggStatePerGroupData;
88 LLVMTypeRef StructAggStatePerTransData;
89
90 LLVMValueRef AttributeTemplate;
91 LLVMValueRef FuncStrlen;
92 LLVMValueRef FuncVarsizeAny;
93 LLVMValueRef FuncSlotGetsomeattrs;
94 LLVMValueRef FuncSlotGetmissingattrs;
95 LLVMValueRef FuncHeapGetsysattr;
96 LLVMValueRef FuncMakeExpandedObjectReadOnlyInternal;
97 LLVMValueRef FuncExecEvalArrayRefSubscript;
98 LLVMValueRef FuncExecAggTransReparent;
99 LLVMValueRef FuncExecAggInitGroup;
100
101
102 static bool llvm_session_initialized = false;
103 static size_t llvm_generation = 0;
104 static const char *llvm_triple = NULL;
105 static const char *llvm_layout = NULL;
106
107
108 static LLVMTargetRef llvm_targetref;
109 #if LLVM_VERSION_MAJOR > 11
110 static LLVMOrcThreadSafeContextRef llvm_ts_context;
111 static LLVMOrcLLJITRef llvm_opt0_orc;
112 static LLVMOrcLLJITRef llvm_opt3_orc;
113 #else /* LLVM_VERSION_MAJOR > 11 */
114 static LLVMOrcJITStackRef llvm_opt0_orc;
115 static LLVMOrcJITStackRef llvm_opt3_orc;
116 #endif /* LLVM_VERSION_MAJOR > 11 */
117
118
119 static void llvm_release_context(JitContext *context);
120 static void llvm_session_initialize(void);
121 static void llvm_shutdown(int code, Datum arg);
122 static void llvm_compile_module(LLVMJitContext *context);
123 static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
124
125 static void llvm_create_types(void);
126 static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
127
128 #if LLVM_VERSION_MAJOR > 11
129 static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
130 static char *llvm_error_message(LLVMErrorRef error);
131 #endif /* LLVM_VERSION_MAJOR > 11 */
132
133 PG_MODULE_MAGIC;
134
135
136 /*
137 * Initialize LLVM JIT provider.
138 */
139 void
_PG_jit_provider_init(JitProviderCallbacks * cb)140 _PG_jit_provider_init(JitProviderCallbacks *cb)
141 {
142 cb->reset_after_error = llvm_reset_after_error;
143 cb->release_context = llvm_release_context;
144 cb->compile_expr = llvm_compile_expr;
145 }
146
147 /*
148 * Create a context for JITing work.
149 *
150 * The context, including subsidiary resources, will be cleaned up either when
151 * the context is explicitly released, or when the lifetime of
152 * CurrentResourceOwner ends (usually the end of the current [sub]xact).
153 */
154 LLVMJitContext *
llvm_create_context(int jitFlags)155 llvm_create_context(int jitFlags)
156 {
157 LLVMJitContext *context;
158
159 llvm_assert_in_fatal_section();
160
161 llvm_session_initialize();
162
163 ResourceOwnerEnlargeJIT(CurrentResourceOwner);
164
165 context = MemoryContextAllocZero(TopMemoryContext,
166 sizeof(LLVMJitContext));
167 context->base.flags = jitFlags;
168
169 /* ensure cleanup */
170 context->base.resowner = CurrentResourceOwner;
171 ResourceOwnerRememberJIT(CurrentResourceOwner, PointerGetDatum(context));
172
173 return context;
174 }
175
176 /*
177 * Release resources required by one llvm context.
178 */
179 static void
llvm_release_context(JitContext * context)180 llvm_release_context(JitContext *context)
181 {
182 LLVMJitContext *llvm_context = (LLVMJitContext *) context;
183
184 /*
185 * When this backend is exiting, don't clean up LLVM. As an error might
186 * have occurred from within LLVM, we do not want to risk reentering. All
187 * resource cleanup is going to happen through process exit.
188 */
189 if (proc_exit_inprogress)
190 return;
191
192 llvm_enter_fatal_on_oom();
193
194 if (llvm_context->module)
195 {
196 LLVMDisposeModule(llvm_context->module);
197 llvm_context->module = NULL;
198 }
199
200 while (llvm_context->handles != NIL)
201 {
202 LLVMJitHandle *jit_handle;
203
204 jit_handle = (LLVMJitHandle *) linitial(llvm_context->handles);
205 llvm_context->handles = list_delete_first(llvm_context->handles);
206
207 #if LLVM_VERSION_MAJOR > 11
208 {
209 LLVMOrcExecutionSessionRef ee;
210 LLVMOrcSymbolStringPoolRef sp;
211
212 LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
213 LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
214
215 /*
216 * Without triggering cleanup of the string pool, we'd leak
217 * memory. It'd be sufficient to do this far less often, but in
218 * experiments the required time was small enough to just always
219 * do it.
220 */
221 ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
222 sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
223 LLVMOrcSymbolStringPoolClearDeadEntries(sp);
224 }
225 #else /* LLVM_VERSION_MAJOR > 11 */
226 {
227 LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
228 }
229 #endif /* LLVM_VERSION_MAJOR > 11 */
230
231 pfree(jit_handle);
232 }
233 }
234
235 /*
236 * Return module which may be modified, e.g. by creating new functions.
237 */
238 LLVMModuleRef
llvm_mutable_module(LLVMJitContext * context)239 llvm_mutable_module(LLVMJitContext *context)
240 {
241 llvm_assert_in_fatal_section();
242
243 /*
244 * If there's no in-progress module, create a new one.
245 */
246 if (!context->module)
247 {
248 context->compiled = false;
249 context->module_generation = llvm_generation++;
250 context->module = LLVMModuleCreateWithName("pg");
251 LLVMSetTarget(context->module, llvm_triple);
252 LLVMSetDataLayout(context->module, llvm_layout);
253 }
254
255 return context->module;
256 }
257
258 /*
259 * Expand function name to be non-conflicting. This should be used by code
260 * generating code, when adding new externally visible function definitions to
261 * a Module.
262 */
263 char *
llvm_expand_funcname(struct LLVMJitContext * context,const char * basename)264 llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
265 {
266 Assert(context->module != NULL);
267
268 context->base.instr.created_functions++;
269
270 /*
271 * Previously we used dots to separate, but turns out some tools, e.g.
272 * GDB, don't like that and truncate name.
273 */
274 return psprintf("%s_%zu_%d",
275 basename,
276 context->module_generation,
277 context->counter++);
278 }
279
280 /*
281 * Return pointer to function funcname, which has to exist. If there's pending
282 * code to be optimized and emitted, do so first.
283 */
284 void *
llvm_get_function(LLVMJitContext * context,const char * funcname)285 llvm_get_function(LLVMJitContext *context, const char *funcname)
286 {
287 #if LLVM_VERSION_MAJOR > 11 || \
288 defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
289 ListCell *lc;
290 #endif
291
292 llvm_assert_in_fatal_section();
293
294 /*
295 * If there is a pending / not emitted module, compile and emit now.
296 * Otherwise we might not find the [correct] function.
297 */
298 if (!context->compiled)
299 {
300 llvm_compile_module(context);
301 }
302
303 /*
304 * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
305 * to mangle here.
306 */
307
308 #if LLVM_VERSION_MAJOR > 11
309 foreach(lc, context->handles)
310 {
311 LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
312 instr_time starttime;
313 instr_time endtime;
314 LLVMErrorRef error;
315 LLVMOrcJITTargetAddress addr;
316
317 INSTR_TIME_SET_CURRENT(starttime);
318
319 addr = 0;
320 error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
321 if (error)
322 elog(ERROR, "failed to look up symbol \"%s\": %s",
323 funcname, llvm_error_message(error));
324
325 /*
326 * LLJIT only actually emits code the first time a symbol is
327 * referenced. Thus add lookup time to emission time. That's counting
328 * a bit more than with older LLVM versions, but unlikely to ever
329 * matter.
330 */
331 INSTR_TIME_SET_CURRENT(endtime);
332 INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
333 endtime, starttime);
334
335 if (addr)
336 return (void *) (uintptr_t) addr;
337 }
338 #elif defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
339 foreach(lc, context->handles)
340 {
341 LLVMOrcTargetAddress addr;
342 LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
343
344 addr = 0;
345 if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
346 elog(ERROR, "failed to look up symbol \"%s\"", funcname);
347 if (addr)
348 return (void *) (uintptr_t) addr;
349 }
350 #elif LLVM_VERSION_MAJOR < 5
351 {
352 LLVMOrcTargetAddress addr;
353
354 if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
355 return (void *) (uintptr_t) addr;
356 if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
357 return (void *) (uintptr_t) addr;
358 }
359 #else
360 {
361 LLVMOrcTargetAddress addr;
362
363 if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
364 elog(ERROR, "failed to look up symbol \"%s\"", funcname);
365 if (addr)
366 return (void *) (uintptr_t) addr;
367 if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
368 elog(ERROR, "failed to look up symbol \"%s\"", funcname);
369 if (addr)
370 return (void *) (uintptr_t) addr;
371 }
372 #endif
373
374 elog(ERROR, "failed to JIT: %s", funcname);
375
376 return NULL;
377 }
378
379 /*
380 * Return declaration for passed function, adding it to the module if
381 * necessary.
382 *
383 * This is used to make functions imported by llvm_create_types() known to the
384 * module that's currently being worked on.
385 */
386 LLVMValueRef
llvm_get_decl(LLVMModuleRef mod,LLVMValueRef v_src)387 llvm_get_decl(LLVMModuleRef mod, LLVMValueRef v_src)
388 {
389 LLVMValueRef v_fn;
390
391 /* don't repeatedly add function */
392 v_fn = LLVMGetNamedFunction(mod, LLVMGetValueName(v_src));
393 if (v_fn)
394 return v_fn;
395
396 v_fn = LLVMAddFunction(mod,
397 LLVMGetValueName(v_src),
398 LLVMGetElementType(LLVMTypeOf(v_src)));
399 llvm_copy_attributes(v_src, v_fn);
400
401 return v_fn;
402 }
403
404 /*
405 * Copy attributes from one function to another, for a specific index (an
406 * index can reference return value, function and parameter attributes).
407 */
408 static void
llvm_copy_attributes_at_index(LLVMValueRef v_from,LLVMValueRef v_to,uint32 index)409 llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
410 {
411 int num_attributes;
412 LLVMAttributeRef *attrs;
413 int attno;
414
415 num_attributes = LLVMGetAttributeCountAtIndexPG(v_from, index);
416
417 /*
418 * Not just for efficiency: LLVM <= 3.9 crashes when
419 * LLVMGetAttributesAtIndex() is called for an index with 0 attributes.
420 */
421 if (num_attributes == 0)
422 return;
423
424 attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
425 LLVMGetAttributesAtIndex(v_from, index, attrs);
426
427 for (attno = 0; attno < num_attributes; attno++)
428 LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
429
430 pfree(attrs);
431 }
432
433 /*
434 * Copy all attributes from one function to another. I.e. function, return and
435 * parameters will be copied.
436 */
437 void
llvm_copy_attributes(LLVMValueRef v_from,LLVMValueRef v_to)438 llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
439 {
440 uint32 param_count;
441 int paramidx;
442
443 /* copy function attributes */
444 llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
445
446 /* and the return value attributes */
447 llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
448
449 /* and each function parameter's attribute */
450 param_count = LLVMCountParams(v_from);
451
452 for (paramidx = 1; paramidx <= param_count; paramidx++)
453 llvm_copy_attributes_at_index(v_from, v_to, paramidx);
454 }
455
456 /*
457 * Return a callable LLVMValueRef for fcinfo.
458 */
459 LLVMValueRef
llvm_function_reference(LLVMJitContext * context,LLVMBuilderRef builder,LLVMModuleRef mod,FunctionCallInfo fcinfo)460 llvm_function_reference(LLVMJitContext *context,
461 LLVMBuilderRef builder,
462 LLVMModuleRef mod,
463 FunctionCallInfo fcinfo)
464 {
465 char *modname;
466 char *basename;
467 char *funcname;
468
469 LLVMValueRef v_fn;
470
471 fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
472
473 if (modname != NULL && basename != NULL)
474 {
475 /* external function in loadable library */
476 funcname = psprintf("pgextern.%s.%s", modname, basename);
477 }
478 else if (basename != NULL)
479 {
480 /* internal function */
481 funcname = psprintf("%s", basename);
482 }
483 else
484 {
485 /*
486 * Function we don't know to handle, return pointer. We do so by
487 * creating a global constant containing a pointer to the function.
488 * Makes IR more readable.
489 */
490 LLVMValueRef v_fn_addr;
491
492 funcname = psprintf("pgoidextern.%u",
493 fcinfo->flinfo->fn_oid);
494 v_fn = LLVMGetNamedGlobal(mod, funcname);
495 if (v_fn != 0)
496 return LLVMBuildLoad(builder, v_fn, "");
497
498 v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
499
500 v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
501 LLVMSetInitializer(v_fn, v_fn_addr);
502 LLVMSetGlobalConstant(v_fn, true);
503 LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
504 LLVMSetUnnamedAddr(v_fn, true);
505
506 return LLVMBuildLoad(builder, v_fn, "");
507 }
508
509 /* check if function already has been added */
510 v_fn = LLVMGetNamedFunction(mod, funcname);
511 if (v_fn != 0)
512 return v_fn;
513
514 v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
515
516 return v_fn;
517 }
518
519 /*
520 * Optimize code in module using the flags set in context.
521 */
522 static void
llvm_optimize_module(LLVMJitContext * context,LLVMModuleRef module)523 llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
524 {
525 LLVMPassManagerBuilderRef llvm_pmb;
526 LLVMPassManagerRef llvm_mpm;
527 LLVMPassManagerRef llvm_fpm;
528 LLVMValueRef func;
529 int compile_optlevel;
530
531 if (context->base.flags & PGJIT_OPT3)
532 compile_optlevel = 3;
533 else
534 compile_optlevel = 0;
535
536 /*
537 * Have to create a new pass manager builder every pass through, as the
538 * inliner has some per-builder state. Otherwise one ends up only inlining
539 * a function the first time though.
540 */
541 llvm_pmb = LLVMPassManagerBuilderCreate();
542 LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
543 llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
544
545 if (context->base.flags & PGJIT_OPT3)
546 {
547 /* TODO: Unscientifically determined threshhold */
548 LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
549 }
550 else
551 {
552 /* we rely on mem2reg heavily, so emit even in the O0 case */
553 LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
554 }
555
556 LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
557
558 /*
559 * Do function level optimization. This could be moved to the point where
560 * functions are emitted, to reduce memory usage a bit.
561 */
562 LLVMInitializeFunctionPassManager(llvm_fpm);
563 for (func = LLVMGetFirstFunction(context->module);
564 func != NULL;
565 func = LLVMGetNextFunction(func))
566 LLVMRunFunctionPassManager(llvm_fpm, func);
567 LLVMFinalizeFunctionPassManager(llvm_fpm);
568 LLVMDisposePassManager(llvm_fpm);
569
570 /*
571 * Perform module level optimization. We do so even in the non-optimized
572 * case, so always-inline functions etc get inlined. It's cheap enough.
573 */
574 llvm_mpm = LLVMCreatePassManager();
575 LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
576 llvm_mpm);
577 /* always use always-inliner pass */
578 if (!(context->base.flags & PGJIT_OPT3))
579 LLVMAddAlwaysInlinerPass(llvm_mpm);
580 /* if doing inlining, but no expensive optimization, add inlining pass */
581 if (context->base.flags & PGJIT_INLINE
582 && !(context->base.flags & PGJIT_OPT3))
583 LLVMAddFunctionInliningPass(llvm_mpm);
584 LLVMRunPassManager(llvm_mpm, context->module);
585 LLVMDisposePassManager(llvm_mpm);
586
587 LLVMPassManagerBuilderDispose(llvm_pmb);
588 }
589
590 /*
591 * Emit code for the currently pending module.
592 */
593 static void
llvm_compile_module(LLVMJitContext * context)594 llvm_compile_module(LLVMJitContext *context)
595 {
596 LLVMJitHandle *handle;
597 MemoryContext oldcontext;
598 instr_time starttime;
599 instr_time endtime;
600 #if LLVM_VERSION_MAJOR > 11
601 LLVMOrcLLJITRef compile_orc;
602 #else
603 LLVMOrcJITStackRef compile_orc;
604 #endif
605
606 if (context->base.flags & PGJIT_OPT3)
607 compile_orc = llvm_opt3_orc;
608 else
609 compile_orc = llvm_opt0_orc;
610
611 /* perform inlining */
612 if (context->base.flags & PGJIT_INLINE)
613 {
614 INSTR_TIME_SET_CURRENT(starttime);
615 llvm_inline(context->module);
616 INSTR_TIME_SET_CURRENT(endtime);
617 INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
618 endtime, starttime);
619 }
620
621 if (jit_dump_bitcode)
622 {
623 char *filename;
624
625 filename = psprintf("%u.%zu.bc",
626 MyProcPid,
627 context->module_generation);
628 LLVMWriteBitcodeToFile(context->module, filename);
629 pfree(filename);
630 }
631
632
633 /* optimize according to the chosen optimization settings */
634 INSTR_TIME_SET_CURRENT(starttime);
635 llvm_optimize_module(context, context->module);
636 INSTR_TIME_SET_CURRENT(endtime);
637 INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
638 endtime, starttime);
639
640 if (jit_dump_bitcode)
641 {
642 char *filename;
643
644 filename = psprintf("%u.%zu.optimized.bc",
645 MyProcPid,
646 context->module_generation);
647 LLVMWriteBitcodeToFile(context->module, filename);
648 pfree(filename);
649 }
650
651 handle = (LLVMJitHandle *)
652 MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
653
654 /*
655 * Emit the code. Note that this can, depending on the optimization
656 * settings, take noticeable resources as code emission executes low-level
657 * instruction combining/selection passes etc. Without optimization a
658 * faster instruction selection mechanism is used.
659 */
660 INSTR_TIME_SET_CURRENT(starttime);
661 #if LLVM_VERSION_MAJOR > 11
662 {
663 LLVMOrcThreadSafeModuleRef ts_module;
664 LLVMErrorRef error;
665 LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
666
667 ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
668
669 handle->lljit = compile_orc;
670 handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
671
672 /*
673 * NB: This doesn't actually emit code. That happens lazily the first
674 * time a symbol defined in the module is requested. Due to that
675 * llvm_get_function() also accounts for emission time.
676 */
677
678 context->module = NULL; /* will be owned by LLJIT */
679 error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
680 handle->resource_tracker,
681 ts_module);
682
683 if (error)
684 elog(ERROR, "failed to JIT module: %s",
685 llvm_error_message(error));
686
687 handle->lljit = compile_orc;
688
689 /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
690 }
691 #elif LLVM_VERSION_MAJOR > 6
692 {
693 handle->stack = compile_orc;
694 if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
695 llvm_resolve_symbol, NULL))
696 elog(ERROR, "failed to JIT module");
697
698 /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
699 }
700 #elif LLVM_VERSION_MAJOR > 4
701 {
702 LLVMSharedModuleRef smod;
703
704 smod = LLVMOrcMakeSharedModule(context->module);
705 handle->stack = compile_orc;
706 if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, smod,
707 llvm_resolve_symbol, NULL))
708 elog(ERROR, "failed to JIT module");
709
710 LLVMOrcDisposeSharedModuleRef(smod);
711 }
712 #else /* LLVM 4.0 and 3.9 */
713 {
714 handle->stack = compile_orc;
715 handle->orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
716 llvm_resolve_symbol, NULL);
717
718 LLVMDisposeModule(context->module);
719 }
720 #endif
721
722 INSTR_TIME_SET_CURRENT(endtime);
723 INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
724 endtime, starttime);
725
726 context->module = NULL;
727 context->compiled = true;
728
729 /* remember emitted code for cleanup and lookups */
730 oldcontext = MemoryContextSwitchTo(TopMemoryContext);
731 context->handles = lappend(context->handles, handle);
732 MemoryContextSwitchTo(oldcontext);
733
734 ereport(DEBUG1,
735 (errmsg("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
736 INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
737 INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
738 INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
739 errhidestmt(true),
740 errhidecontext(true)));
741 }
742
743 /*
744 * Per session initialization.
745 */
746 static void
llvm_session_initialize(void)747 llvm_session_initialize(void)
748 {
749 MemoryContext oldcontext;
750 char *error = NULL;
751 char *cpu = NULL;
752 char *features = NULL;
753 LLVMTargetMachineRef opt0_tm;
754 LLVMTargetMachineRef opt3_tm;
755
756 if (llvm_session_initialized)
757 return;
758
759 oldcontext = MemoryContextSwitchTo(TopMemoryContext);
760
761 LLVMInitializeNativeTarget();
762 LLVMInitializeNativeAsmPrinter();
763 LLVMInitializeNativeAsmParser();
764
765 /*
766 * Synchronize types early, as that also includes inferring the target
767 * triple.
768 */
769 llvm_create_types();
770
771 if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
772 {
773 elog(FATAL, "failed to query triple %s\n", error);
774 }
775
776 /*
777 * We want the generated code to use all available features. Therefore
778 * grab the host CPU string and detect features of the current CPU. The
779 * latter is needed because some CPU architectures default to enabling
780 * features not all CPUs have (weird, huh).
781 */
782 cpu = LLVMGetHostCPUName();
783 features = LLVMGetHostCPUFeatures();
784 elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
785 cpu, features);
786
787 opt0_tm =
788 LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
789 LLVMCodeGenLevelNone,
790 LLVMRelocDefault,
791 LLVMCodeModelJITDefault);
792 opt3_tm =
793 LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
794 LLVMCodeGenLevelAggressive,
795 LLVMRelocDefault,
796 LLVMCodeModelJITDefault);
797
798 LLVMDisposeMessage(cpu);
799 cpu = NULL;
800 LLVMDisposeMessage(features);
801 features = NULL;
802
803 /* force symbols in main binary to be loaded */
804 LLVMLoadLibraryPermanently(NULL);
805
806 #if LLVM_VERSION_MAJOR > 11
807 {
808 llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
809
810 llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
811 opt0_tm = 0;
812
813 llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
814 opt3_tm = 0;
815 }
816 #else /* LLVM_VERSION_MAJOR > 11 */
817 {
818 llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
819 llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
820
821 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
822 if (jit_debugging_support)
823 {
824 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
825
826 LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
827 LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
828 }
829 #endif
830 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
831 if (jit_profiling_support)
832 {
833 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
834
835 LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
836 LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
837 }
838 #endif
839 }
840 #endif /* LLVM_VERSION_MAJOR > 11 */
841
842 before_shmem_exit(llvm_shutdown, 0);
843
844 llvm_session_initialized = true;
845
846 MemoryContextSwitchTo(oldcontext);
847 }
848
849 static void
llvm_shutdown(int code,Datum arg)850 llvm_shutdown(int code, Datum arg)
851 {
852 /*
853 * If llvm_shutdown() is reached while in a fatal-on-oom section an error
854 * has occurred in the middle of LLVM code. It is not safe to call back
855 * into LLVM (which is why a FATAL error was thrown).
856 *
857 * We do need to shutdown LLVM in other shutdown cases, otherwise
858 * e.g. profiling data won't be written out.
859 */
860 if (llvm_in_fatal_on_oom())
861 {
862 Assert(proc_exit_inprogress);
863 return;
864 }
865
866 #if LLVM_VERSION_MAJOR > 11
867 {
868 if (llvm_opt3_orc)
869 {
870 LLVMOrcDisposeLLJIT(llvm_opt3_orc);
871 llvm_opt3_orc = NULL;
872 }
873 if (llvm_opt0_orc)
874 {
875 LLVMOrcDisposeLLJIT(llvm_opt0_orc);
876 llvm_opt0_orc = NULL;
877 }
878 if (llvm_ts_context)
879 {
880 LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
881 llvm_ts_context = NULL;
882 }
883 }
884 #else /* LLVM_VERSION_MAJOR > 11 */
885 {
886 /* unregister profiling support, needs to be flushed to be useful */
887
888 if (llvm_opt3_orc)
889 {
890 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
891 if (jit_profiling_support)
892 LLVMOrcUnregisterPerf(llvm_opt3_orc);
893 #endif
894 LLVMOrcDisposeInstance(llvm_opt3_orc);
895 llvm_opt3_orc = NULL;
896 }
897
898 if (llvm_opt0_orc)
899 {
900 #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
901 if (jit_profiling_support)
902 LLVMOrcUnregisterPerf(llvm_opt0_orc);
903 #endif
904 LLVMOrcDisposeInstance(llvm_opt0_orc);
905 llvm_opt0_orc = NULL;
906 }
907 }
908 #endif /* LLVM_VERSION_MAJOR > 11 */
909 }
910
911 /* helper for llvm_create_types, returning a global var's type */
912 static LLVMTypeRef
load_type(LLVMModuleRef mod,const char * name)913 load_type(LLVMModuleRef mod, const char *name)
914 {
915 LLVMValueRef value;
916 LLVMTypeRef typ;
917
918 /* this'll return a *pointer* to the global */
919 value = LLVMGetNamedGlobal(mod, name);
920 if (!value)
921 elog(ERROR, "type %s is unknown", name);
922
923 /* therefore look at the contained type and return that */
924 typ = LLVMTypeOf(value);
925 Assert(typ != NULL);
926 typ = LLVMGetElementType(typ);
927 Assert(typ != NULL);
928 return typ;
929 }
930
931 /* helper for llvm_create_types, returning a function's return type */
932 static LLVMTypeRef
load_return_type(LLVMModuleRef mod,const char * name)933 load_return_type(LLVMModuleRef mod, const char *name)
934 {
935 LLVMValueRef value;
936 LLVMTypeRef typ;
937
938 /* this'll return a *pointer* to the function */
939 value = LLVMGetNamedFunction(mod, name);
940 if (!value)
941 elog(ERROR, "function %s is unknown", name);
942
943 /* get type of function pointer */
944 typ = LLVMTypeOf(value);
945 Assert(typ != NULL);
946 /* dereference pointer */
947 typ = LLVMGetElementType(typ);
948 Assert(typ != NULL);
949 /* and look at return type */
950 typ = LLVMGetReturnType(typ);
951 Assert(typ != NULL);
952
953 return typ;
954 }
955
956 /*
957 * Load required information, types, function signatures from llvmjit_types.c
958 * and make them available in global variables.
959 *
960 * Those global variables are then used while emitting code.
961 */
962 static void
llvm_create_types(void)963 llvm_create_types(void)
964 {
965 char path[MAXPGPATH];
966 LLVMMemoryBufferRef buf;
967 char *msg;
968 LLVMModuleRef mod = NULL;
969
970 snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
971
972 /* open file */
973 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
974 {
975 elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
976 path, msg);
977 }
978
979 /* eagerly load contents, going to need it all */
980 if (LLVMParseBitcode2(buf, &mod))
981 {
982 elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
983 }
984 LLVMDisposeMemoryBuffer(buf);
985
986 /*
987 * Load triple & layout from clang emitted file so we're guaranteed to be
988 * compatible.
989 */
990 llvm_triple = pstrdup(LLVMGetTarget(mod));
991 llvm_layout = pstrdup(LLVMGetDataLayoutStr(mod));
992
993 TypeSizeT = load_type(mod, "TypeSizeT");
994 TypeParamBool = load_return_type(mod, "FunctionReturningBool");
995 TypeStorageBool = load_type(mod, "TypeStorageBool");
996 TypePGFunction = load_type(mod, "TypePGFunction");
997 StructExprContext = load_type(mod, "StructExprContext");
998 StructExprEvalStep = load_type(mod, "StructExprEvalStep");
999 StructExprState = load_type(mod, "StructExprState");
1000 StructFunctionCallInfoData = load_type(mod, "StructFunctionCallInfoData");
1001 StructMemoryContextData = load_type(mod, "StructMemoryContextData");
1002 StructTupleTableSlot = load_type(mod, "StructTupleTableSlot");
1003 StructHeapTupleData = load_type(mod, "StructHeapTupleData");
1004 StructtupleDesc = load_type(mod, "StructtupleDesc");
1005 StructAggState = load_type(mod, "StructAggState");
1006 StructAggStatePerGroupData = load_type(mod, "StructAggStatePerGroupData");
1007 StructAggStatePerTransData = load_type(mod, "StructAggStatePerTransData");
1008
1009 AttributeTemplate = LLVMGetNamedFunction(mod, "AttributeTemplate");
1010 FuncStrlen = LLVMGetNamedFunction(mod, "strlen");
1011 FuncVarsizeAny = LLVMGetNamedFunction(mod, "varsize_any");
1012 FuncSlotGetsomeattrs = LLVMGetNamedFunction(mod, "slot_getsomeattrs");
1013 FuncSlotGetmissingattrs = LLVMGetNamedFunction(mod, "slot_getmissingattrs");
1014 FuncHeapGetsysattr = LLVMGetNamedFunction(mod, "heap_getsysattr");
1015 FuncMakeExpandedObjectReadOnlyInternal = LLVMGetNamedFunction(mod, "MakeExpandedObjectReadOnlyInternal");
1016 FuncExecEvalArrayRefSubscript = LLVMGetNamedFunction(mod, "ExecEvalArrayRefSubscript");
1017 FuncExecAggTransReparent = LLVMGetNamedFunction(mod, "ExecAggTransReparent");
1018 FuncExecAggInitGroup = LLVMGetNamedFunction(mod, "ExecAggInitGroup");
1019
1020 /*
1021 * Leave the module alive, otherwise references to function would be
1022 * dangling.
1023 */
1024
1025 return;
1026 }
1027
1028 /*
1029 * Split a symbol into module / function parts. If the function is in the
1030 * main binary (or an external library) *modname will be NULL.
1031 */
1032 void
llvm_split_symbol_name(const char * name,char ** modname,char ** funcname)1033 llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1034 {
1035 *modname = NULL;
1036 *funcname = NULL;
1037
1038 /*
1039 * Module function names are pgextern.$module.$funcname
1040 */
1041 if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1042 {
1043 /*
1044 * Symbol names cannot contain a ., therefore we can split based on
1045 * first and last occurance of one.
1046 */
1047 *funcname = rindex(name, '.');
1048 (*funcname)++; /* jump over . */
1049
1050 *modname = pnstrdup(name + strlen("pgextern."),
1051 *funcname - name - strlen("pgextern.") - 1);
1052 Assert(funcname);
1053
1054 *funcname = pstrdup(*funcname);
1055 }
1056 else
1057 {
1058 *modname = NULL;
1059 *funcname = pstrdup(name);
1060 }
1061 }
1062
1063 /*
1064 * Attempt to resolve symbol, so LLVM can emit a reference to it.
1065 */
1066 static uint64_t
llvm_resolve_symbol(const char * symname,void * ctx)1067 llvm_resolve_symbol(const char *symname, void *ctx)
1068 {
1069 uintptr_t addr;
1070 char *funcname;
1071 char *modname;
1072
1073 /*
1074 * macOS prefixes all object level symbols with an underscore. But neither
1075 * dlsym() nor PG's inliner expect that. So undo.
1076 */
1077 #if defined(__darwin__)
1078 if (symname[0] != '_')
1079 elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1080 symname++;
1081 #endif
1082
1083 llvm_split_symbol_name(symname, &modname, &funcname);
1084
1085 /* functions that aren't resolved to names shouldn't ever get here */
1086 Assert(funcname);
1087
1088 if (modname)
1089 addr = (uintptr_t) load_external_function(modname, funcname,
1090 true, NULL);
1091 else
1092 addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1093
1094 pfree(funcname);
1095 if (modname)
1096 pfree(modname);
1097
1098 /* let LLVM will error out - should never happen */
1099 if (!addr)
1100 elog(WARNING, "failed to resolve name %s", symname);
1101
1102 return (uint64_t) addr;
1103 }
1104
1105 #if LLVM_VERSION_MAJOR > 11
1106
1107 static LLVMErrorRef
llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj,void * Ctx,LLVMOrcLookupStateRef * LookupState,LLVMOrcLookupKind Kind,LLVMOrcJITDylibRef JD,LLVMOrcJITDylibLookupFlags JDLookupFlags,LLVMOrcCLookupSet LookupSet,size_t LookupSetSize)1108 llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1109 LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1110 LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1111 LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1112 {
1113 LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1114 LLVMErrorRef error;
1115 LLVMOrcMaterializationUnitRef mu;
1116
1117 for (int i = 0; i < LookupSetSize; i++)
1118 {
1119 const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1120
1121 #if LLVM_VERSION_MAJOR > 12
1122 LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1123 #endif
1124 symbols[i].Name = LookupSet[i].Name;
1125 symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1126 symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1127 }
1128
1129 mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1130 error = LLVMOrcJITDylibDefine(JD, mu);
1131 if (error != LLVMErrorSuccess)
1132 LLVMOrcDisposeMaterializationUnit(mu);
1133
1134 pfree(symbols);
1135
1136 return error;
1137 }
1138
1139 /*
1140 * We cannot throw errors through LLVM (without causing a FATAL at least), so
1141 * just use WARNING here. That's OK anyway, as the error is also reported at
1142 * the top level action (with less detail) and there might be multiple
1143 * invocations of errors with details.
1144 *
1145 * This doesn't really happen during normal operation, but in cases like
1146 * symbol resolution breakage. So just using elog(WARNING) is fine.
1147 */
1148 static void
llvm_log_jit_error(void * ctx,LLVMErrorRef error)1149 llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1150 {
1151 elog(WARNING, "error during JITing: %s",
1152 llvm_error_message(error));
1153 }
1154
1155 /*
1156 * Create our own object layer, so we can add event listeners.
1157 */
1158 static LLVMOrcObjectLayerRef
llvm_create_object_layer(void * Ctx,LLVMOrcExecutionSessionRef ES,const char * Triple)1159 llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1160 {
1161 LLVMOrcObjectLayerRef objlayer =
1162 LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1163
1164 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1165 if (jit_debugging_support)
1166 {
1167 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1168
1169 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1170 }
1171 #endif
1172
1173 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1174 if (jit_profiling_support)
1175 {
1176 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1177
1178 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1179 }
1180 #endif
1181
1182 return objlayer;
1183 }
1184
1185 /*
1186 * Create LLJIT instance, using the passed in target machine. Note that the
1187 * target machine afterwards is owned by the LLJIT instance.
1188 */
1189 static LLVMOrcLLJITRef
llvm_create_jit_instance(LLVMTargetMachineRef tm)1190 llvm_create_jit_instance(LLVMTargetMachineRef tm)
1191 {
1192 LLVMOrcLLJITRef lljit;
1193 LLVMOrcJITTargetMachineBuilderRef tm_builder;
1194 LLVMOrcLLJITBuilderRef lljit_builder;
1195 LLVMErrorRef error;
1196 LLVMOrcDefinitionGeneratorRef main_gen;
1197 LLVMOrcDefinitionGeneratorRef ref_gen;
1198
1199 lljit_builder = LLVMOrcCreateLLJITBuilder();
1200 tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1201 LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1202
1203 LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1204 llvm_create_object_layer,
1205 NULL);
1206
1207 error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1208 if (error)
1209 elog(ERROR, "failed to create lljit instance: %s",
1210 llvm_error_message(error));
1211
1212 LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1213 llvm_log_jit_error, NULL);
1214
1215 /*
1216 * Symbol resolution support for symbols in the postgres binary /
1217 * libraries already loaded.
1218 */
1219 error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1220 LLVMOrcLLJITGetGlobalPrefix(lljit),
1221 0, NULL);
1222 if (error)
1223 elog(ERROR, "failed to create generator: %s",
1224 llvm_error_message(error));
1225 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1226
1227 /*
1228 * Symbol resolution support for "special" functions, e.g. a call into an
1229 * SQL callable function.
1230 */
1231 ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1232 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1233
1234 return lljit;
1235 }
1236
1237 static char *
llvm_error_message(LLVMErrorRef error)1238 llvm_error_message(LLVMErrorRef error)
1239 {
1240 char *orig = LLVMGetErrorMessage(error);
1241 char *msg = pstrdup(orig);
1242
1243 LLVMDisposeErrorMessage(orig);
1244
1245 return msg;
1246 }
1247
1248 #endif /* LLVM_VERSION_MAJOR > 11 */
1249