1 /*
2 * Copyright (c) 2015 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
8 #include "analyze.hpp"
9 #include "codegen.hpp"
10 #include "errmsg.hpp"
11 #include "error.hpp"
12 #include "hash_map.hpp"
13 #include "ir.hpp"
14 #include "os.hpp"
15 #include "target.hpp"
16 #include "util.hpp"
17 #include "zig_llvm.h"
18 #include "stage2.h"
19 #include "dump_analysis.hpp"
20 #include "softfloat.hpp"
21 #include "zigendian.h"
22
23 #include <stdio.h>
24 #include <errno.h>
25 #include <math.h>
26
27 enum ResumeId {
28 ResumeIdManual,
29 ResumeIdReturn,
30 ResumeIdCall,
31 };
32
new_package(const char * root_src_dir,const char * root_src_path,const char * pkg_path)33 static ZigPackage *new_package(const char *root_src_dir, const char *root_src_path, const char *pkg_path) {
34 ZigPackage *entry = heap::c_allocator.create<ZigPackage>();
35 entry->package_table.init(4);
36 buf_init_from_str(&entry->root_src_dir, root_src_dir);
37 buf_init_from_str(&entry->root_src_path, root_src_path);
38 buf_init_from_str(&entry->pkg_path, pkg_path);
39 return entry;
40 }
41
new_anonymous_package()42 ZigPackage *new_anonymous_package() {
43 return new_package("", "", "");
44 }
45
46 static const char *symbols_that_llvm_depends_on[] = {
47 "memcpy",
48 "memset",
49 "sqrt",
50 "powi",
51 "sin",
52 "cos",
53 "pow",
54 "exp",
55 "exp2",
56 "log",
57 "log10",
58 "log2",
59 "fma",
60 "fmaf",
61 "fmal",
62 "fmaq",
63 "fabs",
64 "minnum",
65 "maxnum",
66 "copysign",
67 "floor",
68 "ceil",
69 "trunc",
70 "rint",
71 "nearbyint",
72 "round",
73 // TODO probably all of compiler-rt needs to go here
74 };
75
codegen_set_strip(CodeGen * g,bool strip)76 void codegen_set_strip(CodeGen *g, bool strip) {
77 g->strip_debug_symbols = strip;
78 if (!target_has_debug_info(g->zig_target)) {
79 g->strip_debug_symbols = true;
80 }
81 }
82
83 static void render_const_val(CodeGen *g, ZigValue *const_val, const char *name);
84 static void render_const_val_global(CodeGen *g, ZigValue *const_val, const char *name);
85 static LLVMValueRef gen_const_val(CodeGen *g, ZigValue *const_val, const char *name);
86 static void generate_error_name_table(CodeGen *g);
87 static bool value_is_all_undef(CodeGen *g, ZigValue *const_val);
88 static void gen_undef_init(CodeGen *g, ZigType *ptr_type, ZigType *value_type, LLVMValueRef ptr);
89 static LLVMValueRef build_alloca(CodeGen *g, ZigType *type_entry, const char *name, uint32_t alignment);
90 static LLVMValueRef gen_await_early_return(CodeGen *g, Stage1AirInst *source_instr,
91 LLVMValueRef target_frame_ptr, ZigType *result_type, ZigType *ptr_result_type,
92 LLVMValueRef result_loc, bool non_async);
93
addLLVMAttr(LLVMValueRef val,LLVMAttributeIndex attr_index,const char * attr_name)94 static void addLLVMAttr(LLVMValueRef val, LLVMAttributeIndex attr_index, const char *attr_name) {
95 unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name, strlen(attr_name));
96 assert(kind_id != 0);
97 LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(LLVMGetGlobalContext(), kind_id, 0);
98 LLVMAddAttributeAtIndex(val, attr_index, llvm_attr);
99 }
100
addLLVMAttrStr(LLVMValueRef val,LLVMAttributeIndex attr_index,const char * attr_name,const char * attr_val)101 static void addLLVMAttrStr(LLVMValueRef val, LLVMAttributeIndex attr_index,
102 const char *attr_name, const char *attr_val)
103 {
104 LLVMAttributeRef llvm_attr = LLVMCreateStringAttribute(LLVMGetGlobalContext(),
105 attr_name, (unsigned)strlen(attr_name), attr_val, (unsigned)strlen(attr_val));
106 LLVMAddAttributeAtIndex(val, attr_index, llvm_attr);
107 }
108
addLLVMAttrInt(LLVMValueRef val,LLVMAttributeIndex attr_index,const char * attr_name,uint64_t attr_val)109 static void addLLVMAttrInt(LLVMValueRef val, LLVMAttributeIndex attr_index,
110 const char *attr_name, uint64_t attr_val)
111 {
112 unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name, strlen(attr_name));
113 assert(kind_id != 0);
114 LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(LLVMGetGlobalContext(), kind_id, attr_val);
115 LLVMAddAttributeAtIndex(val, attr_index, llvm_attr);
116 }
117
addLLVMFnAttr(LLVMValueRef fn_val,const char * attr_name)118 static void addLLVMFnAttr(LLVMValueRef fn_val, const char *attr_name) {
119 return addLLVMAttr(fn_val, -1, attr_name);
120 }
121
addLLVMFnAttrStr(LLVMValueRef fn_val,const char * attr_name,const char * attr_val)122 static void addLLVMFnAttrStr(LLVMValueRef fn_val, const char *attr_name, const char *attr_val) {
123 return addLLVMAttrStr(fn_val, -1, attr_name, attr_val);
124 }
125
addLLVMFnAttrInt(LLVMValueRef fn_val,const char * attr_name,uint64_t attr_val)126 static void addLLVMFnAttrInt(LLVMValueRef fn_val, const char *attr_name, uint64_t attr_val) {
127 return addLLVMAttrInt(fn_val, -1, attr_name, attr_val);
128 }
129
addLLVMArgAttr(LLVMValueRef fn_val,unsigned param_index,const char * attr_name)130 static void addLLVMArgAttr(LLVMValueRef fn_val, unsigned param_index, const char *attr_name) {
131 return addLLVMAttr(fn_val, param_index + 1, attr_name);
132 }
133
addLLVMArgAttrInt(LLVMValueRef fn_val,unsigned param_index,const char * attr_name,uint64_t attr_val)134 static void addLLVMArgAttrInt(LLVMValueRef fn_val, unsigned param_index, const char *attr_name, uint64_t attr_val) {
135 return addLLVMAttrInt(fn_val, param_index + 1, attr_name, attr_val);
136 }
137
is_symbol_available(CodeGen * g,const char * name)138 static bool is_symbol_available(CodeGen *g, const char *name) {
139 Buf *buf_name = buf_create_from_str(name);
140 bool result =
141 g->exported_symbol_names.maybe_get(buf_name) == nullptr &&
142 g->external_symbol_names.maybe_get(buf_name) == nullptr;
143 buf_destroy(buf_name);
144 return result;
145 }
146
get_mangled_name(CodeGen * g,const char * original_name)147 static const char *get_mangled_name(CodeGen *g, const char *original_name) {
148 if (is_symbol_available(g, original_name))
149 return original_name;
150
151 int n = 0;
152 for (;; n += 1) {
153 const char *new_name = buf_ptr(buf_sprintf("%s.%d", original_name, n));
154 if (is_symbol_available(g, new_name)) {
155 return new_name;
156 }
157 }
158 }
159
160 // Sync this with emit_error_unless_callconv_allowed_for_target in analyze.cpp
get_llvm_cc(CodeGen * g,CallingConvention cc)161 static ZigLLVM_CallingConv get_llvm_cc(CodeGen *g, CallingConvention cc) {
162 switch (cc) {
163 case CallingConventionUnspecified:
164 case CallingConventionInline:
165 return ZigLLVM_Fast;
166 case CallingConventionC:
167 return ZigLLVM_C;
168 case CallingConventionNaked:
169 zig_unreachable();
170 case CallingConventionStdcall:
171 assert(g->zig_target->arch == ZigLLVM_x86);
172 return ZigLLVM_X86_StdCall;
173 case CallingConventionFastcall:
174 assert(g->zig_target->arch == ZigLLVM_x86);
175 return ZigLLVM_X86_FastCall;
176 case CallingConventionVectorcall:
177 if (g->zig_target->arch == ZigLLVM_x86)
178 return ZigLLVM_X86_VectorCall;
179 if (target_is_arm(g->zig_target) &&
180 target_arch_pointer_bit_width(g->zig_target->arch) == 64)
181 return ZigLLVM_AArch64_VectorCall;
182 zig_unreachable();
183 case CallingConventionThiscall:
184 assert(g->zig_target->arch == ZigLLVM_x86);
185 return ZigLLVM_X86_ThisCall;
186 case CallingConventionAsync:
187 return ZigLLVM_Fast;
188 case CallingConventionAPCS:
189 assert(target_is_arm(g->zig_target));
190 return ZigLLVM_ARM_APCS;
191 case CallingConventionAAPCS:
192 assert(target_is_arm(g->zig_target));
193 return ZigLLVM_ARM_AAPCS;
194 case CallingConventionAAPCSVFP:
195 assert(target_is_arm(g->zig_target));
196 return ZigLLVM_ARM_AAPCS_VFP;
197 case CallingConventionInterrupt:
198 if (g->zig_target->arch == ZigLLVM_x86 ||
199 g->zig_target->arch == ZigLLVM_x86_64)
200 return ZigLLVM_X86_INTR;
201 if (g->zig_target->arch == ZigLLVM_avr)
202 return ZigLLVM_AVR_INTR;
203 if (g->zig_target->arch == ZigLLVM_msp430)
204 return ZigLLVM_MSP430_INTR;
205 zig_unreachable();
206 case CallingConventionSignal:
207 assert(g->zig_target->arch == ZigLLVM_avr);
208 return ZigLLVM_AVR_SIGNAL;
209 case CallingConventionSysV:
210 assert(g->zig_target->arch == ZigLLVM_x86_64);
211 return ZigLLVM_X86_64_SysV;
212 }
213 zig_unreachable();
214 }
215
add_uwtable_attr(CodeGen * g,LLVMValueRef fn_val)216 static void add_uwtable_attr(CodeGen *g, LLVMValueRef fn_val) {
217 if (g->unwind_tables) {
218 addLLVMFnAttr(fn_val, "uwtable");
219 }
220 }
221
to_llvm_linkage(GlobalLinkageId id,bool is_extern)222 static LLVMLinkage to_llvm_linkage(GlobalLinkageId id, bool is_extern) {
223 switch (id) {
224 case GlobalLinkageIdInternal:
225 return LLVMInternalLinkage;
226 case GlobalLinkageIdStrong:
227 return LLVMExternalLinkage;
228 case GlobalLinkageIdWeak:
229 if (is_extern) return LLVMExternalWeakLinkage;
230 return LLVMWeakODRLinkage;
231 case GlobalLinkageIdLinkOnce:
232 return LLVMLinkOnceODRLinkage;
233 }
234 zig_unreachable();
235 }
236
237 struct CalcLLVMFieldIndex {
238 uint32_t offset;
239 uint32_t field_index;
240 };
241
calc_llvm_field_index_add(CodeGen * g,CalcLLVMFieldIndex * calc,ZigType * ty)242 static void calc_llvm_field_index_add(CodeGen *g, CalcLLVMFieldIndex *calc, ZigType *ty) {
243 if (!type_has_bits(g, ty)) return;
244 uint32_t ty_align = get_abi_alignment(g, ty);
245
246 if (calc->offset % ty_align != 0) {
247 uint32_t llvm_align = LLVMABIAlignmentOfType(g->target_data_ref, get_llvm_type(g, ty));
248
249 // Alignment according to Zig.
250 uint32_t adj_offset = calc->offset + (ty_align - (calc->offset % ty_align));
251 // Alignment according to LLVM.
252 uint32_t adj_llvm_offset = (calc->offset % llvm_align) ?
253 calc->offset + (llvm_align - (calc->offset % llvm_align)) :
254 calc->offset;
255 // Cannot under-align structure fields.
256 assert(adj_offset >= adj_llvm_offset);
257
258 // Zig will insert an extra padding field here.
259 if (adj_offset != adj_llvm_offset)
260 calc->field_index += 1;
261
262 calc->offset = adj_offset;
263 }
264 calc->offset += ty->abi_size;
265 calc->field_index += 1;
266 }
267
268 // label (grep this): [fn_frame_struct_layout]
frame_index_trace_arg_calc(CodeGen * g,CalcLLVMFieldIndex * calc,ZigType * return_type)269 static void frame_index_trace_arg_calc(CodeGen *g, CalcLLVMFieldIndex *calc, ZigType *return_type) {
270 calc_llvm_field_index_add(g, calc, g->builtin_types.entry_usize); // function pointer
271 calc_llvm_field_index_add(g, calc, g->builtin_types.entry_usize); // resume index
272 calc_llvm_field_index_add(g, calc, g->builtin_types.entry_usize); // awaiter index
273
274 if (type_has_bits(g, return_type)) {
275 calc_llvm_field_index_add(g, calc, g->builtin_types.entry_usize); // *ReturnType (callee's)
276 calc_llvm_field_index_add(g, calc, g->builtin_types.entry_usize); // *ReturnType (awaiter's)
277 calc_llvm_field_index_add(g, calc, return_type); // ReturnType
278 }
279 }
280
frame_index_trace_arg(CodeGen * g,ZigType * return_type)281 static uint32_t frame_index_trace_arg(CodeGen *g, ZigType *return_type) {
282 CalcLLVMFieldIndex calc = {0};
283 frame_index_trace_arg_calc(g, &calc, return_type);
284 return calc.field_index;
285 }
286
287 // label (grep this): [fn_frame_struct_layout]
frame_index_arg_calc(CodeGen * g,CalcLLVMFieldIndex * calc,ZigType * return_type)288 static void frame_index_arg_calc(CodeGen *g, CalcLLVMFieldIndex *calc, ZigType *return_type) {
289 frame_index_trace_arg_calc(g, calc, return_type);
290
291 if (codegen_fn_has_err_ret_tracing_arg(g, return_type)) {
292 calc_llvm_field_index_add(g, calc, g->builtin_types.entry_usize); // *StackTrace (callee's)
293 calc_llvm_field_index_add(g, calc, g->builtin_types.entry_usize); // *StackTrace (awaiter's)
294 }
295 }
296
297 // label (grep this): [fn_frame_struct_layout]
frame_index_trace_stack(CodeGen * g,ZigFn * fn)298 static uint32_t frame_index_trace_stack(CodeGen *g, ZigFn *fn) {
299 size_t field_index = 6;
300 bool have_stack_trace = codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type);
301 if (have_stack_trace) {
302 field_index += 2;
303 }
304 field_index += fn->type_entry->data.fn.fn_type_id.param_count;
305 ZigType *locals_struct = fn->frame_type->data.frame.locals_struct;
306 TypeStructField *field = locals_struct->data.structure.fields[field_index];
307 return field->gen_index;
308 }
309
310
get_err_ret_trace_arg_index(CodeGen * g,ZigFn * fn_table_entry)311 static uint32_t get_err_ret_trace_arg_index(CodeGen *g, ZigFn *fn_table_entry) {
312 if (!g->have_err_ret_tracing) {
313 return UINT32_MAX;
314 }
315 if (fn_is_async(fn_table_entry)) {
316 return UINT32_MAX;
317 }
318 ZigType *fn_type = fn_table_entry->type_entry;
319 if (!fn_type_can_fail(&fn_type->data.fn.fn_type_id)) {
320 return UINT32_MAX;
321 }
322 ZigType *return_type = fn_type->data.fn.fn_type_id.return_type;
323 bool first_arg_ret = type_has_bits(g, return_type) && handle_is_ptr(g, return_type);
324 return first_arg_ret ? 1 : 0;
325 }
326
maybe_export_dll(CodeGen * g,LLVMValueRef global_value,GlobalLinkageId linkage)327 static void maybe_export_dll(CodeGen *g, LLVMValueRef global_value, GlobalLinkageId linkage) {
328 if (linkage != GlobalLinkageIdInternal && g->zig_target->os == OsWindows && g->dll_export_fns) {
329 LLVMSetDLLStorageClass(global_value, LLVMDLLExportStorageClass);
330 }
331 }
332
maybe_import_dll(CodeGen * g,LLVMValueRef global_value,GlobalLinkageId linkage)333 static void maybe_import_dll(CodeGen *g, LLVMValueRef global_value, GlobalLinkageId linkage) {
334 if (linkage != GlobalLinkageIdInternal && g->zig_target->os == OsWindows) {
335 // TODO come up with a good explanation/understanding for why we never do
336 // DLLImportStorageClass. Empirically it only causes problems. But let's have
337 // this documented and then clean up the code accordingly.
338 //LLVMSetDLLStorageClass(global_value, LLVMDLLImportStorageClass);
339 }
340 }
341
cc_want_sret_attr(CallingConvention cc)342 static bool cc_want_sret_attr(CallingConvention cc) {
343 switch (cc) {
344 case CallingConventionNaked:
345 zig_unreachable();
346 case CallingConventionC:
347 case CallingConventionInterrupt:
348 case CallingConventionSignal:
349 case CallingConventionStdcall:
350 case CallingConventionFastcall:
351 case CallingConventionVectorcall:
352 case CallingConventionThiscall:
353 case CallingConventionAPCS:
354 case CallingConventionAAPCS:
355 case CallingConventionAAPCSVFP:
356 case CallingConventionSysV:
357 return true;
358 case CallingConventionAsync:
359 case CallingConventionUnspecified:
360 case CallingConventionInline:
361 return false;
362 }
363 zig_unreachable();
364 }
365
add_common_fn_attributes(CodeGen * g,LLVMValueRef llvm_fn)366 static void add_common_fn_attributes(CodeGen *g, LLVMValueRef llvm_fn) {
367 if (!g->red_zone) {
368 addLLVMFnAttr(llvm_fn, "noredzone");
369 }
370
371 addLLVMFnAttr(llvm_fn, "nounwind");
372 add_uwtable_attr(g, llvm_fn);
373 addLLVMFnAttr(llvm_fn, "nobuiltin");
374
375 if (g->build_mode == BuildModeSmallRelease) {
376 // Optimize for small code size.
377 addLLVMFnAttr(llvm_fn, "minsize");
378 addLLVMFnAttr(llvm_fn, "optsize");
379 }
380
381 if (g->zig_target->llvm_cpu_name != nullptr) {
382 ZigLLVMAddFunctionAttr(llvm_fn, "target-cpu", g->zig_target->llvm_cpu_name);
383 }
384 if (g->zig_target->llvm_cpu_features != nullptr) {
385 ZigLLVMAddFunctionAttr(llvm_fn, "target-features", g->zig_target->llvm_cpu_features);
386 }
387 }
388
make_fn_llvm_value(CodeGen * g,ZigFn * fn)389 static LLVMValueRef make_fn_llvm_value(CodeGen *g, ZigFn *fn) {
390 const char *unmangled_name = buf_ptr(&fn->symbol_name);
391 const char *symbol_name;
392 GlobalLinkageId linkage;
393 if (fn->body_node == nullptr) {
394 symbol_name = unmangled_name;
395 linkage = GlobalLinkageIdStrong;
396 } else if (fn->export_list.length == 0) {
397 symbol_name = get_mangled_name(g, unmangled_name);
398 linkage = GlobalLinkageIdInternal;
399 } else {
400 GlobalExport *fn_export = &fn->export_list.items[0];
401 symbol_name = buf_ptr(&fn_export->name);
402 linkage = fn_export->linkage;
403 }
404
405 CallingConvention cc = fn->type_entry->data.fn.fn_type_id.cc;
406 bool is_async = fn_is_async(fn);
407
408 ZigType *fn_type = fn->type_entry;
409 // Make the raw_type_ref populated
410 resolve_llvm_types_fn(g, fn);
411 LLVMTypeRef fn_llvm_type = fn->raw_type_ref;
412 LLVMValueRef llvm_fn = nullptr;
413 if (fn->body_node == nullptr) {
414 assert(fn->proto_node->type == NodeTypeFnProto);
415 AstNodeFnProto *fn_proto = &fn->proto_node->data.fn_proto;
416
417 const unsigned fn_addrspace = ZigLLVMDataLayoutGetProgramAddressSpace(g->target_data_ref);
418
419 // The compiler tries to deduplicate extern definitions by looking up
420 // their name, this was introduced to allow the declaration of the same
421 // extern function with differing prototypes.
422 // When Wasm is targeted this check becomes a problem as the user may
423 // declare two (or more) extern functions sharing the same name but
424 // imported from different modules!
425 // To overcome this problem we generate a mangled identifier out of the
426 // import and the function name, this name is only visible within the
427 // compiler as we're telling LLVM (using 'wasm-import-name' and
428 // 'wasm-import-name') what the real function name is and where to find
429 // it.
430 bool use_mangled_name = target_is_wasm(g->zig_target) &&
431 fn_proto->is_extern && fn_proto->lib_name != nullptr;
432 // This is subtle but important to match libc symbols at static link time correctly.
433 // We treat "c" lib_name as a special library indicating that it should be defined
434 // in libc. But if we mangle a libc symbol name here with "c" module name, then wasm-ld cannot resolve
435 // the symbol. This is because at the static link time with wasm-ld, the linker does not
436 // take module names into account, and instead looking for a pure symbol name (i.e. function name)
437 // written into the ".linking" custom section (i.e. it does not use import section).
438 // This is the intended behavior of wasm-ld, because Wasm has a concept of host functions,
439 // which are undefined functions supposed to be resolved by host runtimes *with module names*
440 // at load times even if it is "static linked" with the linker.
441 use_mangled_name = use_mangled_name && (strcmp(buf_ptr(fn_proto->lib_name), "c") != 0);
442 // Pick a weird name to avoid collisions...
443 // This whole function should be burned to the ground.
444 Buf *mangled_symbol_buf = use_mangled_name ?
445 buf_sprintf("%s|%s", unmangled_name, buf_ptr(fn_proto->lib_name)) :
446 nullptr;
447 symbol_name = use_mangled_name ?
448 buf_ptr(mangled_symbol_buf) : unmangled_name;
449
450 LLVMValueRef existing_llvm_fn = LLVMGetNamedFunction(g->module, symbol_name);
451
452 if (existing_llvm_fn) {
453 if (mangled_symbol_buf) buf_destroy(mangled_symbol_buf);
454 return LLVMConstBitCast(existing_llvm_fn, LLVMPointerType(fn_llvm_type, fn_addrspace));
455 } else {
456 Buf *buf_symbol_name = buf_create_from_str(symbol_name);
457 auto entry = g->exported_symbol_names.maybe_get(buf_symbol_name);
458 buf_destroy(buf_symbol_name);
459
460 if (entry == nullptr) {
461 llvm_fn = LLVMAddFunction(g->module, symbol_name, fn_llvm_type);
462
463 if (use_mangled_name) {
464 // Note that "wasm-import-module"ed symbols will not be resolved
465 // in the future version of wasm-ld since the attribute basically means that
466 // "the symbol should be resolved at load time by runtimes", though
467 // the symbol is already mangled here and it is written into "linking" section
468 // used by wasm-ld to match symbols, so it should not be expected by users.
469 // tl;dr is that users should not put the lib_name specifier on extern statements
470 // if they want to link symbols with wasm-ld.
471 addLLVMFnAttrStr(llvm_fn, "wasm-import-name", unmangled_name);
472 addLLVMFnAttrStr(llvm_fn, "wasm-import-module", buf_ptr(fn_proto->lib_name));
473 }
474 } else {
475 assert(entry->value->id == TldIdFn);
476 TldFn *tld_fn = reinterpret_cast<TldFn *>(entry->value);
477 // Make the raw_type_ref populated
478 resolve_llvm_types_fn(g, tld_fn->fn_entry);
479 tld_fn->fn_entry->llvm_value = LLVMAddFunction(g->module, symbol_name,
480 tld_fn->fn_entry->raw_type_ref);
481 llvm_fn = LLVMConstBitCast(tld_fn->fn_entry->llvm_value, LLVMPointerType(fn_llvm_type, fn_addrspace));
482 if (mangled_symbol_buf) buf_destroy(mangled_symbol_buf);
483 return llvm_fn;
484 }
485
486 if (mangled_symbol_buf) buf_destroy(mangled_symbol_buf);
487 }
488 } else {
489 llvm_fn = LLVMAddFunction(g->module, symbol_name, fn_llvm_type);
490
491 for (size_t i = 1; i < fn->export_list.length; i += 1) {
492 GlobalExport *fn_export = &fn->export_list.items[i];
493 LLVMAddAlias(g->module, LLVMTypeOf(llvm_fn), llvm_fn, buf_ptr(&fn_export->name));
494 }
495 }
496
497 if (cc == CallingConventionInline)
498 addLLVMFnAttr(llvm_fn, "alwaysinline");
499
500 if (fn->is_noinline || (cc != CallingConventionInline && fn->alignstack_value != 0))
501 addLLVMFnAttr(llvm_fn, "noinline");
502
503 if (cc == CallingConventionNaked) {
504 addLLVMFnAttr(llvm_fn, "naked");
505 } else {
506 ZigLLVMFunctionSetCallingConv(llvm_fn, get_llvm_cc(g, cc));
507 }
508
509 if (g->tsan_enabled) {
510 addLLVMFnAttr(llvm_fn, "sanitize_thread");
511 }
512
513 bool want_cold = fn->is_cold;
514 if (want_cold) {
515 ZigLLVMAddFunctionAttrCold(llvm_fn);
516 }
517
518
519 LLVMSetLinkage(llvm_fn, to_llvm_linkage(linkage, fn->body_node == nullptr));
520
521 if (linkage == GlobalLinkageIdInternal) {
522 LLVMSetUnnamedAddr(llvm_fn, true);
523 }
524
525 ZigType *return_type = fn_type->data.fn.fn_type_id.return_type;
526 if (return_type->id == ZigTypeIdUnreachable) {
527 addLLVMFnAttr(llvm_fn, "noreturn");
528 }
529
530 if (!calling_convention_allows_zig_types(cc)) {
531 // A simplistic and desperate attempt at making the compiler respect the
532 // target ABI for return types.
533 // This is just enough to avoid miscompiling the test suite, it will be
534 // better in stage2.
535 ZigType *int_type = return_type->id == ZigTypeIdInt ? return_type :
536 return_type->id == ZigTypeIdEnum ? return_type->data.enumeration.tag_int_type :
537 nullptr;
538
539 if (int_type != nullptr) {
540 const bool is_signed = int_type->data.integral.is_signed;
541 const uint32_t bit_width = int_type->data.integral.bit_count;
542 bool should_extend = false;
543
544 // Rough equivalent of Clang's isPromotableIntegerType.
545 switch (bit_width) {
546 case 1: // bool
547 case 8: // {un,}signed char
548 case 16: // {un,}signed short
549 should_extend = true;
550 break;
551 default:
552 break;
553 }
554
555 switch (g->zig_target->arch) {
556 case ZigLLVM_sparcv9:
557 case ZigLLVM_riscv64:
558 case ZigLLVM_ppc64:
559 case ZigLLVM_ppc64le:
560 // Always extend to the register width.
561 should_extend = bit_width < 64;
562 break;
563 default:
564 break;
565 }
566
567 // {zero,sign}-extend the result.
568 if (should_extend) {
569 if (is_signed)
570 addLLVMAttr(llvm_fn, 0, "signext");
571 else
572 addLLVMAttr(llvm_fn, 0, "zeroext");
573 }
574 }
575 }
576
577 if (fn->body_node != nullptr) {
578 maybe_export_dll(g, llvm_fn, linkage);
579
580 bool want_ssp_attrs = g->build_mode != BuildModeFastRelease &&
581 g->build_mode != BuildModeSmallRelease &&
582 g->link_libc &&
583 // WASI-libc does not support stack-protector yet.
584 !target_is_wasm(g->zig_target);
585 if (want_ssp_attrs) {
586 addLLVMFnAttr(llvm_fn, "sspstrong");
587 addLLVMFnAttrStr(llvm_fn, "stack-protector-buffer-size", "4");
588 }
589 if (g->have_stack_probing && !fn->def_scope->safety_off) {
590 addLLVMFnAttrStr(llvm_fn, "probe-stack", "__zig_probe_stack");
591 } else if (g->zig_target->os == OsUefi) {
592 addLLVMFnAttrStr(llvm_fn, "no-stack-arg-probe", "");
593 }
594 } else {
595 maybe_import_dll(g, llvm_fn, linkage);
596 }
597
598 if (fn->alignstack_value != 0) {
599 addLLVMFnAttrInt(llvm_fn, "alignstack", fn->alignstack_value);
600 }
601
602 if (!g->omit_frame_pointer && cc != CallingConventionInline) {
603 ZigLLVMAddFunctionAttr(llvm_fn, "frame-pointer", "all");
604 }
605 if (fn->section_name) {
606 LLVMSetSection(llvm_fn, buf_ptr(fn->section_name));
607 }
608 if (fn->align_bytes > 0) {
609 LLVMSetAlignment(llvm_fn, (unsigned)fn->align_bytes);
610 } else {
611 // We'd like to set the best alignment for the function here, but on Darwin LLVM gives
612 // "Cannot getTypeInfo() on a type that is unsized!" assertion failure when calling
613 // any of the functions for getting alignment. Not specifying the alignment should
614 // use the ABI alignment, which is fine.
615 }
616
617 add_common_fn_attributes(g, llvm_fn);
618
619 if (is_async) {
620 addLLVMArgAttr(llvm_fn, 0, "nonnull");
621 } else {
622 unsigned init_gen_i = 0;
623 if (!type_has_bits(g, return_type)) {
624 // nothing to do
625 } else if (type_is_nonnull_ptr(g, return_type)) {
626 addLLVMAttr(llvm_fn, 0, "nonnull");
627 } else if (want_first_arg_sret(g, &fn_type->data.fn.fn_type_id)) {
628 // Sret pointers must not be address 0
629 addLLVMArgAttr(llvm_fn, 0, "nonnull");
630 ZigLLVMAddSretAttr(llvm_fn, 0, get_llvm_type(g, return_type));
631 if (cc_want_sret_attr(cc)) {
632 addLLVMArgAttr(llvm_fn, 0, "noalias");
633 }
634 init_gen_i = 1;
635 }
636
637 // set parameter attributes
638 FnWalk fn_walk = {};
639 fn_walk.id = FnWalkIdAttrs;
640 fn_walk.data.attrs.fn = fn;
641 fn_walk.data.attrs.llvm_fn = llvm_fn;
642 fn_walk.data.attrs.gen_i = init_gen_i;
643 walk_function_params(g, fn_type, &fn_walk);
644
645 uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn);
646 if (err_ret_trace_arg_index != UINT32_MAX) {
647 // Error return trace memory is in the stack, which is impossible to be at address 0
648 // on any architecture.
649 addLLVMArgAttr(llvm_fn, (unsigned)err_ret_trace_arg_index, "nonnull");
650 }
651 }
652
653 return llvm_fn;
654 }
655
fn_llvm_value(CodeGen * g,ZigFn * fn)656 static LLVMValueRef fn_llvm_value(CodeGen *g, ZigFn *fn) {
657 if (fn->llvm_value)
658 return fn->llvm_value;
659
660 fn->llvm_value = make_fn_llvm_value(g, fn);
661 fn->llvm_name = strdup(LLVMGetValueName(fn->llvm_value));
662 return fn->llvm_value;
663 }
664
node_line_onebased(AstNode * node)665 static uint32_t node_line_onebased(AstNode *node) {
666 RootStruct *root_struct = node->owner->data.structure.root_struct;
667 assert(node->main_token < root_struct->token_count);
668 return root_struct->token_locs[node->main_token].line + 1;
669 }
670
node_column_onebased(AstNode * node)671 static uint32_t node_column_onebased(AstNode *node) {
672 RootStruct *root_struct = node->owner->data.structure.root_struct;
673 assert(node->main_token < root_struct->token_count);
674 return root_struct->token_locs[node->main_token].column + 1;
675 }
676
get_di_scope(CodeGen * g,Scope * scope)677 static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) {
678 if (scope->di_scope)
679 return scope->di_scope;
680
681 ZigType *import = get_scope_import(scope);
682 switch (scope->id) {
683 case ScopeIdCImport:
684 zig_unreachable();
685 case ScopeIdFnDef:
686 {
687 assert(scope->parent);
688 ScopeFnDef *fn_scope = (ScopeFnDef *)scope;
689 ZigFn *fn_table_entry = fn_scope->fn_entry;
690 if (!fn_table_entry->proto_node)
691 return get_di_scope(g, scope->parent);
692 unsigned line_number = node_line_onebased(fn_table_entry->proto_node);
693 unsigned scope_line = line_number;
694 bool is_definition = fn_table_entry->body_node != nullptr;
695 bool is_optimized = g->build_mode != BuildModeDebug;
696 bool is_internal_linkage = (fn_table_entry->body_node != nullptr &&
697 fn_table_entry->export_list.length == 0);
698 unsigned flags = ZigLLVM_DIFlags_StaticMember;
699 ZigLLVMDIScope *fn_di_scope = get_di_scope(g, scope->parent);
700 assert(fn_di_scope != nullptr);
701 assert(fn_table_entry->raw_di_type != nullptr);
702 ZigLLVMDISubprogram *subprogram = ZigLLVMCreateFunction(g->dbuilder,
703 fn_di_scope, buf_ptr(&fn_table_entry->symbol_name), "",
704 import->data.structure.root_struct->di_file, line_number,
705 fn_table_entry->raw_di_type, is_internal_linkage,
706 is_definition, scope_line, flags, is_optimized, nullptr);
707
708 scope->di_scope = ZigLLVMSubprogramToScope(subprogram);
709 if (!g->strip_debug_symbols) {
710 ZigLLVMFnSetSubprogram(fn_llvm_value(g, fn_table_entry), subprogram);
711 }
712 return scope->di_scope;
713 }
714 case ScopeIdDecls:
715 if (scope->parent) {
716 ScopeDecls *decls_scope = (ScopeDecls *)scope;
717 assert(decls_scope->container_type);
718 scope->di_scope = ZigLLVMTypeToScope(get_llvm_di_type(g, decls_scope->container_type));
719 } else {
720 scope->di_scope = ZigLLVMFileToScope(import->data.structure.root_struct->di_file);
721 }
722 return scope->di_scope;
723 case ScopeIdBlock:
724 case ScopeIdDefer:
725 {
726 assert(scope->parent);
727 ZigLLVMDILexicalBlock *di_block = ZigLLVMCreateLexicalBlock(g->dbuilder,
728 get_di_scope(g, scope->parent),
729 import->data.structure.root_struct->di_file,
730 node_line_onebased(scope->source_node),
731 node_column_onebased(scope->source_node));
732 scope->di_scope = ZigLLVMLexicalBlockToScope(di_block);
733 return scope->di_scope;
734 }
735 case ScopeIdVarDecl:
736 case ScopeIdDeferExpr:
737 case ScopeIdLoop:
738 case ScopeIdSuspend:
739 case ScopeIdCompTime:
740 case ScopeIdNoSuspend:
741 case ScopeIdRuntime:
742 case ScopeIdTypeOf:
743 case ScopeIdExpr:
744 return get_di_scope(g, scope->parent);
745 }
746 zig_unreachable();
747 }
748
clear_debug_source_node(CodeGen * g)749 static void clear_debug_source_node(CodeGen *g) {
750 ZigLLVMClearCurrentDebugLocation(g->builder);
751 }
752
get_arithmetic_overflow_fn(CodeGen * g,ZigType * operand_type,const char * signed_name,const char * unsigned_name)753 static LLVMValueRef get_arithmetic_overflow_fn(CodeGen *g, ZigType *operand_type,
754 const char *signed_name, const char *unsigned_name)
755 {
756 ZigType *int_type = (operand_type->id == ZigTypeIdVector) ? operand_type->data.vector.elem_type : operand_type;
757 char fn_name[64];
758
759 assert(int_type->id == ZigTypeIdInt);
760 const char *signed_str = int_type->data.integral.is_signed ? signed_name : unsigned_name;
761
762 LLVMTypeRef param_types[] = {
763 get_llvm_type(g, operand_type),
764 get_llvm_type(g, operand_type),
765 };
766
767 if (operand_type->id == ZigTypeIdVector) {
768 sprintf(fn_name, "llvm.%s.with.overflow.v%" PRIu64 "i%" PRIu32, signed_str,
769 operand_type->data.vector.len, int_type->data.integral.bit_count);
770
771 LLVMTypeRef return_elem_types[] = {
772 get_llvm_type(g, operand_type),
773 LLVMVectorType(LLVMInt1Type(), operand_type->data.vector.len),
774 };
775 LLVMTypeRef return_struct_type = LLVMStructType(return_elem_types, 2, false);
776 LLVMTypeRef fn_type = LLVMFunctionType(return_struct_type, param_types, 2, false);
777 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type);
778 assert(LLVMGetIntrinsicID(fn_val));
779 return fn_val;
780 } else {
781 sprintf(fn_name, "llvm.%s.with.overflow.i%" PRIu32, signed_str, int_type->data.integral.bit_count);
782
783 LLVMTypeRef return_elem_types[] = {
784 get_llvm_type(g, operand_type),
785 LLVMInt1Type(),
786 };
787 LLVMTypeRef return_struct_type = LLVMStructType(return_elem_types, 2, false);
788 LLVMTypeRef fn_type = LLVMFunctionType(return_struct_type, param_types, 2, false);
789 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type);
790 assert(LLVMGetIntrinsicID(fn_val));
791 return fn_val;
792 }
793 }
794
get_int_overflow_fn(CodeGen * g,ZigType * operand_type,AddSubMul add_sub_mul)795 static LLVMValueRef get_int_overflow_fn(CodeGen *g, ZigType *operand_type, AddSubMul add_sub_mul) {
796 ZigType *int_type = (operand_type->id == ZigTypeIdVector) ? operand_type->data.vector.elem_type : operand_type;
797 assert(int_type->id == ZigTypeIdInt);
798
799 ZigLLVMFnKey key = {};
800 key.id = ZigLLVMFnIdOverflowArithmetic;
801 key.data.overflow_arithmetic.is_signed = int_type->data.integral.is_signed;
802 key.data.overflow_arithmetic.add_sub_mul = add_sub_mul;
803 key.data.overflow_arithmetic.bit_count = (uint32_t)int_type->data.integral.bit_count;
804 key.data.overflow_arithmetic.vector_len = (operand_type->id == ZigTypeIdVector) ?
805 operand_type->data.vector.len : 0;
806
807 auto existing_entry = g->llvm_fn_table.maybe_get(key);
808 if (existing_entry)
809 return existing_entry->value;
810
811 LLVMValueRef fn_val;
812 switch (add_sub_mul) {
813 case AddSubMulAdd:
814 fn_val = get_arithmetic_overflow_fn(g, operand_type, "sadd", "uadd");
815 break;
816 case AddSubMulSub:
817 fn_val = get_arithmetic_overflow_fn(g, operand_type, "ssub", "usub");
818 break;
819 case AddSubMulMul:
820 fn_val = get_arithmetic_overflow_fn(g, operand_type, "smul", "umul");
821 break;
822 }
823
824 g->llvm_fn_table.put(key, fn_val);
825 return fn_val;
826 }
827
get_float_fn(CodeGen * g,ZigType * type_entry,ZigLLVMFnId fn_id,BuiltinFnId op)828 static LLVMValueRef get_float_fn(CodeGen *g, ZigType *type_entry, ZigLLVMFnId fn_id, BuiltinFnId op) {
829 assert(type_entry->id == ZigTypeIdFloat ||
830 type_entry->id == ZigTypeIdVector);
831
832 bool is_vector = (type_entry->id == ZigTypeIdVector);
833 ZigType *float_type = is_vector ? type_entry->data.vector.elem_type : type_entry;
834 uint32_t float_bits = float_type->data.floating.bit_count;
835
836 // LLVM incorrectly lowers the fma builtin for f128 to fmal, which is for
837 // `long double`. On some targets this will be correct; on others it will be incorrect.
838 if (fn_id == ZigLLVMFnIdFMA && float_bits == 128 &&
839 !target_long_double_is_f128(g->zig_target))
840 {
841 LLVMValueRef existing_llvm_fn = LLVMGetNamedFunction(g->module, "fmaq");
842 if (existing_llvm_fn != nullptr) return existing_llvm_fn;
843
844 LLVMTypeRef float_type_ref = get_llvm_type(g, type_entry);
845 LLVMTypeRef return_elem_types[3] = { float_type_ref, float_type_ref, float_type_ref };
846 LLVMTypeRef fn_type = LLVMFunctionType(float_type_ref, return_elem_types, 3, false);
847 return LLVMAddFunction(g->module, "fmaq", fn_type);
848 }
849
850 ZigLLVMFnKey key = {};
851 key.id = fn_id;
852 key.data.floating.bit_count = float_bits;
853 key.data.floating.vector_len = is_vector ? (uint32_t)type_entry->data.vector.len : 0;
854 key.data.floating.op = op;
855
856 auto existing_entry = g->llvm_fn_table.maybe_get(key);
857 if (existing_entry)
858 return existing_entry->value;
859
860 const char *name;
861 uint32_t num_args;
862 if (fn_id == ZigLLVMFnIdFMA) {
863 name = "fma";
864 num_args = 3;
865 } else if (fn_id == ZigLLVMFnIdFloatOp) {
866 name = float_op_to_name(op);
867 num_args = 1;
868 } else {
869 zig_unreachable();
870 }
871
872 char fn_name[64];
873 if (is_vector)
874 sprintf(fn_name, "llvm.%s.v%" PRIu32 "f%" PRIu32, name, key.data.floating.vector_len, key.data.floating.bit_count);
875 else
876 sprintf(fn_name, "llvm.%s.f%" PRIu32, name, key.data.floating.bit_count);
877 LLVMTypeRef float_type_ref = get_llvm_type(g, type_entry);
878 LLVMTypeRef return_elem_types[3] = { float_type_ref, float_type_ref, float_type_ref };
879 LLVMTypeRef fn_type = LLVMFunctionType(float_type_ref, return_elem_types, num_args, false);
880 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type);
881 assert(LLVMGetIntrinsicID(fn_val));
882
883 g->llvm_fn_table.put(key, fn_val);
884 return fn_val;
885 }
886
gen_store_untyped(CodeGen * g,LLVMValueRef value,LLVMValueRef ptr,uint32_t alignment,bool is_volatile)887 static LLVMValueRef gen_store_untyped(CodeGen *g, LLVMValueRef value, LLVMValueRef ptr,
888 uint32_t alignment, bool is_volatile)
889 {
890 LLVMValueRef instruction = LLVMBuildStore(g->builder, value, ptr);
891 if (is_volatile) LLVMSetVolatile(instruction, true);
892 if (alignment != 0) {
893 LLVMSetAlignment(instruction, alignment);
894 }
895 return instruction;
896 }
897
gen_store(CodeGen * g,LLVMValueRef value,LLVMValueRef ptr,ZigType * ptr_type)898 static LLVMValueRef gen_store(CodeGen *g, LLVMValueRef value, LLVMValueRef ptr, ZigType *ptr_type) {
899 assert(ptr_type->id == ZigTypeIdPointer);
900 uint32_t alignment = get_ptr_align(g, ptr_type);
901 return gen_store_untyped(g, value, ptr, alignment, ptr_type->data.pointer.is_volatile);
902 }
903
gen_load_untyped(CodeGen * g,LLVMValueRef ptr,uint32_t alignment,bool is_volatile,const char * name)904 static LLVMValueRef gen_load_untyped(CodeGen *g, LLVMValueRef ptr, uint32_t alignment, bool is_volatile,
905 const char *name)
906 {
907 LLVMValueRef result = LLVMBuildLoad(g->builder, ptr, name);
908 if (is_volatile) LLVMSetVolatile(result, true);
909 if (alignment == 0) {
910 LLVMSetAlignment(result, LLVMABIAlignmentOfType(g->target_data_ref, LLVMGetElementType(LLVMTypeOf(ptr))));
911 } else {
912 LLVMSetAlignment(result, alignment);
913 }
914 return result;
915 }
916
gen_load(CodeGen * g,LLVMValueRef ptr,ZigType * ptr_type,const char * name)917 static LLVMValueRef gen_load(CodeGen *g, LLVMValueRef ptr, ZigType *ptr_type, const char *name) {
918 assert(ptr_type->id == ZigTypeIdPointer);
919 uint32_t alignment = get_ptr_align(g, ptr_type);
920 return gen_load_untyped(g, ptr, alignment, ptr_type->data.pointer.is_volatile, name);
921 }
922
get_handle_value(CodeGen * g,LLVMValueRef ptr,ZigType * type,ZigType * ptr_type)923 static LLVMValueRef get_handle_value(CodeGen *g, LLVMValueRef ptr, ZigType *type, ZigType *ptr_type) {
924 if (type_has_bits(g, type)) {
925 if (handle_is_ptr(g, type)) {
926 return ptr;
927 } else {
928 assert(ptr_type->id == ZigTypeIdPointer);
929 return gen_load(g, ptr, ptr_type, "");
930 }
931 } else {
932 return nullptr;
933 }
934 }
935
ir_assert_impl(bool ok,Stage1AirInst * source_instruction,const char * file,unsigned int line)936 static void ir_assert_impl(bool ok, Stage1AirInst *source_instruction, const char *file, unsigned int line) {
937 if (ok) return;
938 src_assert_impl(ok, source_instruction->source_node, file, line);
939 }
940
941 #define ir_assert(OK, SOURCE_INSTRUCTION) ir_assert_impl((OK), (SOURCE_INSTRUCTION), __FILE__, __LINE__)
942
ir_want_fast_math(CodeGen * g,Stage1AirInst * instruction)943 static bool ir_want_fast_math(CodeGen *g, Stage1AirInst *instruction) {
944 // TODO memoize
945 Scope *scope = instruction->scope;
946 while (scope) {
947 if (scope->id == ScopeIdBlock) {
948 ScopeBlock *block_scope = (ScopeBlock *)scope;
949 if (block_scope->fast_math_set_node)
950 return block_scope->fast_math_on;
951 } else if (scope->id == ScopeIdDecls) {
952 ScopeDecls *decls_scope = (ScopeDecls *)scope;
953 if (decls_scope->fast_math_set_node)
954 return decls_scope->fast_math_on;
955 }
956 scope = scope->parent;
957 }
958 return false;
959 }
960
ir_want_runtime_safety_scope(CodeGen * g,Scope * scope)961 static bool ir_want_runtime_safety_scope(CodeGen *g, Scope *scope) {
962 // TODO memoize
963 while (scope) {
964 if (scope->id == ScopeIdBlock) {
965 ScopeBlock *block_scope = (ScopeBlock *)scope;
966 if (block_scope->safety_set_node)
967 return !block_scope->safety_off;
968 } else if (scope->id == ScopeIdDecls) {
969 ScopeDecls *decls_scope = (ScopeDecls *)scope;
970 if (decls_scope->safety_set_node)
971 return !decls_scope->safety_off;
972 }
973 scope = scope->parent;
974 }
975
976 return (g->build_mode != BuildModeFastRelease &&
977 g->build_mode != BuildModeSmallRelease);
978 }
979
ir_want_runtime_safety(CodeGen * g,Stage1AirInst * instruction)980 static bool ir_want_runtime_safety(CodeGen *g, Stage1AirInst *instruction) {
981 return ir_want_runtime_safety_scope(g, instruction->scope);
982 }
983
panic_msg_buf(PanicMsgId msg_id)984 static Buf *panic_msg_buf(PanicMsgId msg_id) {
985 switch (msg_id) {
986 case PanicMsgIdCount:
987 zig_unreachable();
988 case PanicMsgIdBoundsCheckFailure:
989 return buf_create_from_str("index out of bounds");
990 case PanicMsgIdCastNegativeToUnsigned:
991 return buf_create_from_str("attempt to cast negative value to unsigned integer");
992 case PanicMsgIdCastTruncatedData:
993 return buf_create_from_str("integer cast truncated bits");
994 case PanicMsgIdIntegerOverflow:
995 return buf_create_from_str("integer overflow");
996 case PanicMsgIdShlOverflowedBits:
997 return buf_create_from_str("left shift overflowed bits");
998 case PanicMsgIdShrOverflowedBits:
999 return buf_create_from_str("right shift overflowed bits");
1000 case PanicMsgIdDivisionByZero:
1001 return buf_create_from_str("division by zero");
1002 case PanicMsgIdRemainderDivisionByZero:
1003 return buf_create_from_str("remainder division by zero or negative value");
1004 case PanicMsgIdExactDivisionRemainder:
1005 return buf_create_from_str("exact division produced remainder");
1006 case PanicMsgIdUnwrapOptionalFail:
1007 return buf_create_from_str("attempt to use null value");
1008 case PanicMsgIdUnreachable:
1009 return buf_create_from_str("reached unreachable code");
1010 case PanicMsgIdInvalidErrorCode:
1011 return buf_create_from_str("invalid error code");
1012 case PanicMsgIdIncorrectAlignment:
1013 return buf_create_from_str("incorrect alignment");
1014 case PanicMsgIdBadUnionField:
1015 return buf_create_from_str("access of inactive union field");
1016 case PanicMsgIdBadEnumValue:
1017 return buf_create_from_str("invalid enum value");
1018 case PanicMsgIdFloatToInt:
1019 return buf_create_from_str("integer part of floating point value out of bounds");
1020 case PanicMsgIdPtrCastNull:
1021 return buf_create_from_str("cast causes pointer to be null");
1022 case PanicMsgIdBadResume:
1023 return buf_create_from_str("resumed an async function which already returned");
1024 case PanicMsgIdBadAwait:
1025 return buf_create_from_str("async function awaited twice");
1026 case PanicMsgIdBadReturn:
1027 return buf_create_from_str("async function returned twice");
1028 case PanicMsgIdResumedAnAwaitingFn:
1029 return buf_create_from_str("awaiting function resumed");
1030 case PanicMsgIdFrameTooSmall:
1031 return buf_create_from_str("frame too small");
1032 case PanicMsgIdResumedFnPendingAwait:
1033 return buf_create_from_str("resumed an async function which can only be awaited");
1034 case PanicMsgIdBadNoSuspendCall:
1035 return buf_create_from_str("async function called in nosuspend scope suspended");
1036 case PanicMsgIdResumeNotSuspendedFn:
1037 return buf_create_from_str("resumed a non-suspended function");
1038 case PanicMsgIdBadSentinel:
1039 return buf_create_from_str("sentinel mismatch");
1040 case PanicMsgIdShxTooBigRhs:
1041 return buf_create_from_str("shift amount is greater than the type size");
1042 }
1043 zig_unreachable();
1044 }
1045
get_panic_msg_ptr_val(CodeGen * g,PanicMsgId msg_id)1046 static LLVMValueRef get_panic_msg_ptr_val(CodeGen *g, PanicMsgId msg_id) {
1047 ZigValue *val = &g->panic_msg_vals[msg_id];
1048 if (!val->llvm_global) {
1049
1050 Buf *buf_msg = panic_msg_buf(msg_id);
1051 ZigValue *array_val = create_const_str_lit(g, buf_msg)->data.x_ptr.data.ref.pointee;
1052 init_const_slice(g, val, array_val, 0, buf_len(buf_msg), true, nullptr);
1053
1054 render_const_val(g, val, "");
1055 render_const_val_global(g, val, "");
1056
1057 assert(val->llvm_global);
1058 }
1059
1060 ZigType *u8_ptr_type = get_pointer_to_type_extra(g, g->builtin_types.entry_u8, true, false,
1061 PtrLenUnknown, get_abi_alignment(g, g->builtin_types.entry_u8), 0, 0, false);
1062 ZigType *str_type = get_slice_type(g, u8_ptr_type);
1063 return LLVMConstBitCast(val->llvm_global, LLVMPointerType(get_llvm_type(g, str_type), 0));
1064 }
1065
ptr_to_stack_trace_type(CodeGen * g)1066 static ZigType *ptr_to_stack_trace_type(CodeGen *g) {
1067 return get_pointer_to_type(g, get_stack_trace_type(g), false);
1068 }
1069
gen_panic(CodeGen * g,LLVMValueRef msg_arg,LLVMValueRef stack_trace_arg,bool stack_trace_is_llvm_alloca)1070 static void gen_panic(CodeGen *g, LLVMValueRef msg_arg, LLVMValueRef stack_trace_arg,
1071 bool stack_trace_is_llvm_alloca)
1072 {
1073 assert(g->panic_fn != nullptr);
1074 LLVMValueRef fn_val = fn_llvm_value(g, g->panic_fn);
1075 ZigLLVM_CallingConv llvm_cc = get_llvm_cc(g, g->panic_fn->type_entry->data.fn.fn_type_id.cc);
1076 if (stack_trace_arg == nullptr) {
1077 stack_trace_arg = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g)));
1078 }
1079 LLVMValueRef args[] = {
1080 msg_arg,
1081 stack_trace_arg,
1082 };
1083 ZigLLVMBuildCall(g->builder, fn_val, args, 2, llvm_cc, ZigLLVM_CallAttrAuto, "");
1084 if (!stack_trace_is_llvm_alloca) {
1085 // The stack trace argument is not in the stack of the caller, so
1086 // we'd like to set tail call here, but because slices (the type of msg_arg) are
1087 // still passed as pointers (see https://github.com/ziglang/zig/issues/561) we still
1088 // cannot make this a tail call.
1089 //LLVMSetTailCall(call_instruction, true);
1090 }
1091 LLVMBuildUnreachable(g->builder);
1092 }
1093
1094 // TODO update most callsites to call gen_assertion instead of this
gen_safety_crash(CodeGen * g,PanicMsgId msg_id)1095 static void gen_safety_crash(CodeGen *g, PanicMsgId msg_id) {
1096 gen_panic(g, get_panic_msg_ptr_val(g, msg_id), nullptr, false);
1097 }
1098
gen_assertion_scope(CodeGen * g,PanicMsgId msg_id,Scope * source_scope)1099 static void gen_assertion_scope(CodeGen *g, PanicMsgId msg_id, Scope *source_scope) {
1100 if (ir_want_runtime_safety_scope(g, source_scope)) {
1101 gen_safety_crash(g, msg_id);
1102 } else {
1103 LLVMBuildUnreachable(g->builder);
1104 }
1105 }
1106
gen_assertion(CodeGen * g,PanicMsgId msg_id,Stage1AirInst * source_instruction)1107 static void gen_assertion(CodeGen *g, PanicMsgId msg_id, Stage1AirInst *source_instruction) {
1108 return gen_assertion_scope(g, msg_id, source_instruction->scope);
1109 }
1110
gen_wasm_memory_size(CodeGen * g)1111 static LLVMValueRef gen_wasm_memory_size(CodeGen *g) {
1112 if (g->wasm_memory_size)
1113 return g->wasm_memory_size;
1114
1115 // TODO adjust for wasm64 as well
1116 // declare i32 @llvm.wasm.memory.size.i32(i32) nounwind readonly
1117 LLVMTypeRef param_type = LLVMInt32Type();
1118 LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt32Type(), ¶m_type, 1, false);
1119 g->wasm_memory_size = LLVMAddFunction(g->module, "llvm.wasm.memory.size.i32", fn_type);
1120 assert(LLVMGetIntrinsicID(g->wasm_memory_size));
1121
1122 return g->wasm_memory_size;
1123 }
1124
gen_wasm_memory_grow(CodeGen * g)1125 static LLVMValueRef gen_wasm_memory_grow(CodeGen *g) {
1126 if (g->wasm_memory_grow)
1127 return g->wasm_memory_grow;
1128
1129 // TODO adjust for wasm64 as well
1130 // declare i32 @llvm.wasm.memory.grow.i32(i32, i32) nounwind
1131 LLVMTypeRef param_types[] = {
1132 LLVMInt32Type(),
1133 LLVMInt32Type(),
1134 };
1135 LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt32Type(), param_types, 2, false);
1136 g->wasm_memory_grow = LLVMAddFunction(g->module, "llvm.wasm.memory.grow.i32", fn_type);
1137 assert(LLVMGetIntrinsicID(g->wasm_memory_grow));
1138
1139 return g->wasm_memory_grow;
1140 }
1141
gen_prefetch(CodeGen * g)1142 static LLVMValueRef gen_prefetch(CodeGen *g) {
1143 if (g->prefetch)
1144 return g->prefetch;
1145
1146 // declare void @llvm.prefetch(i8*, i32, i32, i32)
1147 LLVMTypeRef param_types[] = {
1148 LLVMPointerType(LLVMInt8Type(), 0),
1149 LLVMInt32Type(),
1150 LLVMInt32Type(),
1151 LLVMInt32Type(),
1152 };
1153 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 4, false);
1154 g->prefetch = LLVMAddFunction(g->module, "llvm.prefetch.p0i8", fn_type);
1155 assert(LLVMGetIntrinsicID(g->prefetch));
1156
1157 return g->prefetch;
1158 }
1159
get_stacksave_fn_val(CodeGen * g)1160 static LLVMValueRef get_stacksave_fn_val(CodeGen *g) {
1161 if (g->stacksave_fn_val)
1162 return g->stacksave_fn_val;
1163
1164 // declare i8* @llvm.stacksave()
1165
1166 LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), nullptr, 0, false);
1167 g->stacksave_fn_val = LLVMAddFunction(g->module, "llvm.stacksave", fn_type);
1168 assert(LLVMGetIntrinsicID(g->stacksave_fn_val));
1169
1170 return g->stacksave_fn_val;
1171 }
1172
get_stackrestore_fn_val(CodeGen * g)1173 static LLVMValueRef get_stackrestore_fn_val(CodeGen *g) {
1174 if (g->stackrestore_fn_val)
1175 return g->stackrestore_fn_val;
1176
1177 // declare void @llvm.stackrestore(i8* %ptr)
1178
1179 LLVMTypeRef param_type = LLVMPointerType(LLVMInt8Type(), 0);
1180 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), ¶m_type, 1, false);
1181 g->stackrestore_fn_val = LLVMAddFunction(g->module, "llvm.stackrestore", fn_type);
1182 assert(LLVMGetIntrinsicID(g->stackrestore_fn_val));
1183
1184 return g->stackrestore_fn_val;
1185 }
1186
get_write_register_fn_val(CodeGen * g)1187 static LLVMValueRef get_write_register_fn_val(CodeGen *g) {
1188 if (g->write_register_fn_val)
1189 return g->write_register_fn_val;
1190
1191 // declare void @llvm.write_register.i64(metadata, i64 @value)
1192 // !0 = !{!"sp\00"}
1193
1194 LLVMTypeRef param_types[] = {
1195 LLVMMetadataTypeInContext(LLVMGetGlobalContext()),
1196 LLVMIntType(g->pointer_size_bytes * 8),
1197 };
1198
1199 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 2, false);
1200 Buf *name = buf_sprintf("llvm.write_register.i%d", g->pointer_size_bytes * 8);
1201 g->write_register_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
1202 assert(LLVMGetIntrinsicID(g->write_register_fn_val));
1203
1204 return g->write_register_fn_val;
1205 }
1206
get_return_address_fn_val(CodeGen * g)1207 static LLVMValueRef get_return_address_fn_val(CodeGen *g) {
1208 if (g->return_address_fn_val)
1209 return g->return_address_fn_val;
1210
1211 ZigType *return_type = get_pointer_to_type(g, g->builtin_types.entry_u8, true);
1212
1213 LLVMTypeRef fn_type = LLVMFunctionType(get_llvm_type(g, return_type),
1214 &g->builtin_types.entry_i32->llvm_type, 1, false);
1215 g->return_address_fn_val = LLVMAddFunction(g->module, "llvm.returnaddress", fn_type);
1216 assert(LLVMGetIntrinsicID(g->return_address_fn_val));
1217
1218 return g->return_address_fn_val;
1219 }
1220
get_add_error_return_trace_addr_fn(CodeGen * g)1221 static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) {
1222 if (g->add_error_return_trace_addr_fn_val != nullptr)
1223 return g->add_error_return_trace_addr_fn_val;
1224
1225 LLVMTypeRef arg_types[] = {
1226 get_llvm_type(g, ptr_to_stack_trace_type(g)),
1227 g->builtin_types.entry_usize->llvm_type,
1228 };
1229 LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false);
1230
1231 const char *fn_name = get_mangled_name(g, "__zig_add_err_ret_trace_addr");
1232 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type_ref);
1233 addLLVMFnAttr(fn_val, "alwaysinline");
1234 LLVMSetLinkage(fn_val, LLVMInternalLinkage);
1235 ZigLLVMFunctionSetCallingConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
1236 add_common_fn_attributes(g, fn_val);
1237 // Error return trace memory is in the stack, which is impossible to be at address 0
1238 // on any architecture.
1239 addLLVMArgAttr(fn_val, (unsigned)0, "nonnull");
1240 if (!g->omit_frame_pointer) {
1241 ZigLLVMAddFunctionAttr(fn_val, "frame-pointer", "all");
1242 }
1243
1244 LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
1245 LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
1246 LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
1247 LLVMPositionBuilderAtEnd(g->builder, entry_block);
1248 ZigLLVMClearCurrentDebugLocation(g->builder);
1249
1250 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
1251
1252 // stack_trace.instruction_addresses[stack_trace.index & (stack_trace.instruction_addresses.len - 1)] = return_address;
1253
1254 LLVMValueRef err_ret_trace_ptr = LLVMGetParam(fn_val, 0);
1255 LLVMValueRef address_value = LLVMGetParam(fn_val, 1);
1256
1257 size_t index_field_index = g->stack_trace_type->data.structure.fields[0]->gen_index;
1258 LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)index_field_index, "");
1259 size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1]->gen_index;
1260 LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)addresses_field_index, "");
1261
1262 ZigType *slice_type = g->stack_trace_type->data.structure.fields[1]->type_entry;
1263 size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index]->gen_index;
1264 LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, "");
1265 size_t len_field_index = slice_type->data.structure.fields[slice_len_index]->gen_index;
1266 LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, "");
1267
1268 LLVMValueRef len_value = gen_load_untyped(g, len_field_ptr, 0, false, "");
1269 LLVMValueRef index_val = gen_load_untyped(g, index_field_ptr, 0, false, "");
1270 LLVMValueRef len_val_minus_one = LLVMBuildSub(g->builder, len_value, LLVMConstInt(usize_type_ref, 1, false), "");
1271 LLVMValueRef masked_val = LLVMBuildAnd(g->builder, index_val, len_val_minus_one, "");
1272 LLVMValueRef address_indices[] = {
1273 masked_val,
1274 };
1275
1276 LLVMValueRef ptr_value = gen_load_untyped(g, ptr_field_ptr, 0, false, "");
1277 LLVMValueRef address_slot = LLVMBuildInBoundsGEP(g->builder, ptr_value, address_indices, 1, "");
1278
1279 gen_store_untyped(g, address_value, address_slot, 0, false);
1280
1281 // stack_trace.index += 1;
1282 LLVMValueRef index_plus_one_val = LLVMBuildNUWAdd(g->builder, index_val, LLVMConstInt(usize_type_ref, 1, false), "");
1283 gen_store_untyped(g, index_plus_one_val, index_field_ptr, 0, false);
1284
1285 // return;
1286 LLVMBuildRetVoid(g->builder);
1287
1288 LLVMPositionBuilderAtEnd(g->builder, prev_block);
1289 if (!g->strip_debug_symbols) {
1290 LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
1291 }
1292
1293 g->add_error_return_trace_addr_fn_val = fn_val;
1294 return fn_val;
1295 }
1296
get_return_err_fn(CodeGen * g)1297 static LLVMValueRef get_return_err_fn(CodeGen *g) {
1298 if (g->return_err_fn != nullptr)
1299 return g->return_err_fn;
1300
1301 assert(g->err_tag_type != nullptr);
1302
1303 LLVMTypeRef arg_types[] = {
1304 // error return trace pointer
1305 get_llvm_type(g, ptr_to_stack_trace_type(g)),
1306 };
1307 LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 1, false);
1308
1309 const char *fn_name = get_mangled_name(g, "__zig_return_error");
1310 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type_ref);
1311 addLLVMFnAttr(fn_val, "noinline"); // so that we can look at return address
1312 addLLVMFnAttr(fn_val, "cold");
1313 LLVMSetLinkage(fn_val, LLVMInternalLinkage);
1314 ZigLLVMFunctionSetCallingConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
1315 add_common_fn_attributes(g, fn_val);
1316 if (!g->omit_frame_pointer) {
1317 ZigLLVMAddFunctionAttr(fn_val, "frame-pointer", "all");
1318 }
1319
1320 // this is above the ZigLLVMClearCurrentDebugLocation
1321 LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g);
1322
1323 LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
1324 LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
1325 LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
1326 LLVMPositionBuilderAtEnd(g->builder, entry_block);
1327 ZigLLVMClearCurrentDebugLocation(g->builder);
1328
1329 LLVMValueRef err_ret_trace_ptr = LLVMGetParam(fn_val, 0);
1330
1331 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
1332 LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, g->builtin_types.entry_i32));
1333 LLVMValueRef return_address_ptr = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, "");
1334 LLVMValueRef return_address = LLVMBuildPtrToInt(g->builder, return_address_ptr, usize_type_ref, "");
1335
1336 LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return");
1337 LLVMBasicBlockRef dest_non_null_block = LLVMAppendBasicBlock(fn_val, "DestNonNull");
1338
1339 LLVMValueRef null_dest_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, err_ret_trace_ptr,
1340 LLVMConstNull(LLVMTypeOf(err_ret_trace_ptr)), "");
1341 LLVMBuildCondBr(g->builder, null_dest_bit, return_block, dest_non_null_block);
1342
1343 LLVMPositionBuilderAtEnd(g->builder, return_block);
1344 LLVMBuildRetVoid(g->builder);
1345
1346 LLVMPositionBuilderAtEnd(g->builder, dest_non_null_block);
1347 LLVMValueRef args[] = { err_ret_trace_ptr, return_address };
1348 ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2,
1349 get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_CallAttrAlwaysInline, "");
1350 LLVMBuildRetVoid(g->builder);
1351
1352 LLVMPositionBuilderAtEnd(g->builder, prev_block);
1353 if (!g->strip_debug_symbols) {
1354 LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
1355 }
1356
1357 g->return_err_fn = fn_val;
1358 return fn_val;
1359 }
1360
get_safety_crash_err_fn(CodeGen * g)1361 static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) {
1362 if (g->safety_crash_err_fn != nullptr)
1363 return g->safety_crash_err_fn;
1364
1365 static const char *unwrap_err_msg_text = "attempt to unwrap error: ";
1366
1367 g->generate_error_name_table = true;
1368 generate_error_name_table(g);
1369 assert(g->err_name_table != nullptr);
1370
1371 // Generate the constant part of the error message
1372 LLVMValueRef msg_prefix_init = LLVMConstString(unwrap_err_msg_text, strlen(unwrap_err_msg_text), 1);
1373 LLVMValueRef msg_prefix = LLVMAddGlobal(g->module, LLVMTypeOf(msg_prefix_init), "");
1374 LLVMSetInitializer(msg_prefix, msg_prefix_init);
1375 LLVMSetLinkage(msg_prefix, LLVMPrivateLinkage);
1376 LLVMSetGlobalConstant(msg_prefix, true);
1377
1378 const char *fn_name = get_mangled_name(g, "__zig_fail_unwrap");
1379 LLVMTypeRef fn_type_ref;
1380 if (g->have_err_ret_tracing) {
1381 LLVMTypeRef arg_types[] = {
1382 get_llvm_type(g, get_pointer_to_type(g, get_stack_trace_type(g), false)),
1383 get_llvm_type(g, g->err_tag_type),
1384 };
1385 fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false);
1386 } else {
1387 LLVMTypeRef arg_types[] = {
1388 get_llvm_type(g, g->err_tag_type),
1389 };
1390 fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 1, false);
1391 }
1392 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type_ref);
1393 addLLVMFnAttr(fn_val, "noreturn");
1394 addLLVMFnAttr(fn_val, "cold");
1395 LLVMSetLinkage(fn_val, LLVMInternalLinkage);
1396 ZigLLVMFunctionSetCallingConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
1397 add_common_fn_attributes(g, fn_val);
1398 if (!g->omit_frame_pointer) {
1399 ZigLLVMAddFunctionAttr(fn_val, "frame-pointer", "all");
1400 }
1401 // Not setting alignment here. See the comment above about
1402 // "Cannot getTypeInfo() on a type that is unsized!"
1403 // assertion failure on Darwin.
1404
1405 LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
1406 LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
1407 LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
1408 LLVMPositionBuilderAtEnd(g->builder, entry_block);
1409 ZigLLVMClearCurrentDebugLocation(g->builder);
1410
1411 ZigType *usize_ty = g->builtin_types.entry_usize;
1412 ZigType *u8_ptr_type = get_pointer_to_type_extra(g, g->builtin_types.entry_u8, true, false,
1413 PtrLenUnknown, get_abi_alignment(g, g->builtin_types.entry_u8), 0, 0, false);
1414 ZigType *str_type = get_slice_type(g, u8_ptr_type);
1415
1416 // Allocate a buffer to hold the fully-formatted error message
1417 const size_t err_buf_len = strlen(unwrap_err_msg_text) + g->largest_err_name_len;
1418 LLVMValueRef max_msg_len = LLVMConstInt(usize_ty->llvm_type, err_buf_len, 0);
1419 LLVMValueRef msg_buffer = LLVMBuildArrayAlloca(g->builder, LLVMInt8Type(), max_msg_len, "msg_buffer");
1420
1421 // Allocate a []u8 slice for the message
1422 LLVMValueRef msg_slice = build_alloca(g, str_type, "msg_slice", 0);
1423
1424 LLVMValueRef err_ret_trace_arg;
1425 LLVMValueRef err_val;
1426 if (g->have_err_ret_tracing) {
1427 err_ret_trace_arg = LLVMGetParam(fn_val, 0);
1428 err_val = LLVMGetParam(fn_val, 1);
1429 } else {
1430 err_ret_trace_arg = nullptr;
1431 err_val = LLVMGetParam(fn_val, 0);
1432 }
1433
1434 // Fetch the error name from the global table
1435 LLVMValueRef err_table_indices[] = {
1436 LLVMConstNull(usize_ty->llvm_type),
1437 err_val,
1438 };
1439 LLVMValueRef err_name_val = LLVMBuildInBoundsGEP(g->builder, g->err_name_table, err_table_indices, 2, "");
1440
1441 LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, err_name_val, slice_ptr_index, "");
1442 LLVMValueRef err_name_ptr = gen_load_untyped(g, ptr_field_ptr, 0, false, "");
1443
1444 LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, err_name_val, slice_len_index, "");
1445 LLVMValueRef err_name_len = gen_load_untyped(g, len_field_ptr, 0, false, "");
1446
1447 LLVMValueRef msg_prefix_len = LLVMConstInt(usize_ty->llvm_type, strlen(unwrap_err_msg_text), false);
1448 // Points to the beginning of msg_buffer
1449 LLVMValueRef msg_buffer_ptr_indices[] = {
1450 LLVMConstNull(usize_ty->llvm_type),
1451 };
1452 LLVMValueRef msg_buffer_ptr = LLVMBuildInBoundsGEP(g->builder, msg_buffer, msg_buffer_ptr_indices, 1, "");
1453 // Points to the beginning of the constant prefix message
1454 LLVMValueRef msg_prefix_ptr_indices[] = {
1455 LLVMConstNull(usize_ty->llvm_type),
1456 };
1457 LLVMValueRef msg_prefix_ptr = LLVMConstInBoundsGEP(msg_prefix, msg_prefix_ptr_indices, 1);
1458
1459 // Build the message using the prefix...
1460 ZigLLVMBuildMemCpy(g->builder, msg_buffer_ptr, 1, msg_prefix_ptr, 1, msg_prefix_len, false);
1461 // ..and append the error name
1462 LLVMValueRef msg_buffer_ptr_after_indices[] = {
1463 msg_prefix_len,
1464 };
1465 LLVMValueRef msg_buffer_ptr_after = LLVMBuildInBoundsGEP(g->builder, msg_buffer, msg_buffer_ptr_after_indices, 1, "");
1466 ZigLLVMBuildMemCpy(g->builder, msg_buffer_ptr_after, 1, err_name_ptr, 1, err_name_len, false);
1467
1468 // Set the slice pointer
1469 LLVMValueRef msg_slice_ptr_field_ptr = LLVMBuildStructGEP(g->builder, msg_slice, slice_ptr_index, "");
1470 gen_store_untyped(g, msg_buffer_ptr, msg_slice_ptr_field_ptr, 0, false);
1471
1472 // Set the slice length
1473 LLVMValueRef slice_len = LLVMBuildNUWAdd(g->builder, msg_prefix_len, err_name_len, "");
1474 LLVMValueRef msg_slice_len_field_ptr = LLVMBuildStructGEP(g->builder, msg_slice, slice_len_index, "");
1475 gen_store_untyped(g, slice_len, msg_slice_len_field_ptr, 0, false);
1476
1477 // Call panic()
1478 gen_panic(g, msg_slice, err_ret_trace_arg, false);
1479
1480 LLVMPositionBuilderAtEnd(g->builder, prev_block);
1481 if (!g->strip_debug_symbols) {
1482 LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
1483 }
1484
1485 g->safety_crash_err_fn = fn_val;
1486 return fn_val;
1487 }
1488
get_cur_err_ret_trace_val(CodeGen * g,Scope * scope,bool * is_llvm_alloca)1489 static LLVMValueRef get_cur_err_ret_trace_val(CodeGen *g, Scope *scope, bool *is_llvm_alloca) {
1490 if (!g->have_err_ret_tracing) {
1491 *is_llvm_alloca = false;
1492 return nullptr;
1493 }
1494 if (g->cur_err_ret_trace_val_stack != nullptr) {
1495 *is_llvm_alloca = !fn_is_async(g->cur_fn);
1496 return g->cur_err_ret_trace_val_stack;
1497 }
1498 *is_llvm_alloca = false;
1499 return g->cur_err_ret_trace_val_arg;
1500 }
1501
gen_safety_crash_for_err(CodeGen * g,LLVMValueRef err_val,Scope * scope)1502 static void gen_safety_crash_for_err(CodeGen *g, LLVMValueRef err_val, Scope *scope) {
1503 LLVMValueRef safety_crash_err_fn = get_safety_crash_err_fn(g);
1504 LLVMValueRef call_instruction;
1505 bool is_llvm_alloca = false;
1506 if (g->have_err_ret_tracing) {
1507 LLVMValueRef err_ret_trace_val = get_cur_err_ret_trace_val(g, scope, &is_llvm_alloca);
1508 if (err_ret_trace_val == nullptr) {
1509 err_ret_trace_val = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g)));
1510 }
1511 LLVMValueRef args[] = {
1512 err_ret_trace_val,
1513 err_val,
1514 };
1515 call_instruction = ZigLLVMBuildCall(g->builder, safety_crash_err_fn, args, 2,
1516 get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_CallAttrAuto, "");
1517 } else {
1518 LLVMValueRef args[] = {
1519 err_val,
1520 };
1521 call_instruction = ZigLLVMBuildCall(g->builder, safety_crash_err_fn, args, 1,
1522 get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_CallAttrAuto, "");
1523 }
1524 if (!is_llvm_alloca) {
1525 LLVMSetTailCall(call_instruction, true);
1526 }
1527 LLVMBuildUnreachable(g->builder);
1528 }
1529
add_bounds_check(CodeGen * g,LLVMValueRef target_val,LLVMIntPredicate lower_pred,LLVMValueRef lower_value,LLVMIntPredicate upper_pred,LLVMValueRef upper_value)1530 static void add_bounds_check(CodeGen *g, LLVMValueRef target_val,
1531 LLVMIntPredicate lower_pred, LLVMValueRef lower_value,
1532 LLVMIntPredicate upper_pred, LLVMValueRef upper_value)
1533 {
1534 if (!lower_value && !upper_value) {
1535 return;
1536 }
1537 if (upper_value && !lower_value) {
1538 lower_value = upper_value;
1539 lower_pred = upper_pred;
1540 upper_value = nullptr;
1541 }
1542
1543 LLVMBasicBlockRef bounds_check_fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "BoundsCheckFail");
1544 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "BoundsCheckOk");
1545 LLVMBasicBlockRef lower_ok_block = upper_value ?
1546 LLVMAppendBasicBlock(g->cur_fn_val, "FirstBoundsCheckOk") : ok_block;
1547
1548 LLVMValueRef lower_ok_val = LLVMBuildICmp(g->builder, lower_pred, target_val, lower_value, "");
1549 LLVMBuildCondBr(g->builder, lower_ok_val, lower_ok_block, bounds_check_fail_block);
1550
1551 LLVMPositionBuilderAtEnd(g->builder, bounds_check_fail_block);
1552 gen_safety_crash(g, PanicMsgIdBoundsCheckFailure);
1553
1554 if (upper_value) {
1555 LLVMPositionBuilderAtEnd(g->builder, lower_ok_block);
1556 LLVMValueRef upper_ok_val = LLVMBuildICmp(g->builder, upper_pred, target_val, upper_value, "");
1557 LLVMBuildCondBr(g->builder, upper_ok_val, ok_block, bounds_check_fail_block);
1558 }
1559
1560 LLVMPositionBuilderAtEnd(g->builder, ok_block);
1561 }
1562
add_sentinel_check(CodeGen * g,LLVMValueRef sentinel_elem_ptr,ZigValue * sentinel)1563 static void add_sentinel_check(CodeGen *g, LLVMValueRef sentinel_elem_ptr, ZigValue *sentinel) {
1564 LLVMValueRef expected_sentinel = gen_const_val(g, sentinel, "");
1565
1566 LLVMValueRef actual_sentinel = gen_load_untyped(g, sentinel_elem_ptr, 0, false, "");
1567 LLVMValueRef ok_bit;
1568 if (sentinel->type->id == ZigTypeIdFloat) {
1569 ok_bit = LLVMBuildFCmp(g->builder, LLVMRealOEQ, actual_sentinel, expected_sentinel, "");
1570 } else {
1571 ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, actual_sentinel, expected_sentinel, "");
1572 }
1573
1574 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "SentinelFail");
1575 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "SentinelOk");
1576 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
1577
1578 LLVMPositionBuilderAtEnd(g->builder, fail_block);
1579 gen_safety_crash(g, PanicMsgIdBadSentinel);
1580
1581 LLVMPositionBuilderAtEnd(g->builder, ok_block);
1582 }
1583
gen_assert_zero(CodeGen * g,LLVMValueRef expr_val,ZigType * int_type)1584 static LLVMValueRef gen_assert_zero(CodeGen *g, LLVMValueRef expr_val, ZigType *int_type) {
1585 LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, int_type));
1586 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, expr_val, zero, "");
1587 if (int_type->id == ZigTypeIdVector) {
1588 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
1589 }
1590 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "CastShortenOk");
1591 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "CastShortenFail");
1592 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
1593
1594 LLVMPositionBuilderAtEnd(g->builder, fail_block);
1595 gen_safety_crash(g, PanicMsgIdCastTruncatedData);
1596
1597 LLVMPositionBuilderAtEnd(g->builder, ok_block);
1598 return nullptr;
1599 }
1600
gen_widen_or_shorten(CodeGen * g,bool want_runtime_safety,ZigType * actual_type,ZigType * wanted_type,LLVMValueRef expr_val)1601 static LLVMValueRef gen_widen_or_shorten(CodeGen *g, bool want_runtime_safety, ZigType *actual_type,
1602 ZigType *wanted_type, LLVMValueRef expr_val)
1603 {
1604 assert(actual_type->id == wanted_type->id);
1605 assert(expr_val != nullptr);
1606
1607 ZigType *scalar_actual_type = (actual_type->id == ZigTypeIdVector) ?
1608 actual_type->data.vector.elem_type : actual_type;
1609 ZigType *scalar_wanted_type = (wanted_type->id == ZigTypeIdVector) ?
1610 wanted_type->data.vector.elem_type : wanted_type;
1611
1612 uint64_t actual_bits;
1613 uint64_t wanted_bits;
1614 if (scalar_actual_type->id == ZigTypeIdFloat) {
1615 actual_bits = scalar_actual_type->data.floating.bit_count;
1616 wanted_bits = scalar_wanted_type->data.floating.bit_count;
1617 } else if (scalar_actual_type->id == ZigTypeIdInt) {
1618 actual_bits = scalar_actual_type->data.integral.bit_count;
1619 wanted_bits = scalar_wanted_type->data.integral.bit_count;
1620 } else {
1621 zig_unreachable();
1622 }
1623
1624 if (scalar_actual_type->id == ZigTypeIdInt && want_runtime_safety && (
1625 // negative to unsigned
1626 (!scalar_wanted_type->data.integral.is_signed && scalar_actual_type->data.integral.is_signed) ||
1627 // unsigned would become negative
1628 (scalar_wanted_type->data.integral.is_signed && !scalar_actual_type->data.integral.is_signed && actual_bits == wanted_bits)))
1629 {
1630 LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, actual_type));
1631 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntSGE, expr_val, zero, "");
1632
1633 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "SignCastOk");
1634 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "SignCastFail");
1635 if (actual_type->id == ZigTypeIdVector) {
1636 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
1637 }
1638 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
1639
1640 LLVMPositionBuilderAtEnd(g->builder, fail_block);
1641 gen_safety_crash(g, scalar_actual_type->data.integral.is_signed ? PanicMsgIdCastNegativeToUnsigned : PanicMsgIdCastTruncatedData);
1642
1643 LLVMPositionBuilderAtEnd(g->builder, ok_block);
1644 }
1645
1646 if (actual_bits == wanted_bits) {
1647 return expr_val;
1648 } else if (actual_bits < wanted_bits) {
1649 if (scalar_actual_type->id == ZigTypeIdFloat) {
1650 return LLVMBuildFPExt(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
1651 } else if (scalar_actual_type->id == ZigTypeIdInt) {
1652 if (scalar_actual_type->data.integral.is_signed) {
1653 return LLVMBuildSExt(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
1654 } else {
1655 return LLVMBuildZExt(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
1656 }
1657 } else {
1658 zig_unreachable();
1659 }
1660 } else if (actual_bits > wanted_bits) {
1661 if (scalar_actual_type->id == ZigTypeIdFloat) {
1662 return LLVMBuildFPTrunc(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
1663 } else if (scalar_actual_type->id == ZigTypeIdInt) {
1664 if (wanted_bits == 0) {
1665 if (!want_runtime_safety)
1666 return nullptr;
1667
1668 return gen_assert_zero(g, expr_val, actual_type);
1669 }
1670 LLVMValueRef trunc_val = LLVMBuildTrunc(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
1671 if (!want_runtime_safety) {
1672 return trunc_val;
1673 }
1674 LLVMValueRef orig_val;
1675 if (scalar_wanted_type->data.integral.is_signed) {
1676 orig_val = LLVMBuildSExt(g->builder, trunc_val, get_llvm_type(g, actual_type), "");
1677 } else {
1678 orig_val = LLVMBuildZExt(g->builder, trunc_val, get_llvm_type(g, actual_type), "");
1679 }
1680 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, expr_val, orig_val, "");
1681 if (actual_type->id == ZigTypeIdVector) {
1682 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
1683 }
1684 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "CastShortenOk");
1685 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "CastShortenFail");
1686 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
1687
1688 LLVMPositionBuilderAtEnd(g->builder, fail_block);
1689 gen_safety_crash(g, PanicMsgIdCastTruncatedData);
1690
1691 LLVMPositionBuilderAtEnd(g->builder, ok_block);
1692 return trunc_val;
1693 } else {
1694 zig_unreachable();
1695 }
1696 } else {
1697 zig_unreachable();
1698 }
1699 }
1700
1701 typedef LLVMValueRef (*BuildBinOpFunc)(LLVMBuilderRef, LLVMValueRef, LLVMValueRef, const char *);
1702 // These are lookup table using the AddSubMul enum as the lookup.
1703 // If AddSubMul ever changes, then these tables will be out of
1704 // date.
1705 static const BuildBinOpFunc float_op[3] = { LLVMBuildFAdd, LLVMBuildFSub, LLVMBuildFMul };
1706 static const BuildBinOpFunc wrap_op[3] = { LLVMBuildAdd, LLVMBuildSub, LLVMBuildMul };
1707 static const BuildBinOpFunc signed_op[3] = { LLVMBuildNSWAdd, LLVMBuildNSWSub, LLVMBuildNSWMul };
1708 static const BuildBinOpFunc unsigned_op[3] = { LLVMBuildNUWAdd, LLVMBuildNUWSub, LLVMBuildNUWMul };
1709
gen_overflow_op(CodeGen * g,ZigType * operand_type,AddSubMul op,LLVMValueRef val1,LLVMValueRef val2)1710 static LLVMValueRef gen_overflow_op(CodeGen *g, ZigType *operand_type, AddSubMul op,
1711 LLVMValueRef val1, LLVMValueRef val2)
1712 {
1713 LLVMValueRef fn_val = get_int_overflow_fn(g, operand_type, op);
1714 LLVMValueRef params[] = {
1715 val1,
1716 val2,
1717 };
1718 LLVMValueRef result_struct = LLVMBuildCall(g->builder, fn_val, params, 2, "");
1719 LLVMValueRef result = LLVMBuildExtractValue(g->builder, result_struct, 0, "");
1720 LLVMValueRef overflow_bit = LLVMBuildExtractValue(g->builder, result_struct, 1, "");
1721 if (operand_type->id == ZigTypeIdVector) {
1722 overflow_bit = ZigLLVMBuildOrReduce(g->builder, overflow_bit);
1723 }
1724
1725 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowFail");
1726 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowOk");
1727 LLVMBuildCondBr(g->builder, overflow_bit, fail_block, ok_block);
1728
1729 LLVMPositionBuilderAtEnd(g->builder, fail_block);
1730 gen_safety_crash(g, PanicMsgIdIntegerOverflow);
1731
1732 LLVMPositionBuilderAtEnd(g->builder, ok_block);
1733 return result;
1734 }
1735
cmp_op_to_int_predicate(IrBinOp cmp_op,bool is_signed)1736 static LLVMIntPredicate cmp_op_to_int_predicate(IrBinOp cmp_op, bool is_signed) {
1737 switch (cmp_op) {
1738 case IrBinOpCmpEq:
1739 return LLVMIntEQ;
1740 case IrBinOpCmpNotEq:
1741 return LLVMIntNE;
1742 case IrBinOpCmpLessThan:
1743 return is_signed ? LLVMIntSLT : LLVMIntULT;
1744 case IrBinOpCmpGreaterThan:
1745 return is_signed ? LLVMIntSGT : LLVMIntUGT;
1746 case IrBinOpCmpLessOrEq:
1747 return is_signed ? LLVMIntSLE : LLVMIntULE;
1748 case IrBinOpCmpGreaterOrEq:
1749 return is_signed ? LLVMIntSGE : LLVMIntUGE;
1750 default:
1751 zig_unreachable();
1752 }
1753 }
1754
cmp_op_to_real_predicate(IrBinOp cmp_op)1755 static LLVMRealPredicate cmp_op_to_real_predicate(IrBinOp cmp_op) {
1756 switch (cmp_op) {
1757 case IrBinOpCmpEq:
1758 return LLVMRealOEQ;
1759 case IrBinOpCmpNotEq:
1760 return LLVMRealUNE;
1761 case IrBinOpCmpLessThan:
1762 return LLVMRealOLT;
1763 case IrBinOpCmpGreaterThan:
1764 return LLVMRealOGT;
1765 case IrBinOpCmpLessOrEq:
1766 return LLVMRealOLE;
1767 case IrBinOpCmpGreaterOrEq:
1768 return LLVMRealOGE;
1769 default:
1770 zig_unreachable();
1771 }
1772 }
1773
gen_assign_raw(CodeGen * g,LLVMValueRef ptr,ZigType * ptr_type,LLVMValueRef value)1774 static void gen_assign_raw(CodeGen *g, LLVMValueRef ptr, ZigType *ptr_type,
1775 LLVMValueRef value)
1776 {
1777 assert(ptr_type->id == ZigTypeIdPointer);
1778 ZigType *child_type = ptr_type->data.pointer.child_type;
1779
1780 if (!type_has_bits(g, child_type))
1781 return;
1782
1783 if (handle_is_ptr(g, child_type)) {
1784 assert(LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMPointerTypeKind);
1785 assert(LLVMGetTypeKind(LLVMTypeOf(ptr)) == LLVMPointerTypeKind);
1786
1787 LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
1788
1789 LLVMValueRef src_ptr = LLVMBuildBitCast(g->builder, value, ptr_u8, "");
1790 LLVMValueRef dest_ptr = LLVMBuildBitCast(g->builder, ptr, ptr_u8, "");
1791
1792 ZigType *usize = g->builtin_types.entry_usize;
1793 uint64_t size_bytes = LLVMStoreSizeOfType(g->target_data_ref, get_llvm_type(g, child_type));
1794 uint64_t src_align_bytes = get_abi_alignment(g, child_type);
1795 uint64_t dest_align_bytes = get_ptr_align(g, ptr_type);
1796 assert(size_bytes > 0);
1797 assert(src_align_bytes > 0);
1798 assert(dest_align_bytes > 0);
1799
1800 ZigLLVMBuildMemCpy(g->builder, dest_ptr, dest_align_bytes, src_ptr, src_align_bytes,
1801 LLVMConstInt(usize->llvm_type, size_bytes, false),
1802 ptr_type->data.pointer.is_volatile);
1803 return;
1804 }
1805
1806 assert(ptr_type->data.pointer.vector_index != VECTOR_INDEX_RUNTIME);
1807 if (ptr_type->data.pointer.vector_index != VECTOR_INDEX_NONE) {
1808 LLVMValueRef index_val = LLVMConstInt(LLVMInt32Type(),
1809 ptr_type->data.pointer.vector_index, false);
1810 LLVMValueRef loaded_vector = gen_load(g, ptr, ptr_type, "");
1811 LLVMValueRef new_vector = LLVMBuildInsertElement(g->builder, loaded_vector, value,
1812 index_val, "");
1813 gen_store(g, new_vector, ptr, ptr_type);
1814 return;
1815 }
1816
1817 uint32_t host_int_bytes = ptr_type->data.pointer.host_int_bytes;
1818 if (host_int_bytes == 0) {
1819 gen_store(g, value, ptr, ptr_type);
1820 return;
1821 }
1822
1823 bool big_endian = g->is_big_endian;
1824
1825 LLVMTypeRef int_ptr_ty = LLVMPointerType(LLVMIntType(host_int_bytes * 8), 0);
1826 LLVMValueRef int_ptr = LLVMBuildBitCast(g->builder, ptr, int_ptr_ty, "");
1827 LLVMValueRef containing_int = gen_load(g, int_ptr, ptr_type, "");
1828 uint32_t host_bit_count = LLVMGetIntTypeWidth(LLVMTypeOf(containing_int));
1829 assert(host_bit_count == host_int_bytes * 8);
1830 uint32_t size_in_bits = type_size_bits(g, child_type);
1831
1832 uint32_t bit_offset = ptr_type->data.pointer.bit_offset_in_host;
1833 uint32_t shift_amt = big_endian ? host_bit_count - bit_offset - size_in_bits : bit_offset;
1834 LLVMValueRef shift_amt_val = LLVMConstInt(LLVMTypeOf(containing_int), shift_amt, false);
1835
1836 // Convert to equally-sized integer type in order to perform the bit
1837 // operations on the value to store
1838 LLVMTypeRef value_bits_type = LLVMIntType(size_in_bits);
1839 LLVMValueRef value_bits = LLVMBuildBitCast(g->builder, value, value_bits_type, "");
1840
1841 LLVMValueRef mask_val = LLVMConstAllOnes(value_bits_type);
1842 mask_val = LLVMConstZExt(mask_val, LLVMTypeOf(containing_int));
1843 mask_val = LLVMConstShl(mask_val, shift_amt_val);
1844 mask_val = LLVMConstNot(mask_val);
1845
1846 LLVMValueRef anded_containing_int = LLVMBuildAnd(g->builder, containing_int, mask_val, "");
1847 LLVMValueRef extended_value = LLVMBuildZExt(g->builder, value_bits, LLVMTypeOf(containing_int), "");
1848 LLVMValueRef shifted_value = LLVMBuildShl(g->builder, extended_value, shift_amt_val, "");
1849 LLVMValueRef ored_value = LLVMBuildOr(g->builder, shifted_value, anded_containing_int, "");
1850
1851 gen_store(g, ored_value, int_ptr, ptr_type);
1852 }
1853
gen_var_debug_decl(CodeGen * g,ZigVar * var)1854 static void gen_var_debug_decl(CodeGen *g, ZigVar *var) {
1855 if (g->strip_debug_symbols) return;
1856 assert(var->di_loc_var != nullptr);
1857 AstNode *source_node = var->decl_node;
1858 ZigLLVMDILocation *debug_loc = ZigLLVMGetDebugLoc(node_line_onebased(source_node),
1859 node_column_onebased(source_node), get_di_scope(g, var->parent_scope));
1860 ZigLLVMInsertDeclareAtEnd(g->dbuilder, var->value_ref, var->di_loc_var, debug_loc,
1861 LLVMGetInsertBlock(g->builder));
1862 }
1863
ir_llvm_value(CodeGen * g,Stage1AirInst * instruction)1864 static LLVMValueRef ir_llvm_value(CodeGen *g, Stage1AirInst *instruction) {
1865 Error err;
1866
1867 bool value_has_bits;
1868 if ((err = type_has_bits2(g, instruction->value->type, &value_has_bits)))
1869 codegen_report_errors_and_exit(g);
1870
1871 if (!value_has_bits)
1872 return nullptr;
1873
1874 if (!instruction->llvm_value) {
1875 if (instruction->id == Stage1AirInstIdAwait) {
1876 Stage1AirInstAwait *await = reinterpret_cast<Stage1AirInstAwait*>(instruction);
1877 if (await->result_loc != nullptr) {
1878 return get_handle_value(g, ir_llvm_value(g, await->result_loc),
1879 await->result_loc->value->type->data.pointer.child_type, await->result_loc->value->type);
1880 }
1881 }
1882 if (instruction->spill != nullptr) {
1883 ZigType *ptr_type = instruction->spill->value->type;
1884 ir_assert(ptr_type->id == ZigTypeIdPointer, instruction);
1885 return get_handle_value(g, ir_llvm_value(g, instruction->spill),
1886 ptr_type->data.pointer.child_type, instruction->spill->value->type);
1887 }
1888 ir_assert(instruction->value->special != ConstValSpecialRuntime, instruction);
1889 assert(instruction->value->type);
1890 render_const_val(g, instruction->value, "");
1891 // we might have to do some pointer casting here due to the way union
1892 // values are rendered with a type other than the one we expect
1893 if (handle_is_ptr(g, instruction->value->type)) {
1894 render_const_val_global(g, instruction->value, "");
1895 ZigType *ptr_type = get_pointer_to_type(g, instruction->value->type, true);
1896 instruction->llvm_value = LLVMBuildBitCast(g->builder, instruction->value->llvm_global, get_llvm_type(g, ptr_type), "");
1897 } else {
1898 instruction->llvm_value = LLVMBuildBitCast(g->builder, instruction->value->llvm_value,
1899 get_llvm_type(g, instruction->value->type), "");
1900 }
1901 assert(instruction->llvm_value);
1902 }
1903 return instruction->llvm_value;
1904 }
1905
codegen_report_errors_and_exit(CodeGen * g)1906 void codegen_report_errors_and_exit(CodeGen *g) {
1907 // Clear progress indicator before printing errors
1908 if (g->sub_progress_node != nullptr) {
1909 stage2_progress_end(g->sub_progress_node);
1910 g->sub_progress_node = nullptr;
1911 }
1912 if (g->main_progress_node != nullptr) {
1913 stage2_progress_end(g->main_progress_node);
1914 g->main_progress_node = nullptr;
1915 }
1916
1917 assert(g->errors.length != 0);
1918 for (size_t i = 0; i < g->errors.length; i += 1) {
1919 ErrorMsg *err = g->errors.at(i);
1920 print_err_msg(err, g->err_color);
1921 }
1922 exit(1);
1923 }
1924
report_errors_and_maybe_exit(CodeGen * g)1925 static void report_errors_and_maybe_exit(CodeGen *g) {
1926 if (g->errors.length != 0) {
1927 codegen_report_errors_and_exit(g);
1928 }
1929 }
1930
1931 ATTRIBUTE_NORETURN
give_up_with_c_abi_error(CodeGen * g,AstNode * source_node)1932 static void give_up_with_c_abi_error(CodeGen *g, AstNode *source_node) {
1933 ErrorMsg *msg = add_node_error(g, source_node,
1934 buf_sprintf("TODO: support C ABI for more targets. https://github.com/ziglang/zig/issues/1481"));
1935 add_error_note(g, msg, source_node,
1936 buf_sprintf("pointers, integers, floats, bools, and enums work on all targets"));
1937 codegen_report_errors_and_exit(g);
1938 }
1939
build_alloca(CodeGen * g,ZigType * type_entry,const char * name,uint32_t alignment)1940 static LLVMValueRef build_alloca(CodeGen *g, ZigType *type_entry, const char *name, uint32_t alignment) {
1941 LLVMValueRef result = LLVMBuildAlloca(g->builder, get_llvm_type(g, type_entry), name);
1942 LLVMSetAlignment(result, (alignment == 0) ? get_abi_alignment(g, type_entry) : alignment);
1943 return result;
1944 }
1945
iter_function_params_c_abi(CodeGen * g,ZigType * fn_type,FnWalk * fn_walk,size_t src_i)1946 static bool iter_function_params_c_abi(CodeGen *g, ZigType *fn_type, FnWalk *fn_walk, size_t src_i) {
1947 // Initialized from the type for some walks, but because of C var args,
1948 // initialized based on callsite instructions for that one.
1949 FnTypeParamInfo *param_info = nullptr;
1950 ZigType *ty;
1951 ZigType *dest_ty = nullptr;
1952 AstNode *source_node = nullptr;
1953 LLVMValueRef val;
1954 LLVMValueRef llvm_fn;
1955 unsigned di_arg_index;
1956 ZigVar *var;
1957 switch (fn_walk->id) {
1958 case FnWalkIdAttrs:
1959 if (src_i >= fn_type->data.fn.fn_type_id.param_count)
1960 return false;
1961 param_info = &fn_type->data.fn.fn_type_id.param_info[src_i];
1962 ty = param_info->type;
1963 source_node = fn_walk->data.attrs.fn->proto_node;
1964 llvm_fn = fn_walk->data.attrs.llvm_fn;
1965 break;
1966 case FnWalkIdCall: {
1967 if (src_i >= fn_walk->data.call.inst->arg_count)
1968 return false;
1969 Stage1AirInst *arg = fn_walk->data.call.inst->args[src_i];
1970 ty = arg->value->type;
1971 source_node = arg->source_node;
1972 val = ir_llvm_value(g, arg);
1973 break;
1974 }
1975 case FnWalkIdTypes:
1976 if (src_i >= fn_type->data.fn.fn_type_id.param_count)
1977 return false;
1978 param_info = &fn_type->data.fn.fn_type_id.param_info[src_i];
1979 ty = param_info->type;
1980 break;
1981 case FnWalkIdVars:
1982 assert(src_i < fn_type->data.fn.fn_type_id.param_count);
1983 param_info = &fn_type->data.fn.fn_type_id.param_info[src_i];
1984 ty = param_info->type;
1985 var = fn_walk->data.vars.var;
1986 source_node = var->decl_node;
1987 llvm_fn = fn_walk->data.vars.llvm_fn;
1988 break;
1989 case FnWalkIdInits:
1990 if (src_i >= fn_type->data.fn.fn_type_id.param_count)
1991 return false;
1992 param_info = &fn_type->data.fn.fn_type_id.param_info[src_i];
1993 ty = param_info->type;
1994 var = fn_walk->data.inits.fn->variable_list.at(src_i);
1995 source_node = fn_walk->data.inits.fn->proto_node;
1996 llvm_fn = fn_walk->data.inits.llvm_fn;
1997 break;
1998 }
1999
2000 if (type_is_c_abi_int_bail(g, ty) || ty->id == ZigTypeIdFloat || ty->id == ZigTypeIdVector ||
2001 ty->id == ZigTypeIdInt // TODO investigate if we need to change this
2002 ) {
2003 switch (fn_walk->id) {
2004 case FnWalkIdAttrs: {
2005 ZigType *ptr_type = get_codegen_ptr_type_bail(g, ty);
2006 if (ptr_type != nullptr) {
2007 if (type_is_nonnull_ptr(g, ty)) {
2008 addLLVMArgAttr(llvm_fn, fn_walk->data.attrs.gen_i, "nonnull");
2009 }
2010 if (ptr_type->id == ZigTypeIdPointer && ptr_type->data.pointer.is_const) {
2011 addLLVMArgAttr(llvm_fn, fn_walk->data.attrs.gen_i, "readonly");
2012 }
2013 if (param_info->is_noalias) {
2014 addLLVMArgAttr(llvm_fn, fn_walk->data.attrs.gen_i, "noalias");
2015 }
2016 }
2017 fn_walk->data.attrs.gen_i += 1;
2018 break;
2019 }
2020 case FnWalkIdCall:
2021 fn_walk->data.call.gen_param_values->append(val);
2022 break;
2023 case FnWalkIdTypes:
2024 fn_walk->data.types.gen_param_types->append(get_llvm_type(g, ty));
2025 fn_walk->data.types.param_di_types->append(get_llvm_di_type(g, ty));
2026 break;
2027 case FnWalkIdVars: {
2028 var->value_ref = build_alloca(g, ty, var->name, var->align_bytes);
2029 di_arg_index = fn_walk->data.vars.gen_i;
2030 fn_walk->data.vars.gen_i += 1;
2031 dest_ty = ty;
2032 goto var_ok;
2033 }
2034 case FnWalkIdInits:
2035 clear_debug_source_node(g);
2036 gen_store_untyped(g, LLVMGetParam(llvm_fn, fn_walk->data.inits.gen_i), var->value_ref, var->align_bytes, false);
2037 if (var->decl_node) {
2038 gen_var_debug_decl(g, var);
2039 }
2040 fn_walk->data.inits.gen_i += 1;
2041 break;
2042 }
2043 return true;
2044 }
2045
2046 {
2047 // Arrays are just pointers
2048 if (ty->id == ZigTypeIdArray) {
2049 assert(handle_is_ptr(g, ty));
2050 switch (fn_walk->id) {
2051 case FnWalkIdAttrs:
2052 // arrays passed to C ABI functions may not be at address 0
2053 addLLVMArgAttr(llvm_fn, fn_walk->data.attrs.gen_i, "nonnull");
2054 addLLVMArgAttrInt(llvm_fn, fn_walk->data.attrs.gen_i, "align", get_abi_alignment(g, ty));
2055 fn_walk->data.attrs.gen_i += 1;
2056 break;
2057 case FnWalkIdCall:
2058 fn_walk->data.call.gen_param_values->append(val);
2059 break;
2060 case FnWalkIdTypes: {
2061 ZigType *gen_type = get_pointer_to_type(g, ty, true);
2062 fn_walk->data.types.gen_param_types->append(get_llvm_type(g, gen_type));
2063 fn_walk->data.types.param_di_types->append(get_llvm_di_type(g, gen_type));
2064 break;
2065 }
2066 case FnWalkIdVars: {
2067 var->value_ref = LLVMGetParam(llvm_fn, fn_walk->data.vars.gen_i);
2068 di_arg_index = fn_walk->data.vars.gen_i;
2069 dest_ty = get_pointer_to_type(g, ty, false);
2070 fn_walk->data.vars.gen_i += 1;
2071 goto var_ok;
2072 }
2073 case FnWalkIdInits:
2074 if (var->decl_node) {
2075 gen_var_debug_decl(g, var);
2076 }
2077 fn_walk->data.inits.gen_i += 1;
2078 break;
2079 }
2080 return true;
2081 }
2082
2083 X64CABIClass abi_class = type_c_abi_x86_64_class(g, ty);
2084 size_t ty_size = type_size(g, ty);
2085 if (abi_class == X64CABIClass_MEMORY || abi_class == X64CABIClass_MEMORY_nobyval) {
2086 assert(handle_is_ptr(g, ty));
2087 switch (fn_walk->id) {
2088 case FnWalkIdAttrs:
2089 if (abi_class != X64CABIClass_MEMORY_nobyval) {
2090 ZigLLVMAddByValAttr(llvm_fn, fn_walk->data.attrs.gen_i, get_llvm_type(g, ty));
2091 addLLVMArgAttrInt(llvm_fn, fn_walk->data.attrs.gen_i, "align", get_abi_alignment(g, ty));
2092 } else if (g->zig_target->arch == ZigLLVM_aarch64 ||
2093 g->zig_target->arch == ZigLLVM_aarch64_be)
2094 {
2095 // no attrs needed
2096 } else {
2097 if (source_node != nullptr) {
2098 give_up_with_c_abi_error(g, source_node);
2099 }
2100 // otherwise allow codegen code to report a compile error
2101 return false;
2102 }
2103
2104 // Byvalue parameters must not have address 0
2105 addLLVMArgAttr(llvm_fn, fn_walk->data.attrs.gen_i, "nonnull");
2106 fn_walk->data.attrs.gen_i += 1;
2107 break;
2108 case FnWalkIdCall:
2109 fn_walk->data.call.gen_param_values->append(val);
2110 break;
2111 case FnWalkIdTypes: {
2112 ZigType *gen_type = get_pointer_to_type(g, ty, true);
2113 fn_walk->data.types.gen_param_types->append(get_llvm_type(g, gen_type));
2114 fn_walk->data.types.param_di_types->append(get_llvm_di_type(g, gen_type));
2115 break;
2116 }
2117 case FnWalkIdVars: {
2118 di_arg_index = fn_walk->data.vars.gen_i;
2119 var->value_ref = LLVMGetParam(llvm_fn, fn_walk->data.vars.gen_i);
2120 dest_ty = get_pointer_to_type(g, ty, false);
2121 fn_walk->data.vars.gen_i += 1;
2122 goto var_ok;
2123 }
2124 case FnWalkIdInits:
2125 if (var->decl_node) {
2126 gen_var_debug_decl(g, var);
2127 }
2128 fn_walk->data.inits.gen_i += 1;
2129 break;
2130 }
2131 return true;
2132 } else if (abi_class == X64CABIClass_INTEGER) {
2133 switch (fn_walk->id) {
2134 case FnWalkIdAttrs:
2135 fn_walk->data.attrs.gen_i += 1;
2136 break;
2137 case FnWalkIdCall: {
2138 LLVMTypeRef ptr_to_int_type_ref = LLVMPointerType(LLVMIntType((unsigned)ty_size * 8), 0);
2139 LLVMValueRef bitcasted = LLVMBuildBitCast(g->builder, val, ptr_to_int_type_ref, "");
2140 LLVMValueRef loaded = LLVMBuildLoad(g->builder, bitcasted, "");
2141 fn_walk->data.call.gen_param_values->append(loaded);
2142 break;
2143 }
2144 case FnWalkIdTypes: {
2145 ZigType *gen_type = get_int_type(g, false, ty_size * 8);
2146 fn_walk->data.types.gen_param_types->append(get_llvm_type(g, gen_type));
2147 fn_walk->data.types.param_di_types->append(get_llvm_di_type(g, gen_type));
2148 break;
2149 }
2150 case FnWalkIdVars: {
2151 di_arg_index = fn_walk->data.vars.gen_i;
2152 var->value_ref = build_alloca(g, ty, var->name, var->align_bytes);
2153 fn_walk->data.vars.gen_i += 1;
2154 dest_ty = ty;
2155 goto var_ok;
2156 }
2157 case FnWalkIdInits: {
2158 clear_debug_source_node(g);
2159 if (!fn_is_async(fn_walk->data.inits.fn)) {
2160 LLVMValueRef arg = LLVMGetParam(llvm_fn, fn_walk->data.inits.gen_i);
2161 LLVMTypeRef ptr_to_int_type_ref = LLVMPointerType(LLVMIntType((unsigned)ty_size * 8), 0);
2162 LLVMValueRef bitcasted = LLVMBuildBitCast(g->builder, var->value_ref, ptr_to_int_type_ref, "");
2163 gen_store_untyped(g, arg, bitcasted, var->align_bytes, false);
2164 }
2165 if (var->decl_node) {
2166 gen_var_debug_decl(g, var);
2167 }
2168 fn_walk->data.inits.gen_i += 1;
2169 break;
2170 }
2171 }
2172 return true;
2173 } else if (abi_class == X64CABIClass_AGG) {
2174 // The SystemV ABI says that we have to setup 1 register per eightbyte.
2175 // So two f32 can be passed in one f64, but 3 f32 have to be passed in 2 FP registers.
2176 // Similarly, two i32 can be passed in one i64, but 3 i32 have to be passed in 2 registers.
2177 // LLVM does not allow us to control registers in this way, nor to request specific
2178 // ABI conventions. So we have to trick it into allocating the right registers, based
2179 // on how clang does it.
2180
2181 // First, we get the LLVM type corresponding to the C abi for the struct, then
2182 // we pass each field as an argument.
2183
2184 // Example:
2185 // extern struct {
2186 // x: f32,
2187 // y: f32,
2188 // z: i32,
2189 // };
2190 // LLVM abi type: { double, i32 }
2191 // const ptr = (*abi_type)*Struct;
2192 // FP Register 1: abi_type[0]
2193 // Register 1: abi_type[1]
2194
2195 // However, if the struct fits in one register, then we'll pass it as such
2196 size_t number_of_regs = (size_t)ceilf((float)ty_size / (float)8);
2197
2198 LLVMTypeRef abi_type = get_llvm_c_abi_type(g, ty);
2199
2200 assert(ty_size <= 16);
2201
2202 switch (fn_walk->id) {
2203 case FnWalkIdAttrs: {
2204 fn_walk->data.attrs.gen_i += number_of_regs;
2205 break;
2206 }
2207 case FnWalkIdCall: {
2208 LLVMValueRef abi_ptr_to_struct = LLVMBuildBitCast(g->builder, val, LLVMPointerType(abi_type, 0), "");
2209 if (number_of_regs == 1) {
2210 LLVMValueRef loaded = LLVMBuildLoad(g->builder, abi_ptr_to_struct, "");
2211 fn_walk->data.call.gen_param_values->append(loaded);
2212 break;
2213 }
2214 for (uint32_t i = 0; i < number_of_regs; i += 1) {
2215 LLVMValueRef zero = LLVMConstInt(LLVMInt32Type(), 0, false);
2216 LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, false);
2217 LLVMValueRef indices[] = { zero, index };
2218 LLVMValueRef adjusted_ptr_to_struct = LLVMBuildInBoundsGEP(g->builder, abi_ptr_to_struct, indices, 2, "");
2219 LLVMValueRef loaded = LLVMBuildLoad(g->builder, adjusted_ptr_to_struct, "");
2220 fn_walk->data.call.gen_param_values->append(loaded);
2221 }
2222 break;
2223 }
2224 case FnWalkIdTypes: {
2225 if (number_of_regs == 1) {
2226 fn_walk->data.types.gen_param_types->append(abi_type);
2227 fn_walk->data.types.param_di_types->append(get_llvm_di_type(g, g->builtin_types.entry_f64));
2228 break;
2229 }
2230 for (uint32_t i = 0; i < number_of_regs; i += 1) {
2231 fn_walk->data.types.gen_param_types->append(LLVMStructGetTypeAtIndex(abi_type, i));
2232 fn_walk->data.types.param_di_types->append(get_llvm_di_type(g, g->builtin_types.entry_f64));
2233 }
2234 break;
2235 }
2236 case FnWalkIdVars: {
2237 var->value_ref = build_alloca(g, ty, var->name, var->align_bytes);
2238 di_arg_index = fn_walk->data.vars.gen_i;
2239 fn_walk->data.vars.gen_i += 1;
2240 dest_ty = ty;
2241 goto var_ok;
2242 }
2243 case FnWalkIdInits: {
2244 // since we're representing the struct differently as an arg, and potentially
2245 // splitting it, we have to do some work to put it back together.
2246 // the one reg case is straightforward, but if we used two registers we have
2247 // to iterate through the struct abi repr fields and load them one by one.
2248 if (number_of_regs == 1) {
2249 LLVMValueRef arg = LLVMGetParam(llvm_fn, fn_walk->data.inits.gen_i);
2250 LLVMTypeRef ptr_to_int_type_ref = LLVMPointerType(abi_type, 0);
2251 LLVMValueRef bitcasted = LLVMBuildBitCast(g->builder, var->value_ref, ptr_to_int_type_ref, "");
2252 gen_store_untyped(g, arg, bitcasted, var->align_bytes, false);
2253 } else {
2254 LLVMValueRef abi_ptr_to_struct = LLVMBuildBitCast(g->builder, var->value_ref, LLVMPointerType(abi_type, 0), "");
2255 for (uint32_t i = 0; i < number_of_regs; i += 1) {
2256 LLVMValueRef arg = LLVMGetParam(llvm_fn, fn_walk->data.inits.gen_i + i);
2257 LLVMValueRef zero = LLVMConstInt(LLVMInt32Type(), 0, false);
2258 LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, false);
2259 LLVMValueRef indices[] = { zero, index };
2260 LLVMValueRef adjusted_ptr_to_struct = LLVMBuildInBoundsGEP(g->builder, abi_ptr_to_struct, indices, 2, "");
2261 LLVMBuildStore(g->builder, arg, adjusted_ptr_to_struct);
2262 }
2263 fn_walk->data.inits.gen_i += 1;
2264 }
2265 if (var->decl_node) {
2266 gen_var_debug_decl(g, var);
2267 }
2268 fn_walk->data.inits.gen_i += 1;
2269 break;
2270 }
2271 }
2272 return true;
2273 }
2274 }
2275 if (source_node != nullptr) {
2276 give_up_with_c_abi_error(g, source_node);
2277 }
2278 // otherwise allow codegen code to report a compile error
2279 return false;
2280
2281 var_ok:
2282 if (dest_ty != nullptr && var->decl_node) {
2283 // arg index + 1 because the 0 index is return value
2284 var->di_loc_var = ZigLLVMCreateParameterVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
2285 var->name, fn_walk->data.vars.import->data.structure.root_struct->di_file,
2286 node_line_onebased(var->decl_node),
2287 get_llvm_di_type(g, dest_ty), !g->strip_debug_symbols, 0, di_arg_index + 1);
2288 }
2289 return true;
2290 }
2291
walk_function_params(CodeGen * g,ZigType * fn_type,FnWalk * fn_walk)2292 void walk_function_params(CodeGen *g, ZigType *fn_type, FnWalk *fn_walk) {
2293 CallingConvention cc = fn_type->data.fn.fn_type_id.cc;
2294 if (!calling_convention_allows_zig_types(cc)) {
2295 size_t src_i = 0;
2296 for (;;) {
2297 if (!iter_function_params_c_abi(g, fn_type, fn_walk, src_i))
2298 break;
2299 src_i += 1;
2300 }
2301 return;
2302 }
2303 if (fn_walk->id == FnWalkIdCall) {
2304 Stage1AirInstCall *instruction = fn_walk->data.call.inst;
2305 bool is_var_args = fn_walk->data.call.is_var_args;
2306 for (size_t call_i = 0; call_i < instruction->arg_count; call_i += 1) {
2307 Stage1AirInst *param_instruction = instruction->args[call_i];
2308 ZigType *param_type = param_instruction->value->type;
2309 if (is_var_args || type_has_bits(g, param_type)) {
2310 LLVMValueRef param_value = ir_llvm_value(g, param_instruction);
2311 assert(param_value);
2312 fn_walk->data.call.gen_param_values->append(param_value);
2313 fn_walk->data.call.gen_param_types->append(param_type);
2314 }
2315 }
2316 return;
2317 }
2318 size_t next_var_i = 0;
2319 for (size_t param_i = 0; param_i < fn_type->data.fn.fn_type_id.param_count; param_i += 1) {
2320 FnGenParamInfo *gen_info = &fn_type->data.fn.gen_param_info[param_i];
2321 size_t gen_index = gen_info->gen_index;
2322
2323 if (gen_index == SIZE_MAX) {
2324 continue;
2325 }
2326
2327 switch (fn_walk->id) {
2328 case FnWalkIdAttrs: {
2329 LLVMValueRef llvm_fn = fn_walk->data.attrs.llvm_fn;
2330 bool is_byval = gen_info->is_byval;
2331 FnTypeParamInfo *param_info = &fn_type->data.fn.fn_type_id.param_info[param_i];
2332
2333 ZigType *param_type = gen_info->type;
2334 if (param_info->is_noalias) {
2335 addLLVMArgAttr(llvm_fn, (unsigned)gen_index, "noalias");
2336 }
2337 if ((param_type->id == ZigTypeIdPointer && param_type->data.pointer.is_const) || is_byval) {
2338 addLLVMArgAttr(llvm_fn, (unsigned)gen_index, "readonly");
2339 }
2340 if (get_codegen_ptr_type_bail(g, param_type) != nullptr) {
2341 addLLVMArgAttrInt(llvm_fn, (unsigned)gen_index, "align", get_ptr_align(g, param_type));
2342 }
2343 if (type_is_nonnull_ptr(g, param_type)) {
2344 addLLVMArgAttr(llvm_fn, (unsigned)gen_index, "nonnull");
2345 }
2346 break;
2347 }
2348 case FnWalkIdInits: {
2349 ZigFn *fn_table_entry = fn_walk->data.inits.fn;
2350 LLVMValueRef llvm_fn = fn_table_entry->llvm_value;
2351 ZigVar *variable = fn_table_entry->variable_list.at(next_var_i);
2352 assert(variable->src_arg_index != SIZE_MAX);
2353 next_var_i += 1;
2354
2355 assert(variable);
2356 assert(variable->value_ref);
2357
2358 if (!handle_is_ptr(g, variable->var_type) && !fn_is_async(fn_walk->data.inits.fn)) {
2359 clear_debug_source_node(g);
2360 ZigType *fn_type = fn_table_entry->type_entry;
2361 unsigned gen_arg_index = fn_type->data.fn.gen_param_info[variable->src_arg_index].gen_index;
2362 gen_store_untyped(g, LLVMGetParam(llvm_fn, gen_arg_index),
2363 variable->value_ref, variable->align_bytes, false);
2364 }
2365
2366 if (variable->decl_node) {
2367 gen_var_debug_decl(g, variable);
2368 }
2369 break;
2370 }
2371 case FnWalkIdCall:
2372 // handled before for loop
2373 zig_unreachable();
2374 case FnWalkIdTypes:
2375 // Not called for non-c-abi
2376 zig_unreachable();
2377 case FnWalkIdVars:
2378 // iter_function_params_c_abi is called directly for this one
2379 zig_unreachable();
2380 }
2381 }
2382 }
2383
get_merge_err_ret_traces_fn_val(CodeGen * g)2384 static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) {
2385 if (g->merge_err_ret_traces_fn_val)
2386 return g->merge_err_ret_traces_fn_val;
2387
2388 assert(g->stack_trace_type != nullptr);
2389
2390 LLVMTypeRef param_types[] = {
2391 get_llvm_type(g, ptr_to_stack_trace_type(g)),
2392 get_llvm_type(g, ptr_to_stack_trace_type(g)),
2393 };
2394 LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), param_types, 2, false);
2395
2396 const char *fn_name = get_mangled_name(g, "__zig_merge_error_return_traces");
2397 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type_ref);
2398 LLVMSetLinkage(fn_val, LLVMInternalLinkage);
2399 ZigLLVMFunctionSetCallingConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
2400 add_common_fn_attributes(g, fn_val);
2401 addLLVMArgAttr(fn_val, (unsigned)0, "noalias");
2402 addLLVMArgAttr(fn_val, (unsigned)0, "writeonly");
2403
2404 addLLVMArgAttr(fn_val, (unsigned)1, "noalias");
2405 addLLVMArgAttr(fn_val, (unsigned)1, "readonly");
2406 if (!g->omit_frame_pointer) {
2407 ZigLLVMAddFunctionAttr(fn_val, "frame-pointer", "all");
2408 }
2409
2410 // this is above the ZigLLVMClearCurrentDebugLocation
2411 LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g);
2412
2413 LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
2414 LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
2415 LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
2416 LLVMPositionBuilderAtEnd(g->builder, entry_block);
2417 ZigLLVMClearCurrentDebugLocation(g->builder);
2418
2419 // if (dest_stack_trace == null or src_stack_trace == null) return;
2420 // var frame_index: usize = undefined;
2421 // var frames_left: usize = undefined;
2422 // if (src_stack_trace.index < src_stack_trace.instruction_addresses.len) {
2423 // frame_index = 0;
2424 // frames_left = src_stack_trace.index;
2425 // if (frames_left == 0) return;
2426 // } else {
2427 // frame_index = (src_stack_trace.index + 1) % src_stack_trace.instruction_addresses.len;
2428 // frames_left = src_stack_trace.instruction_addresses.len;
2429 // }
2430 // while (true) {
2431 // __zig_add_err_ret_trace_addr(dest_stack_trace, src_stack_trace.instruction_addresses[frame_index]);
2432 // frames_left -= 1;
2433 // if (frames_left == 0) return;
2434 // frame_index = (frame_index + 1) % src_stack_trace.instruction_addresses.len;
2435 // }
2436 LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return");
2437 LLVMBasicBlockRef non_null_block = LLVMAppendBasicBlock(fn_val, "NonNull");
2438
2439 LLVMValueRef frame_index_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frame_index");
2440 LLVMValueRef frames_left_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frames_left");
2441
2442 LLVMValueRef dest_stack_trace_ptr = LLVMGetParam(fn_val, 0);
2443 LLVMValueRef src_stack_trace_ptr = LLVMGetParam(fn_val, 1);
2444
2445 LLVMValueRef null_dest_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, dest_stack_trace_ptr,
2446 LLVMConstNull(LLVMTypeOf(dest_stack_trace_ptr)), "");
2447 LLVMValueRef null_src_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_stack_trace_ptr,
2448 LLVMConstNull(LLVMTypeOf(src_stack_trace_ptr)), "");
2449 LLVMValueRef null_bit = LLVMBuildOr(g->builder, null_dest_bit, null_src_bit, "");
2450 LLVMBuildCondBr(g->builder, null_bit, return_block, non_null_block);
2451
2452 LLVMPositionBuilderAtEnd(g->builder, non_null_block);
2453 size_t src_index_field_index = g->stack_trace_type->data.structure.fields[0]->gen_index;
2454 size_t src_addresses_field_index = g->stack_trace_type->data.structure.fields[1]->gen_index;
2455 LLVMValueRef src_index_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr,
2456 (unsigned)src_index_field_index, "");
2457 LLVMValueRef src_addresses_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr,
2458 (unsigned)src_addresses_field_index, "");
2459 ZigType *slice_type = g->stack_trace_type->data.structure.fields[1]->type_entry;
2460 size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index]->gen_index;
2461 LLVMValueRef src_ptr_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)ptr_field_index, "");
2462 size_t len_field_index = slice_type->data.structure.fields[slice_len_index]->gen_index;
2463 LLVMValueRef src_len_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)len_field_index, "");
2464 LLVMValueRef src_index_val = LLVMBuildLoad(g->builder, src_index_field_ptr, "");
2465 LLVMValueRef src_ptr_val = LLVMBuildLoad(g->builder, src_ptr_field_ptr, "");
2466 LLVMValueRef src_len_val = LLVMBuildLoad(g->builder, src_len_field_ptr, "");
2467 LLVMValueRef no_wrap_bit = LLVMBuildICmp(g->builder, LLVMIntULT, src_index_val, src_len_val, "");
2468 LLVMBasicBlockRef no_wrap_block = LLVMAppendBasicBlock(fn_val, "NoWrap");
2469 LLVMBasicBlockRef yes_wrap_block = LLVMAppendBasicBlock(fn_val, "YesWrap");
2470 LLVMBasicBlockRef loop_block = LLVMAppendBasicBlock(fn_val, "Loop");
2471 LLVMBuildCondBr(g->builder, no_wrap_bit, no_wrap_block, yes_wrap_block);
2472
2473 LLVMPositionBuilderAtEnd(g->builder, no_wrap_block);
2474 LLVMValueRef usize_zero = LLVMConstNull(g->builtin_types.entry_usize->llvm_type);
2475 LLVMBuildStore(g->builder, usize_zero, frame_index_ptr);
2476 LLVMBuildStore(g->builder, src_index_val, frames_left_ptr);
2477 LLVMValueRef frames_left_eq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_index_val, usize_zero, "");
2478 LLVMBuildCondBr(g->builder, frames_left_eq_zero_bit, return_block, loop_block);
2479
2480 LLVMPositionBuilderAtEnd(g->builder, yes_wrap_block);
2481 LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 1, false);
2482 LLVMValueRef plus_one = LLVMBuildNUWAdd(g->builder, src_index_val, usize_one, "");
2483 LLVMValueRef mod_len = LLVMBuildURem(g->builder, plus_one, src_len_val, "");
2484 LLVMBuildStore(g->builder, mod_len, frame_index_ptr);
2485 LLVMBuildStore(g->builder, src_len_val, frames_left_ptr);
2486 LLVMBuildBr(g->builder, loop_block);
2487
2488 LLVMPositionBuilderAtEnd(g->builder, loop_block);
2489 LLVMValueRef ptr_index = LLVMBuildLoad(g->builder, frame_index_ptr, "");
2490 LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, "");
2491 LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, "");
2492 LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val};
2493 ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_CallAttrAlwaysInline, "");
2494 LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, "");
2495 LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, "");
2496 LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, "");
2497 LLVMBasicBlockRef continue_block = LLVMAppendBasicBlock(fn_val, "Continue");
2498 LLVMBuildCondBr(g->builder, done_bit, return_block, continue_block);
2499
2500 LLVMPositionBuilderAtEnd(g->builder, return_block);
2501 LLVMBuildRetVoid(g->builder);
2502
2503 LLVMPositionBuilderAtEnd(g->builder, continue_block);
2504 LLVMBuildStore(g->builder, new_frames_left, frames_left_ptr);
2505 LLVMValueRef prev_index = LLVMBuildLoad(g->builder, frame_index_ptr, "");
2506 LLVMValueRef index_plus_one = LLVMBuildNUWAdd(g->builder, prev_index, usize_one, "");
2507 LLVMValueRef index_mod_len = LLVMBuildURem(g->builder, index_plus_one, src_len_val, "");
2508 LLVMBuildStore(g->builder, index_mod_len, frame_index_ptr);
2509 LLVMBuildBr(g->builder, loop_block);
2510
2511 LLVMPositionBuilderAtEnd(g->builder, prev_block);
2512 if (!g->strip_debug_symbols) {
2513 LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
2514 }
2515
2516 g->merge_err_ret_traces_fn_val = fn_val;
2517 return fn_val;
2518
2519 }
ir_render_save_err_ret_addr(CodeGen * g,Stage1Air * executable,Stage1AirInstSaveErrRetAddr * save_err_ret_addr_instruction)2520 static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, Stage1Air *executable,
2521 Stage1AirInstSaveErrRetAddr *save_err_ret_addr_instruction)
2522 {
2523 assert(g->have_err_ret_tracing);
2524
2525 LLVMValueRef return_err_fn = get_return_err_fn(g);
2526 bool is_llvm_alloca;
2527 LLVMValueRef my_err_trace_val = get_cur_err_ret_trace_val(g, save_err_ret_addr_instruction->base.scope,
2528 &is_llvm_alloca);
2529 ZigLLVMBuildCall(g->builder, return_err_fn, &my_err_trace_val, 1,
2530 get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_CallAttrAuto, "");
2531
2532 ZigType *ret_type = g->cur_fn->type_entry->data.fn.fn_type_id.return_type;
2533 if (fn_is_async(g->cur_fn) && codegen_fn_has_err_ret_tracing_arg(g, ret_type)) {
2534 LLVMValueRef trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
2535 frame_index_trace_arg(g, ret_type), "");
2536 LLVMBuildStore(g->builder, my_err_trace_val, trace_ptr_ptr);
2537 }
2538
2539 return nullptr;
2540 }
2541
gen_assert_resume_id(CodeGen * g,Stage1AirInst * source_instr,ResumeId resume_id,PanicMsgId msg_id,LLVMBasicBlockRef end_bb)2542 static void gen_assert_resume_id(CodeGen *g, Stage1AirInst *source_instr, ResumeId resume_id, PanicMsgId msg_id,
2543 LLVMBasicBlockRef end_bb)
2544 {
2545 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
2546
2547 if (ir_want_runtime_safety(g, source_instr)) {
2548 // Write a value to the resume index which indicates the function was resumed while not suspended.
2549 LLVMBuildStore(g->builder, g->cur_bad_not_suspended_index, g->cur_async_resume_index_ptr);
2550 }
2551
2552 LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
2553 if (end_bb == nullptr) end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume");
2554 LLVMValueRef expected_value = LLVMConstSub(LLVMConstAllOnes(usize_type_ref),
2555 LLVMConstInt(usize_type_ref, resume_id, false));
2556 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, LLVMGetParam(g->cur_fn_val, 1), expected_value, "");
2557 LLVMBuildCondBr(g->builder, ok_bit, end_bb, bad_resume_block);
2558
2559 LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
2560 gen_assertion(g, msg_id, source_instr);
2561
2562 LLVMPositionBuilderAtEnd(g->builder, end_bb);
2563 }
2564
gen_resume(CodeGen * g,LLVMValueRef fn_val,LLVMValueRef target_frame_ptr,ResumeId resume_id)2565 static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef target_frame_ptr, ResumeId resume_id) {
2566 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
2567 if (fn_val == nullptr) {
2568 LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_fn_ptr_index, "");
2569 fn_val = LLVMBuildLoad(g->builder, fn_ptr_ptr, "");
2570 }
2571 LLVMValueRef arg_val = LLVMConstSub(LLVMConstAllOnes(usize_type_ref),
2572 LLVMConstInt(usize_type_ref, resume_id, false));
2573 LLVMValueRef args[] = {target_frame_ptr, arg_val};
2574 return ZigLLVMBuildCall(g->builder, fn_val, args, 2, ZigLLVM_Fast, ZigLLVM_CallAttrAuto, "");
2575 }
2576
gen_suspend_begin(CodeGen * g,const char * name_hint)2577 static LLVMBasicBlockRef gen_suspend_begin(CodeGen *g, const char *name_hint) {
2578 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
2579 LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, name_hint);
2580 size_t new_block_index = g->cur_resume_block_count;
2581 g->cur_resume_block_count += 1;
2582 LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false);
2583 LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb);
2584 LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr);
2585 return resume_bb;
2586 }
2587
2588 // Be careful setting tail call. According to LLVM lang ref,
2589 // tail and musttail imply that the callee does not access allocas from the caller.
2590 // This works for async functions since the locals are spilled.
2591 // http://llvm.org/docs/LangRef.html#id320
set_tail_call_if_appropriate(CodeGen * g,LLVMValueRef call_inst)2592 static void set_tail_call_if_appropriate(CodeGen *g, LLVMValueRef call_inst) {
2593 LLVMSetTailCall(call_inst, true);
2594 }
2595
gen_maybe_atomic_op(CodeGen * g,LLVMAtomicRMWBinOp op,LLVMValueRef ptr,LLVMValueRef val,LLVMAtomicOrdering order)2596 static LLVMValueRef gen_maybe_atomic_op(CodeGen *g, LLVMAtomicRMWBinOp op, LLVMValueRef ptr, LLVMValueRef val,
2597 LLVMAtomicOrdering order)
2598 {
2599 if (g->is_single_threaded) {
2600 LLVMValueRef loaded = LLVMBuildLoad(g->builder, ptr, "");
2601 LLVMValueRef modified;
2602 switch (op) {
2603 case LLVMAtomicRMWBinOpXchg:
2604 modified = val;
2605 break;
2606 case LLVMAtomicRMWBinOpXor:
2607 modified = LLVMBuildXor(g->builder, loaded, val, "");
2608 break;
2609 default:
2610 zig_unreachable();
2611 }
2612 LLVMBuildStore(g->builder, modified, ptr);
2613 return loaded;
2614 } else {
2615 return LLVMBuildAtomicRMW(g->builder, op, ptr, val, order, false);
2616 }
2617 }
2618
gen_async_return(CodeGen * g,Stage1AirInstReturn * instruction)2619 static void gen_async_return(CodeGen *g, Stage1AirInstReturn *instruction) {
2620 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
2621
2622 ZigType *operand_type = (instruction->operand != nullptr) ? instruction->operand->value->type : nullptr;
2623 bool operand_has_bits = (operand_type != nullptr) && type_has_bits(g, operand_type);
2624 ZigType *ret_type = g->cur_fn->type_entry->data.fn.fn_type_id.return_type;
2625 bool ret_type_has_bits = type_has_bits(g, ret_type);
2626
2627 if (operand_has_bits && instruction->operand != nullptr) {
2628 bool need_store = instruction->operand->value->special != ConstValSpecialRuntime || !handle_is_ptr(g, ret_type);
2629 if (need_store) {
2630 // It didn't get written to the result ptr. We do that now.
2631 ZigType *ret_ptr_type = get_pointer_to_type(g, ret_type, true);
2632 gen_assign_raw(g, g->cur_ret_ptr, ret_ptr_type, ir_llvm_value(g, instruction->operand));
2633 }
2634 }
2635
2636 // Whether we tail resume the awaiter, or do an early return, we are done and will not be resumed.
2637 if (ir_want_runtime_safety(g, &instruction->base)) {
2638 LLVMValueRef new_resume_index = LLVMConstAllOnes(usize_type_ref);
2639 LLVMBuildStore(g->builder, new_resume_index, g->cur_async_resume_index_ptr);
2640 }
2641
2642 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
2643 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
2644
2645 LLVMValueRef prev_val = gen_maybe_atomic_op(g, LLVMAtomicRMWBinOpXor, g->cur_async_awaiter_ptr,
2646 all_ones, LLVMAtomicOrderingAcquire);
2647
2648 LLVMBasicBlockRef bad_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadReturn");
2649 LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
2650 LLVMBasicBlockRef resume_them_block = LLVMAppendBasicBlock(g->cur_fn_val, "ResumeThem");
2651
2652 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, resume_them_block, 2);
2653
2654 LLVMAddCase(switch_instr, zero, early_return_block);
2655 LLVMAddCase(switch_instr, all_ones, bad_return_block);
2656
2657 // Something has gone horribly wrong, and this is an invalid second return.
2658 LLVMPositionBuilderAtEnd(g->builder, bad_return_block);
2659 gen_assertion(g, PanicMsgIdBadReturn, &instruction->base);
2660
2661 // There is no awaiter yet, but we're completely done.
2662 LLVMPositionBuilderAtEnd(g->builder, early_return_block);
2663 LLVMBuildRetVoid(g->builder);
2664
2665 // We need to resume the caller by tail calling them,
2666 // but first write through the result pointer and possibly
2667 // error return trace pointer.
2668 LLVMPositionBuilderAtEnd(g->builder, resume_them_block);
2669
2670 if (ret_type_has_bits) {
2671 // If the awaiter result pointer is non-null, we need to copy the result to there.
2672 LLVMBasicBlockRef copy_block = LLVMAppendBasicBlock(g->cur_fn_val, "CopyResult");
2673 LLVMBasicBlockRef copy_end_block = LLVMAppendBasicBlock(g->cur_fn_val, "CopyResultEnd");
2674 LLVMValueRef awaiter_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_ret_start + 1, "");
2675 LLVMValueRef awaiter_ret_ptr = LLVMBuildLoad(g->builder, awaiter_ret_ptr_ptr, "");
2676 LLVMValueRef zero_ptr = LLVMConstNull(LLVMTypeOf(awaiter_ret_ptr));
2677 LLVMValueRef need_copy_bit = LLVMBuildICmp(g->builder, LLVMIntNE, awaiter_ret_ptr, zero_ptr, "");
2678 LLVMBuildCondBr(g->builder, need_copy_bit, copy_block, copy_end_block);
2679
2680 LLVMPositionBuilderAtEnd(g->builder, copy_block);
2681 LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
2682 LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, awaiter_ret_ptr, ptr_u8, "");
2683 LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, g->cur_ret_ptr, ptr_u8, "");
2684 bool is_volatile = false;
2685 uint32_t abi_align = get_abi_alignment(g, ret_type);
2686 LLVMValueRef byte_count_val = LLVMConstInt(usize_type_ref, type_size(g, ret_type), false);
2687 ZigLLVMBuildMemCpy(g->builder,
2688 dest_ptr_casted, abi_align,
2689 src_ptr_casted, abi_align, byte_count_val, is_volatile);
2690 LLVMBuildBr(g->builder, copy_end_block);
2691
2692 LLVMPositionBuilderAtEnd(g->builder, copy_end_block);
2693 if (codegen_fn_has_err_ret_tracing_arg(g, ret_type)) {
2694 LLVMValueRef awaiter_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
2695 frame_index_trace_arg(g, ret_type) + 1, "");
2696 LLVMValueRef dest_trace_ptr = LLVMBuildLoad(g->builder, awaiter_trace_ptr_ptr, "");
2697 bool is_llvm_alloca;
2698 LLVMValueRef my_err_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope, &is_llvm_alloca);
2699 LLVMValueRef args[] = { dest_trace_ptr, my_err_trace_val };
2700 ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2,
2701 get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_CallAttrAuto, "");
2702 }
2703 }
2704
2705 // Resume the caller by tail calling them.
2706 ZigType *any_frame_type = get_any_frame_type(g, ret_type);
2707 LLVMValueRef their_frame_ptr = LLVMBuildIntToPtr(g->builder, prev_val, get_llvm_type(g, any_frame_type), "");
2708 LLVMValueRef call_inst = gen_resume(g, nullptr, their_frame_ptr, ResumeIdReturn);
2709 set_tail_call_if_appropriate(g, call_inst);
2710 LLVMBuildRetVoid(g->builder);
2711 }
2712
gen_convert_to_c_abi(CodeGen * g,LLVMValueRef location,LLVMValueRef value)2713 static LLVMValueRef gen_convert_to_c_abi(CodeGen *g, LLVMValueRef location, LLVMValueRef value) {
2714 ZigType *return_type = g->cur_fn->type_entry->data.fn.gen_return_type;
2715 size_t size = type_size(g, return_type);
2716
2717 LLVMTypeRef abi_return_type = get_llvm_c_abi_type(g, return_type);
2718 LLVMTypeRef abi_return_type_pointer = LLVMPointerType(abi_return_type, 0);
2719
2720 if (size < 8) {
2721 LLVMValueRef bitcast = LLVMBuildBitCast(g->builder, value, abi_return_type_pointer, "");
2722 return LLVMBuildLoad(g->builder, bitcast, "");
2723 } else {
2724 LLVMTypeRef i8ptr = LLVMPointerType(LLVMInt8Type(), 0);
2725 LLVMValueRef bc_location = LLVMBuildBitCast(g->builder, location, i8ptr, "");
2726 LLVMValueRef bc_value = LLVMBuildBitCast(g->builder, value, i8ptr, "");
2727
2728 LLVMValueRef len = LLVMConstInt(LLVMInt64Type(), size, false);
2729 ZigLLVMBuildMemCpy(g->builder, bc_location, 8, bc_value, return_type->abi_align, len, false);
2730 return LLVMBuildLoad(g->builder, location, "");
2731 }
2732 }
2733
ir_render_return(CodeGen * g,Stage1Air * executable,Stage1AirInstReturn * instruction)2734 static LLVMValueRef ir_render_return(CodeGen *g, Stage1Air *executable, Stage1AirInstReturn *instruction) {
2735 if (fn_is_async(g->cur_fn)) {
2736 gen_async_return(g, instruction);
2737 return nullptr;
2738 }
2739
2740 FnTypeId *fn_type_id = &g->cur_fn->type_entry->data.fn.fn_type_id;
2741
2742 if (want_first_arg_sret(g, fn_type_id)) {
2743 if (instruction->operand == nullptr) {
2744 LLVMBuildRetVoid(g->builder);
2745 return nullptr;
2746 }
2747 assert(g->cur_ret_ptr);
2748 ir_assert(instruction->operand->value->special != ConstValSpecialRuntime, &instruction->base);
2749 LLVMValueRef value = ir_llvm_value(g, instruction->operand);
2750 ZigType *return_type = instruction->operand->value->type;
2751 gen_assign_raw(g, g->cur_ret_ptr, get_pointer_to_type(g, return_type, false), value);
2752 LLVMBuildRetVoid(g->builder);
2753 } else if (fn_returns_c_abi_small_struct(fn_type_id)) {
2754 LLVMValueRef location = g->cur_fn->abi_return_value;
2755 if (instruction->operand == nullptr) {
2756 LLVMValueRef converted = gen_convert_to_c_abi(g, location, g->cur_ret_ptr);
2757 LLVMBuildRet(g->builder, converted);
2758 } else {
2759 LLVMValueRef value = ir_llvm_value(g, instruction->operand);
2760 LLVMValueRef converted = gen_convert_to_c_abi(g, location, value);
2761 LLVMBuildRet(g->builder, converted);
2762 }
2763 } else if (g->cur_fn->type_entry->data.fn.fn_type_id.cc != CallingConventionAsync &&
2764 handle_is_ptr(g, g->cur_fn->type_entry->data.fn.fn_type_id.return_type))
2765 {
2766 if (instruction->operand == nullptr) {
2767 LLVMValueRef by_val_value = gen_load_untyped(g, g->cur_ret_ptr, 0, false, "");
2768 LLVMBuildRet(g->builder, by_val_value);
2769 } else {
2770 LLVMValueRef value = ir_llvm_value(g, instruction->operand);
2771 LLVMValueRef by_val_value = gen_load_untyped(g, value, 0, false, "");
2772 LLVMBuildRet(g->builder, by_val_value);
2773 }
2774 } else if (instruction->operand == nullptr) {
2775 if (g->cur_ret_ptr == nullptr) {
2776 LLVMBuildRetVoid(g->builder);
2777 } else {
2778 LLVMValueRef by_val_value = gen_load_untyped(g, g->cur_ret_ptr, 0, false, "");
2779 LLVMBuildRet(g->builder, by_val_value);
2780 }
2781 } else {
2782 LLVMValueRef value = ir_llvm_value(g, instruction->operand);
2783 LLVMBuildRet(g->builder, value);
2784 }
2785 return nullptr;
2786 }
2787
gen_overflow_shl_op(CodeGen * g,ZigType * operand_type,LLVMValueRef val1,LLVMValueRef val2)2788 static LLVMValueRef gen_overflow_shl_op(CodeGen *g, ZigType *operand_type,
2789 LLVMValueRef val1, LLVMValueRef val2)
2790 {
2791 // for unsigned left shifting, we do the lossy shift, then logically shift
2792 // right the same number of bits
2793 // if the values don't match, we have an overflow
2794 // for signed left shifting we do the same except arithmetic shift right
2795 ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ?
2796 operand_type->data.vector.elem_type : operand_type;
2797
2798 assert(scalar_type->id == ZigTypeIdInt);
2799
2800 LLVMValueRef result = LLVMBuildShl(g->builder, val1, val2, "");
2801 LLVMValueRef orig_val;
2802 if (scalar_type->data.integral.is_signed) {
2803 orig_val = LLVMBuildAShr(g->builder, result, val2, "");
2804 } else {
2805 orig_val = LLVMBuildLShr(g->builder, result, val2, "");
2806 }
2807 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, val1, orig_val, "");
2808
2809 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowOk");
2810 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowFail");
2811 if (operand_type->id == ZigTypeIdVector) {
2812 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
2813 }
2814 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
2815
2816 LLVMPositionBuilderAtEnd(g->builder, fail_block);
2817 gen_safety_crash(g, PanicMsgIdShlOverflowedBits);
2818
2819 LLVMPositionBuilderAtEnd(g->builder, ok_block);
2820 return result;
2821 }
2822
gen_overflow_shr_op(CodeGen * g,ZigType * operand_type,LLVMValueRef val1,LLVMValueRef val2)2823 static LLVMValueRef gen_overflow_shr_op(CodeGen *g, ZigType *operand_type,
2824 LLVMValueRef val1, LLVMValueRef val2)
2825 {
2826 ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ?
2827 operand_type->data.vector.elem_type : operand_type;
2828
2829 assert(scalar_type->id == ZigTypeIdInt);
2830
2831 LLVMValueRef result;
2832 if (scalar_type->data.integral.is_signed) {
2833 result = LLVMBuildAShr(g->builder, val1, val2, "");
2834 } else {
2835 result = LLVMBuildLShr(g->builder, val1, val2, "");
2836 }
2837 LLVMValueRef orig_val = LLVMBuildShl(g->builder, result, val2, "");
2838 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, val1, orig_val, "");
2839
2840 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowOk");
2841 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowFail");
2842 if (operand_type->id == ZigTypeIdVector) {
2843 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
2844 }
2845 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
2846
2847 LLVMPositionBuilderAtEnd(g->builder, fail_block);
2848 gen_safety_crash(g, PanicMsgIdShrOverflowedBits);
2849
2850 LLVMPositionBuilderAtEnd(g->builder, ok_block);
2851 return result;
2852 }
2853
gen_float_op(CodeGen * g,LLVMValueRef val,ZigType * type_entry,BuiltinFnId op)2854 static LLVMValueRef gen_float_op(CodeGen *g, LLVMValueRef val, ZigType *type_entry, BuiltinFnId op) {
2855 assert(type_entry->id == ZigTypeIdFloat || type_entry->id == ZigTypeIdVector);
2856 LLVMValueRef floor_fn = get_float_fn(g, type_entry, ZigLLVMFnIdFloatOp, op);
2857 return LLVMBuildCall(g->builder, floor_fn, &val, 1, "");
2858 }
2859
2860 enum DivKind {
2861 DivKindFloat,
2862 DivKindTrunc,
2863 DivKindFloor,
2864 DivKindExact,
2865 };
2866
bigint_to_llvm_const(LLVMTypeRef type_ref,BigInt * bigint)2867 static LLVMValueRef bigint_to_llvm_const(LLVMTypeRef type_ref, BigInt *bigint) {
2868 if (bigint->digit_count == 0) {
2869 return LLVMConstNull(type_ref);
2870 }
2871
2872 if (LLVMGetTypeKind(type_ref) == LLVMVectorTypeKind) {
2873 const unsigned vector_len = LLVMGetVectorSize(type_ref);
2874 LLVMTypeRef elem_type = LLVMGetElementType(type_ref);
2875
2876 LLVMValueRef *values = heap::c_allocator.allocate_nonzero<LLVMValueRef>(vector_len);
2877 // Create a vector with all the elements having the same value
2878 for (unsigned i = 0; i < vector_len; i++) {
2879 values[i] = bigint_to_llvm_const(elem_type, bigint);
2880 }
2881 LLVMValueRef result = LLVMConstVector(values, vector_len);
2882 heap::c_allocator.deallocate(values, vector_len);
2883 return result;
2884 }
2885
2886 LLVMValueRef unsigned_val;
2887 if (bigint->digit_count == 1) {
2888 unsigned_val = LLVMConstInt(type_ref, bigint_ptr(bigint)[0], false);
2889 } else {
2890 unsigned_val = LLVMConstIntOfArbitraryPrecision(type_ref, bigint->digit_count, bigint_ptr(bigint));
2891 }
2892 if (bigint->is_negative) {
2893 return LLVMConstNeg(unsigned_val);
2894 } else {
2895 return unsigned_val;
2896 }
2897 }
2898
gen_div(CodeGen * g,bool want_runtime_safety,bool want_fast_math,LLVMValueRef val1,LLVMValueRef val2,ZigType * operand_type,DivKind div_kind)2899 static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast_math,
2900 LLVMValueRef val1, LLVMValueRef val2, ZigType *operand_type, DivKind div_kind)
2901 {
2902 ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ?
2903 operand_type->data.vector.elem_type : operand_type;
2904
2905 ZigLLVMSetFastMath(g->builder, want_fast_math);
2906
2907 LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, operand_type));
2908 if (want_runtime_safety && (want_fast_math || scalar_type->id != ZigTypeIdFloat)) {
2909 // Safety check: divisor != 0
2910 LLVMValueRef is_zero_bit;
2911 if (scalar_type->id == ZigTypeIdInt) {
2912 is_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, val2, zero, "");
2913 } else if (scalar_type->id == ZigTypeIdFloat) {
2914 is_zero_bit = LLVMBuildFCmp(g->builder, LLVMRealOEQ, val2, zero, "");
2915 } else {
2916 zig_unreachable();
2917 }
2918
2919 if (operand_type->id == ZigTypeIdVector) {
2920 is_zero_bit = ZigLLVMBuildOrReduce(g->builder, is_zero_bit);
2921 }
2922
2923 LLVMBasicBlockRef div_zero_fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivZeroFail");
2924 LLVMBasicBlockRef div_zero_ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivZeroOk");
2925 LLVMBuildCondBr(g->builder, is_zero_bit, div_zero_fail_block, div_zero_ok_block);
2926
2927 LLVMPositionBuilderAtEnd(g->builder, div_zero_fail_block);
2928 gen_safety_crash(g, PanicMsgIdDivisionByZero);
2929
2930 LLVMPositionBuilderAtEnd(g->builder, div_zero_ok_block);
2931
2932 // Safety check: check for overflow (dividend = minInt and divisor = -1)
2933 if (scalar_type->id == ZigTypeIdInt && scalar_type->data.integral.is_signed) {
2934 LLVMValueRef neg_1_value = LLVMConstAllOnes(get_llvm_type(g, operand_type));
2935 BigInt int_min_bi = {0};
2936 eval_min_max_value_int(g, scalar_type, &int_min_bi, false);
2937 LLVMValueRef int_min_value = bigint_to_llvm_const(get_llvm_type(g, operand_type), &int_min_bi);
2938
2939 LLVMBasicBlockRef overflow_fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivOverflowFail");
2940 LLVMBasicBlockRef overflow_ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivOverflowOk");
2941 LLVMValueRef num_is_int_min = LLVMBuildICmp(g->builder, LLVMIntEQ, val1, int_min_value, "");
2942 LLVMValueRef den_is_neg_1 = LLVMBuildICmp(g->builder, LLVMIntEQ, val2, neg_1_value, "");
2943 LLVMValueRef overflow_fail_bit = LLVMBuildAnd(g->builder, num_is_int_min, den_is_neg_1, "");
2944 if (operand_type->id == ZigTypeIdVector) {
2945 overflow_fail_bit = ZigLLVMBuildOrReduce(g->builder, overflow_fail_bit);
2946 }
2947 LLVMBuildCondBr(g->builder, overflow_fail_bit, overflow_fail_block, overflow_ok_block);
2948
2949 LLVMPositionBuilderAtEnd(g->builder, overflow_fail_block);
2950 gen_safety_crash(g, PanicMsgIdIntegerOverflow);
2951
2952 LLVMPositionBuilderAtEnd(g->builder, overflow_ok_block);
2953 }
2954 }
2955
2956 if (scalar_type->id == ZigTypeIdFloat) {
2957 LLVMValueRef result = LLVMBuildFDiv(g->builder, val1, val2, "");
2958 switch (div_kind) {
2959 case DivKindFloat:
2960 return result;
2961 case DivKindExact:
2962 if (want_runtime_safety) {
2963 // Safety check: a / b == floor(a / b)
2964 LLVMValueRef floored = gen_float_op(g, result, operand_type, BuiltinFnIdFloor);
2965
2966 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactOk");
2967 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactFail");
2968 LLVMValueRef ok_bit = LLVMBuildFCmp(g->builder, LLVMRealOEQ, floored, result, "");
2969 if (operand_type->id == ZigTypeIdVector) {
2970 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
2971 }
2972 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
2973
2974 LLVMPositionBuilderAtEnd(g->builder, fail_block);
2975 gen_safety_crash(g, PanicMsgIdExactDivisionRemainder);
2976
2977 LLVMPositionBuilderAtEnd(g->builder, ok_block);
2978 }
2979 return result;
2980 case DivKindTrunc:
2981 return gen_float_op(g, result, operand_type, BuiltinFnIdTrunc);
2982 case DivKindFloor:
2983 return gen_float_op(g, result, operand_type, BuiltinFnIdFloor);
2984 }
2985 zig_unreachable();
2986 }
2987
2988 assert(scalar_type->id == ZigTypeIdInt);
2989
2990 switch (div_kind) {
2991 case DivKindFloat:
2992 zig_unreachable();
2993 case DivKindTrunc:
2994 if (scalar_type->data.integral.is_signed) {
2995 return LLVMBuildSDiv(g->builder, val1, val2, "");
2996 } else {
2997 return LLVMBuildUDiv(g->builder, val1, val2, "");
2998 }
2999 case DivKindExact:
3000 if (want_runtime_safety) {
3001 // Safety check: a % b == 0
3002 LLVMValueRef remainder_val;
3003 if (scalar_type->data.integral.is_signed) {
3004 remainder_val = LLVMBuildSRem(g->builder, val1, val2, "");
3005 } else {
3006 remainder_val = LLVMBuildURem(g->builder, val1, val2, "");
3007 }
3008
3009 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactOk");
3010 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactFail");
3011 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, remainder_val, zero, "");
3012 if (operand_type->id == ZigTypeIdVector) {
3013 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
3014 }
3015 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
3016
3017 LLVMPositionBuilderAtEnd(g->builder, fail_block);
3018 gen_safety_crash(g, PanicMsgIdExactDivisionRemainder);
3019
3020 LLVMPositionBuilderAtEnd(g->builder, ok_block);
3021 }
3022 if (scalar_type->data.integral.is_signed) {
3023 return LLVMBuildExactSDiv(g->builder, val1, val2, "");
3024 } else {
3025 return LLVMBuildExactUDiv(g->builder, val1, val2, "");
3026 }
3027 case DivKindFloor:
3028 {
3029 if (!scalar_type->data.integral.is_signed) {
3030 return LLVMBuildUDiv(g->builder, val1, val2, "");
3031 }
3032 // const d = @divTrunc(a, b);
3033 // const r = @rem(a, b);
3034 // return if (r == 0) d else d - ((a < 0) ^ (b < 0));
3035
3036 LLVMValueRef div_trunc = LLVMBuildSDiv(g->builder, val1, val2, "");
3037 LLVMValueRef rem = LLVMBuildSRem(g->builder, val1, val2, "");
3038 LLVMValueRef rem_eq_0 = LLVMBuildICmp(g->builder, LLVMIntEQ, rem, zero, "");
3039 LLVMValueRef a_lt_0 = LLVMBuildICmp(g->builder, LLVMIntSLT, val1, zero, "");
3040 LLVMValueRef b_lt_0 = LLVMBuildICmp(g->builder, LLVMIntSLT, val2, zero, "");
3041 LLVMValueRef a_b_xor = LLVMBuildXor(g->builder, a_lt_0, b_lt_0, "");
3042 LLVMValueRef a_b_xor_ext = LLVMBuildZExt(g->builder, a_b_xor, LLVMTypeOf(div_trunc), "");
3043 LLVMValueRef d_sub_xor = LLVMBuildSub(g->builder, div_trunc, a_b_xor_ext, "");
3044 return LLVMBuildSelect(g->builder, rem_eq_0, div_trunc, d_sub_xor, "");
3045 }
3046 }
3047 zig_unreachable();
3048 }
3049
3050 enum RemKind {
3051 RemKindRem,
3052 RemKindMod,
3053 };
3054
gen_rem(CodeGen * g,bool want_runtime_safety,bool want_fast_math,LLVMValueRef val1,LLVMValueRef val2,ZigType * operand_type,RemKind rem_kind)3055 static LLVMValueRef gen_rem(CodeGen *g, bool want_runtime_safety, bool want_fast_math,
3056 LLVMValueRef val1, LLVMValueRef val2, ZigType *operand_type, RemKind rem_kind)
3057 {
3058 ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ?
3059 operand_type->data.vector.elem_type : operand_type;
3060
3061 ZigLLVMSetFastMath(g->builder, want_fast_math);
3062
3063 LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, operand_type));
3064 if (want_runtime_safety) {
3065 // Safety check: divisor != 0
3066 LLVMValueRef is_zero_bit;
3067 if (scalar_type->id == ZigTypeIdInt) {
3068 LLVMIntPredicate pred = scalar_type->data.integral.is_signed ? LLVMIntSLE : LLVMIntEQ;
3069 is_zero_bit = LLVMBuildICmp(g->builder, pred, val2, zero, "");
3070 } else if (scalar_type->id == ZigTypeIdFloat) {
3071 is_zero_bit = LLVMBuildFCmp(g->builder, LLVMRealOEQ, val2, zero, "");
3072 } else {
3073 zig_unreachable();
3074 }
3075
3076 if (operand_type->id == ZigTypeIdVector) {
3077 is_zero_bit = ZigLLVMBuildOrReduce(g->builder, is_zero_bit);
3078 }
3079
3080 LLVMBasicBlockRef rem_zero_ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "RemZeroOk");
3081 LLVMBasicBlockRef rem_zero_fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "RemZeroFail");
3082 LLVMBuildCondBr(g->builder, is_zero_bit, rem_zero_fail_block, rem_zero_ok_block);
3083
3084 LLVMPositionBuilderAtEnd(g->builder, rem_zero_fail_block);
3085 gen_safety_crash(g, PanicMsgIdRemainderDivisionByZero);
3086
3087 LLVMPositionBuilderAtEnd(g->builder, rem_zero_ok_block);
3088 }
3089
3090 if (scalar_type->id == ZigTypeIdFloat) {
3091 if (rem_kind == RemKindRem) {
3092 return LLVMBuildFRem(g->builder, val1, val2, "");
3093 } else {
3094 LLVMValueRef a = LLVMBuildFRem(g->builder, val1, val2, "");
3095 LLVMValueRef b = LLVMBuildFAdd(g->builder, a, val2, "");
3096 LLVMValueRef c = LLVMBuildFRem(g->builder, b, val2, "");
3097 LLVMValueRef ltz = LLVMBuildFCmp(g->builder, LLVMRealOLT, val1, zero, "");
3098 return LLVMBuildSelect(g->builder, ltz, c, a, "");
3099 }
3100 } else {
3101 assert(scalar_type->id == ZigTypeIdInt);
3102 if (scalar_type->data.integral.is_signed) {
3103 if (rem_kind == RemKindRem) {
3104 return LLVMBuildSRem(g->builder, val1, val2, "");
3105 } else {
3106 LLVMValueRef a = LLVMBuildSRem(g->builder, val1, val2, "");
3107 LLVMValueRef b = LLVMBuildNSWAdd(g->builder, a, val2, "");
3108 LLVMValueRef c = LLVMBuildSRem(g->builder, b, val2, "");
3109 LLVMValueRef ltz = LLVMBuildICmp(g->builder, LLVMIntSLT, val1, zero, "");
3110 return LLVMBuildSelect(g->builder, ltz, c, a, "");
3111 }
3112 } else {
3113 return LLVMBuildURem(g->builder, val1, val2, "");
3114 }
3115 }
3116
3117 }
3118
gen_shift_rhs_check(CodeGen * g,ZigType * lhs_type,ZigType * rhs_type,LLVMValueRef value)3119 static void gen_shift_rhs_check(CodeGen *g, ZigType *lhs_type, ZigType *rhs_type, LLVMValueRef value) {
3120 // We only check if the rhs value of the shift expression is greater or
3121 // equal to the number of bits of the lhs if it's not a power of two,
3122 // otherwise the check is useful as the allowed values are limited by the
3123 // operand type itself
3124 if (!is_power_of_2(lhs_type->data.integral.bit_count)) {
3125 BigInt bit_count_bi = {0};
3126 bigint_init_unsigned(&bit_count_bi, lhs_type->data.integral.bit_count);
3127 LLVMValueRef bit_count_value = bigint_to_llvm_const(get_llvm_type(g, rhs_type),
3128 &bit_count_bi);
3129
3130 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "CheckFail");
3131 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "CheckOk");
3132 LLVMValueRef less_than_bit = LLVMBuildICmp(g->builder, LLVMIntULT, value, bit_count_value, "");
3133 if (rhs_type->id == ZigTypeIdVector) {
3134 less_than_bit = ZigLLVMBuildOrReduce(g->builder, less_than_bit);
3135 }
3136 LLVMBuildCondBr(g->builder, less_than_bit, ok_block, fail_block);
3137
3138 LLVMPositionBuilderAtEnd(g->builder, fail_block);
3139 gen_safety_crash(g, PanicMsgIdShxTooBigRhs);
3140
3141 LLVMPositionBuilderAtEnd(g->builder, ok_block);
3142 }
3143 }
3144
ir_render_bin_op(CodeGen * g,Stage1Air * executable,Stage1AirInstBinOp * bin_op_instruction)3145 static LLVMValueRef ir_render_bin_op(CodeGen *g, Stage1Air *executable,
3146 Stage1AirInstBinOp *bin_op_instruction)
3147 {
3148 IrBinOp op_id = bin_op_instruction->op_id;
3149 Stage1AirInst *op1 = bin_op_instruction->op1;
3150 Stage1AirInst *op2 = bin_op_instruction->op2;
3151
3152 ZigType *operand_type = op1->value->type;
3153 ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ? operand_type->data.vector.elem_type : operand_type;
3154
3155 bool want_runtime_safety = bin_op_instruction->safety_check_on &&
3156 ir_want_runtime_safety(g, &bin_op_instruction->base);
3157
3158 LLVMValueRef op1_value = ir_llvm_value(g, op1);
3159 LLVMValueRef op2_value = ir_llvm_value(g, op2);
3160
3161
3162 switch (op_id) {
3163 case IrBinOpInvalid:
3164 case IrBinOpArrayCat:
3165 case IrBinOpArrayMult:
3166 case IrBinOpRemUnspecified:
3167 zig_unreachable();
3168 case IrBinOpBoolOr:
3169 return LLVMBuildOr(g->builder, op1_value, op2_value, "");
3170 case IrBinOpBoolAnd:
3171 return LLVMBuildAnd(g->builder, op1_value, op2_value, "");
3172 case IrBinOpCmpEq:
3173 case IrBinOpCmpNotEq:
3174 case IrBinOpCmpLessThan:
3175 case IrBinOpCmpGreaterThan:
3176 case IrBinOpCmpLessOrEq:
3177 case IrBinOpCmpGreaterOrEq:
3178 if (scalar_type->id == ZigTypeIdFloat) {
3179 ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, &bin_op_instruction->base));
3180 LLVMRealPredicate pred = cmp_op_to_real_predicate(op_id);
3181 return LLVMBuildFCmp(g->builder, pred, op1_value, op2_value, "");
3182 } else if (scalar_type->id == ZigTypeIdInt) {
3183 LLVMIntPredicate pred = cmp_op_to_int_predicate(op_id, scalar_type->data.integral.is_signed);
3184 return LLVMBuildICmp(g->builder, pred, op1_value, op2_value, "");
3185 } else if (scalar_type->id == ZigTypeIdEnum ||
3186 scalar_type->id == ZigTypeIdErrorSet ||
3187 scalar_type->id == ZigTypeIdBool ||
3188 get_codegen_ptr_type_bail(g, scalar_type) != nullptr)
3189 {
3190 LLVMIntPredicate pred = cmp_op_to_int_predicate(op_id, false);
3191 return LLVMBuildICmp(g->builder, pred, op1_value, op2_value, "");
3192 } else {
3193 zig_unreachable();
3194 }
3195 case IrBinOpMult:
3196 case IrBinOpMultWrap:
3197 case IrBinOpAdd:
3198 case IrBinOpAddWrap:
3199 case IrBinOpSub:
3200 case IrBinOpSubWrap: {
3201 bool is_wrapping = (op_id == IrBinOpSubWrap || op_id == IrBinOpAddWrap || op_id == IrBinOpMultWrap);
3202 AddSubMul add_sub_mul =
3203 op_id == IrBinOpAdd || op_id == IrBinOpAddWrap ? AddSubMulAdd :
3204 op_id == IrBinOpSub || op_id == IrBinOpSubWrap ? AddSubMulSub :
3205 AddSubMulMul;
3206
3207 if (scalar_type->id == ZigTypeIdPointer) {
3208 LLVMValueRef subscript_value;
3209 if (operand_type->id == ZigTypeIdVector)
3210 zig_panic("TODO: Implement vector operations on pointers.");
3211
3212 switch (add_sub_mul) {
3213 case AddSubMulAdd:
3214 subscript_value = op2_value;
3215 break;
3216 case AddSubMulSub:
3217 subscript_value = LLVMBuildNeg(g->builder, op2_value, "");
3218 break;
3219 case AddSubMulMul:
3220 zig_unreachable();
3221 }
3222
3223 // TODO runtime safety
3224 return LLVMBuildInBoundsGEP(g->builder, op1_value, &subscript_value, 1, "");
3225 } else if (scalar_type->id == ZigTypeIdFloat) {
3226 ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, &bin_op_instruction->base));
3227 return float_op[add_sub_mul](g->builder, op1_value, op2_value, "");
3228 } else if (scalar_type->id == ZigTypeIdInt) {
3229 if (is_wrapping) {
3230 return wrap_op[add_sub_mul](g->builder, op1_value, op2_value, "");
3231 } else if (want_runtime_safety) {
3232 return gen_overflow_op(g, operand_type, add_sub_mul, op1_value, op2_value);
3233 } else if (scalar_type->data.integral.is_signed) {
3234 return signed_op[add_sub_mul](g->builder, op1_value, op2_value, "");
3235 } else {
3236 return unsigned_op[add_sub_mul](g->builder, op1_value, op2_value, "");
3237 }
3238 } else {
3239 zig_unreachable();
3240 }
3241 }
3242 case IrBinOpBinOr:
3243 return LLVMBuildOr(g->builder, op1_value, op2_value, "");
3244 case IrBinOpBinXor:
3245 return LLVMBuildXor(g->builder, op1_value, op2_value, "");
3246 case IrBinOpBinAnd:
3247 return LLVMBuildAnd(g->builder, op1_value, op2_value, "");
3248 case IrBinOpBitShiftLeftLossy:
3249 case IrBinOpBitShiftLeftExact:
3250 {
3251 assert(scalar_type->id == ZigTypeIdInt);
3252 LLVMValueRef op2_casted = LLVMBuildZExt(g->builder, op2_value,
3253 LLVMTypeOf(op1_value), "");
3254
3255 if (want_runtime_safety) {
3256 gen_shift_rhs_check(g, scalar_type, op2->value->type, op2_value);
3257 }
3258
3259 bool is_sloppy = (op_id == IrBinOpBitShiftLeftLossy);
3260 if (is_sloppy) {
3261 return LLVMBuildShl(g->builder, op1_value, op2_casted, "");
3262 } else if (want_runtime_safety) {
3263 return gen_overflow_shl_op(g, operand_type, op1_value, op2_casted);
3264 } else if (scalar_type->data.integral.is_signed) {
3265 return ZigLLVMBuildNSWShl(g->builder, op1_value, op2_casted, "");
3266 } else {
3267 return ZigLLVMBuildNUWShl(g->builder, op1_value, op2_casted, "");
3268 }
3269 }
3270 case IrBinOpBitShiftRightLossy:
3271 case IrBinOpBitShiftRightExact:
3272 {
3273 assert(scalar_type->id == ZigTypeIdInt);
3274 LLVMValueRef op2_casted = LLVMBuildZExt(g->builder, op2_value,
3275 LLVMTypeOf(op1_value), "");
3276
3277 if (want_runtime_safety) {
3278 gen_shift_rhs_check(g, scalar_type, op2->value->type, op2_value);
3279 }
3280
3281 bool is_sloppy = (op_id == IrBinOpBitShiftRightLossy);
3282 if (is_sloppy) {
3283 if (scalar_type->data.integral.is_signed) {
3284 return LLVMBuildAShr(g->builder, op1_value, op2_casted, "");
3285 } else {
3286 return LLVMBuildLShr(g->builder, op1_value, op2_casted, "");
3287 }
3288 } else if (want_runtime_safety) {
3289 return gen_overflow_shr_op(g, operand_type, op1_value, op2_casted);
3290 } else if (scalar_type->data.integral.is_signed) {
3291 return ZigLLVMBuildAShrExact(g->builder, op1_value, op2_casted, "");
3292 } else {
3293 return ZigLLVMBuildLShrExact(g->builder, op1_value, op2_casted, "");
3294 }
3295 }
3296 case IrBinOpDivUnspecified:
3297 return gen_div(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
3298 op1_value, op2_value, operand_type, DivKindFloat);
3299 case IrBinOpDivExact:
3300 return gen_div(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
3301 op1_value, op2_value, operand_type, DivKindExact);
3302 case IrBinOpDivTrunc:
3303 return gen_div(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
3304 op1_value, op2_value, operand_type, DivKindTrunc);
3305 case IrBinOpDivFloor:
3306 return gen_div(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
3307 op1_value, op2_value, operand_type, DivKindFloor);
3308 case IrBinOpRemRem:
3309 return gen_rem(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
3310 op1_value, op2_value, operand_type, RemKindRem);
3311 case IrBinOpRemMod:
3312 return gen_rem(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
3313 op1_value, op2_value, operand_type, RemKindMod);
3314 case IrBinOpMaximum:
3315 if (scalar_type->id == ZigTypeIdFloat) {
3316 return ZigLLVMBuildMaxNum(g->builder, op1_value, op2_value, "");
3317 } else if (scalar_type->id == ZigTypeIdInt) {
3318 if (scalar_type->data.integral.is_signed) {
3319 return ZigLLVMBuildSMax(g->builder, op1_value, op2_value, "");
3320 } else {
3321 return ZigLLVMBuildUMax(g->builder, op1_value, op2_value, "");
3322 }
3323 } else {
3324 zig_unreachable();
3325 }
3326 case IrBinOpMinimum:
3327 if (scalar_type->id == ZigTypeIdFloat) {
3328 return ZigLLVMBuildMinNum(g->builder, op1_value, op2_value, "");
3329 } else if (scalar_type->id == ZigTypeIdInt) {
3330 if (scalar_type->data.integral.is_signed) {
3331 return ZigLLVMBuildSMin(g->builder, op1_value, op2_value, "");
3332 } else {
3333 return ZigLLVMBuildUMin(g->builder, op1_value, op2_value, "");
3334 }
3335 } else {
3336 zig_unreachable();
3337 }
3338 case IrBinOpAddSat:
3339 if (scalar_type->id == ZigTypeIdInt) {
3340 if (scalar_type->data.integral.is_signed) {
3341 return ZigLLVMBuildSAddSat(g->builder, op1_value, op2_value, "");
3342 } else {
3343 return ZigLLVMBuildUAddSat(g->builder, op1_value, op2_value, "");
3344 }
3345 } else {
3346 zig_unreachable();
3347 }
3348 case IrBinOpSubSat:
3349 if (scalar_type->id == ZigTypeIdInt) {
3350 if (scalar_type->data.integral.is_signed) {
3351 return ZigLLVMBuildSSubSat(g->builder, op1_value, op2_value, "");
3352 } else {
3353 return ZigLLVMBuildUSubSat(g->builder, op1_value, op2_value, "");
3354 }
3355 } else {
3356 zig_unreachable();
3357 }
3358 case IrBinOpMultSat:
3359 if (scalar_type->id == ZigTypeIdInt) {
3360 if (scalar_type->data.integral.is_signed) {
3361 return ZigLLVMBuildSMulFixSat(g->builder, op1_value, op2_value, "");
3362 } else {
3363 return ZigLLVMBuildUMulFixSat(g->builder, op1_value, op2_value, "");
3364 }
3365 } else {
3366 zig_unreachable();
3367 }
3368 case IrBinOpShlSat:
3369 if (scalar_type->id == ZigTypeIdInt) {
3370 if (scalar_type->data.integral.is_signed) {
3371 return ZigLLVMBuildSShlSat(g->builder, op1_value, op2_value, "");
3372 } else {
3373 return ZigLLVMBuildUShlSat(g->builder, op1_value, op2_value, "");
3374 }
3375 } else {
3376 zig_unreachable();
3377 }
3378 }
3379 zig_unreachable();
3380 }
3381
add_error_range_check(CodeGen * g,ZigType * err_set_type,ZigType * int_type,LLVMValueRef target_val)3382 static void add_error_range_check(CodeGen *g, ZigType *err_set_type, ZigType *int_type, LLVMValueRef target_val) {
3383 assert(err_set_type->id == ZigTypeIdErrorSet);
3384
3385 if (type_is_global_error_set(err_set_type)) {
3386 LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, int_type));
3387 LLVMValueRef neq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntNE, target_val, zero, "");
3388 LLVMValueRef ok_bit;
3389
3390 BigInt biggest_possible_err_val = {0};
3391 eval_min_max_value_int(g, int_type, &biggest_possible_err_val, true);
3392
3393 if (bigint_fits_in_bits(&biggest_possible_err_val, 64, false) &&
3394 bigint_as_usize(&biggest_possible_err_val) < g->errors_by_index.length)
3395 {
3396 ok_bit = neq_zero_bit;
3397 } else {
3398 LLVMValueRef error_value_count = LLVMConstInt(get_llvm_type(g, int_type), g->errors_by_index.length, false);
3399 LLVMValueRef in_bounds_bit = LLVMBuildICmp(g->builder, LLVMIntULT, target_val, error_value_count, "");
3400 ok_bit = LLVMBuildAnd(g->builder, neq_zero_bit, in_bounds_bit, "");
3401 }
3402
3403 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "IntToErrOk");
3404 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "IntToErrFail");
3405
3406 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
3407
3408 LLVMPositionBuilderAtEnd(g->builder, fail_block);
3409 gen_safety_crash(g, PanicMsgIdInvalidErrorCode);
3410
3411 LLVMPositionBuilderAtEnd(g->builder, ok_block);
3412 } else {
3413 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "IntToErrOk");
3414 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "IntToErrFail");
3415
3416 uint32_t err_count = err_set_type->data.error_set.err_count;
3417 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, target_val, fail_block, err_count);
3418 for (uint32_t i = 0; i < err_count; i += 1) {
3419 LLVMValueRef case_value = LLVMConstInt(get_llvm_type(g, g->err_tag_type),
3420 err_set_type->data.error_set.errors[i]->value, false);
3421 LLVMAddCase(switch_instr, case_value, ok_block);
3422 }
3423
3424 LLVMPositionBuilderAtEnd(g->builder, fail_block);
3425 gen_safety_crash(g, PanicMsgIdInvalidErrorCode);
3426
3427 LLVMPositionBuilderAtEnd(g->builder, ok_block);
3428 }
3429 }
3430
ir_render_cast(CodeGen * g,Stage1Air * executable,Stage1AirInstCast * cast_instruction)3431 static LLVMValueRef ir_render_cast(CodeGen *g, Stage1Air *executable,
3432 Stage1AirInstCast *cast_instruction)
3433 {
3434 Error err;
3435 ZigType *actual_type = cast_instruction->value->value->type;
3436 ZigType *wanted_type = cast_instruction->base.value->type;
3437 bool wanted_type_has_bits;
3438 if ((err = type_has_bits2(g, wanted_type, &wanted_type_has_bits)))
3439 codegen_report_errors_and_exit(g);
3440 if (!wanted_type_has_bits)
3441 return nullptr;
3442 LLVMValueRef expr_val = ir_llvm_value(g, cast_instruction->value);
3443 ir_assert(expr_val, &cast_instruction->base);
3444
3445 switch (cast_instruction->cast_op) {
3446 case CastOpNoCast:
3447 case CastOpNumLitToConcrete:
3448 zig_unreachable();
3449 case CastOpNoop:
3450 if (actual_type->id == ZigTypeIdPointer && wanted_type->id == ZigTypeIdPointer &&
3451 actual_type->data.pointer.child_type->id == ZigTypeIdArray &&
3452 wanted_type->data.pointer.child_type->id == ZigTypeIdArray)
3453 {
3454 return LLVMBuildBitCast(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3455 } else {
3456 return expr_val;
3457 }
3458 case CastOpIntToFloat:
3459 assert(actual_type->id == ZigTypeIdInt);
3460 if (actual_type->data.integral.is_signed) {
3461 return LLVMBuildSIToFP(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3462 } else {
3463 return LLVMBuildUIToFP(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3464 }
3465 case CastOpFloatToInt: {
3466 assert(wanted_type->id == ZigTypeIdInt);
3467 ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, &cast_instruction->base));
3468
3469 bool want_safety = ir_want_runtime_safety(g, &cast_instruction->base);
3470
3471 LLVMValueRef result;
3472 if (wanted_type->data.integral.is_signed) {
3473 result = LLVMBuildFPToSI(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3474 } else {
3475 result = LLVMBuildFPToUI(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3476 }
3477
3478 if (want_safety) {
3479 LLVMValueRef back_to_float;
3480 if (wanted_type->data.integral.is_signed) {
3481 back_to_float = LLVMBuildSIToFP(g->builder, result, LLVMTypeOf(expr_val), "");
3482 } else {
3483 back_to_float = LLVMBuildUIToFP(g->builder, result, LLVMTypeOf(expr_val), "");
3484 }
3485 LLVMValueRef difference = LLVMBuildFSub(g->builder, expr_val, back_to_float, "");
3486 LLVMValueRef one_pos = LLVMConstReal(LLVMTypeOf(expr_val), 1.0f);
3487 LLVMValueRef one_neg = LLVMConstReal(LLVMTypeOf(expr_val), -1.0f);
3488 LLVMValueRef ok_bit_pos = LLVMBuildFCmp(g->builder, LLVMRealOLT, difference, one_pos, "");
3489 LLVMValueRef ok_bit_neg = LLVMBuildFCmp(g->builder, LLVMRealOGT, difference, one_neg, "");
3490 LLVMValueRef ok_bit = LLVMBuildAnd(g->builder, ok_bit_pos, ok_bit_neg, "");
3491 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "FloatCheckOk");
3492 LLVMBasicBlockRef bad_block = LLVMAppendBasicBlock(g->cur_fn_val, "FloatCheckFail");
3493 LLVMBuildCondBr(g->builder, ok_bit, ok_block, bad_block);
3494 LLVMPositionBuilderAtEnd(g->builder, bad_block);
3495 gen_safety_crash(g, PanicMsgIdFloatToInt);
3496 LLVMPositionBuilderAtEnd(g->builder, ok_block);
3497 }
3498 return result;
3499 }
3500 case CastOpBoolToInt:
3501 assert(wanted_type->id == ZigTypeIdInt);
3502 assert(actual_type->id == ZigTypeIdBool);
3503 return LLVMBuildZExt(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3504 case CastOpErrSet:
3505 if (ir_want_runtime_safety(g, &cast_instruction->base)) {
3506 add_error_range_check(g, wanted_type, g->err_tag_type, expr_val);
3507 }
3508 return expr_val;
3509 case CastOpBitCast:
3510 return LLVMBuildBitCast(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3511 }
3512 zig_unreachable();
3513 }
3514
ir_render_ptr_of_array_to_slice(CodeGen * g,Stage1Air * executable,Stage1AirInstPtrOfArrayToSlice * instruction)3515 static LLVMValueRef ir_render_ptr_of_array_to_slice(CodeGen *g, Stage1Air *executable,
3516 Stage1AirInstPtrOfArrayToSlice *instruction)
3517 {
3518 ZigType *actual_type = instruction->operand->value->type;
3519 ZigType *slice_type = instruction->base.value->type;
3520 ZigType *slice_ptr_type = slice_type->data.structure.fields[slice_ptr_index]->type_entry;
3521 size_t ptr_index = slice_type->data.structure.fields[slice_ptr_index]->gen_index;
3522 size_t len_index = slice_type->data.structure.fields[slice_len_index]->gen_index;
3523
3524 LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
3525
3526 assert(actual_type->id == ZigTypeIdPointer);
3527 ZigType *array_type = actual_type->data.pointer.child_type;
3528 assert(array_type->id == ZigTypeIdArray);
3529
3530 if (type_has_bits(g, actual_type)) {
3531 LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, result_loc, ptr_index, "");
3532 LLVMValueRef indices[] = {
3533 LLVMConstNull(g->builtin_types.entry_usize->llvm_type),
3534 LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 0, false),
3535 };
3536 LLVMValueRef expr_val = ir_llvm_value(g, instruction->operand);
3537 LLVMValueRef slice_start_ptr = LLVMBuildInBoundsGEP(g->builder, expr_val, indices, 2, "");
3538 gen_store_untyped(g, slice_start_ptr, ptr_field_ptr, 0, false);
3539 } else if (ir_want_runtime_safety(g, &instruction->base) && ptr_index != SIZE_MAX) {
3540 LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, result_loc, ptr_index, "");
3541 gen_undef_init(g, slice_ptr_type, slice_ptr_type, ptr_field_ptr);
3542 }
3543
3544 LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, result_loc, len_index, "");
3545 LLVMValueRef len_value = LLVMConstInt(g->builtin_types.entry_usize->llvm_type,
3546 array_type->data.array.len, false);
3547 gen_store_untyped(g, len_value, len_field_ptr, 0, false);
3548
3549 return result_loc;
3550 }
3551
ir_render_ptr_cast(CodeGen * g,Stage1Air * executable,Stage1AirInstPtrCast * instruction)3552 static LLVMValueRef ir_render_ptr_cast(CodeGen *g, Stage1Air *executable,
3553 Stage1AirInstPtrCast *instruction)
3554 {
3555 ZigType *wanted_type = instruction->base.value->type;
3556 if (!type_has_bits(g, wanted_type)) {
3557 return nullptr;
3558 }
3559 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
3560 LLVMValueRef result_ptr = LLVMBuildBitCast(g->builder, ptr, get_llvm_type(g, wanted_type), "");
3561 bool want_safety_check = instruction->safety_check_on && ir_want_runtime_safety(g, &instruction->base);
3562 if (!want_safety_check || ptr_allows_addr_zero(wanted_type))
3563 return result_ptr;
3564
3565 LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(result_ptr));
3566 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntNE, result_ptr, zero, "");
3567 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "PtrCastFail");
3568 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "PtrCastOk");
3569 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
3570
3571 LLVMPositionBuilderAtEnd(g->builder, fail_block);
3572 gen_safety_crash(g, PanicMsgIdPtrCastNull);
3573
3574 LLVMPositionBuilderAtEnd(g->builder, ok_block);
3575 return result_ptr;
3576 }
3577
ir_render_bit_cast(CodeGen * g,Stage1Air * executable,Stage1AirInstBitCast * instruction)3578 static LLVMValueRef ir_render_bit_cast(CodeGen *g, Stage1Air *executable,
3579 Stage1AirInstBitCast *instruction)
3580 {
3581 ZigType *wanted_type = instruction->base.value->type;
3582 ZigType *actual_type = instruction->operand->value->type;
3583 LLVMValueRef value = ir_llvm_value(g, instruction->operand);
3584
3585 bool wanted_is_ptr = handle_is_ptr(g, wanted_type);
3586 bool actual_is_ptr = handle_is_ptr(g, actual_type);
3587 if (wanted_is_ptr == actual_is_ptr) {
3588 // We either bitcast the value directly or bitcast the pointer which does a pointer cast
3589 LLVMTypeRef wanted_type_ref = wanted_is_ptr ?
3590 LLVMPointerType(get_llvm_type(g, wanted_type), 0) : get_llvm_type(g, wanted_type);
3591 return LLVMBuildBitCast(g->builder, value, wanted_type_ref, "");
3592 } else if (actual_is_ptr) {
3593 // A scalar is wanted but we got a pointer
3594 LLVMTypeRef wanted_ptr_type_ref = LLVMPointerType(get_llvm_type(g, wanted_type), 0);
3595 LLVMValueRef bitcasted_ptr = LLVMBuildBitCast(g->builder, value, wanted_ptr_type_ref, "");
3596 uint32_t alignment = get_abi_alignment(g, actual_type);
3597 return gen_load_untyped(g, bitcasted_ptr, alignment, false, "");
3598 } else {
3599 // A pointer is wanted but we got a scalar
3600 assert(actual_type->id == ZigTypeIdPointer);
3601 LLVMTypeRef wanted_ptr_type_ref = LLVMPointerType(get_llvm_type(g, wanted_type), 0);
3602 return LLVMBuildBitCast(g->builder, value, wanted_ptr_type_ref, "");
3603 }
3604 }
3605
ir_render_widen_or_shorten(CodeGen * g,Stage1Air * executable,Stage1AirInstWidenOrShorten * instruction)3606 static LLVMValueRef ir_render_widen_or_shorten(CodeGen *g, Stage1Air *executable,
3607 Stage1AirInstWidenOrShorten *instruction)
3608 {
3609 ZigType *actual_type = instruction->target->value->type;
3610 // TODO instead of this logic, use the Noop instruction to change the type from
3611 // enum_tag to the underlying int type
3612 ZigType *int_type;
3613 if (actual_type->id == ZigTypeIdEnum) {
3614 int_type = actual_type->data.enumeration.tag_int_type;
3615 } else {
3616 int_type = actual_type;
3617 }
3618 LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
3619 return gen_widen_or_shorten(g, ir_want_runtime_safety(g, &instruction->base), int_type,
3620 instruction->base.value->type, target_val);
3621 }
3622
ir_render_int_to_ptr(CodeGen * g,Stage1Air * executable,Stage1AirInstIntToPtr * instruction)3623 static LLVMValueRef ir_render_int_to_ptr(CodeGen *g, Stage1Air *executable, Stage1AirInstIntToPtr *instruction) {
3624 ZigType *wanted_type = instruction->base.value->type;
3625 LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
3626 const uint32_t align_bytes = get_ptr_align(g, wanted_type);
3627
3628 if (ir_want_runtime_safety(g, &instruction->base)) {
3629 ZigType *usize = g->builtin_types.entry_usize;
3630 LLVMValueRef zero = LLVMConstNull(usize->llvm_type);
3631
3632 if (!ptr_allows_addr_zero(wanted_type)) {
3633 LLVMValueRef is_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, target_val, zero, "");
3634 LLVMBasicBlockRef bad_block = LLVMAppendBasicBlock(g->cur_fn_val, "PtrToIntBad");
3635 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "PtrToIntOk");
3636 LLVMBuildCondBr(g->builder, is_zero_bit, bad_block, ok_block);
3637
3638 LLVMPositionBuilderAtEnd(g->builder, bad_block);
3639 gen_safety_crash(g, PanicMsgIdPtrCastNull);
3640
3641 LLVMPositionBuilderAtEnd(g->builder, ok_block);
3642 }
3643
3644 if (align_bytes > 1) {
3645 LLVMValueRef alignment_minus_1 = LLVMConstInt(usize->llvm_type, align_bytes - 1, false);
3646 LLVMValueRef anded_val = LLVMBuildAnd(g->builder, target_val, alignment_minus_1, "");
3647 LLVMValueRef is_ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, anded_val, zero, "");
3648 LLVMBasicBlockRef bad_block = LLVMAppendBasicBlock(g->cur_fn_val, "PtrToIntAlignBad");
3649 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "PtrToIntAlignOk");
3650 LLVMBuildCondBr(g->builder, is_ok_bit, ok_block, bad_block);
3651
3652 LLVMPositionBuilderAtEnd(g->builder, bad_block);
3653 gen_safety_crash(g, PanicMsgIdIncorrectAlignment);
3654
3655 LLVMPositionBuilderAtEnd(g->builder, ok_block);
3656 }
3657 }
3658 return LLVMBuildIntToPtr(g->builder, target_val, get_llvm_type(g, wanted_type), "");
3659 }
3660
ir_render_ptr_to_int(CodeGen * g,Stage1Air * executable,Stage1AirInstPtrToInt * instruction)3661 static LLVMValueRef ir_render_ptr_to_int(CodeGen *g, Stage1Air *executable, Stage1AirInstPtrToInt *instruction) {
3662 ZigType *wanted_type = instruction->base.value->type;
3663 LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
3664 return LLVMBuildPtrToInt(g->builder, target_val, get_llvm_type(g, wanted_type), "");
3665 }
3666
ir_render_int_to_enum(CodeGen * g,Stage1Air * executable,Stage1AirInstIntToEnum * instruction)3667 static LLVMValueRef ir_render_int_to_enum(CodeGen *g, Stage1Air *executable, Stage1AirInstIntToEnum *instruction) {
3668 ZigType *wanted_type = instruction->base.value->type;
3669 assert(wanted_type->id == ZigTypeIdEnum);
3670 ZigType *tag_int_type = wanted_type->data.enumeration.tag_int_type;
3671
3672 LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
3673 LLVMValueRef tag_int_value = gen_widen_or_shorten(g, ir_want_runtime_safety(g, &instruction->base),
3674 instruction->target->value->type, tag_int_type, target_val);
3675
3676 if (ir_want_runtime_safety(g, &instruction->base) && !wanted_type->data.enumeration.non_exhaustive) {
3677 LLVMBasicBlockRef bad_value_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadValue");
3678 LLVMBasicBlockRef ok_value_block = LLVMAppendBasicBlock(g->cur_fn_val, "OkValue");
3679 size_t field_count = wanted_type->data.enumeration.src_field_count;
3680 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, tag_int_value, bad_value_block, field_count);
3681
3682 HashMap<BigInt, Buf *, bigint_hash, bigint_eql> occupied_tag_values = {};
3683 occupied_tag_values.init(field_count);
3684
3685 for (size_t field_i = 0; field_i < field_count; field_i += 1) {
3686 TypeEnumField *type_enum_field = &wanted_type->data.enumeration.fields[field_i];
3687
3688 Buf *name = type_enum_field->name;
3689 auto entry = occupied_tag_values.put_unique(type_enum_field->value, name);
3690 if (entry != nullptr) {
3691 continue;
3692 }
3693
3694 LLVMValueRef this_tag_int_value = bigint_to_llvm_const(get_llvm_type(g, tag_int_type),
3695 &type_enum_field->value);
3696 LLVMAddCase(switch_instr, this_tag_int_value, ok_value_block);
3697 }
3698 occupied_tag_values.deinit();
3699 LLVMPositionBuilderAtEnd(g->builder, bad_value_block);
3700 gen_safety_crash(g, PanicMsgIdBadEnumValue);
3701
3702 LLVMPositionBuilderAtEnd(g->builder, ok_value_block);
3703 }
3704 return tag_int_value;
3705 }
3706
ir_render_int_to_err(CodeGen * g,Stage1Air * executable,Stage1AirInstIntToErr * instruction)3707 static LLVMValueRef ir_render_int_to_err(CodeGen *g, Stage1Air *executable, Stage1AirInstIntToErr *instruction) {
3708 ZigType *wanted_type = instruction->base.value->type;
3709 assert(wanted_type->id == ZigTypeIdErrorSet);
3710
3711 ZigType *actual_type = instruction->target->value->type;
3712 assert(actual_type->id == ZigTypeIdInt);
3713 assert(!actual_type->data.integral.is_signed);
3714
3715 LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
3716
3717 if (ir_want_runtime_safety(g, &instruction->base)) {
3718 add_error_range_check(g, wanted_type, actual_type, target_val);
3719 }
3720
3721 return gen_widen_or_shorten(g, false, actual_type, g->err_tag_type, target_val);
3722 }
3723
ir_render_err_to_int(CodeGen * g,Stage1Air * executable,Stage1AirInstErrToInt * instruction)3724 static LLVMValueRef ir_render_err_to_int(CodeGen *g, Stage1Air *executable, Stage1AirInstErrToInt *instruction) {
3725 ZigType *wanted_type = instruction->base.value->type;
3726 assert(wanted_type->id == ZigTypeIdInt);
3727 assert(!wanted_type->data.integral.is_signed);
3728
3729 ZigType *actual_type = instruction->target->value->type;
3730 LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
3731
3732 if (actual_type->id == ZigTypeIdErrorSet) {
3733 return gen_widen_or_shorten(g, ir_want_runtime_safety(g, &instruction->base),
3734 g->err_tag_type, wanted_type, target_val);
3735 } else if (actual_type->id == ZigTypeIdErrorUnion) {
3736 // this should have been a compile time constant
3737 assert(type_has_bits(g, actual_type->data.error_union.err_set_type));
3738
3739 if (!type_has_bits(g, actual_type->data.error_union.payload_type)) {
3740 return gen_widen_or_shorten(g, ir_want_runtime_safety(g, &instruction->base),
3741 g->err_tag_type, wanted_type, target_val);
3742 } else {
3743 zig_panic("TODO err to int when error union payload type not void");
3744 }
3745 } else {
3746 zig_unreachable();
3747 }
3748 }
3749
ir_render_unreachable(CodeGen * g,Stage1Air * executable,Stage1AirInstUnreachable * unreachable_instruction)3750 static LLVMValueRef ir_render_unreachable(CodeGen *g, Stage1Air *executable,
3751 Stage1AirInstUnreachable *unreachable_instruction)
3752 {
3753 if (ir_want_runtime_safety(g, &unreachable_instruction->base)) {
3754 gen_safety_crash(g, PanicMsgIdUnreachable);
3755 } else {
3756 LLVMBuildUnreachable(g->builder);
3757 }
3758 return nullptr;
3759 }
3760
ir_render_cond_br(CodeGen * g,Stage1Air * executable,Stage1AirInstCondBr * cond_br_instruction)3761 static LLVMValueRef ir_render_cond_br(CodeGen *g, Stage1Air *executable,
3762 Stage1AirInstCondBr *cond_br_instruction)
3763 {
3764 LLVMBuildCondBr(g->builder,
3765 ir_llvm_value(g, cond_br_instruction->condition),
3766 cond_br_instruction->then_block->llvm_block,
3767 cond_br_instruction->else_block->llvm_block);
3768 return nullptr;
3769 }
3770
ir_render_br(CodeGen * g,Stage1Air * executable,Stage1AirInstBr * br_instruction)3771 static LLVMValueRef ir_render_br(CodeGen *g, Stage1Air *executable, Stage1AirInstBr *br_instruction) {
3772 LLVMBuildBr(g->builder, br_instruction->dest_block->llvm_block);
3773 return nullptr;
3774 }
3775
ir_render_binary_not(CodeGen * g,Stage1Air * executable,Stage1AirInstBinaryNot * inst)3776 static LLVMValueRef ir_render_binary_not(CodeGen *g, Stage1Air *executable,
3777 Stage1AirInstBinaryNot *inst)
3778 {
3779 LLVMValueRef operand = ir_llvm_value(g, inst->operand);
3780 return LLVMBuildNot(g->builder, operand, "");
3781 }
3782
ir_gen_negation(CodeGen * g,Stage1AirInst * inst,Stage1AirInst * operand,bool wrapping)3783 static LLVMValueRef ir_gen_negation(CodeGen *g, Stage1AirInst *inst, Stage1AirInst *operand, bool wrapping) {
3784 LLVMValueRef llvm_operand = ir_llvm_value(g, operand);
3785 ZigType *operand_type = operand->value->type;
3786 ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ?
3787 operand_type->data.vector.elem_type : operand_type;
3788
3789 if (scalar_type->id == ZigTypeIdFloat) {
3790 ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, inst));
3791 return LLVMBuildFNeg(g->builder, llvm_operand, "");
3792 } else if (scalar_type->id == ZigTypeIdInt) {
3793 if (wrapping) {
3794 return LLVMBuildNeg(g->builder, llvm_operand, "");
3795 } else if (ir_want_runtime_safety(g, inst)) {
3796 LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(llvm_operand));
3797 return gen_overflow_op(g, operand_type, AddSubMulSub, zero, llvm_operand);
3798 } else if (scalar_type->data.integral.is_signed) {
3799 return LLVMBuildNSWNeg(g->builder, llvm_operand, "");
3800 } else {
3801 zig_unreachable();
3802 }
3803 } else {
3804 zig_unreachable();
3805 }
3806 }
3807
ir_render_negation(CodeGen * g,Stage1Air * executable,Stage1AirInstNegation * inst)3808 static LLVMValueRef ir_render_negation(CodeGen *g, Stage1Air *executable,
3809 Stage1AirInstNegation *inst)
3810 {
3811 return ir_gen_negation(g, &inst->base, inst->operand, inst->wrapping);
3812 }
3813
ir_render_bool_not(CodeGen * g,Stage1Air * executable,Stage1AirInstBoolNot * instruction)3814 static LLVMValueRef ir_render_bool_not(CodeGen *g, Stage1Air *executable, Stage1AirInstBoolNot *instruction) {
3815 LLVMValueRef value = ir_llvm_value(g, instruction->value);
3816 LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(value));
3817 return LLVMBuildICmp(g->builder, LLVMIntEQ, value, zero, "");
3818 }
3819
render_decl_var(CodeGen * g,ZigVar * var)3820 static void render_decl_var(CodeGen *g, ZigVar *var) {
3821 if (!type_has_bits(g, var->var_type))
3822 return;
3823
3824 var->value_ref = ir_llvm_value(g, var->ptr_instruction);
3825 gen_var_debug_decl(g, var);
3826 }
3827
ir_render_decl_var(CodeGen * g,Stage1Air * executable,Stage1AirInstDeclVar * instruction)3828 static LLVMValueRef ir_render_decl_var(CodeGen *g, Stage1Air *executable, Stage1AirInstDeclVar *instruction) {
3829 instruction->var->ptr_instruction = instruction->var_ptr;
3830 instruction->var->did_the_decl_codegen = true;
3831 render_decl_var(g, instruction->var);
3832 return nullptr;
3833 }
3834
ir_render_load_ptr(CodeGen * g,Stage1Air * executable,Stage1AirInstLoadPtr * instruction)3835 static LLVMValueRef ir_render_load_ptr(CodeGen *g, Stage1Air *executable,
3836 Stage1AirInstLoadPtr *instruction)
3837 {
3838 ZigType *child_type = instruction->base.value->type;
3839 if (!type_has_bits(g, child_type))
3840 return nullptr;
3841
3842 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
3843 ZigType *ptr_type = instruction->ptr->value->type;
3844 assert(ptr_type->id == ZigTypeIdPointer);
3845
3846 ir_assert(ptr_type->data.pointer.vector_index != VECTOR_INDEX_RUNTIME, &instruction->base);
3847 if (ptr_type->data.pointer.vector_index != VECTOR_INDEX_NONE) {
3848 LLVMValueRef index_val = LLVMConstInt(LLVMInt32Type(),
3849 ptr_type->data.pointer.vector_index, false);
3850 LLVMValueRef loaded_vector = LLVMBuildLoad(g->builder, ptr, "");
3851 return LLVMBuildExtractElement(g->builder, loaded_vector, index_val, "");
3852 }
3853
3854 uint32_t host_int_bytes = ptr_type->data.pointer.host_int_bytes;
3855 if (host_int_bytes == 0)
3856 return get_handle_value(g, ptr, child_type, ptr_type);
3857
3858 bool big_endian = g->is_big_endian;
3859
3860 LLVMTypeRef int_ptr_ty = LLVMPointerType(LLVMIntType(host_int_bytes * 8), 0);
3861 LLVMValueRef int_ptr = LLVMBuildBitCast(g->builder, ptr, int_ptr_ty, "");
3862 LLVMValueRef containing_int = gen_load(g, int_ptr, ptr_type, "");
3863
3864 uint32_t host_bit_count = LLVMGetIntTypeWidth(LLVMTypeOf(containing_int));
3865 assert(host_bit_count == host_int_bytes * 8);
3866 uint32_t size_in_bits = type_size_bits(g, child_type);
3867
3868 uint32_t bit_offset = ptr_type->data.pointer.bit_offset_in_host;
3869 uint32_t shift_amt = big_endian ? host_bit_count - bit_offset - size_in_bits : bit_offset;
3870
3871 LLVMValueRef shift_amt_val = LLVMConstInt(LLVMTypeOf(containing_int), shift_amt, false);
3872 LLVMValueRef shifted_value = LLVMBuildLShr(g->builder, containing_int, shift_amt_val, "");
3873
3874 if (handle_is_ptr(g, child_type)) {
3875 LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
3876 LLVMTypeRef same_size_int = LLVMIntType(size_in_bits);
3877 LLVMValueRef truncated_int = LLVMBuildTrunc(g->builder, shifted_value, same_size_int, "");
3878 LLVMValueRef bitcasted_ptr = LLVMBuildBitCast(g->builder, result_loc,
3879 LLVMPointerType(same_size_int, 0), "");
3880 LLVMBuildStore(g->builder, truncated_int, bitcasted_ptr);
3881 return result_loc;
3882 }
3883
3884 if (child_type->id == ZigTypeIdFloat) {
3885 LLVMTypeRef same_size_int = LLVMIntType(size_in_bits);
3886 LLVMValueRef truncated_int = LLVMBuildTrunc(g->builder, shifted_value, same_size_int, "");
3887 return LLVMBuildBitCast(g->builder, truncated_int, get_llvm_type(g, child_type), "");
3888 }
3889
3890 return LLVMBuildTrunc(g->builder, shifted_value, get_llvm_type(g, child_type), "");
3891 }
3892
value_is_all_undef_array(CodeGen * g,ZigValue * const_val,size_t len)3893 static bool value_is_all_undef_array(CodeGen *g, ZigValue *const_val, size_t len) {
3894 switch (const_val->data.x_array.special) {
3895 case ConstArraySpecialUndef:
3896 return true;
3897 case ConstArraySpecialBuf:
3898 return false;
3899 case ConstArraySpecialNone:
3900 for (size_t i = 0; i < len; i += 1) {
3901 if (!value_is_all_undef(g, &const_val->data.x_array.data.s_none.elements[i]))
3902 return false;
3903 }
3904 return true;
3905 }
3906 zig_unreachable();
3907 }
3908
value_is_all_undef(CodeGen * g,ZigValue * const_val)3909 static bool value_is_all_undef(CodeGen *g, ZigValue *const_val) {
3910 Error err;
3911 if (const_val->special == ConstValSpecialLazy &&
3912 (err = ir_resolve_lazy(g, nullptr, const_val)))
3913 codegen_report_errors_and_exit(g);
3914
3915 switch (const_val->special) {
3916 case ConstValSpecialLazy:
3917 zig_unreachable();
3918 case ConstValSpecialRuntime:
3919 return false;
3920 case ConstValSpecialUndef:
3921 return true;
3922 case ConstValSpecialStatic:
3923 if (const_val->type->id == ZigTypeIdStruct) {
3924 for (size_t i = 0; i < const_val->type->data.structure.src_field_count; i += 1) {
3925 TypeStructField *field = const_val->type->data.structure.fields[i];
3926 if (field->is_comptime) {
3927 // Comptime fields are part of the type, may be uninitialized,
3928 // and should not be inspected.
3929 continue;
3930 }
3931 if (!value_is_all_undef(g, const_val->data.x_struct.fields[i]))
3932 return false;
3933 }
3934 return true;
3935 } else if (const_val->type->id == ZigTypeIdArray) {
3936 return value_is_all_undef_array(g, const_val, const_val->type->data.array.len);
3937 } else if (const_val->type->id == ZigTypeIdVector) {
3938 return value_is_all_undef_array(g, const_val, const_val->type->data.vector.len);
3939 } else {
3940 return false;
3941 }
3942 }
3943 zig_unreachable();
3944 }
3945
gen_valgrind_client_request(CodeGen * g,LLVMValueRef default_value,LLVMValueRef request,LLVMValueRef a1,LLVMValueRef a2,LLVMValueRef a3,LLVMValueRef a4,LLVMValueRef a5)3946 static LLVMValueRef gen_valgrind_client_request(CodeGen *g, LLVMValueRef default_value, LLVMValueRef request,
3947 LLVMValueRef a1, LLVMValueRef a2, LLVMValueRef a3, LLVMValueRef a4, LLVMValueRef a5)
3948 {
3949 if (!target_has_valgrind_support(g->zig_target)) {
3950 return default_value;
3951 }
3952 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
3953 bool asm_has_side_effects = true;
3954 bool asm_is_alignstack = false;
3955 if (g->zig_target->arch == ZigLLVM_x86_64) {
3956 if (g->zig_target->os == OsLinux || target_os_is_darwin(g->zig_target->os) || g->zig_target->os == OsSolaris ||
3957 (g->zig_target->os == OsWindows && g->zig_target->abi != ZigLLVM_MSVC))
3958 {
3959 if (g->cur_fn->valgrind_client_request_array == nullptr) {
3960 LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
3961 LLVMBasicBlockRef entry_block = LLVMGetEntryBasicBlock(g->cur_fn->llvm_value);
3962 LLVMValueRef first_inst = LLVMGetFirstInstruction(entry_block);
3963 LLVMPositionBuilderBefore(g->builder, first_inst);
3964 LLVMTypeRef array_type_ref = LLVMArrayType(usize_type_ref, 6);
3965 g->cur_fn->valgrind_client_request_array = LLVMBuildAlloca(g->builder, array_type_ref, "");
3966 LLVMPositionBuilderAtEnd(g->builder, prev_block);
3967 }
3968 LLVMValueRef array_ptr = g->cur_fn->valgrind_client_request_array;
3969 LLVMValueRef array_elements[] = {request, a1, a2, a3, a4, a5};
3970 LLVMValueRef zero = LLVMConstInt(usize_type_ref, 0, false);
3971 for (unsigned i = 0; i < 6; i += 1) {
3972 LLVMValueRef indexes[] = {
3973 zero,
3974 LLVMConstInt(usize_type_ref, i, false),
3975 };
3976 LLVMValueRef elem_ptr = LLVMBuildInBoundsGEP(g->builder, array_ptr, indexes, 2, "");
3977 LLVMBuildStore(g->builder, array_elements[i], elem_ptr);
3978 }
3979
3980 Buf *asm_template = buf_create_from_str(
3981 "rolq $$3, %rdi ; rolq $$13, %rdi\n"
3982 "rolq $$61, %rdi ; rolq $$51, %rdi\n"
3983 "xchgq %rbx,%rbx\n"
3984 );
3985 Buf *asm_constraints = buf_create_from_str(
3986 "={rdx},{rax},0,~{cc},~{memory}"
3987 );
3988 unsigned input_and_output_count = 2;
3989 LLVMValueRef array_ptr_as_usize = LLVMBuildPtrToInt(g->builder, array_ptr, usize_type_ref, "");
3990 LLVMValueRef param_values[] = { array_ptr_as_usize, default_value };
3991 LLVMTypeRef param_types[] = {usize_type_ref, usize_type_ref};
3992 LLVMTypeRef function_type = LLVMFunctionType(usize_type_ref, param_types,
3993 input_and_output_count, false);
3994 LLVMValueRef asm_fn = LLVMGetInlineAsm(function_type, buf_ptr(asm_template), buf_len(asm_template),
3995 buf_ptr(asm_constraints), buf_len(asm_constraints), asm_has_side_effects, asm_is_alignstack,
3996 LLVMInlineAsmDialectATT, false);
3997 return LLVMBuildCall(g->builder, asm_fn, param_values, input_and_output_count, "");
3998 }
3999 }
4000 zig_unreachable();
4001 }
4002
gen_valgrind_undef(CodeGen * g,LLVMValueRef dest_ptr,LLVMValueRef byte_count)4003 static void gen_valgrind_undef(CodeGen *g, LLVMValueRef dest_ptr, LLVMValueRef byte_count) {
4004 static const uint32_t VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
4005 ZigType *usize = g->builtin_types.entry_usize;
4006 LLVMValueRef zero = LLVMConstInt(usize->llvm_type, 0, false);
4007 LLVMValueRef req = LLVMConstInt(usize->llvm_type, VG_USERREQ__MAKE_MEM_UNDEFINED, false);
4008 LLVMValueRef ptr_as_usize = LLVMBuildPtrToInt(g->builder, dest_ptr, usize->llvm_type, "");
4009 gen_valgrind_client_request(g, zero, req, ptr_as_usize, byte_count, zero, zero, zero);
4010 }
4011
gen_undef_init(CodeGen * g,ZigType * ptr_type,ZigType * value_type,LLVMValueRef ptr)4012 static void gen_undef_init(CodeGen *g, ZigType *ptr_type, ZigType *value_type, LLVMValueRef ptr) {
4013 assert(type_has_bits(g, value_type));
4014
4015 uint64_t ptr_align_bytes = get_ptr_align(g, ptr_type);
4016 assert(ptr_align_bytes > 0);
4017 uint64_t size_bytes = LLVMStoreSizeOfType(g->target_data_ref, get_llvm_type(g, value_type));
4018 assert(size_bytes > 0);
4019
4020 if (ptr_type->data.pointer.host_int_bytes == 0) {
4021 // memset uninitialized memory to 0xaa
4022 LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
4023 LLVMValueRef fill_char = LLVMConstInt(LLVMInt8Type(), 0xaa, false);
4024 LLVMValueRef dest_ptr = LLVMBuildBitCast(g->builder, ptr, ptr_u8, "");
4025 ZigType *usize = g->builtin_types.entry_usize;
4026 LLVMValueRef byte_count = LLVMConstInt(usize->llvm_type, size_bytes, false);
4027 ZigLLVMBuildMemSet(g->builder, dest_ptr, fill_char, byte_count, ptr_align_bytes, false);
4028 // then tell valgrind that the memory is undefined even though we just memset it
4029 if (g->valgrind_enabled) {
4030 gen_valgrind_undef(g, dest_ptr, byte_count);
4031 }
4032 return;
4033 }
4034
4035 // This is a pointer into a packed struct, we can't use memset here.
4036 // The jury is still out on what pattern should be written here so clear the
4037 // old value and call it a day. Generating a 0xAA...AA mask for this n-bit
4038 // value is left as an exercise for the (bored) reader.
4039 LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, value_type));
4040 gen_assign_raw(g, ptr, ptr_type, zero);
4041 }
4042
ir_render_store_ptr(CodeGen * g,Stage1Air * executable,Stage1AirInstStorePtr * instruction)4043 static LLVMValueRef ir_render_store_ptr(CodeGen *g, Stage1Air *executable, Stage1AirInstStorePtr *instruction) {
4044 Error err;
4045
4046 ZigType *ptr_type = instruction->ptr->value->type;
4047 assert(ptr_type->id == ZigTypeIdPointer);
4048 bool ptr_type_has_bits;
4049 if ((err = type_has_bits2(g, ptr_type, &ptr_type_has_bits)))
4050 codegen_report_errors_and_exit(g);
4051 if (!ptr_type_has_bits)
4052 return nullptr;
4053 if (instruction->ptr->ref_count == 0) {
4054 // In this case, this StorePtr instruction should be elided. Something happened like this:
4055 // var t = true;
4056 // const x = if (t) Num.Two else unreachable;
4057 // The if condition is a runtime value, so the StorePtr for `x = Num.Two` got generated
4058 // (this instruction being rendered) but because of `else unreachable` the result ended
4059 // up being a comptime const value.
4060 return nullptr;
4061 }
4062
4063 bool have_init_expr = !value_is_all_undef(g, instruction->value->value);
4064 if (have_init_expr) {
4065 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
4066 LLVMValueRef value = ir_llvm_value(g, instruction->value);
4067 gen_assign_raw(g, ptr, ptr_type, value);
4068 } else if (ir_want_runtime_safety(g, &instruction->base)) {
4069 gen_undef_init(g, ptr_type, instruction->value->value->type,
4070 ir_llvm_value(g, instruction->ptr));
4071 }
4072 return nullptr;
4073 }
4074
ir_render_vector_store_elem(CodeGen * g,Stage1Air * executable,Stage1AirInstVectorStoreElem * instruction)4075 static LLVMValueRef ir_render_vector_store_elem(CodeGen *g, Stage1Air *executable,
4076 Stage1AirInstVectorStoreElem *instruction)
4077 {
4078 LLVMValueRef vector_ptr = ir_llvm_value(g, instruction->vector_ptr);
4079 LLVMValueRef index = ir_llvm_value(g, instruction->index);
4080 LLVMValueRef value = ir_llvm_value(g, instruction->value);
4081
4082 LLVMValueRef loaded_vector = gen_load(g, vector_ptr, instruction->vector_ptr->value->type, "");
4083 LLVMValueRef modified_vector = LLVMBuildInsertElement(g->builder, loaded_vector, value, index, "");
4084 gen_store(g, modified_vector, vector_ptr, instruction->vector_ptr->value->type);
4085 return nullptr;
4086 }
4087
ir_render_var_ptr(CodeGen * g,Stage1Air * executable,Stage1AirInstVarPtr * instruction)4088 static LLVMValueRef ir_render_var_ptr(CodeGen *g, Stage1Air *executable, Stage1AirInstVarPtr *instruction) {
4089 Error err;
4090
4091 ZigType *ptr_type = instruction->base.value->type;
4092 assert(ptr_type->id == ZigTypeIdPointer);
4093 bool ptr_type_has_bits;
4094 if ((err = type_has_bits2(g, ptr_type, &ptr_type_has_bits)))
4095 codegen_report_errors_and_exit(g);
4096
4097 if (!ptr_type_has_bits) {
4098 return nullptr;
4099 }
4100
4101 // The extra bitcasts are needed in case the LLVM value is an unnamed
4102 // struct, as it happens when rendering container types with extra alignment
4103 // fields.
4104 if (instruction->base.value->special != ConstValSpecialRuntime) {
4105 return LLVMBuildBitCast(g->builder, ir_llvm_value(g, &instruction->base),
4106 get_llvm_type(g, ptr_type), "");
4107 }
4108
4109 ZigVar *var = instruction->var;
4110 assert(var->value_ref);
4111 return LLVMBuildBitCast(g->builder, var->value_ref,
4112 get_llvm_type(g, ptr_type), "");
4113 }
4114
ir_render_return_ptr(CodeGen * g,Stage1Air * executable,Stage1AirInstReturnPtr * instruction)4115 static LLVMValueRef ir_render_return_ptr(CodeGen *g, Stage1Air *executable,
4116 Stage1AirInstReturnPtr *instruction)
4117 {
4118 if (!type_has_bits(g, instruction->base.value->type))
4119 return nullptr;
4120 ir_assert(g->cur_ret_ptr != nullptr, &instruction->base);
4121 return g->cur_ret_ptr;
4122 }
4123
ir_render_elem_ptr(CodeGen * g,Stage1Air * executable,Stage1AirInstElemPtr * instruction)4124 static LLVMValueRef ir_render_elem_ptr(CodeGen *g, Stage1Air *executable, Stage1AirInstElemPtr *instruction) {
4125 LLVMValueRef array_ptr_ptr = ir_llvm_value(g, instruction->array_ptr);
4126 ZigType *array_ptr_type = instruction->array_ptr->value->type;
4127 assert(array_ptr_type->id == ZigTypeIdPointer);
4128 ZigType *array_type = array_ptr_type->data.pointer.child_type;
4129 LLVMValueRef subscript_value = ir_llvm_value(g, instruction->elem_index);
4130 assert(subscript_value);
4131
4132 if (!type_has_bits(g, array_type))
4133 return nullptr;
4134
4135 bool safety_check_on = ir_want_runtime_safety(g, &instruction->base) && instruction->safety_check_on;
4136
4137 if (array_type->id == ZigTypeIdArray ||
4138 (array_type->id == ZigTypeIdPointer && array_type->data.pointer.ptr_len == PtrLenSingle))
4139 {
4140 LLVMValueRef array_ptr = get_handle_value(g, array_ptr_ptr, array_type, array_ptr_type);
4141 if (array_type->id == ZigTypeIdPointer) {
4142 assert(array_type->data.pointer.child_type->id == ZigTypeIdArray);
4143 array_type = array_type->data.pointer.child_type;
4144 }
4145
4146 assert(array_type->data.array.len != 0 || array_type->data.array.sentinel != nullptr);
4147
4148 if (safety_check_on) {
4149 uint64_t extra_len_from_sentinel = (array_type->data.array.sentinel != nullptr) ? 1 : 0;
4150 uint64_t full_len = array_type->data.array.len + extra_len_from_sentinel;
4151 LLVMValueRef end = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, full_len, false);
4152 add_bounds_check(g, subscript_value, LLVMIntEQ, nullptr, LLVMIntULT, end);
4153 }
4154 if (array_ptr_type->data.pointer.host_int_bytes != 0) {
4155 return array_ptr_ptr;
4156 }
4157 ZigType *child_type = array_type->data.array.child_type;
4158 if (child_type->id == ZigTypeIdStruct &&
4159 child_type->data.structure.layout == ContainerLayoutPacked)
4160 {
4161 ZigType *ptr_type = instruction->base.value->type;
4162 size_t host_int_bytes = ptr_type->data.pointer.host_int_bytes;
4163 if (host_int_bytes != 0) {
4164 uint32_t size_in_bits = type_size_bits(g, ptr_type->data.pointer.child_type);
4165 LLVMTypeRef ptr_u8_type_ref = LLVMPointerType(LLVMInt8Type(), 0);
4166 LLVMValueRef u8_array_ptr = LLVMBuildBitCast(g->builder, array_ptr, ptr_u8_type_ref, "");
4167 assert(size_in_bits % 8 == 0);
4168 LLVMValueRef elem_size_bytes = LLVMConstInt(g->builtin_types.entry_usize->llvm_type,
4169 size_in_bits / 8, false);
4170 LLVMValueRef byte_offset = LLVMBuildNUWMul(g->builder, subscript_value, elem_size_bytes, "");
4171 LLVMValueRef indices[] = {
4172 byte_offset
4173 };
4174 LLVMValueRef elem_byte_ptr = LLVMBuildInBoundsGEP(g->builder, u8_array_ptr, indices, 1, "");
4175 return LLVMBuildBitCast(g->builder, elem_byte_ptr, LLVMPointerType(get_llvm_type(g, child_type), 0), "");
4176 }
4177 }
4178 LLVMValueRef indices[] = {
4179 LLVMConstNull(g->builtin_types.entry_usize->llvm_type),
4180 subscript_value
4181 };
4182 return LLVMBuildInBoundsGEP(g->builder, array_ptr, indices, 2, "");
4183 } else if (array_type->id == ZigTypeIdPointer) {
4184 LLVMValueRef array_ptr = get_handle_value(g, array_ptr_ptr, array_type, array_ptr_type);
4185 assert(LLVMGetTypeKind(LLVMTypeOf(array_ptr)) == LLVMPointerTypeKind);
4186 LLVMValueRef indices[] = {
4187 subscript_value
4188 };
4189 return LLVMBuildInBoundsGEP(g->builder, array_ptr, indices, 1, "");
4190 } else if (array_type->id == ZigTypeIdStruct) {
4191 LLVMValueRef array_ptr = get_handle_value(g, array_ptr_ptr, array_type, array_ptr_type);
4192 assert(array_type->data.structure.special == StructSpecialSlice);
4193
4194 ZigType *ptr_type = array_type->data.structure.fields[slice_ptr_index]->type_entry;
4195 if (!type_has_bits(g, ptr_type)) {
4196 if (safety_check_on) {
4197 assert(LLVMGetTypeKind(LLVMTypeOf(array_ptr)) == LLVMIntegerTypeKind);
4198 add_bounds_check(g, subscript_value, LLVMIntEQ, nullptr, LLVMIntULT, array_ptr);
4199 }
4200 return nullptr;
4201 }
4202
4203 assert(LLVMGetTypeKind(LLVMTypeOf(array_ptr)) == LLVMPointerTypeKind);
4204 assert(LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(array_ptr))) == LLVMStructTypeKind);
4205
4206 if (safety_check_on) {
4207 size_t len_index = array_type->data.structure.fields[slice_len_index]->gen_index;
4208 assert(len_index != SIZE_MAX);
4209 LLVMValueRef len_ptr = LLVMBuildStructGEP(g->builder, array_ptr, (unsigned)len_index, "");
4210 LLVMValueRef len = gen_load_untyped(g, len_ptr, 0, false, "");
4211 LLVMIntPredicate upper_op = (ptr_type->data.pointer.sentinel != nullptr) ? LLVMIntULE : LLVMIntULT;
4212 add_bounds_check(g, subscript_value, LLVMIntEQ, nullptr, upper_op, len);
4213 }
4214
4215 size_t ptr_index = array_type->data.structure.fields[slice_ptr_index]->gen_index;
4216 assert(ptr_index != SIZE_MAX);
4217 LLVMValueRef ptr_ptr = LLVMBuildStructGEP(g->builder, array_ptr, (unsigned)ptr_index, "");
4218 LLVMValueRef ptr = gen_load_untyped(g, ptr_ptr, 0, false, "");
4219 return LLVMBuildInBoundsGEP(g->builder, ptr, &subscript_value, 1, "");
4220 } else if (array_type->id == ZigTypeIdVector) {
4221 return array_ptr_ptr;
4222 } else {
4223 zig_unreachable();
4224 }
4225 }
4226
get_new_stack_addr(CodeGen * g,LLVMValueRef new_stack)4227 static LLVMValueRef get_new_stack_addr(CodeGen *g, LLVMValueRef new_stack) {
4228 LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, new_stack, (unsigned)slice_ptr_index, "");
4229 LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, new_stack, (unsigned)slice_len_index, "");
4230
4231 LLVMValueRef ptr_value = gen_load_untyped(g, ptr_field_ptr, 0, false, "");
4232 LLVMValueRef len_value = gen_load_untyped(g, len_field_ptr, 0, false, "");
4233
4234 LLVMValueRef ptr_addr = LLVMBuildPtrToInt(g->builder, ptr_value, LLVMTypeOf(len_value), "");
4235 LLVMValueRef end_addr = LLVMBuildNUWAdd(g->builder, ptr_addr, len_value, "");
4236 const unsigned alignment_factor = ZigLLVMDataLayoutGetStackAlignment(g->target_data_ref);
4237 LLVMValueRef align_amt = LLVMConstInt(LLVMTypeOf(end_addr), alignment_factor, false);
4238 LLVMValueRef align_adj = LLVMBuildURem(g->builder, end_addr, align_amt, "");
4239 return LLVMBuildNUWSub(g->builder, end_addr, align_adj, "");
4240 }
4241
gen_set_stack_pointer(CodeGen * g,LLVMValueRef aligned_end_addr)4242 static void gen_set_stack_pointer(CodeGen *g, LLVMValueRef aligned_end_addr) {
4243 LLVMValueRef write_register_fn_val = get_write_register_fn_val(g);
4244
4245 if (g->sp_md_node == nullptr) {
4246 Buf *sp_reg_name = buf_create_from_str(arch_stack_pointer_register_name(g->zig_target->arch));
4247 LLVMValueRef str_node = LLVMMDString(buf_ptr(sp_reg_name), buf_len(sp_reg_name) + 1);
4248 g->sp_md_node = LLVMMDNode(&str_node, 1);
4249 }
4250
4251 LLVMValueRef params[] = {
4252 g->sp_md_node,
4253 aligned_end_addr,
4254 };
4255
4256 LLVMBuildCall(g->builder, write_register_fn_val, params, 2, "");
4257 }
4258
render_async_spills(CodeGen * g)4259 static void render_async_spills(CodeGen *g) {
4260 ZigType *fn_type = g->cur_fn->type_entry;
4261 ZigType *import = get_scope_import(&g->cur_fn->fndef_scope->base);
4262
4263 CalcLLVMFieldIndex arg_calc = {0};
4264 frame_index_arg_calc(g, &arg_calc, fn_type->data.fn.fn_type_id.return_type);
4265 for (size_t var_i = 0; var_i < g->cur_fn->variable_list.length; var_i += 1) {
4266 ZigVar *var = g->cur_fn->variable_list.at(var_i);
4267
4268 if (!type_has_bits(g, var->var_type)) {
4269 continue;
4270 }
4271 if (ir_get_var_is_comptime(var))
4272 continue;
4273 switch (type_requires_comptime(g, var->var_type)) {
4274 case ReqCompTimeInvalid:
4275 zig_unreachable();
4276 case ReqCompTimeYes:
4277 continue;
4278 case ReqCompTimeNo:
4279 break;
4280 }
4281 if (var->src_arg_index == SIZE_MAX) {
4282 continue;
4283 }
4284
4285 calc_llvm_field_index_add(g, &arg_calc, var->var_type);
4286 var->value_ref = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, arg_calc.field_index - 1, var->name);
4287 if (var->decl_node) {
4288 var->di_loc_var = ZigLLVMCreateAutoVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
4289 var->name, import->data.structure.root_struct->di_file,
4290 node_line_onebased(var->decl_node),
4291 get_llvm_di_type(g, var->var_type), !g->strip_debug_symbols, 0);
4292 gen_var_debug_decl(g, var);
4293 }
4294 }
4295
4296 ZigType *frame_type = g->cur_fn->frame_type->data.frame.locals_struct;
4297
4298 for (size_t alloca_i = 0; alloca_i < g->cur_fn->alloca_gen_list.length; alloca_i += 1) {
4299 Stage1AirInstAlloca *instruction = g->cur_fn->alloca_gen_list.at(alloca_i);
4300 if (instruction->field_index == SIZE_MAX)
4301 continue;
4302
4303 size_t gen_index = frame_type->data.structure.fields[instruction->field_index]->gen_index;
4304 instruction->base.llvm_value = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, gen_index,
4305 instruction->name_hint);
4306 }
4307 }
4308
render_async_var_decls(CodeGen * g,Scope * scope)4309 static void render_async_var_decls(CodeGen *g, Scope *scope) {
4310 for (;;) {
4311 switch (scope->id) {
4312 case ScopeIdCImport:
4313 zig_unreachable();
4314 case ScopeIdFnDef:
4315 return;
4316 case ScopeIdVarDecl: {
4317 ZigVar *var = reinterpret_cast<ScopeVarDecl *>(scope)->var;
4318 if (var->did_the_decl_codegen) {
4319 render_decl_var(g, var);
4320 }
4321 }
4322 ZIG_FALLTHROUGH;
4323
4324 case ScopeIdDecls:
4325 case ScopeIdBlock:
4326 case ScopeIdDefer:
4327 case ScopeIdDeferExpr:
4328 case ScopeIdLoop:
4329 case ScopeIdSuspend:
4330 case ScopeIdCompTime:
4331 case ScopeIdNoSuspend:
4332 case ScopeIdRuntime:
4333 case ScopeIdTypeOf:
4334 case ScopeIdExpr:
4335 scope = scope->parent;
4336 continue;
4337 }
4338 }
4339 }
4340
gen_frame_size(CodeGen * g,LLVMValueRef fn_val)4341 static LLVMValueRef gen_frame_size(CodeGen *g, LLVMValueRef fn_val) {
4342 assert(g->need_frame_size_prefix_data);
4343 LLVMTypeRef usize_llvm_type = g->builtin_types.entry_usize->llvm_type;
4344 LLVMTypeRef ptr_usize_llvm_type = LLVMPointerType(usize_llvm_type, 0);
4345 LLVMValueRef casted_fn_val = LLVMBuildBitCast(g->builder, fn_val, ptr_usize_llvm_type, "");
4346 LLVMValueRef negative_one = LLVMConstInt(LLVMInt32Type(), -1, true);
4347 LLVMValueRef prefix_ptr = LLVMBuildInBoundsGEP(g->builder, casted_fn_val, &negative_one, 1, "");
4348 LLVMValueRef load_inst = LLVMBuildLoad(g->builder, prefix_ptr, "");
4349
4350 // Some architectures (e.g SPARCv9) has different alignment requirements between a
4351 // function/usize pointer and also require all loads to be aligned.
4352 // On those architectures, not explicitly setting the alignment will lead into @frameSize
4353 // generating usize-aligned load instruction that could crash if the function pointer
4354 // happens to be not usize-aligned.
4355 LLVMSetAlignment(load_inst, 1);
4356 return load_inst;
4357 }
4358
gen_init_stack_trace(CodeGen * g,LLVMValueRef trace_field_ptr,LLVMValueRef addrs_field_ptr)4359 static void gen_init_stack_trace(CodeGen *g, LLVMValueRef trace_field_ptr, LLVMValueRef addrs_field_ptr) {
4360 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
4361 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
4362
4363 LLVMValueRef index_ptr = LLVMBuildStructGEP(g->builder, trace_field_ptr, 0, "");
4364 LLVMBuildStore(g->builder, zero, index_ptr);
4365
4366 LLVMValueRef addrs_slice_ptr = LLVMBuildStructGEP(g->builder, trace_field_ptr, 1, "");
4367 LLVMValueRef addrs_ptr_ptr = LLVMBuildStructGEP(g->builder, addrs_slice_ptr, slice_ptr_index, "");
4368 LLVMValueRef indices[] = { LLVMConstNull(usize_type_ref), LLVMConstNull(usize_type_ref) };
4369 LLVMValueRef trace_field_addrs_as_ptr = LLVMBuildInBoundsGEP(g->builder, addrs_field_ptr, indices, 2, "");
4370 LLVMBuildStore(g->builder, trace_field_addrs_as_ptr, addrs_ptr_ptr);
4371
4372 LLVMValueRef addrs_len_ptr = LLVMBuildStructGEP(g->builder, addrs_slice_ptr, slice_len_index, "");
4373 LLVMBuildStore(g->builder, LLVMConstInt(usize_type_ref, stack_trace_ptr_count, false), addrs_len_ptr);
4374 }
4375
ir_render_call(CodeGen * g,Stage1Air * executable,Stage1AirInstCall * instruction)4376 static LLVMValueRef ir_render_call(CodeGen *g, Stage1Air *executable, Stage1AirInstCall *instruction) {
4377 Error err;
4378
4379 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
4380
4381 LLVMValueRef fn_val;
4382 ZigType *fn_type;
4383 bool callee_is_async;
4384 if (instruction->fn_entry) {
4385 fn_val = fn_llvm_value(g, instruction->fn_entry);
4386 fn_type = instruction->fn_entry->type_entry;
4387 callee_is_async = fn_is_async(instruction->fn_entry);
4388 } else {
4389 assert(instruction->fn_ref);
4390 fn_val = ir_llvm_value(g, instruction->fn_ref);
4391 fn_type = instruction->fn_ref->value->type;
4392 callee_is_async = fn_type->data.fn.fn_type_id.cc == CallingConventionAsync;
4393 }
4394
4395 FnTypeId *fn_type_id = &fn_type->data.fn.fn_type_id;
4396
4397 ZigType *src_return_type = fn_type_id->return_type;
4398 bool ret_has_bits = type_has_bits(g, src_return_type);
4399
4400 CallingConvention cc = fn_type->data.fn.fn_type_id.cc;
4401
4402 bool first_arg_ret = ret_has_bits && want_first_arg_sret(g, fn_type_id);
4403 bool prefix_arg_err_ret_stack = codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type);
4404 bool is_var_args = fn_type_id->is_var_args;
4405 ZigList<LLVMValueRef> gen_param_values = {};
4406 ZigList<ZigType *> gen_param_types = {};
4407 LLVMValueRef result_loc = instruction->result_loc ? ir_llvm_value(g, instruction->result_loc) : nullptr;
4408 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
4409 bool need_frame_ptr_ptr_spill = false;
4410 ZigType *anyframe_type = nullptr;
4411 LLVMValueRef frame_result_loc_uncasted = nullptr;
4412 LLVMValueRef frame_result_loc;
4413 LLVMValueRef awaiter_init_val;
4414 LLVMValueRef ret_ptr;
4415 if (callee_is_async) {
4416 if (instruction->new_stack == nullptr) {
4417 if (instruction->modifier == CallModifierAsync) {
4418 frame_result_loc = result_loc;
4419 } else {
4420 ir_assert(instruction->frame_result_loc != nullptr, &instruction->base);
4421 frame_result_loc_uncasted = ir_llvm_value(g, instruction->frame_result_loc);
4422 ir_assert(instruction->fn_entry != nullptr, &instruction->base);
4423 frame_result_loc = LLVMBuildBitCast(g->builder, frame_result_loc_uncasted,
4424 LLVMPointerType(get_llvm_type(g, instruction->fn_entry->frame_type), 0), "");
4425 }
4426 } else {
4427 if (instruction->new_stack->value->type->id == ZigTypeIdPointer &&
4428 instruction->new_stack->value->type->data.pointer.child_type->id == ZigTypeIdFnFrame)
4429 {
4430 frame_result_loc = ir_llvm_value(g, instruction->new_stack);
4431 } else {
4432 LLVMValueRef frame_slice_ptr = ir_llvm_value(g, instruction->new_stack);
4433 if (ir_want_runtime_safety(g, &instruction->base)) {
4434 LLVMValueRef given_len_ptr = LLVMBuildStructGEP(g->builder, frame_slice_ptr, slice_len_index, "");
4435 LLVMValueRef given_frame_len = LLVMBuildLoad(g->builder, given_len_ptr, "");
4436 LLVMValueRef actual_frame_len = gen_frame_size(g, fn_val);
4437
4438 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "FrameSizeCheckFail");
4439 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "FrameSizeCheckOk");
4440
4441 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntUGE, given_frame_len, actual_frame_len, "");
4442 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
4443
4444 LLVMPositionBuilderAtEnd(g->builder, fail_block);
4445 gen_safety_crash(g, PanicMsgIdFrameTooSmall);
4446
4447 LLVMPositionBuilderAtEnd(g->builder, ok_block);
4448 }
4449 need_frame_ptr_ptr_spill = true;
4450 LLVMValueRef frame_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_slice_ptr, slice_ptr_index, "");
4451 LLVMValueRef frame_ptr = LLVMBuildLoad(g->builder, frame_ptr_ptr, "");
4452 if (instruction->fn_entry == nullptr) {
4453 anyframe_type = get_any_frame_type(g, src_return_type);
4454 frame_result_loc = LLVMBuildBitCast(g->builder, frame_ptr, get_llvm_type(g, anyframe_type), "");
4455 } else {
4456 ZigType *frame_type = get_fn_frame_type(g, instruction->fn_entry);
4457 if ((err = type_resolve(g, frame_type, ResolveStatusLLVMFull)))
4458 codegen_report_errors_and_exit(g);
4459 ZigType *ptr_frame_type = get_pointer_to_type(g, frame_type, false);
4460 frame_result_loc = LLVMBuildBitCast(g->builder, frame_ptr,
4461 get_llvm_type(g, ptr_frame_type), "");
4462 }
4463 }
4464 }
4465 if (instruction->modifier == CallModifierAsync) {
4466 if (instruction->new_stack == nullptr) {
4467 awaiter_init_val = zero;
4468
4469 if (ret_has_bits) {
4470 // Use the result location which is inside the frame if this is an async call.
4471 ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
4472 }
4473 } else {
4474 awaiter_init_val = zero;
4475
4476 if (ret_has_bits) {
4477 if (result_loc != nullptr) {
4478 // Use the result location provided to the @asyncCall builtin
4479 ret_ptr = result_loc;
4480 } else {
4481 // no result location provided to @asyncCall - use the one inside the frame.
4482 ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
4483 }
4484 }
4485 }
4486
4487 // even if prefix_arg_err_ret_stack is true, let the async function do its own
4488 // initialization.
4489 } else {
4490 if (instruction->modifier == CallModifierNoSuspend && !fn_is_async(g->cur_fn)) {
4491 // Async function called as a normal function, and calling function is not async.
4492 // This is allowed because it was called with `nosuspend` which asserts that it will
4493 // never suspend.
4494 awaiter_init_val = zero;
4495 } else {
4496 // async function called as a normal function
4497 awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_frame_ptr, usize_type_ref, ""); // caller's own frame pointer
4498 }
4499 if (ret_has_bits) {
4500 if (result_loc == nullptr) {
4501 // return type is a scalar, but we still need a pointer to it. Use the async fn frame.
4502 ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
4503 } else {
4504 // Use the call instruction's result location.
4505 ret_ptr = result_loc;
4506 }
4507
4508 // Store a zero in the awaiter's result ptr to indicate we do not need a copy made.
4509 LLVMValueRef awaiter_ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 1, "");
4510 LLVMValueRef zero_ptr = LLVMConstNull(LLVMGetElementType(LLVMTypeOf(awaiter_ret_ptr)));
4511 LLVMBuildStore(g->builder, zero_ptr, awaiter_ret_ptr);
4512 }
4513
4514 if (prefix_arg_err_ret_stack) {
4515 LLVMValueRef err_ret_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
4516 frame_index_trace_arg(g, src_return_type) + 1, "");
4517 bool is_llvm_alloca;
4518 LLVMValueRef my_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope,
4519 &is_llvm_alloca);
4520 LLVMBuildStore(g->builder, my_err_ret_trace_val, err_ret_trace_ptr_ptr);
4521 }
4522 }
4523
4524 assert(frame_result_loc != nullptr);
4525
4526 LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_fn_ptr_index, "");
4527 LLVMValueRef bitcasted_fn_val = LLVMBuildBitCast(g->builder, fn_val,
4528 LLVMGetElementType(LLVMTypeOf(fn_ptr_ptr)), "");
4529 LLVMBuildStore(g->builder, bitcasted_fn_val, fn_ptr_ptr);
4530
4531 LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_resume_index, "");
4532 LLVMBuildStore(g->builder, zero, resume_index_ptr);
4533
4534 LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_awaiter_index, "");
4535 LLVMBuildStore(g->builder, awaiter_init_val, awaiter_ptr);
4536
4537 if (ret_has_bits) {
4538 LLVMValueRef ret_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start, "");
4539 LLVMBuildStore(g->builder, ret_ptr, ret_ptr_ptr);
4540 }
4541 } else if (instruction->modifier == CallModifierAsync) {
4542 // Async call of blocking function
4543 if (instruction->new_stack != nullptr) {
4544 zig_panic("TODO @asyncCall of non-async function");
4545 }
4546 frame_result_loc = result_loc;
4547 awaiter_init_val = LLVMConstAllOnes(usize_type_ref);
4548
4549 LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_awaiter_index, "");
4550 LLVMBuildStore(g->builder, awaiter_init_val, awaiter_ptr);
4551
4552 if (ret_has_bits) {
4553 ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
4554 LLVMValueRef ret_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start, "");
4555 LLVMBuildStore(g->builder, ret_ptr, ret_ptr_ptr);
4556
4557 if (first_arg_ret) {
4558 gen_param_values.append(ret_ptr);
4559 }
4560 if (prefix_arg_err_ret_stack) {
4561 // Set up the callee stack trace pointer pointing into the frame.
4562 // Then we have to wire up the StackTrace pointers.
4563 // Await is responsible for merging error return traces.
4564 uint32_t trace_field_index_start = frame_index_trace_arg(g, src_return_type);
4565 LLVMValueRef callee_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
4566 trace_field_index_start, "");
4567 LLVMValueRef trace_field_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
4568 trace_field_index_start + 2, "");
4569 LLVMValueRef addrs_field_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
4570 trace_field_index_start + 3, "");
4571
4572 LLVMBuildStore(g->builder, trace_field_ptr, callee_trace_ptr_ptr);
4573
4574 gen_init_stack_trace(g, trace_field_ptr, addrs_field_ptr);
4575
4576 bool is_llvm_alloca;
4577 gen_param_values.append(get_cur_err_ret_trace_val(g, instruction->base.scope, &is_llvm_alloca));
4578 }
4579 }
4580 } else {
4581 if (first_arg_ret) {
4582 gen_param_values.append(result_loc);
4583 }
4584 if (prefix_arg_err_ret_stack) {
4585 bool is_llvm_alloca;
4586 gen_param_values.append(get_cur_err_ret_trace_val(g, instruction->base.scope, &is_llvm_alloca));
4587 }
4588 }
4589 FnWalk fn_walk = {};
4590 fn_walk.id = FnWalkIdCall;
4591 fn_walk.data.call.inst = instruction;
4592 fn_walk.data.call.is_var_args = is_var_args;
4593 fn_walk.data.call.gen_param_values = &gen_param_values;
4594 fn_walk.data.call.gen_param_types = &gen_param_types;
4595 walk_function_params(g, fn_type, &fn_walk);
4596
4597 ZigLLVM_CallAttr call_attr;
4598 switch (instruction->modifier) {
4599 case CallModifierBuiltin:
4600 case CallModifierCompileTime:
4601 zig_unreachable();
4602 case CallModifierNone:
4603 case CallModifierNoSuspend:
4604 case CallModifierAsync:
4605 call_attr = ZigLLVM_CallAttrAuto;
4606 break;
4607 case CallModifierNeverTail:
4608 call_attr = ZigLLVM_CallAttrNeverTail;
4609 break;
4610 case CallModifierNeverInline:
4611 call_attr = ZigLLVM_CallAttrNeverInline;
4612 break;
4613 case CallModifierAlwaysTail:
4614 call_attr = ZigLLVM_CallAttrAlwaysTail;
4615 break;
4616 case CallModifierAlwaysInline:
4617 ir_assert(instruction->fn_entry != nullptr, &instruction->base);
4618 call_attr = ZigLLVM_CallAttrAlwaysInline;
4619 break;
4620 }
4621
4622 ZigLLVM_CallingConv llvm_cc = get_llvm_cc(g, cc);
4623 LLVMValueRef result;
4624
4625 if (callee_is_async) {
4626 CalcLLVMFieldIndex arg_calc_start = {0};
4627 frame_index_arg_calc(g, &arg_calc_start, fn_type->data.fn.fn_type_id.return_type);
4628
4629 LLVMValueRef casted_frame;
4630 if (instruction->new_stack != nullptr && instruction->fn_entry == nullptr) {
4631 // We need the frame type to be a pointer to a struct that includes the args
4632
4633 // Count ahead to determine how many llvm struct fields we need.
4634 CalcLLVMFieldIndex arg_calc = arg_calc_start;
4635 for (size_t i = 0; i < gen_param_types.length; i += 1) {
4636 calc_llvm_field_index_add(g, &arg_calc, gen_param_types.at(i));
4637 }
4638 size_t field_count = arg_calc.field_index;
4639
4640 LLVMTypeRef *field_types = heap::c_allocator.allocate_nonzero<LLVMTypeRef>(field_count);
4641 LLVMGetStructElementTypes(LLVMGetElementType(LLVMTypeOf(frame_result_loc)), field_types);
4642 assert(LLVMCountStructElementTypes(LLVMGetElementType(LLVMTypeOf(frame_result_loc))) == arg_calc_start.field_index);
4643
4644 arg_calc = arg_calc_start;
4645 for (size_t arg_i = 0; arg_i < gen_param_values.length; arg_i += 1) {
4646 CalcLLVMFieldIndex prev = arg_calc;
4647 // Use the declared argument type and not the value one to be
4648 // consistent with the assignment operation below.
4649 calc_llvm_field_index_add(g, &arg_calc, gen_param_types.at(arg_i));
4650 field_types[arg_calc.field_index - 1] = get_llvm_type(g, gen_param_types.at(arg_i));
4651 if (arg_calc.field_index - prev.field_index > 1) {
4652 // Padding field
4653 uint32_t pad_bytes = arg_calc.offset - prev.offset - gen_param_types.at(arg_i)->abi_size;
4654 LLVMTypeRef pad_llvm_type = LLVMArrayType(LLVMInt8Type(), pad_bytes);
4655 field_types[arg_calc.field_index - 2] = pad_llvm_type;
4656 }
4657 }
4658 LLVMTypeRef frame_with_args_type = LLVMStructType(field_types, field_count, false);
4659 heap::c_allocator.deallocate(field_types, field_count);
4660 LLVMTypeRef ptr_frame_with_args_type = LLVMPointerType(frame_with_args_type, 0);
4661
4662 casted_frame = LLVMBuildBitCast(g->builder, frame_result_loc, ptr_frame_with_args_type, "");
4663 } else {
4664 casted_frame = frame_result_loc;
4665 }
4666
4667 CalcLLVMFieldIndex arg_calc = arg_calc_start;
4668 for (size_t arg_i = 0; arg_i < gen_param_values.length; arg_i += 1) {
4669 calc_llvm_field_index_add(g, &arg_calc, gen_param_types.at(arg_i));
4670 LLVMValueRef arg_ptr = LLVMBuildStructGEP(g->builder, casted_frame, arg_calc.field_index - 1, "");
4671 gen_assign_raw(g, arg_ptr, get_pointer_to_type(g, gen_param_types.at(arg_i), true),
4672 gen_param_values.at(arg_i));
4673 }
4674 gen_param_types.deinit();
4675
4676 if (instruction->modifier == CallModifierAsync) {
4677 gen_resume(g, fn_val, frame_result_loc, ResumeIdCall);
4678 if (instruction->new_stack != nullptr) {
4679 return LLVMBuildBitCast(g->builder, frame_result_loc,
4680 get_llvm_type(g, instruction->base.value->type), "");
4681 }
4682 return nullptr;
4683 } else if (instruction->modifier == CallModifierNoSuspend && !fn_is_async(g->cur_fn)) {
4684 gen_resume(g, fn_val, frame_result_loc, ResumeIdCall);
4685
4686 if (ir_want_runtime_safety(g, &instruction->base)) {
4687 LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
4688 frame_awaiter_index, "");
4689 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
4690 LLVMValueRef prev_val = gen_maybe_atomic_op(g, LLVMAtomicRMWBinOpXchg, awaiter_ptr,
4691 all_ones, LLVMAtomicOrderingRelease);
4692 LLVMValueRef ok_val = LLVMBuildICmp(g->builder, LLVMIntEQ, prev_val, all_ones, "");
4693
4694 LLVMBasicBlockRef bad_block = LLVMAppendBasicBlock(g->cur_fn_val, "NoSuspendPanic");
4695 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "NoSuspendOk");
4696 LLVMBuildCondBr(g->builder, ok_val, ok_block, bad_block);
4697
4698 // The async function suspended, but this nosuspend call asserted it wouldn't.
4699 LLVMPositionBuilderAtEnd(g->builder, bad_block);
4700 gen_safety_crash(g, PanicMsgIdBadNoSuspendCall);
4701
4702 LLVMPositionBuilderAtEnd(g->builder, ok_block);
4703 }
4704
4705 ZigType *result_type = instruction->base.value->type;
4706 ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true);
4707 return gen_await_early_return(g, &instruction->base, frame_result_loc,
4708 result_type, ptr_result_type, result_loc, true);
4709 } else {
4710 ZigType *ptr_result_type = get_pointer_to_type(g, src_return_type, true);
4711
4712 LLVMBasicBlockRef call_bb = gen_suspend_begin(g, "CallResume");
4713
4714 LLVMValueRef call_inst = gen_resume(g, fn_val, frame_result_loc, ResumeIdCall);
4715 set_tail_call_if_appropriate(g, call_inst);
4716 LLVMBuildRetVoid(g->builder);
4717
4718 LLVMPositionBuilderAtEnd(g->builder, call_bb);
4719 gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedAnAwaitingFn, nullptr);
4720 render_async_var_decls(g, instruction->base.scope);
4721
4722 if (!type_has_bits(g, src_return_type))
4723 return nullptr;
4724
4725 if (result_loc != nullptr) {
4726 if (instruction->result_loc->id == Stage1AirInstIdReturnPtr) {
4727 instruction->base.spill = nullptr;
4728 return g->cur_ret_ptr;
4729 } else {
4730 return get_handle_value(g, result_loc, src_return_type, ptr_result_type);
4731 }
4732 }
4733
4734 if (need_frame_ptr_ptr_spill) {
4735 LLVMValueRef frame_slice_ptr = ir_llvm_value(g, instruction->new_stack);
4736 LLVMValueRef frame_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_slice_ptr, slice_ptr_index, "");
4737 frame_result_loc_uncasted = LLVMBuildLoad(g->builder, frame_ptr_ptr, "");
4738 }
4739 if (frame_result_loc_uncasted != nullptr) {
4740 if (instruction->fn_entry != nullptr) {
4741 frame_result_loc = LLVMBuildBitCast(g->builder, frame_result_loc_uncasted,
4742 LLVMPointerType(get_llvm_type(g, instruction->fn_entry->frame_type), 0), "");
4743 } else {
4744 frame_result_loc = LLVMBuildBitCast(g->builder, frame_result_loc_uncasted,
4745 get_llvm_type(g, anyframe_type), "");
4746 }
4747 }
4748
4749 LLVMValueRef result_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
4750 return LLVMBuildLoad(g->builder, result_ptr, "");
4751 }
4752 } else {
4753 gen_param_types.deinit();
4754 }
4755
4756 if (instruction->new_stack == nullptr || instruction->is_async_call_builtin) {
4757 result = ZigLLVMBuildCall(g->builder, fn_val,
4758 gen_param_values.items, (unsigned)gen_param_values.length, llvm_cc, call_attr, "");
4759 } else if (instruction->modifier == CallModifierAsync) {
4760 zig_panic("TODO @asyncCall of non-async function");
4761 } else {
4762 LLVMValueRef new_stack_addr = get_new_stack_addr(g, ir_llvm_value(g, instruction->new_stack));
4763 LLVMValueRef old_stack_ref;
4764 if (src_return_type->id != ZigTypeIdUnreachable) {
4765 LLVMValueRef stacksave_fn_val = get_stacksave_fn_val(g);
4766 old_stack_ref = LLVMBuildCall(g->builder, stacksave_fn_val, nullptr, 0, "");
4767 }
4768 gen_set_stack_pointer(g, new_stack_addr);
4769 result = ZigLLVMBuildCall(g->builder, fn_val,
4770 gen_param_values.items, (unsigned)gen_param_values.length, llvm_cc, call_attr, "");
4771 if (src_return_type->id != ZigTypeIdUnreachable) {
4772 LLVMValueRef stackrestore_fn_val = get_stackrestore_fn_val(g);
4773 LLVMBuildCall(g->builder, stackrestore_fn_val, &old_stack_ref, 1, "");
4774 }
4775 }
4776
4777 if (src_return_type->id == ZigTypeIdUnreachable) {
4778 return LLVMBuildUnreachable(g->builder);
4779 } else if (!ret_has_bits) {
4780 return nullptr;
4781 } else if (first_arg_ret) {
4782 ZigLLVMSetCallSret(result, get_llvm_type(g, src_return_type));
4783 return result_loc;
4784 } else if (fn_returns_c_abi_small_struct(fn_type_id)) {
4785 LLVMTypeRef abi_type = get_llvm_c_abi_type(g, src_return_type);
4786 LLVMTypeRef abi_type_ptr = LLVMPointerType(abi_type, 0);
4787 LLVMValueRef bitcast = LLVMBuildBitCast(g->builder, result_loc, abi_type_ptr, "");
4788 LLVMBuildStore(g->builder, result, bitcast);
4789 return result_loc;
4790 } else if (handle_is_ptr(g, src_return_type)) {
4791 LLVMValueRef store_instr = LLVMBuildStore(g->builder, result, result_loc);
4792 LLVMSetAlignment(store_instr, get_ptr_align(g, instruction->result_loc->value->type));
4793 return result_loc;
4794 } else if (!callee_is_async && instruction->modifier == CallModifierAsync) {
4795 LLVMBuildStore(g->builder, result, ret_ptr);
4796 return result_loc;
4797 } else {
4798 return result;
4799 }
4800 }
4801
ir_render_struct_field_ptr(CodeGen * g,Stage1Air * executable,Stage1AirInstStructFieldPtr * instruction)4802 static LLVMValueRef ir_render_struct_field_ptr(CodeGen *g, Stage1Air *executable,
4803 Stage1AirInstStructFieldPtr *instruction)
4804 {
4805 Error err;
4806
4807 if (instruction->base.value->special != ConstValSpecialRuntime)
4808 return nullptr;
4809
4810 LLVMValueRef struct_ptr = ir_llvm_value(g, instruction->struct_ptr);
4811 // not necessarily a pointer. could be ZigTypeIdStruct
4812 ZigType *struct_ptr_type = instruction->struct_ptr->value->type;
4813 TypeStructField *field = instruction->field;
4814
4815 if (!type_has_bits(g, field->type_entry))
4816 return nullptr;
4817
4818 if (struct_ptr_type->id == ZigTypeIdPointer &&
4819 struct_ptr_type->data.pointer.host_int_bytes != 0)
4820 {
4821 return struct_ptr;
4822 }
4823
4824 ZigType *struct_type;
4825 if (struct_ptr_type->id == ZigTypeIdPointer) {
4826 if (struct_ptr_type->data.pointer.inferred_struct_field != nullptr) {
4827 struct_type = struct_ptr_type->data.pointer.inferred_struct_field->inferred_struct_type;
4828 } else {
4829 struct_type = struct_ptr_type->data.pointer.child_type;
4830 }
4831 } else {
4832 struct_type = struct_ptr_type;
4833 }
4834
4835 if ((err = type_resolve(g, struct_type, ResolveStatusLLVMFull)))
4836 codegen_report_errors_and_exit(g);
4837
4838 ir_assert(field->gen_index != SIZE_MAX, &instruction->base);
4839 LLVMValueRef field_ptr_val = LLVMBuildStructGEP(g->builder, struct_ptr, (unsigned)field->gen_index, "");
4840 ZigType *res_type = instruction->base.value->type;
4841 ir_assert(res_type->id == ZigTypeIdPointer, &instruction->base);
4842 if (res_type->data.pointer.host_int_bytes != 0) {
4843 // We generate packed structs with get_llvm_type_of_n_bytes, which is
4844 // u8 for 1 byte or [n]u8 for multiple bytes. But the pointer to the type
4845 // is supposed to be a pointer to the integer. So we bitcast it here.
4846 LLVMTypeRef int_elem_type = LLVMIntType(8*res_type->data.pointer.host_int_bytes);
4847 LLVMTypeRef integer_ptr_type = LLVMPointerType(int_elem_type, 0);
4848 return LLVMBuildBitCast(g->builder, field_ptr_val, integer_ptr_type, "");
4849 }
4850 return field_ptr_val;
4851 }
4852
ir_render_union_field_ptr(CodeGen * g,Stage1Air * executable,Stage1AirInstUnionFieldPtr * instruction)4853 static LLVMValueRef ir_render_union_field_ptr(CodeGen *g, Stage1Air *executable,
4854 Stage1AirInstUnionFieldPtr *instruction)
4855 {
4856 if (instruction->base.value->special != ConstValSpecialRuntime)
4857 return nullptr;
4858
4859 ZigType *union_ptr_type = instruction->union_ptr->value->type;
4860 assert(union_ptr_type->id == ZigTypeIdPointer);
4861 ZigType *union_type = union_ptr_type->data.pointer.child_type;
4862 assert(union_type->id == ZigTypeIdUnion);
4863
4864 TypeUnionField *field = instruction->field;
4865
4866 if (!type_has_bits(g, field->type_entry)) {
4867 ZigType *tag_type = union_type->data.unionation.tag_type;
4868 if (!instruction->initializing || tag_type == nullptr || !type_has_bits(g, tag_type))
4869 return nullptr;
4870
4871 // The field has no bits but we still have to change the discriminant
4872 // value here
4873 LLVMValueRef union_ptr = ir_llvm_value(g, instruction->union_ptr);
4874
4875 LLVMTypeRef tag_type_ref = get_llvm_type(g, tag_type);
4876 LLVMValueRef tag_field_ptr = nullptr;
4877 if (union_type->data.unionation.gen_field_count == 0) {
4878 assert(union_type->data.unionation.gen_tag_index == SIZE_MAX);
4879 // The whole union is collapsed into the discriminant
4880 tag_field_ptr = LLVMBuildBitCast(g->builder, union_ptr,
4881 LLVMPointerType(tag_type_ref, 0), "");
4882 } else {
4883 assert(union_type->data.unionation.gen_tag_index != SIZE_MAX);
4884 tag_field_ptr = LLVMBuildStructGEP(g->builder, union_ptr,
4885 union_type->data.unionation.gen_tag_index, "");
4886 }
4887
4888 LLVMValueRef tag_value = bigint_to_llvm_const(tag_type_ref,
4889 &field->enum_field->value);
4890 assert(tag_field_ptr != nullptr);
4891 gen_store_untyped(g, tag_value, tag_field_ptr, 0, false);
4892
4893 return nullptr;
4894 }
4895
4896 LLVMValueRef union_ptr = ir_llvm_value(g, instruction->union_ptr);
4897 LLVMTypeRef field_type_ref = LLVMPointerType(get_llvm_type(g, field->type_entry), 0);
4898
4899 if (union_type->data.unionation.gen_tag_index == SIZE_MAX) {
4900 LLVMValueRef union_field_ptr = LLVMBuildStructGEP(g->builder, union_ptr, 0, "");
4901 LLVMValueRef bitcasted_union_field_ptr = LLVMBuildBitCast(g->builder, union_field_ptr, field_type_ref, "");
4902 return bitcasted_union_field_ptr;
4903 }
4904
4905 if (instruction->initializing) {
4906 LLVMValueRef tag_field_ptr = LLVMBuildStructGEP(g->builder, union_ptr, union_type->data.unionation.gen_tag_index, "");
4907 LLVMValueRef tag_value = bigint_to_llvm_const(get_llvm_type(g, union_type->data.unionation.tag_type),
4908 &field->enum_field->value);
4909 gen_store_untyped(g, tag_value, tag_field_ptr, 0, false);
4910 } else if (instruction->safety_check_on && ir_want_runtime_safety(g, &instruction->base)) {
4911 LLVMValueRef tag_field_ptr = LLVMBuildStructGEP(g->builder, union_ptr, union_type->data.unionation.gen_tag_index, "");
4912 LLVMValueRef tag_value = gen_load_untyped(g, tag_field_ptr, 0, false, "");
4913
4914
4915 LLVMValueRef expected_tag_value = bigint_to_llvm_const(get_llvm_type(g, union_type->data.unionation.tag_type),
4916 &field->enum_field->value);
4917 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnionCheckOk");
4918 LLVMBasicBlockRef bad_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnionCheckFail");
4919 LLVMValueRef ok_val = LLVMBuildICmp(g->builder, LLVMIntEQ, tag_value, expected_tag_value, "");
4920 LLVMBuildCondBr(g->builder, ok_val, ok_block, bad_block);
4921
4922 LLVMPositionBuilderAtEnd(g->builder, bad_block);
4923 gen_safety_crash(g, PanicMsgIdBadUnionField);
4924
4925 LLVMPositionBuilderAtEnd(g->builder, ok_block);
4926 }
4927
4928 LLVMValueRef union_field_ptr = LLVMBuildStructGEP(g->builder, union_ptr,
4929 union_type->data.unionation.gen_union_index, "");
4930 LLVMValueRef bitcasted_union_field_ptr = LLVMBuildBitCast(g->builder, union_field_ptr, field_type_ref, "");
4931 return bitcasted_union_field_ptr;
4932 }
4933
find_asm_index(CodeGen * g,AstNode * node,AsmToken * tok,Buf * src_template)4934 static size_t find_asm_index(CodeGen *g, AstNode *node, AsmToken *tok, Buf *src_template) {
4935 const char *ptr = buf_ptr(src_template) + tok->start + 2;
4936 size_t len = tok->end - tok->start - 2;
4937 size_t result = 0;
4938 for (size_t i = 0; i < node->data.asm_expr.output_list.length; i += 1, result += 1) {
4939 AsmOutput *asm_output = node->data.asm_expr.output_list.at(i);
4940 if (buf_eql_mem(asm_output->asm_symbolic_name, ptr, len)) {
4941 return result;
4942 }
4943 }
4944 for (size_t i = 0; i < node->data.asm_expr.input_list.length; i += 1, result += 1) {
4945 AsmInput *asm_input = node->data.asm_expr.input_list.at(i);
4946 if (buf_eql_mem(asm_input->asm_symbolic_name, ptr, len)) {
4947 return result;
4948 }
4949 }
4950 return SIZE_MAX;
4951 }
4952
ir_render_asm_gen(CodeGen * g,Stage1Air * executable,Stage1AirInstAsm * instruction)4953 static LLVMValueRef ir_render_asm_gen(CodeGen *g, Stage1Air *executable, Stage1AirInstAsm *instruction) {
4954 AstNode *asm_node = instruction->base.source_node;
4955 assert(asm_node->type == NodeTypeAsmExpr);
4956 AstNodeAsmExpr *asm_expr = &asm_node->data.asm_expr;
4957
4958 Buf *src_template = instruction->asm_template;
4959
4960 Buf llvm_template = BUF_INIT;
4961 buf_resize(&llvm_template, 0);
4962
4963 for (size_t token_i = 0; token_i < instruction->token_list_len; token_i += 1) {
4964 AsmToken *asm_token = &instruction->token_list[token_i];
4965 switch (asm_token->id) {
4966 case AsmTokenIdTemplate:
4967 for (size_t offset = asm_token->start; offset < asm_token->end; offset += 1) {
4968 uint8_t c = *((uint8_t*)(buf_ptr(src_template) + offset));
4969 if (c == '$') {
4970 buf_append_str(&llvm_template, "$$");
4971 } else {
4972 buf_append_char(&llvm_template, c);
4973 }
4974 }
4975 break;
4976 case AsmTokenIdPercent:
4977 buf_append_char(&llvm_template, '%');
4978 break;
4979 case AsmTokenIdVar:
4980 {
4981 size_t index = find_asm_index(g, asm_node, asm_token, src_template);
4982 assert(index < SIZE_MAX);
4983 buf_appendf(&llvm_template, "$%" ZIG_PRI_usize "", index);
4984 break;
4985 }
4986 case AsmTokenIdUniqueId:
4987 buf_append_str(&llvm_template, "${:uid}");
4988 break;
4989 }
4990 }
4991
4992 Buf constraint_buf = BUF_INIT;
4993 buf_resize(&constraint_buf, 0);
4994
4995 assert(instruction->return_count == 0 || instruction->return_count == 1);
4996
4997 size_t total_constraint_count = asm_expr->output_list.length +
4998 asm_expr->input_list.length +
4999 asm_expr->clobber_list.length;
5000 size_t input_and_output_count = asm_expr->output_list.length +
5001 asm_expr->input_list.length -
5002 instruction->return_count;
5003 size_t total_index = 0;
5004 size_t param_index = 0;
5005 LLVMTypeRef *param_types = heap::c_allocator.allocate<LLVMTypeRef>(input_and_output_count);
5006 LLVMValueRef *param_values = heap::c_allocator.allocate<LLVMValueRef>(input_and_output_count);
5007 for (size_t i = 0; i < asm_expr->output_list.length; i += 1, total_index += 1) {
5008 AsmOutput *asm_output = asm_expr->output_list.at(i);
5009 bool is_return = (asm_output->return_type != nullptr);
5010 assert(*buf_ptr(asm_output->constraint) == '=');
5011 // LLVM uses commas internally to separate different constraints,
5012 // alternative constraints are achieved with pipes.
5013 // We still allow the user to use commas in a way that is similar
5014 // to GCC's inline assembly.
5015 // http://llvm.org/docs/LangRef.html#constraint-codes
5016 buf_replace(asm_output->constraint, ',', '|');
5017
5018 if (is_return) {
5019 buf_appendf(&constraint_buf, "=%s", buf_ptr(asm_output->constraint) + 1);
5020 } else {
5021 buf_appendf(&constraint_buf, "=*%s", buf_ptr(asm_output->constraint) + 1);
5022 }
5023 if (total_index + 1 < total_constraint_count) {
5024 buf_append_char(&constraint_buf, ',');
5025 }
5026
5027 if (!is_return) {
5028 ZigVar *variable = instruction->output_vars[i];
5029 assert(variable);
5030 param_types[param_index] = LLVMTypeOf(variable->value_ref);
5031 param_values[param_index] = variable->value_ref;
5032 param_index += 1;
5033 }
5034 }
5035 for (size_t i = 0; i < asm_expr->input_list.length; i += 1, total_index += 1, param_index += 1) {
5036 AsmInput *asm_input = asm_expr->input_list.at(i);
5037 buf_replace(asm_input->constraint, ',', '|');
5038 Stage1AirInst *ir_input = instruction->input_list[i];
5039 buf_append_buf(&constraint_buf, asm_input->constraint);
5040 if (total_index + 1 < total_constraint_count) {
5041 buf_append_char(&constraint_buf, ',');
5042 }
5043
5044 ZigType *const type = ir_input->value->type;
5045 LLVMTypeRef type_ref = get_llvm_type(g, type);
5046 LLVMValueRef value_ref = ir_llvm_value(g, ir_input);
5047 // Handle integers of non pot bitsize by widening them.
5048 if (type->id == ZigTypeIdInt) {
5049 const size_t bitsize = type->data.integral.bit_count;
5050 if (bitsize < 8 || !is_power_of_2(bitsize)) {
5051 const bool is_signed = type->data.integral.is_signed;
5052 const size_t wider_bitsize = bitsize < 8 ? 8 : round_to_next_power_of_2(bitsize);
5053 ZigType *const wider_type = get_int_type(g, is_signed, wider_bitsize);
5054 type_ref = get_llvm_type(g, wider_type);
5055 value_ref = gen_widen_or_shorten(g, false, type, wider_type, value_ref);
5056 }
5057 } else if (handle_is_ptr(g, type)) {
5058 ZigType *gen_type = get_pointer_to_type(g, type, true);
5059 type_ref = get_llvm_type(g, gen_type);
5060 }
5061
5062 param_types[param_index] = type_ref;
5063 param_values[param_index] = value_ref;
5064 }
5065 for (size_t i = 0; i < asm_expr->clobber_list.length; i += 1, total_index += 1) {
5066 Buf *clobber_buf = asm_expr->clobber_list.at(i);
5067 buf_appendf(&constraint_buf, "~{%s}", buf_ptr(clobber_buf));
5068 if (total_index + 1 < total_constraint_count) {
5069 buf_append_char(&constraint_buf, ',');
5070 }
5071 }
5072
5073 // Add some architecture-specific clobbers.
5074 const char *arch_clobbers = nullptr;
5075 switch (g->zig_target->arch) {
5076 case ZigLLVM_x86:
5077 case ZigLLVM_x86_64:
5078 arch_clobbers = "~{dirflag},~{fpsr},~{flags}";
5079 break;
5080 case ZigLLVM_mips:
5081 case ZigLLVM_mipsel:
5082 case ZigLLVM_mips64:
5083 case ZigLLVM_mips64el:
5084 arch_clobbers = "~{$1}";
5085 break;
5086 default:
5087 break;
5088 }
5089
5090 if (arch_clobbers != nullptr) {
5091 if (buf_len(&constraint_buf))
5092 buf_append_char(&constraint_buf, ',');
5093 buf_append_str(&constraint_buf, arch_clobbers);
5094 }
5095
5096 LLVMTypeRef ret_type;
5097 if (instruction->return_count == 0) {
5098 ret_type = LLVMVoidType();
5099 } else {
5100 ret_type = get_llvm_type(g, instruction->base.value->type);
5101 }
5102 LLVMTypeRef function_type = LLVMFunctionType(ret_type, param_types, (unsigned)input_and_output_count, false);
5103 heap::c_allocator.deallocate(param_types, input_and_output_count);
5104
5105 bool is_volatile = instruction->has_side_effects || (asm_expr->output_list.length == 0);
5106 LLVMValueRef asm_fn = LLVMGetInlineAsm(function_type, buf_ptr(&llvm_template), buf_len(&llvm_template),
5107 buf_ptr(&constraint_buf), buf_len(&constraint_buf), is_volatile, false, LLVMInlineAsmDialectATT, false);
5108
5109 LLVMValueRef built_call = LLVMBuildCall(g->builder, asm_fn, param_values, (unsigned)input_and_output_count, "");
5110 heap::c_allocator.deallocate(param_values, input_and_output_count);
5111 return built_call;
5112 }
5113
gen_non_null_bit(CodeGen * g,ZigType * maybe_type,LLVMValueRef maybe_handle)5114 static LLVMValueRef gen_non_null_bit(CodeGen *g, ZigType *maybe_type, LLVMValueRef maybe_handle) {
5115 assert(maybe_type->id == ZigTypeIdOptional ||
5116 (maybe_type->id == ZigTypeIdPointer && maybe_type->data.pointer.allow_zero));
5117
5118 ZigType *child_type = maybe_type->data.maybe.child_type;
5119 if (!type_has_bits(g, child_type))
5120 return maybe_handle;
5121
5122 bool is_scalar = !handle_is_ptr(g, maybe_type);
5123 if (is_scalar)
5124 return LLVMBuildICmp(g->builder, LLVMIntNE, maybe_handle, LLVMConstNull(get_llvm_type(g, maybe_type)), "");
5125
5126 LLVMValueRef maybe_field_ptr = LLVMBuildStructGEP(g->builder, maybe_handle, maybe_null_index, "");
5127 return gen_load_untyped(g, maybe_field_ptr, 0, false, "");
5128 }
5129
ir_render_test_non_null(CodeGen * g,Stage1Air * executable,Stage1AirInstTestNonNull * instruction)5130 static LLVMValueRef ir_render_test_non_null(CodeGen *g, Stage1Air *executable,
5131 Stage1AirInstTestNonNull *instruction)
5132 {
5133 return gen_non_null_bit(g, instruction->value->value->type, ir_llvm_value(g, instruction->value));
5134 }
5135
ir_render_optional_unwrap_ptr(CodeGen * g,Stage1Air * executable,Stage1AirInstOptionalUnwrapPtr * instruction)5136 static LLVMValueRef ir_render_optional_unwrap_ptr(CodeGen *g, Stage1Air *executable,
5137 Stage1AirInstOptionalUnwrapPtr *instruction)
5138 {
5139 if (instruction->base.value->special != ConstValSpecialRuntime)
5140 return nullptr;
5141
5142 ZigType *ptr_type = instruction->base_ptr->value->type;
5143 assert(ptr_type->id == ZigTypeIdPointer);
5144 ZigType *maybe_type = ptr_type->data.pointer.child_type;
5145 assert(maybe_type->id == ZigTypeIdOptional);
5146 ZigType *child_type = maybe_type->data.maybe.child_type;
5147 LLVMValueRef base_ptr = ir_llvm_value(g, instruction->base_ptr);
5148 if (instruction->safety_check_on && ir_want_runtime_safety(g, &instruction->base)) {
5149 LLVMValueRef maybe_handle = get_handle_value(g, base_ptr, maybe_type, ptr_type);
5150 LLVMValueRef non_null_bit = gen_non_null_bit(g, maybe_type, maybe_handle);
5151 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnwrapOptionalFail");
5152 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnwrapOptionalOk");
5153 LLVMBuildCondBr(g->builder, non_null_bit, ok_block, fail_block);
5154
5155 LLVMPositionBuilderAtEnd(g->builder, fail_block);
5156 gen_safety_crash(g, PanicMsgIdUnwrapOptionalFail);
5157
5158 LLVMPositionBuilderAtEnd(g->builder, ok_block);
5159 }
5160 if (!type_has_bits(g, child_type)) {
5161 if (instruction->initializing) {
5162 LLVMValueRef non_null_bit = LLVMConstInt(LLVMInt1Type(), 1, false);
5163 gen_store_untyped(g, non_null_bit, base_ptr, 0, false);
5164 }
5165 return nullptr;
5166 } else {
5167 bool is_scalar = !handle_is_ptr(g, maybe_type);
5168 if (is_scalar) {
5169 return base_ptr;
5170 } else {
5171 LLVMValueRef optional_struct_ref = get_handle_value(g, base_ptr, maybe_type, ptr_type);
5172 if (instruction->initializing) {
5173 LLVMValueRef non_null_bit_ptr = LLVMBuildStructGEP(g->builder, optional_struct_ref,
5174 maybe_null_index, "");
5175 LLVMValueRef non_null_bit = LLVMConstInt(LLVMInt1Type(), 1, false);
5176 gen_store_untyped(g, non_null_bit, non_null_bit_ptr, 0, false);
5177 }
5178 return LLVMBuildStructGEP(g->builder, optional_struct_ref, maybe_child_index, "");
5179 }
5180 }
5181 }
5182
get_int_builtin_fn(CodeGen * g,ZigType * expr_type,BuiltinFnId fn_id)5183 static LLVMValueRef get_int_builtin_fn(CodeGen *g, ZigType *expr_type, BuiltinFnId fn_id) {
5184 bool is_vector = expr_type->id == ZigTypeIdVector;
5185 ZigType *int_type = is_vector ? expr_type->data.vector.elem_type : expr_type;
5186 assert(int_type->id == ZigTypeIdInt);
5187 uint32_t vector_len = is_vector ? expr_type->data.vector.len : 0;
5188 ZigLLVMFnKey key = {};
5189 const char *fn_name;
5190 uint32_t n_args;
5191 if (fn_id == BuiltinFnIdCtz) {
5192 fn_name = "cttz";
5193 n_args = 2;
5194 key.id = ZigLLVMFnIdCtz;
5195 key.data.ctz.bit_count = (uint32_t)int_type->data.integral.bit_count;
5196 key.data.ctz.vector_len = vector_len;
5197 } else if (fn_id == BuiltinFnIdClz) {
5198 fn_name = "ctlz";
5199 n_args = 2;
5200 key.id = ZigLLVMFnIdClz;
5201 key.data.clz.bit_count = (uint32_t)int_type->data.integral.bit_count;
5202 key.data.clz.vector_len = vector_len;
5203 } else if (fn_id == BuiltinFnIdPopCount) {
5204 fn_name = "ctpop";
5205 n_args = 1;
5206 key.id = ZigLLVMFnIdPopCount;
5207 key.data.pop_count.bit_count = (uint32_t)int_type->data.integral.bit_count;
5208 key.data.pop_count.vector_len = vector_len;
5209 } else if (fn_id == BuiltinFnIdBswap) {
5210 fn_name = "bswap";
5211 n_args = 1;
5212 key.id = ZigLLVMFnIdBswap;
5213 key.data.bswap.bit_count = (uint32_t)int_type->data.integral.bit_count;
5214 key.data.bswap.vector_len = vector_len;
5215 } else if (fn_id == BuiltinFnIdBitReverse) {
5216 fn_name = "bitreverse";
5217 n_args = 1;
5218 key.id = ZigLLVMFnIdBitReverse;
5219 key.data.bit_reverse.bit_count = (uint32_t)int_type->data.integral.bit_count;
5220 key.data.bit_reverse.vector_len = vector_len;
5221 } else {
5222 zig_unreachable();
5223 }
5224
5225 auto existing_entry = g->llvm_fn_table.maybe_get(key);
5226 if (existing_entry)
5227 return existing_entry->value;
5228
5229 char llvm_name[64];
5230 if (is_vector)
5231 sprintf(llvm_name, "llvm.%s.v%" PRIu32 "i%" PRIu32, fn_name, vector_len, int_type->data.integral.bit_count);
5232 else
5233 sprintf(llvm_name, "llvm.%s.i%" PRIu32, fn_name, int_type->data.integral.bit_count);
5234 LLVMTypeRef param_types[] = {
5235 get_llvm_type(g, expr_type),
5236 LLVMInt1Type(),
5237 };
5238 LLVMTypeRef fn_type = LLVMFunctionType(get_llvm_type(g, expr_type), param_types, n_args, false);
5239 LLVMValueRef fn_val = LLVMAddFunction(g->module, llvm_name, fn_type);
5240 assert(LLVMGetIntrinsicID(fn_val));
5241
5242 g->llvm_fn_table.put(key, fn_val);
5243
5244 return fn_val;
5245 }
5246
ir_render_clz(CodeGen * g,Stage1Air * executable,Stage1AirInstClz * instruction)5247 static LLVMValueRef ir_render_clz(CodeGen *g, Stage1Air *executable, Stage1AirInstClz *instruction) {
5248 ZigType *int_type = instruction->op->value->type;
5249 LLVMValueRef fn_val = get_int_builtin_fn(g, int_type, BuiltinFnIdClz);
5250 LLVMValueRef operand = ir_llvm_value(g, instruction->op);
5251 LLVMValueRef params[] {
5252 operand,
5253 LLVMConstNull(LLVMInt1Type()),
5254 };
5255 LLVMValueRef wrong_size_int = LLVMBuildCall(g->builder, fn_val, params, 2, "");
5256 return gen_widen_or_shorten(g, false, int_type, instruction->base.value->type, wrong_size_int);
5257 }
5258
ir_render_ctz(CodeGen * g,Stage1Air * executable,Stage1AirInstCtz * instruction)5259 static LLVMValueRef ir_render_ctz(CodeGen *g, Stage1Air *executable, Stage1AirInstCtz *instruction) {
5260 ZigType *int_type = instruction->op->value->type;
5261 LLVMValueRef fn_val = get_int_builtin_fn(g, int_type, BuiltinFnIdCtz);
5262 LLVMValueRef operand = ir_llvm_value(g, instruction->op);
5263 LLVMValueRef params[] {
5264 operand,
5265 LLVMConstNull(LLVMInt1Type()),
5266 };
5267 LLVMValueRef wrong_size_int = LLVMBuildCall(g->builder, fn_val, params, 2, "");
5268 return gen_widen_or_shorten(g, false, int_type, instruction->base.value->type, wrong_size_int);
5269 }
5270
ir_render_shuffle_vector(CodeGen * g,Stage1Air * executable,Stage1AirInstShuffleVector * instruction)5271 static LLVMValueRef ir_render_shuffle_vector(CodeGen *g, Stage1Air *executable, Stage1AirInstShuffleVector *instruction) {
5272 uint64_t len_a = instruction->a->value->type->data.vector.len;
5273 uint64_t len_mask = instruction->mask->value->type->data.vector.len;
5274
5275 // LLVM uses integers larger than the length of the first array to
5276 // index into the second array. This was deemed unnecessarily fragile
5277 // when changing code, so Zig uses negative numbers to index the
5278 // second vector. These start at -1 and go down, and are easiest to use
5279 // with the ~ operator. Here we convert between the two formats.
5280 Stage1AirInst *mask = instruction->mask;
5281 LLVMValueRef *values = heap::c_allocator.allocate<LLVMValueRef>(len_mask);
5282 for (uint64_t i = 0; i < len_mask; i++) {
5283 if (mask->value->data.x_array.data.s_none.elements[i].special == ConstValSpecialUndef) {
5284 values[i] = LLVMGetUndef(LLVMInt32Type());
5285 } else {
5286 int32_t v = bigint_as_signed(&mask->value->data.x_array.data.s_none.elements[i].data.x_bigint);
5287 uint32_t index_val = (v >= 0) ? (uint32_t)v : (uint32_t)~v + (uint32_t)len_a;
5288 values[i] = LLVMConstInt(LLVMInt32Type(), index_val, false);
5289 }
5290 }
5291
5292 LLVMValueRef llvm_mask_value = LLVMConstVector(values, len_mask);
5293 heap::c_allocator.deallocate(values, len_mask);
5294
5295 return LLVMBuildShuffleVector(g->builder,
5296 ir_llvm_value(g, instruction->a),
5297 ir_llvm_value(g, instruction->b),
5298 llvm_mask_value, "");
5299 }
5300
ir_render_select(CodeGen * g,Stage1Air * executable,Stage1AirInstSelect * instruction)5301 static LLVMValueRef ir_render_select(CodeGen *g, Stage1Air *executable, Stage1AirInstSelect *instruction) {
5302 LLVMValueRef pred = ir_llvm_value(g, instruction->pred);
5303 LLVMValueRef a = ir_llvm_value(g, instruction->a);
5304 LLVMValueRef b = ir_llvm_value(g, instruction->b);
5305 return LLVMBuildSelect(g->builder, pred, a, b, "");
5306 }
5307
ir_render_splat(CodeGen * g,Stage1Air * executable,Stage1AirInstSplat * instruction)5308 static LLVMValueRef ir_render_splat(CodeGen *g, Stage1Air *executable, Stage1AirInstSplat *instruction) {
5309 ZigType *result_type = instruction->base.value->type;
5310 ir_assert(result_type->id == ZigTypeIdVector, &instruction->base);
5311 uint32_t len = result_type->data.vector.len;
5312 LLVMTypeRef op_llvm_type = LLVMVectorType(get_llvm_type(g, instruction->scalar->value->type), 1);
5313 LLVMTypeRef mask_llvm_type = LLVMVectorType(LLVMInt32Type(), len);
5314 LLVMValueRef undef_vector = LLVMGetUndef(op_llvm_type);
5315 LLVMValueRef op_vector = LLVMBuildInsertElement(g->builder, undef_vector,
5316 ir_llvm_value(g, instruction->scalar), LLVMConstInt(LLVMInt32Type(), 0, false), "");
5317 return LLVMBuildShuffleVector(g->builder, op_vector, undef_vector, LLVMConstNull(mask_llvm_type), "");
5318 }
5319
ir_render_pop_count(CodeGen * g,Stage1Air * executable,Stage1AirInstPopCount * instruction)5320 static LLVMValueRef ir_render_pop_count(CodeGen *g, Stage1Air *executable, Stage1AirInstPopCount *instruction) {
5321 ZigType *int_type = instruction->op->value->type;
5322 LLVMValueRef fn_val = get_int_builtin_fn(g, int_type, BuiltinFnIdPopCount);
5323 LLVMValueRef operand = ir_llvm_value(g, instruction->op);
5324 LLVMValueRef wrong_size_int = LLVMBuildCall(g->builder, fn_val, &operand, 1, "");
5325 return gen_widen_or_shorten(g, false, int_type, instruction->base.value->type, wrong_size_int);
5326 }
5327
ir_render_switch_br(CodeGen * g,Stage1Air * executable,Stage1AirInstSwitchBr * instruction)5328 static LLVMValueRef ir_render_switch_br(CodeGen *g, Stage1Air *executable, Stage1AirInstSwitchBr *instruction) {
5329 ZigType *target_type = instruction->target_value->value->type;
5330 LLVMBasicBlockRef else_block = instruction->else_block->llvm_block;
5331
5332 LLVMValueRef target_value = ir_llvm_value(g, instruction->target_value);
5333 if (target_type->id == ZigTypeIdPointer) {
5334 const ZigType *usize = g->builtin_types.entry_usize;
5335 target_value = LLVMBuildPtrToInt(g->builder, target_value, usize->llvm_type, "");
5336 }
5337
5338 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, target_value, else_block,
5339 (unsigned)instruction->case_count);
5340
5341 for (size_t i = 0; i < instruction->case_count; i += 1) {
5342 Stage1AirInstSwitchBrCase *this_case = &instruction->cases[i];
5343
5344 LLVMValueRef case_value = ir_llvm_value(g, this_case->value);
5345 if (target_type->id == ZigTypeIdPointer) {
5346 const ZigType *usize = g->builtin_types.entry_usize;
5347 case_value = LLVMBuildPtrToInt(g->builder, case_value, usize->llvm_type, "");
5348 }
5349
5350 LLVMAddCase(switch_instr, case_value, this_case->block->llvm_block);
5351 }
5352
5353 return nullptr;
5354 }
5355
ir_render_phi(CodeGen * g,Stage1Air * executable,Stage1AirInstPhi * instruction)5356 static LLVMValueRef ir_render_phi(CodeGen *g, Stage1Air *executable, Stage1AirInstPhi *instruction) {
5357 if (!type_has_bits(g, instruction->base.value->type))
5358 return nullptr;
5359
5360 LLVMTypeRef phi_type;
5361 if (handle_is_ptr(g, instruction->base.value->type)) {
5362 phi_type = LLVMPointerType(get_llvm_type(g,instruction->base.value->type), 0);
5363 } else {
5364 phi_type = get_llvm_type(g, instruction->base.value->type);
5365 }
5366
5367 LLVMValueRef phi = LLVMBuildPhi(g->builder, phi_type, "");
5368 LLVMValueRef *incoming_values = heap::c_allocator.allocate<LLVMValueRef>(instruction->incoming_count);
5369 LLVMBasicBlockRef *incoming_blocks = heap::c_allocator.allocate<LLVMBasicBlockRef>(instruction->incoming_count);
5370 for (size_t i = 0; i < instruction->incoming_count; i += 1) {
5371 incoming_values[i] = ir_llvm_value(g, instruction->incoming_values[i]);
5372 incoming_blocks[i] = instruction->incoming_blocks[i]->llvm_exit_block;
5373 }
5374 LLVMAddIncoming(phi, incoming_values, incoming_blocks, (unsigned)instruction->incoming_count);
5375 heap::c_allocator.deallocate(incoming_values, instruction->incoming_count);
5376 heap::c_allocator.deallocate(incoming_blocks, instruction->incoming_count);
5377 return phi;
5378 }
5379
ir_render_ref(CodeGen * g,Stage1Air * executable,Stage1AirInstRef * instruction)5380 static LLVMValueRef ir_render_ref(CodeGen *g, Stage1Air *executable, Stage1AirInstRef *instruction) {
5381 if (!type_has_bits(g, instruction->base.value->type)) {
5382 return nullptr;
5383 }
5384 if (instruction->operand->id == Stage1AirInstIdCall) {
5385 Stage1AirInstCall *call = reinterpret_cast<Stage1AirInstCall *>(instruction->operand);
5386 if (call->result_loc != nullptr) {
5387 return ir_llvm_value(g, call->result_loc);
5388 }
5389 }
5390 LLVMValueRef value = ir_llvm_value(g, instruction->operand);
5391 if (handle_is_ptr(g, instruction->operand->value->type)) {
5392 return value;
5393 } else {
5394 LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
5395 gen_store_untyped(g, value, result_loc, 0, false);
5396 return result_loc;
5397 }
5398 }
5399
ir_render_err_name(CodeGen * g,Stage1Air * executable,Stage1AirInstErrName * instruction)5400 static LLVMValueRef ir_render_err_name(CodeGen *g, Stage1Air *executable, Stage1AirInstErrName *instruction) {
5401 assert(g->generate_error_name_table);
5402 assert(g->errors_by_index.length > 0);
5403
5404 LLVMValueRef err_val = ir_llvm_value(g, instruction->value);
5405 if (ir_want_runtime_safety(g, &instruction->base)) {
5406 LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(err_val));
5407 LLVMValueRef end_val = LLVMConstInt(LLVMTypeOf(err_val), g->errors_by_index.length, false);
5408 add_bounds_check(g, err_val, LLVMIntNE, zero, LLVMIntULT, end_val);
5409 }
5410
5411 LLVMValueRef indices[] = {
5412 LLVMConstNull(g->builtin_types.entry_usize->llvm_type),
5413 err_val,
5414 };
5415 return LLVMBuildInBoundsGEP(g->builder, g->err_name_table, indices, 2, "");
5416 }
5417
get_enum_tag_name_function(CodeGen * g,ZigType * enum_type)5418 static LLVMValueRef get_enum_tag_name_function(CodeGen *g, ZigType *enum_type) {
5419 assert(enum_type->id == ZigTypeIdEnum);
5420 if (enum_type->data.enumeration.name_function)
5421 return enum_type->data.enumeration.name_function;
5422
5423 ZigType *u8_ptr_type = get_pointer_to_type_extra2(g, g->builtin_types.entry_u8, false, false,
5424 PtrLenUnknown, get_abi_alignment(g, g->builtin_types.entry_u8), 0, 0, false,
5425 VECTOR_INDEX_NONE, nullptr, g->intern.for_zero_byte());
5426 ZigType *u8_slice_type = get_slice_type(g, u8_ptr_type);
5427 ZigType *tag_int_type = enum_type->data.enumeration.tag_int_type;
5428
5429 LLVMTypeRef tag_int_llvm_type = get_llvm_type(g, tag_int_type);
5430 LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMPointerType(get_llvm_type(g, u8_slice_type), 0),
5431 &tag_int_llvm_type, 1, false);
5432
5433 const char *fn_name = get_mangled_name(g,
5434 buf_ptr(buf_sprintf("__zig_tag_name_%s", buf_ptr(&enum_type->name))));
5435 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type_ref);
5436 LLVMSetLinkage(fn_val, LLVMInternalLinkage);
5437 ZigLLVMFunctionSetCallingConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
5438 add_common_fn_attributes(g, fn_val);
5439 if (!g->omit_frame_pointer) {
5440 ZigLLVMAddFunctionAttr(fn_val, "frame-pointer", "all");
5441 }
5442
5443 LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
5444 LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
5445 ZigFn *prev_cur_fn = g->cur_fn;
5446 LLVMValueRef prev_cur_fn_val = g->cur_fn_val;
5447
5448 LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
5449 LLVMPositionBuilderAtEnd(g->builder, entry_block);
5450 ZigLLVMClearCurrentDebugLocation(g->builder);
5451 g->cur_fn = nullptr;
5452 g->cur_fn_val = fn_val;
5453
5454 size_t field_count = enum_type->data.enumeration.src_field_count;
5455 LLVMBasicBlockRef bad_value_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadValue");
5456 LLVMValueRef tag_int_value = LLVMGetParam(fn_val, 0);
5457 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, tag_int_value, bad_value_block, field_count);
5458
5459
5460 ZigType *usize = g->builtin_types.entry_usize;
5461 LLVMValueRef array_ptr_indices[] = {
5462 LLVMConstNull(usize->llvm_type),
5463 LLVMConstNull(usize->llvm_type),
5464 };
5465
5466 HashMap<BigInt, Buf *, bigint_hash, bigint_eql> occupied_tag_values = {};
5467 occupied_tag_values.init(field_count);
5468
5469 for (size_t field_i = 0; field_i < field_count; field_i += 1) {
5470 TypeEnumField *type_enum_field = &enum_type->data.enumeration.fields[field_i];
5471
5472 Buf *name = type_enum_field->name;
5473 auto entry = occupied_tag_values.put_unique(type_enum_field->value, name);
5474 if (entry != nullptr) {
5475 continue;
5476 }
5477
5478 LLVMValueRef str_init = LLVMConstString(buf_ptr(name), (unsigned)buf_len(name), false);
5479 LLVMValueRef str_global = LLVMAddGlobal(g->module, LLVMTypeOf(str_init), "");
5480 LLVMSetInitializer(str_global, str_init);
5481 LLVMSetLinkage(str_global, LLVMPrivateLinkage);
5482 LLVMSetGlobalConstant(str_global, true);
5483 LLVMSetUnnamedAddr(str_global, true);
5484 LLVMSetAlignment(str_global, LLVMABIAlignmentOfType(g->target_data_ref, LLVMTypeOf(str_init)));
5485
5486 LLVMValueRef fields[] = {
5487 LLVMConstGEP(str_global, array_ptr_indices, 2),
5488 LLVMConstInt(g->builtin_types.entry_usize->llvm_type, buf_len(name), false),
5489 };
5490 LLVMValueRef slice_init_value = LLVMConstNamedStruct(get_llvm_type(g, u8_slice_type), fields, 2);
5491
5492 LLVMValueRef slice_global = LLVMAddGlobal(g->module, LLVMTypeOf(slice_init_value), "");
5493 LLVMSetInitializer(slice_global, slice_init_value);
5494 LLVMSetLinkage(slice_global, LLVMPrivateLinkage);
5495 LLVMSetGlobalConstant(slice_global, true);
5496 LLVMSetUnnamedAddr(slice_global, true);
5497 LLVMSetAlignment(slice_global, LLVMABIAlignmentOfType(g->target_data_ref, LLVMTypeOf(slice_init_value)));
5498
5499 LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(g->cur_fn_val, "Name");
5500 LLVMValueRef this_tag_int_value = bigint_to_llvm_const(get_llvm_type(g, tag_int_type),
5501 &enum_type->data.enumeration.fields[field_i].value);
5502 LLVMAddCase(switch_instr, this_tag_int_value, return_block);
5503
5504 LLVMPositionBuilderAtEnd(g->builder, return_block);
5505 LLVMBuildRet(g->builder, slice_global);
5506 }
5507 occupied_tag_values.deinit();
5508
5509 LLVMPositionBuilderAtEnd(g->builder, bad_value_block);
5510 if (g->build_mode == BuildModeDebug || g->build_mode == BuildModeSafeRelease) {
5511 gen_safety_crash(g, PanicMsgIdBadEnumValue);
5512 } else {
5513 LLVMBuildUnreachable(g->builder);
5514 }
5515
5516 g->cur_fn = prev_cur_fn;
5517 g->cur_fn_val = prev_cur_fn_val;
5518 LLVMPositionBuilderAtEnd(g->builder, prev_block);
5519 if (!g->strip_debug_symbols) {
5520 LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
5521 }
5522
5523 enum_type->data.enumeration.name_function = fn_val;
5524 return fn_val;
5525 }
5526
ir_render_enum_tag_name(CodeGen * g,Stage1Air * executable,Stage1AirInstTagName * instruction)5527 static LLVMValueRef ir_render_enum_tag_name(CodeGen *g, Stage1Air *executable,
5528 Stage1AirInstTagName *instruction)
5529 {
5530 ZigType *enum_type = instruction->target->value->type;
5531 assert(enum_type->id == ZigTypeIdEnum);
5532
5533 LLVMValueRef enum_name_function = get_enum_tag_name_function(g, enum_type);
5534
5535 LLVMValueRef enum_tag_value = ir_llvm_value(g, instruction->target);
5536 return ZigLLVMBuildCall(g->builder, enum_name_function, &enum_tag_value, 1,
5537 get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_CallAttrAuto, "");
5538 }
5539
ir_render_field_parent_ptr(CodeGen * g,Stage1Air * executable,Stage1AirInstFieldParentPtr * instruction)5540 static LLVMValueRef ir_render_field_parent_ptr(CodeGen *g, Stage1Air *executable,
5541 Stage1AirInstFieldParentPtr *instruction)
5542 {
5543 ZigType *container_ptr_type = instruction->base.value->type;
5544 assert(container_ptr_type->id == ZigTypeIdPointer);
5545
5546 ZigType *container_type = container_ptr_type->data.pointer.child_type;
5547
5548 size_t byte_offset = LLVMOffsetOfElement(g->target_data_ref,
5549 get_llvm_type(g, container_type), instruction->field->gen_index);
5550
5551 LLVMValueRef field_ptr_val = ir_llvm_value(g, instruction->field_ptr);
5552
5553 if (byte_offset == 0) {
5554 return LLVMBuildBitCast(g->builder, field_ptr_val, get_llvm_type(g, container_ptr_type), "");
5555 } else {
5556 ZigType *usize = g->builtin_types.entry_usize;
5557
5558 LLVMValueRef field_ptr_int = LLVMBuildPtrToInt(g->builder, field_ptr_val, usize->llvm_type, "");
5559
5560 LLVMValueRef base_ptr_int = LLVMBuildNUWSub(g->builder, field_ptr_int,
5561 LLVMConstInt(usize->llvm_type, byte_offset, false), "");
5562
5563 return LLVMBuildIntToPtr(g->builder, base_ptr_int, get_llvm_type(g, container_ptr_type), "");
5564 }
5565 }
5566
ir_render_align_cast(CodeGen * g,Stage1Air * executable,Stage1AirInstAlignCast * instruction)5567 static LLVMValueRef ir_render_align_cast(CodeGen *g, Stage1Air *executable, Stage1AirInstAlignCast *instruction) {
5568 LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
5569 assert(target_val);
5570
5571 bool want_runtime_safety = ir_want_runtime_safety(g, &instruction->base);
5572 if (!want_runtime_safety) {
5573 return target_val;
5574 }
5575
5576 ZigType *target_type = instruction->base.value->type;
5577 uint32_t align_bytes;
5578 LLVMValueRef ptr_val;
5579
5580 if (target_type->id == ZigTypeIdPointer) {
5581 align_bytes = get_ptr_align(g, target_type);
5582 ptr_val = target_val;
5583 } else if (target_type->id == ZigTypeIdFn) {
5584 align_bytes = target_type->data.fn.fn_type_id.alignment;
5585 ptr_val = target_val;
5586 } else if (target_type->id == ZigTypeIdOptional &&
5587 target_type->data.maybe.child_type->id == ZigTypeIdPointer)
5588 {
5589 align_bytes = get_ptr_align(g, target_type->data.maybe.child_type);
5590 ptr_val = target_val;
5591 } else if (target_type->id == ZigTypeIdOptional &&
5592 target_type->data.maybe.child_type->id == ZigTypeIdFn)
5593 {
5594 align_bytes = target_type->data.maybe.child_type->data.fn.fn_type_id.alignment;
5595 ptr_val = target_val;
5596 } else if (target_type->id == ZigTypeIdStruct &&
5597 target_type->data.structure.special == StructSpecialSlice)
5598 {
5599 ZigType *slice_ptr_type = target_type->data.structure.fields[slice_ptr_index]->type_entry;
5600 align_bytes = get_ptr_align(g, slice_ptr_type);
5601
5602 size_t ptr_index = target_type->data.structure.fields[slice_ptr_index]->gen_index;
5603 LLVMValueRef ptr_val_ptr = LLVMBuildStructGEP(g->builder, target_val, (unsigned)ptr_index, "");
5604 ptr_val = gen_load_untyped(g, ptr_val_ptr, 0, false, "");
5605 } else {
5606 zig_unreachable();
5607 }
5608
5609 assert(align_bytes != 1);
5610
5611 ZigType *usize = g->builtin_types.entry_usize;
5612 LLVMValueRef ptr_as_int_val = LLVMBuildPtrToInt(g->builder, ptr_val, usize->llvm_type, "");
5613 LLVMValueRef alignment_minus_1 = LLVMConstInt(usize->llvm_type, align_bytes - 1, false);
5614 LLVMValueRef anded_val = LLVMBuildAnd(g->builder, ptr_as_int_val, alignment_minus_1, "");
5615 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, anded_val, LLVMConstNull(usize->llvm_type), "");
5616
5617 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "AlignCastOk");
5618 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "AlignCastFail");
5619
5620 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
5621
5622 LLVMPositionBuilderAtEnd(g->builder, fail_block);
5623 gen_safety_crash(g, PanicMsgIdIncorrectAlignment);
5624
5625 LLVMPositionBuilderAtEnd(g->builder, ok_block);
5626
5627 return target_val;
5628 }
5629
ir_render_error_return_trace(CodeGen * g,Stage1Air * executable,Stage1AirInstErrorReturnTrace * instruction)5630 static LLVMValueRef ir_render_error_return_trace(CodeGen *g, Stage1Air *executable,
5631 Stage1AirInstErrorReturnTrace *instruction)
5632 {
5633 bool is_llvm_alloca;
5634 LLVMValueRef cur_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope, &is_llvm_alloca);
5635 if (cur_err_ret_trace_val == nullptr) {
5636 return LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g)));
5637 }
5638 return cur_err_ret_trace_val;
5639 }
5640
to_LLVMAtomicOrdering(AtomicOrder atomic_order)5641 static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) {
5642 switch (atomic_order) {
5643 case AtomicOrderUnordered: return LLVMAtomicOrderingUnordered;
5644 case AtomicOrderMonotonic: return LLVMAtomicOrderingMonotonic;
5645 case AtomicOrderAcquire: return LLVMAtomicOrderingAcquire;
5646 case AtomicOrderRelease: return LLVMAtomicOrderingRelease;
5647 case AtomicOrderAcqRel: return LLVMAtomicOrderingAcquireRelease;
5648 case AtomicOrderSeqCst: return LLVMAtomicOrderingSequentiallyConsistent;
5649 }
5650 zig_unreachable();
5651 }
5652
to_LLVMAtomicRMWBinOp(AtomicRmwOp op,bool is_signed,bool is_float)5653 static LLVMAtomicRMWBinOp to_LLVMAtomicRMWBinOp(AtomicRmwOp op, bool is_signed, bool is_float) {
5654 switch (op) {
5655 case AtomicRmwOp_xchg: return LLVMAtomicRMWBinOpXchg;
5656 case AtomicRmwOp_add:
5657 return is_float ? LLVMAtomicRMWBinOpFAdd : LLVMAtomicRMWBinOpAdd;
5658 case AtomicRmwOp_sub:
5659 return is_float ? LLVMAtomicRMWBinOpFSub : LLVMAtomicRMWBinOpSub;
5660 case AtomicRmwOp_and: return LLVMAtomicRMWBinOpAnd;
5661 case AtomicRmwOp_nand: return LLVMAtomicRMWBinOpNand;
5662 case AtomicRmwOp_or: return LLVMAtomicRMWBinOpOr;
5663 case AtomicRmwOp_xor: return LLVMAtomicRMWBinOpXor;
5664 case AtomicRmwOp_max:
5665 return is_signed ? LLVMAtomicRMWBinOpMax : LLVMAtomicRMWBinOpUMax;
5666 case AtomicRmwOp_min:
5667 return is_signed ? LLVMAtomicRMWBinOpMin : LLVMAtomicRMWBinOpUMin;
5668 }
5669 zig_unreachable();
5670 }
5671
get_atomic_abi_type(CodeGen * g,Stage1AirInst * instruction,bool RMWXchg)5672 static LLVMTypeRef get_atomic_abi_type(CodeGen *g, Stage1AirInst *instruction, bool RMWXchg) {
5673 // If the operand type of an atomic operation is not byte sized we need to
5674 // widen it before using it and then truncate the result.
5675 // RMW exchange of floating-point values is bitcasted to same-sized integer
5676 // types to work around a LLVM deficiency when targeting ARM/AArch64.
5677
5678 ir_assert(instruction->value->type->id == ZigTypeIdPointer, instruction);
5679 ZigType *operand_type = instruction->value->type->data.pointer.child_type;
5680 if (operand_type->id == ZigTypeIdInt || operand_type->id == ZigTypeIdEnum) {
5681 if (operand_type->id == ZigTypeIdEnum) {
5682 operand_type = operand_type->data.enumeration.tag_int_type;
5683 }
5684 auto bit_count = operand_type->data.integral.bit_count;
5685 bool is_signed = operand_type->data.integral.is_signed;
5686
5687 ir_assert(bit_count != 0, instruction);
5688 if (!is_power_of_2(bit_count) || bit_count % 8) {
5689 return get_llvm_type(g, get_int_type(g, is_signed, operand_type->abi_size * 8));
5690 } else {
5691 return nullptr;
5692 }
5693 } else if (operand_type->id == ZigTypeIdFloat) {
5694 return RMWXchg ? LLVMIntType(operand_type->abi_size * 8) : nullptr;
5695 } else if (operand_type->id == ZigTypeIdBool) {
5696 return g->builtin_types.entry_u8->llvm_type;
5697 } else {
5698 ir_assert(get_codegen_ptr_type_bail(g, operand_type) != nullptr, instruction);
5699 return nullptr;
5700 }
5701 }
5702
ir_render_cmpxchg(CodeGen * g,Stage1Air * executable,Stage1AirInstCmpxchg * instruction)5703 static LLVMValueRef ir_render_cmpxchg(CodeGen *g, Stage1Air *executable, Stage1AirInstCmpxchg *instruction) {
5704 LLVMValueRef ptr_val = ir_llvm_value(g, instruction->ptr);
5705 LLVMValueRef cmp_val = ir_llvm_value(g, instruction->cmp_value);
5706 LLVMValueRef new_val = ir_llvm_value(g, instruction->new_value);
5707
5708 ZigType *operand_type = instruction->new_value->value->type;
5709 LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr, false);
5710 if (actual_abi_type != nullptr) {
5711 // operand needs widening and truncating
5712 ptr_val = LLVMBuildBitCast(g->builder, ptr_val,
5713 LLVMPointerType(actual_abi_type, 0), "");
5714 if (operand_type->data.integral.is_signed) {
5715 cmp_val = LLVMBuildSExt(g->builder, cmp_val, actual_abi_type, "");
5716 new_val = LLVMBuildSExt(g->builder, new_val, actual_abi_type, "");
5717 } else {
5718 cmp_val = LLVMBuildZExt(g->builder, cmp_val, actual_abi_type, "");
5719 new_val = LLVMBuildZExt(g->builder, new_val, actual_abi_type, "");
5720 }
5721 }
5722
5723 LLVMAtomicOrdering success_order = to_LLVMAtomicOrdering(instruction->success_order);
5724 LLVMAtomicOrdering failure_order = to_LLVMAtomicOrdering(instruction->failure_order);
5725
5726 LLVMValueRef result_val = LLVMBuildAtomicCmpXchg(g->builder, ptr_val, cmp_val, new_val,
5727 success_order, failure_order, g->is_single_threaded);
5728 LLVMSetWeak(result_val, instruction->is_weak);
5729
5730 ZigType *optional_type = instruction->base.value->type;
5731 assert(optional_type->id == ZigTypeIdOptional);
5732 ZigType *child_type = optional_type->data.maybe.child_type;
5733
5734 if (!handle_is_ptr(g, optional_type)) {
5735 LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, "");
5736 if (actual_abi_type != nullptr) {
5737 payload_val = LLVMBuildTrunc(g->builder, payload_val, get_llvm_type(g, operand_type), "");
5738 }
5739 LLVMValueRef success_bit = LLVMBuildExtractValue(g->builder, result_val, 1, "");
5740 return LLVMBuildSelect(g->builder, success_bit, LLVMConstNull(get_llvm_type(g, child_type)), payload_val, "");
5741 }
5742
5743 // When the cmpxchg is discarded, the result location will have no bits.
5744 if (!type_has_bits(g, instruction->result_loc->value->type)) {
5745 return nullptr;
5746 }
5747
5748 LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
5749 ir_assert(result_loc != nullptr, &instruction->base);
5750 ir_assert(type_has_bits(g, child_type), &instruction->base);
5751
5752 LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, "");
5753 if (actual_abi_type != nullptr) {
5754 payload_val = LLVMBuildTrunc(g->builder, payload_val, get_llvm_type(g, operand_type), "");
5755 }
5756 LLVMValueRef val_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_child_index, "");
5757 gen_assign_raw(g, val_ptr, get_pointer_to_type(g, child_type, false), payload_val);
5758
5759 LLVMValueRef success_bit = LLVMBuildExtractValue(g->builder, result_val, 1, "");
5760 LLVMValueRef nonnull_bit = LLVMBuildNot(g->builder, success_bit, "");
5761 LLVMValueRef maybe_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_null_index, "");
5762 gen_store_untyped(g, nonnull_bit, maybe_ptr, 0, false);
5763 return result_loc;
5764 }
5765
ir_render_reduce(CodeGen * g,Stage1Air * executable,Stage1AirInstReduce * instruction)5766 static LLVMValueRef ir_render_reduce(CodeGen *g, Stage1Air *executable, Stage1AirInstReduce *instruction) {
5767 LLVMValueRef value = ir_llvm_value(g, instruction->value);
5768
5769 ZigType *value_type = instruction->value->value->type;
5770 assert(value_type->id == ZigTypeIdVector);
5771 ZigType *scalar_type = value_type->data.vector.elem_type;
5772
5773 ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, &instruction->base));
5774
5775 LLVMValueRef result_val;
5776 switch (instruction->op) {
5777 case ReduceOp_and:
5778 assert(scalar_type->id == ZigTypeIdInt || scalar_type->id == ZigTypeIdBool);
5779 result_val = ZigLLVMBuildAndReduce(g->builder, value);
5780 break;
5781 case ReduceOp_or:
5782 assert(scalar_type->id == ZigTypeIdInt || scalar_type->id == ZigTypeIdBool);
5783 result_val = ZigLLVMBuildOrReduce(g->builder, value);
5784 break;
5785 case ReduceOp_xor:
5786 assert(scalar_type->id == ZigTypeIdInt || scalar_type->id == ZigTypeIdBool);
5787 result_val = ZigLLVMBuildXorReduce(g->builder, value);
5788 break;
5789 case ReduceOp_min: {
5790 if (scalar_type->id == ZigTypeIdInt) {
5791 const bool is_signed = scalar_type->data.integral.is_signed;
5792 result_val = ZigLLVMBuildIntMinReduce(g->builder, value, is_signed);
5793 } else if (scalar_type->id == ZigTypeIdFloat) {
5794 result_val = ZigLLVMBuildFPMinReduce(g->builder, value);
5795 } else zig_unreachable();
5796 } break;
5797 case ReduceOp_max: {
5798 if (scalar_type->id == ZigTypeIdInt) {
5799 const bool is_signed = scalar_type->data.integral.is_signed;
5800 result_val = ZigLLVMBuildIntMaxReduce(g->builder, value, is_signed);
5801 } else if (scalar_type->id == ZigTypeIdFloat) {
5802 result_val = ZigLLVMBuildFPMaxReduce(g->builder, value);
5803 } else zig_unreachable();
5804 } break;
5805 case ReduceOp_add: {
5806 if (scalar_type->id == ZigTypeIdInt) {
5807 result_val = ZigLLVMBuildAddReduce(g->builder, value);
5808 } else if (scalar_type->id == ZigTypeIdFloat) {
5809 LLVMValueRef neutral_value = LLVMConstReal(
5810 get_llvm_type(g, scalar_type), -0.0);
5811 result_val = ZigLLVMBuildFPAddReduce(g->builder, neutral_value, value);
5812 } else zig_unreachable();
5813 } break;
5814 case ReduceOp_mul: {
5815 if (scalar_type->id == ZigTypeIdInt) {
5816 result_val = ZigLLVMBuildMulReduce(g->builder, value);
5817 } else if (scalar_type->id == ZigTypeIdFloat) {
5818 LLVMValueRef neutral_value = LLVMConstReal(
5819 get_llvm_type(g, scalar_type), 1.0);
5820 result_val = ZigLLVMBuildFPMulReduce(g->builder, neutral_value, value);
5821 } else zig_unreachable();
5822 } break;
5823 default:
5824 zig_unreachable();
5825 }
5826
5827 return result_val;
5828 }
5829
ir_render_fence(CodeGen * g,Stage1Air * executable,Stage1AirInstFence * instruction)5830 static LLVMValueRef ir_render_fence(CodeGen *g, Stage1Air *executable, Stage1AirInstFence *instruction) {
5831 LLVMAtomicOrdering atomic_order = to_LLVMAtomicOrdering(instruction->order);
5832 LLVMBuildFence(g->builder, atomic_order, false, "");
5833 return nullptr;
5834 }
5835
ir_render_truncate(CodeGen * g,Stage1Air * executable,Stage1AirInstTruncate * instruction)5836 static LLVMValueRef ir_render_truncate(CodeGen *g, Stage1Air *executable, Stage1AirInstTruncate *instruction) {
5837 LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
5838 ZigType *dest_type = instruction->base.value->type;
5839 ZigType *src_type = instruction->target->value->type;
5840 if (dest_type == src_type) {
5841 // no-op
5842 return target_val;
5843 } if (src_type->data.integral.bit_count == dest_type->data.integral.bit_count) {
5844 return LLVMBuildBitCast(g->builder, target_val, get_llvm_type(g, dest_type), "");
5845 } else {
5846 LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
5847 return LLVMBuildTrunc(g->builder, target_val, get_llvm_type(g, dest_type), "");
5848 }
5849 }
5850
ir_render_memset(CodeGen * g,Stage1Air * executable,Stage1AirInstMemset * instruction)5851 static LLVMValueRef ir_render_memset(CodeGen *g, Stage1Air *executable, Stage1AirInstMemset *instruction) {
5852 LLVMValueRef dest_ptr = ir_llvm_value(g, instruction->dest_ptr);
5853 LLVMValueRef len_val = ir_llvm_value(g, instruction->count);
5854
5855 LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
5856 LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, dest_ptr, ptr_u8, "");
5857
5858 ZigType *ptr_type = instruction->dest_ptr->value->type;
5859 assert(ptr_type->id == ZigTypeIdPointer);
5860
5861 bool val_is_undef = value_is_all_undef(g, instruction->byte->value);
5862 LLVMValueRef fill_char;
5863 if (val_is_undef) {
5864 if (ir_want_runtime_safety_scope(g, instruction->base.scope)) {
5865 fill_char = LLVMConstInt(LLVMInt8Type(), 0xaa, false);
5866 } else {
5867 return nullptr;
5868 }
5869 } else {
5870 fill_char = ir_llvm_value(g, instruction->byte);
5871 }
5872 ZigLLVMBuildMemSet(g->builder, dest_ptr_casted, fill_char, len_val, get_ptr_align(g, ptr_type),
5873 ptr_type->data.pointer.is_volatile);
5874
5875 if (val_is_undef && g->valgrind_enabled) {
5876 gen_valgrind_undef(g, dest_ptr_casted, len_val);
5877 }
5878 return nullptr;
5879 }
5880
ir_render_memcpy(CodeGen * g,Stage1Air * executable,Stage1AirInstMemcpy * instruction)5881 static LLVMValueRef ir_render_memcpy(CodeGen *g, Stage1Air *executable, Stage1AirInstMemcpy *instruction) {
5882 LLVMValueRef dest_ptr = ir_llvm_value(g, instruction->dest_ptr);
5883 LLVMValueRef src_ptr = ir_llvm_value(g, instruction->src_ptr);
5884 LLVMValueRef len_val = ir_llvm_value(g, instruction->count);
5885
5886 LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
5887
5888 LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, dest_ptr, ptr_u8, "");
5889 LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, src_ptr, ptr_u8, "");
5890
5891 ZigType *dest_ptr_type = instruction->dest_ptr->value->type;
5892 ZigType *src_ptr_type = instruction->src_ptr->value->type;
5893
5894 assert(dest_ptr_type->id == ZigTypeIdPointer);
5895 assert(src_ptr_type->id == ZigTypeIdPointer);
5896
5897 bool is_volatile = (dest_ptr_type->data.pointer.is_volatile || src_ptr_type->data.pointer.is_volatile);
5898 ZigLLVMBuildMemCpy(g->builder, dest_ptr_casted, get_ptr_align(g, dest_ptr_type),
5899 src_ptr_casted, get_ptr_align(g, src_ptr_type), len_val, is_volatile);
5900 return nullptr;
5901 }
5902
ir_render_wasm_memory_size(CodeGen * g,Stage1Air * executable,Stage1AirInstWasmMemorySize * instruction)5903 static LLVMValueRef ir_render_wasm_memory_size(CodeGen *g, Stage1Air *executable, Stage1AirInstWasmMemorySize *instruction) {
5904 // TODO adjust for wasm64
5905 LLVMValueRef param = ir_llvm_value(g, instruction->index);
5906 LLVMValueRef val = LLVMBuildCall(g->builder, gen_wasm_memory_size(g), ¶m, 1, "");
5907 return val;
5908 }
5909
ir_render_wasm_memory_grow(CodeGen * g,Stage1Air * executable,Stage1AirInstWasmMemoryGrow * instruction)5910 static LLVMValueRef ir_render_wasm_memory_grow(CodeGen *g, Stage1Air *executable, Stage1AirInstWasmMemoryGrow *instruction) {
5911 // TODO adjust for wasm64
5912 LLVMValueRef params[] = {
5913 ir_llvm_value(g, instruction->index),
5914 ir_llvm_value(g, instruction->delta),
5915 };
5916 LLVMValueRef val = LLVMBuildCall(g->builder, gen_wasm_memory_grow(g), params, 2, "");
5917 return val;
5918 }
5919
ir_render_prefetch(CodeGen * g,Stage1Air * executable,Stage1AirInstPrefetch * instruction)5920 static LLVMValueRef ir_render_prefetch(CodeGen *g, Stage1Air *executable, Stage1AirInstPrefetch *instruction) {
5921 static_assert(PrefetchRwRead == 0, "");
5922 static_assert(PrefetchRwWrite == 1, "");
5923 assert(instruction->rw == PrefetchRwRead || instruction->rw == PrefetchRwWrite);
5924
5925 assert(instruction->locality >= 0 && instruction->locality <= 3);
5926
5927 static_assert(PrefetchCacheInstruction == 0, "");
5928 static_assert(PrefetchCacheData == 1, "");
5929 assert(instruction->cache == PrefetchCacheData || instruction->cache == PrefetchCacheInstruction);
5930
5931 // LLVM fails during codegen of instruction cache prefetchs for these architectures.
5932 // This is an LLVM bug as the prefetch intrinsic should be a noop if not supported by the target.
5933 // To work around this, simply don't emit llvm.prefetch in this case.
5934 // See https://bugs.llvm.org/show_bug.cgi?id=21037
5935 if (instruction->cache == PrefetchCacheInstruction) {
5936 switch (g->zig_target->arch) {
5937 case ZigLLVM_x86:
5938 case ZigLLVM_x86_64:
5939 return nullptr;
5940 default:
5941 break;
5942 }
5943 }
5944
5945 // Another case of the same LLVM bug described above
5946 if (instruction->rw == PrefetchRwWrite && instruction->cache == PrefetchCacheInstruction) {
5947 switch (g->zig_target->arch) {
5948 case ZigLLVM_arm:
5949 return nullptr;
5950 default:
5951 break;
5952 }
5953
5954 }
5955
5956 LLVMValueRef params[] = {
5957 LLVMBuildBitCast(g->builder, ir_llvm_value(g, instruction->ptr), LLVMPointerType(LLVMInt8Type(), 0), ""),
5958 LLVMConstInt(LLVMInt32Type(), instruction->rw, false),
5959 LLVMConstInt(LLVMInt32Type(), instruction->locality, false),
5960 LLVMConstInt(LLVMInt32Type(), instruction->cache, false),
5961 };
5962 LLVMValueRef val = LLVMBuildCall(g->builder, gen_prefetch(g), params, 4, "");
5963 return val;
5964 }
5965
ir_render_slice(CodeGen * g,Stage1Air * executable,Stage1AirInstSlice * instruction)5966 static LLVMValueRef ir_render_slice(CodeGen *g, Stage1Air *executable, Stage1AirInstSlice *instruction) {
5967 Error err;
5968
5969 LLVMValueRef array_ptr_ptr = ir_llvm_value(g, instruction->ptr);
5970 ZigType *array_ptr_type = instruction->ptr->value->type;
5971 assert(array_ptr_type->id == ZigTypeIdPointer);
5972 ZigType *array_type = array_ptr_type->data.pointer.child_type;
5973 LLVMValueRef array_ptr = get_handle_value(g, array_ptr_ptr, array_type, array_ptr_type);
5974
5975 bool want_runtime_safety = instruction->safety_check_on && ir_want_runtime_safety(g, &instruction->base);
5976
5977 // The result is either a slice or a pointer to an array
5978 ZigType *result_type = instruction->base.value->type;
5979
5980 // This is not whether the result type has a sentinel, but whether there should be a sentinel check,
5981 // e.g. if they used [a..b :s] syntax.
5982 ZigValue *sentinel = instruction->sentinel;
5983
5984 LLVMValueRef slice_start_ptr = nullptr;
5985 LLVMValueRef len_value = nullptr;
5986
5987 if (array_type->id == ZigTypeIdArray ||
5988 (array_type->id == ZigTypeIdPointer && array_type->data.pointer.ptr_len == PtrLenSingle))
5989 {
5990 if (array_type->id == ZigTypeIdPointer) {
5991 array_type = array_type->data.pointer.child_type;
5992 }
5993 LLVMValueRef start_val = ir_llvm_value(g, instruction->start);
5994 LLVMValueRef end_val;
5995 if (instruction->end) {
5996 end_val = ir_llvm_value(g, instruction->end);
5997 } else {
5998 end_val = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, array_type->data.array.len, false);
5999 }
6000
6001 if (want_runtime_safety) {
6002 // Safety check: start <= end
6003 if (instruction->start->value->special == ConstValSpecialRuntime || instruction->end) {
6004 add_bounds_check(g, start_val, LLVMIntEQ, nullptr, LLVMIntULE, end_val);
6005 }
6006
6007 // Safety check: the last element of the slice (the sentinel if
6008 // requested) must be inside the array
6009 // XXX: Overflow is not checked here...
6010 const size_t full_len = array_type->data.array.len +
6011 (array_type->data.array.sentinel != nullptr);
6012 LLVMValueRef array_end = LLVMConstInt(g->builtin_types.entry_usize->llvm_type,
6013 full_len, false);
6014
6015 LLVMValueRef check_end_val = end_val;
6016 if (sentinel != nullptr) {
6017 LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 1, false);
6018 check_end_val = LLVMBuildNUWAdd(g->builder, end_val, usize_one, "");
6019 }
6020 add_bounds_check(g, check_end_val, LLVMIntEQ, nullptr, LLVMIntULE, array_end);
6021 }
6022
6023 bool value_has_bits;
6024 if ((err = type_has_bits2(g, array_type, &value_has_bits)))
6025 codegen_report_errors_and_exit(g);
6026
6027 if (value_has_bits) {
6028 if (want_runtime_safety && sentinel != nullptr) {
6029 LLVMValueRef indices[] = {
6030 LLVMConstNull(g->builtin_types.entry_usize->llvm_type),
6031 end_val,
6032 };
6033 LLVMValueRef sentinel_elem_ptr = LLVMBuildInBoundsGEP(g->builder, array_ptr, indices, 2, "");
6034 add_sentinel_check(g, sentinel_elem_ptr, sentinel);
6035 }
6036
6037 LLVMValueRef indices[] = {
6038 LLVMConstNull(g->builtin_types.entry_usize->llvm_type),
6039 start_val,
6040 };
6041 slice_start_ptr = LLVMBuildInBoundsGEP(g->builder, array_ptr, indices, 2, "");
6042 }
6043
6044 len_value = LLVMBuildNUWSub(g->builder, end_val, start_val, "");
6045 } else if (array_type->id == ZigTypeIdPointer) {
6046 assert(array_type->data.pointer.ptr_len != PtrLenSingle);
6047 LLVMValueRef start_val = ir_llvm_value(g, instruction->start);
6048 LLVMValueRef end_val = ir_llvm_value(g, instruction->end);
6049
6050 if (want_runtime_safety) {
6051 // Safety check: start <= end
6052 add_bounds_check(g, start_val, LLVMIntEQ, nullptr, LLVMIntULE, end_val);
6053 }
6054
6055 bool value_has_bits;
6056 if ((err = type_has_bits2(g, array_type, &value_has_bits)))
6057 codegen_report_errors_and_exit(g);
6058
6059 if (value_has_bits) {
6060 if (want_runtime_safety && sentinel != nullptr) {
6061 LLVMValueRef sentinel_elem_ptr = LLVMBuildInBoundsGEP(g->builder, array_ptr, &end_val, 1, "");
6062 add_sentinel_check(g, sentinel_elem_ptr, sentinel);
6063 }
6064
6065 slice_start_ptr = LLVMBuildInBoundsGEP(g->builder, array_ptr, &start_val, 1, "");
6066 }
6067
6068 len_value = LLVMBuildNUWSub(g->builder, end_val, start_val, "");
6069 } else if (array_type->id == ZigTypeIdStruct) {
6070 assert(array_type->data.structure.special == StructSpecialSlice);
6071 assert(LLVMGetTypeKind(LLVMTypeOf(array_ptr)) == LLVMPointerTypeKind);
6072 assert(LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(array_ptr))) == LLVMStructTypeKind);
6073
6074 const size_t gen_len_index = array_type->data.structure.fields[slice_len_index]->gen_index;
6075 assert(gen_len_index != SIZE_MAX);
6076
6077 LLVMValueRef prev_end = nullptr;
6078 if (!instruction->end || want_runtime_safety) {
6079 LLVMValueRef src_len_ptr = LLVMBuildStructGEP(g->builder, array_ptr, gen_len_index, "");
6080 prev_end = gen_load_untyped(g, src_len_ptr, 0, false, "");
6081 }
6082
6083 LLVMValueRef start_val = ir_llvm_value(g, instruction->start);
6084 LLVMValueRef end_val;
6085 if (instruction->end) {
6086 end_val = ir_llvm_value(g, instruction->end);
6087 } else {
6088 end_val = prev_end;
6089 }
6090
6091 ZigType *ptr_field_type = array_type->data.structure.fields[slice_ptr_index]->type_entry;
6092
6093 if (want_runtime_safety) {
6094 assert(prev_end);
6095 // Safety check: start <= end
6096 add_bounds_check(g, start_val, LLVMIntEQ, nullptr, LLVMIntULE, end_val);
6097
6098 // Safety check: the sentinel counts as one more element
6099 // XXX: Overflow is not checked here...
6100 LLVMValueRef check_prev_end = prev_end;
6101 if (ptr_field_type->data.pointer.sentinel != nullptr) {
6102 LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 1, false);
6103 check_prev_end = LLVMBuildNUWAdd(g->builder, prev_end, usize_one, "");
6104 }
6105 LLVMValueRef check_end_val = end_val;
6106 if (sentinel != nullptr) {
6107 LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 1, false);
6108 check_end_val = LLVMBuildNUWAdd(g->builder, end_val, usize_one, "");
6109 }
6110
6111 add_bounds_check(g, check_end_val, LLVMIntEQ, nullptr, LLVMIntULE, check_prev_end);
6112 }
6113
6114 bool ptr_has_bits;
6115 if ((err = type_has_bits2(g, ptr_field_type, &ptr_has_bits)))
6116 codegen_report_errors_and_exit(g);
6117
6118 if (ptr_has_bits) {
6119 const size_t gen_ptr_index = array_type->data.structure.fields[slice_ptr_index]->gen_index;
6120 assert(gen_ptr_index != SIZE_MAX);
6121
6122 LLVMValueRef src_ptr_ptr = LLVMBuildStructGEP(g->builder, array_ptr, gen_ptr_index, "");
6123 LLVMValueRef src_ptr = gen_load_untyped(g, src_ptr_ptr, 0, false, "");
6124
6125 if (sentinel != nullptr) {
6126 LLVMValueRef sentinel_elem_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr, &end_val, 1, "");
6127 add_sentinel_check(g, sentinel_elem_ptr, sentinel);
6128 }
6129
6130 slice_start_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr, &start_val, 1, "");
6131 }
6132
6133 len_value = LLVMBuildNUWSub(g->builder, end_val, start_val, "");
6134 } else {
6135 zig_unreachable();
6136 }
6137
6138 bool result_has_bits;
6139 if ((err = type_has_bits2(g, result_type, &result_has_bits)))
6140 codegen_report_errors_and_exit(g);
6141
6142 // Nothing to do, we're only interested in the bound checks emitted above
6143 if (!result_has_bits)
6144 return nullptr;
6145
6146 // The starting pointer for the slice may be null in case of zero-sized
6147 // arrays, the length value is always defined.
6148 assert(len_value != nullptr);
6149
6150 // The slice decays into a pointer to an array, the size is tracked in the
6151 // type itself
6152 if (result_type->id == ZigTypeIdPointer) {
6153 ir_assert(instruction->result_loc == nullptr, &instruction->base);
6154 LLVMTypeRef result_ptr_type = get_llvm_type(g, result_type);
6155
6156 if (slice_start_ptr != nullptr) {
6157 return LLVMBuildBitCast(g->builder, slice_start_ptr, result_ptr_type, "");
6158 }
6159
6160 return LLVMGetUndef(result_ptr_type);
6161 }
6162
6163 ir_assert(instruction->result_loc != nullptr, &instruction->base);
6164 // Create a new slice
6165 LLVMValueRef tmp_struct_ptr = ir_llvm_value(g, instruction->result_loc);
6166
6167 ZigType *slice_ptr_type = result_type->data.structure.fields[slice_ptr_index]->type_entry;
6168
6169 // The slice may not have a pointer at all if it points to a zero-sized type
6170 const size_t gen_ptr_index = result_type->data.structure.fields[slice_ptr_index]->gen_index;
6171 if (gen_ptr_index != SIZE_MAX) {
6172 LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, gen_ptr_index, "");
6173 if (slice_start_ptr != nullptr) {
6174 gen_store_untyped(g, slice_start_ptr, ptr_field_ptr, 0, false);
6175 } else if (want_runtime_safety) {
6176 gen_undef_init(g, slice_ptr_type, slice_ptr_type, ptr_field_ptr);
6177 } else {
6178 gen_store_untyped(g, LLVMGetUndef(get_llvm_type(g, slice_ptr_type)), ptr_field_ptr, 0, false);
6179 }
6180 }
6181
6182 const size_t gen_len_index = result_type->data.structure.fields[slice_len_index]->gen_index;
6183 assert(gen_len_index != SIZE_MAX);
6184
6185 LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, gen_len_index, "");
6186 gen_store_untyped(g, len_value, len_field_ptr, 0, false);
6187
6188 return tmp_struct_ptr;
6189 }
6190
get_trap_fn_val(CodeGen * g)6191 static LLVMValueRef get_trap_fn_val(CodeGen *g) {
6192 if (g->trap_fn_val)
6193 return g->trap_fn_val;
6194
6195 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), nullptr, 0, false);
6196 g->trap_fn_val = LLVMAddFunction(g->module, "llvm.debugtrap", fn_type);
6197 assert(LLVMGetIntrinsicID(g->trap_fn_val));
6198
6199 return g->trap_fn_val;
6200 }
6201
6202
ir_render_breakpoint(CodeGen * g,Stage1Air * executable,Stage1AirInstBreakpoint * instruction)6203 static LLVMValueRef ir_render_breakpoint(CodeGen *g, Stage1Air *executable, Stage1AirInstBreakpoint *instruction) {
6204 LLVMBuildCall(g->builder, get_trap_fn_val(g), nullptr, 0, "");
6205 return nullptr;
6206 }
6207
ir_render_return_address(CodeGen * g,Stage1Air * executable,Stage1AirInstReturnAddress * instruction)6208 static LLVMValueRef ir_render_return_address(CodeGen *g, Stage1Air *executable,
6209 Stage1AirInstReturnAddress *instruction)
6210 {
6211 if ((target_is_wasm(g->zig_target) && g->zig_target->os != OsEmscripten) || target_is_bpf(g->zig_target)) {
6212 // I got this error from LLVM 10:
6213 // "Non-Emscripten WebAssembly hasn't implemented __builtin_return_address"
6214 return LLVMConstNull(get_llvm_type(g, instruction->base.value->type));
6215 }
6216
6217 LLVMValueRef zero = LLVMConstNull(g->builtin_types.entry_i32->llvm_type);
6218 LLVMValueRef ptr_val = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, "");
6219 return LLVMBuildPtrToInt(g->builder, ptr_val, g->builtin_types.entry_usize->llvm_type, "");
6220 }
6221
get_frame_address_fn_val(CodeGen * g)6222 static LLVMValueRef get_frame_address_fn_val(CodeGen *g) {
6223 if (g->frame_address_fn_val)
6224 return g->frame_address_fn_val;
6225
6226 ZigType *return_type = get_pointer_to_type(g, g->builtin_types.entry_u8, true);
6227
6228 LLVMTypeRef fn_type = LLVMFunctionType(get_llvm_type(g, return_type),
6229 &g->builtin_types.entry_i32->llvm_type, 1, false);
6230 g->frame_address_fn_val = LLVMAddFunction(g->module, "llvm.frameaddress.p0i8", fn_type);
6231 assert(LLVMGetIntrinsicID(g->frame_address_fn_val));
6232
6233 return g->frame_address_fn_val;
6234 }
6235
ir_render_frame_address(CodeGen * g,Stage1Air * executable,Stage1AirInstFrameAddress * instruction)6236 static LLVMValueRef ir_render_frame_address(CodeGen *g, Stage1Air *executable,
6237 Stage1AirInstFrameAddress *instruction)
6238 {
6239 LLVMValueRef zero = LLVMConstNull(g->builtin_types.entry_i32->llvm_type);
6240 LLVMValueRef ptr_val = LLVMBuildCall(g->builder, get_frame_address_fn_val(g), &zero, 1, "");
6241 return LLVMBuildPtrToInt(g->builder, ptr_val, g->builtin_types.entry_usize->llvm_type, "");
6242 }
6243
ir_render_handle(CodeGen * g,Stage1Air * executable,Stage1AirInstFrameHandle * instruction)6244 static LLVMValueRef ir_render_handle(CodeGen *g, Stage1Air *executable, Stage1AirInstFrameHandle *instruction) {
6245 return g->cur_frame_ptr;
6246 }
6247
render_shl_with_overflow(CodeGen * g,Stage1AirInstOverflowOp * instruction)6248 static LLVMValueRef render_shl_with_overflow(CodeGen *g, Stage1AirInstOverflowOp *instruction) {
6249 ZigType *int_type = instruction->result_ptr_type;
6250 assert(int_type->id == ZigTypeIdInt);
6251
6252 LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);
6253 LLVMValueRef op2 = ir_llvm_value(g, instruction->op2);
6254 LLVMValueRef ptr_result = ir_llvm_value(g, instruction->result_ptr);
6255
6256 LLVMValueRef op2_casted = gen_widen_or_shorten(g, false, instruction->op2->value->type,
6257 instruction->op1->value->type, op2);
6258
6259 LLVMValueRef result = LLVMBuildShl(g->builder, op1, op2_casted, "");
6260 LLVMValueRef orig_val;
6261 if (int_type->data.integral.is_signed) {
6262 orig_val = LLVMBuildAShr(g->builder, result, op2_casted, "");
6263 } else {
6264 orig_val = LLVMBuildLShr(g->builder, result, op2_casted, "");
6265 }
6266 LLVMValueRef overflow_bit = LLVMBuildICmp(g->builder, LLVMIntNE, op1, orig_val, "");
6267
6268 gen_store(g, result, ptr_result, instruction->result_ptr->value->type);
6269
6270 return overflow_bit;
6271 }
6272
ir_render_overflow_op(CodeGen * g,Stage1Air * executable,Stage1AirInstOverflowOp * instruction)6273 static LLVMValueRef ir_render_overflow_op(CodeGen *g, Stage1Air *executable, Stage1AirInstOverflowOp *instruction) {
6274 AddSubMul add_sub_mul;
6275 switch (instruction->op) {
6276 case IrOverflowOpAdd:
6277 add_sub_mul = AddSubMulAdd;
6278 break;
6279 case IrOverflowOpSub:
6280 add_sub_mul = AddSubMulSub;
6281 break;
6282 case IrOverflowOpMul:
6283 add_sub_mul = AddSubMulMul;
6284 break;
6285 case IrOverflowOpShl:
6286 return render_shl_with_overflow(g, instruction);
6287 }
6288
6289 ZigType *int_type = instruction->result_ptr_type;
6290 assert(int_type->id == ZigTypeIdInt);
6291
6292 LLVMValueRef fn_val = get_int_overflow_fn(g, int_type, add_sub_mul);
6293
6294 LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);
6295 LLVMValueRef op2 = ir_llvm_value(g, instruction->op2);
6296 LLVMValueRef ptr_result = ir_llvm_value(g, instruction->result_ptr);
6297
6298 LLVMValueRef params[] = {
6299 op1,
6300 op2,
6301 };
6302
6303 LLVMValueRef result_struct = LLVMBuildCall(g->builder, fn_val, params, 2, "");
6304 LLVMValueRef result = LLVMBuildExtractValue(g->builder, result_struct, 0, "");
6305 LLVMValueRef overflow_bit = LLVMBuildExtractValue(g->builder, result_struct, 1, "");
6306 gen_store(g, result, ptr_result, instruction->result_ptr->value->type);
6307
6308 return overflow_bit;
6309 }
6310
ir_render_test_err(CodeGen * g,Stage1Air * executable,Stage1AirInstTestErr * instruction)6311 static LLVMValueRef ir_render_test_err(CodeGen *g, Stage1Air *executable, Stage1AirInstTestErr *instruction) {
6312 ZigType *err_union_type = instruction->err_union->value->type;
6313 ZigType *payload_type = err_union_type->data.error_union.payload_type;
6314 LLVMValueRef err_union_handle = ir_llvm_value(g, instruction->err_union);
6315
6316 LLVMValueRef err_val;
6317 if (type_has_bits(g, payload_type)) {
6318 LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, err_union_handle, err_union_err_index, "");
6319 err_val = gen_load_untyped(g, err_val_ptr, 0, false, "");
6320 } else {
6321 err_val = err_union_handle;
6322 }
6323
6324 LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, g->err_tag_type));
6325 return LLVMBuildICmp(g->builder, LLVMIntNE, err_val, zero, "");
6326 }
6327
ir_render_unwrap_err_code(CodeGen * g,Stage1Air * executable,Stage1AirInstUnwrapErrCode * instruction)6328 static LLVMValueRef ir_render_unwrap_err_code(CodeGen *g, Stage1Air *executable,
6329 Stage1AirInstUnwrapErrCode *instruction)
6330 {
6331 if (instruction->base.value->special != ConstValSpecialRuntime)
6332 return nullptr;
6333
6334 ZigType *ptr_type = instruction->err_union_ptr->value->type;
6335 assert(ptr_type->id == ZigTypeIdPointer);
6336 ZigType *err_union_type = ptr_type->data.pointer.child_type;
6337 ZigType *payload_type = err_union_type->data.error_union.payload_type;
6338 LLVMValueRef err_union_ptr = ir_llvm_value(g, instruction->err_union_ptr);
6339 if (!type_has_bits(g, payload_type)) {
6340 return err_union_ptr;
6341 } else {
6342 // TODO assign undef to the payload
6343 LLVMValueRef err_union_handle = get_handle_value(g, err_union_ptr, err_union_type, ptr_type);
6344 return LLVMBuildStructGEP(g->builder, err_union_handle, err_union_err_index, "");
6345 }
6346 }
6347
ir_render_unwrap_err_payload(CodeGen * g,Stage1Air * executable,Stage1AirInstUnwrapErrPayload * instruction)6348 static LLVMValueRef ir_render_unwrap_err_payload(CodeGen *g, Stage1Air *executable,
6349 Stage1AirInstUnwrapErrPayload *instruction)
6350 {
6351 Error err;
6352
6353 if (instruction->base.value->special != ConstValSpecialRuntime)
6354 return nullptr;
6355
6356 bool want_safety = instruction->safety_check_on && ir_want_runtime_safety(g, &instruction->base) &&
6357 g->errors_by_index.length > 1;
6358
6359 ZigType *ptr_type = instruction->value->value->type;
6360 assert(ptr_type->id == ZigTypeIdPointer);
6361 ZigType *err_union_type = ptr_type->data.pointer.child_type;
6362 ZigType *payload_type = err_union_type->data.error_union.payload_type;
6363 LLVMValueRef err_union_ptr = ir_llvm_value(g, instruction->value);
6364
6365 LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, g->err_tag_type));
6366 bool value_has_bits;
6367 if ((err = type_has_bits2(g, instruction->base.value->type, &value_has_bits)))
6368 codegen_report_errors_and_exit(g);
6369 if (!want_safety && !value_has_bits) {
6370 if (instruction->initializing) {
6371 gen_store_untyped(g, zero, err_union_ptr, 0, false);
6372 }
6373 return nullptr;
6374 }
6375
6376
6377 LLVMValueRef err_union_handle = get_handle_value(g, err_union_ptr, err_union_type, ptr_type);
6378
6379 if (!type_has_bits(g, err_union_type->data.error_union.err_set_type)) {
6380 return err_union_handle;
6381 }
6382
6383 if (want_safety) {
6384 LLVMValueRef err_val;
6385 if (type_has_bits(g, payload_type)) {
6386 LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, err_union_handle, err_union_err_index, "");
6387 err_val = gen_load_untyped(g, err_val_ptr, 0, false, "");
6388 } else {
6389 err_val = err_union_handle;
6390 }
6391 LLVMValueRef cond_val = LLVMBuildICmp(g->builder, LLVMIntEQ, err_val, zero, "");
6392 LLVMBasicBlockRef err_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnwrapErrError");
6393 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnwrapErrOk");
6394 LLVMBuildCondBr(g->builder, cond_val, ok_block, err_block);
6395
6396 LLVMPositionBuilderAtEnd(g->builder, err_block);
6397 gen_safety_crash_for_err(g, err_val, instruction->base.scope);
6398
6399 LLVMPositionBuilderAtEnd(g->builder, ok_block);
6400 }
6401
6402 if (type_has_bits(g, payload_type)) {
6403 if (instruction->initializing) {
6404 LLVMValueRef err_tag_ptr = LLVMBuildStructGEP(g->builder, err_union_handle, err_union_err_index, "");
6405 LLVMValueRef ok_err_val = LLVMConstNull(get_llvm_type(g, g->err_tag_type));
6406 gen_store_untyped(g, ok_err_val, err_tag_ptr, 0, false);
6407 }
6408 return LLVMBuildStructGEP(g->builder, err_union_handle, err_union_payload_index, "");
6409 } else {
6410 if (instruction->initializing) {
6411 gen_store_untyped(g, zero, err_union_ptr, 0, false);
6412 }
6413 return nullptr;
6414 }
6415 }
6416
ir_render_optional_wrap(CodeGen * g,Stage1Air * executable,Stage1AirInstOptionalWrap * instruction)6417 static LLVMValueRef ir_render_optional_wrap(CodeGen *g, Stage1Air *executable, Stage1AirInstOptionalWrap *instruction) {
6418 ZigType *wanted_type = instruction->base.value->type;
6419
6420 assert(wanted_type->id == ZigTypeIdOptional);
6421
6422 ZigType *child_type = wanted_type->data.maybe.child_type;
6423
6424 if (!type_has_bits(g, child_type)) {
6425 LLVMValueRef result = LLVMConstAllOnes(LLVMInt1Type());
6426 if (instruction->result_loc != nullptr) {
6427 LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
6428 gen_store_untyped(g, result, result_loc, 0, false);
6429 }
6430 return result;
6431 }
6432
6433 LLVMValueRef payload_val = ir_llvm_value(g, instruction->operand);
6434 if (!handle_is_ptr(g, wanted_type)) {
6435 if (instruction->result_loc != nullptr) {
6436 LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
6437 gen_store_untyped(g, payload_val, result_loc, 0, false);
6438 }
6439 return payload_val;
6440 }
6441
6442 LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
6443
6444 LLVMValueRef val_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_child_index, "");
6445 // child_type and instruction->value->value->type may differ by constness
6446 gen_assign_raw(g, val_ptr, get_pointer_to_type(g, child_type, false), payload_val);
6447 LLVMValueRef maybe_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_null_index, "");
6448 gen_store_untyped(g, LLVMConstAllOnes(LLVMInt1Type()), maybe_ptr, 0, false);
6449
6450 return result_loc;
6451 }
6452
ir_render_err_wrap_code(CodeGen * g,Stage1Air * executable,Stage1AirInstErrWrapCode * instruction)6453 static LLVMValueRef ir_render_err_wrap_code(CodeGen *g, Stage1Air *executable, Stage1AirInstErrWrapCode *instruction) {
6454 ZigType *wanted_type = instruction->base.value->type;
6455
6456 assert(wanted_type->id == ZigTypeIdErrorUnion);
6457
6458 LLVMValueRef err_val = ir_llvm_value(g, instruction->operand);
6459
6460 if (!handle_is_ptr(g, wanted_type))
6461 return err_val;
6462
6463 LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
6464
6465 LLVMValueRef err_tag_ptr = LLVMBuildStructGEP(g->builder, result_loc, err_union_err_index, "");
6466 gen_store_untyped(g, err_val, err_tag_ptr, 0, false);
6467
6468 // TODO store undef to the payload
6469
6470 return result_loc;
6471 }
6472
ir_render_err_wrap_payload(CodeGen * g,Stage1Air * executable,Stage1AirInstErrWrapPayload * instruction)6473 static LLVMValueRef ir_render_err_wrap_payload(CodeGen *g, Stage1Air *executable, Stage1AirInstErrWrapPayload *instruction) {
6474 ZigType *wanted_type = instruction->base.value->type;
6475
6476 assert(wanted_type->id == ZigTypeIdErrorUnion);
6477
6478 ZigType *payload_type = wanted_type->data.error_union.payload_type;
6479 ZigType *err_set_type = wanted_type->data.error_union.err_set_type;
6480
6481 if (!type_has_bits(g, err_set_type)) {
6482 return ir_llvm_value(g, instruction->operand);
6483 }
6484
6485 LLVMValueRef ok_err_val = LLVMConstNull(get_llvm_type(g, g->err_tag_type));
6486
6487 if (!type_has_bits(g, payload_type))
6488 return ok_err_val;
6489
6490
6491 LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
6492
6493 LLVMValueRef payload_val = ir_llvm_value(g, instruction->operand);
6494
6495 LLVMValueRef err_tag_ptr = LLVMBuildStructGEP(g->builder, result_loc, err_union_err_index, "");
6496 gen_store_untyped(g, ok_err_val, err_tag_ptr, 0, false);
6497
6498 LLVMValueRef payload_ptr = LLVMBuildStructGEP(g->builder, result_loc, err_union_payload_index, "");
6499 gen_assign_raw(g, payload_ptr, get_pointer_to_type(g, payload_type, false), payload_val);
6500
6501 return result_loc;
6502 }
6503
ir_render_union_tag(CodeGen * g,Stage1Air * executable,Stage1AirInstUnionTag * instruction)6504 static LLVMValueRef ir_render_union_tag(CodeGen *g, Stage1Air *executable, Stage1AirInstUnionTag *instruction) {
6505 ZigType *union_type = instruction->value->value->type;
6506
6507 ZigType *tag_type = union_type->data.unionation.tag_type;
6508 if (!type_has_bits(g, tag_type))
6509 return nullptr;
6510
6511 LLVMValueRef union_val = ir_llvm_value(g, instruction->value);
6512 if (union_type->data.unionation.gen_field_count == 0)
6513 return union_val;
6514
6515 assert(union_type->data.unionation.gen_tag_index != SIZE_MAX);
6516 LLVMValueRef tag_field_ptr = LLVMBuildStructGEP(g->builder, union_val,
6517 union_type->data.unionation.gen_tag_index, "");
6518 ZigType *ptr_type = get_pointer_to_type(g, tag_type, false);
6519 return get_handle_value(g, tag_field_ptr, tag_type, ptr_type);
6520 }
6521
ir_render_panic(CodeGen * g,Stage1Air * executable,Stage1AirInstPanic * instruction)6522 static LLVMValueRef ir_render_panic(CodeGen *g, Stage1Air *executable, Stage1AirInstPanic *instruction) {
6523 bool is_llvm_alloca;
6524 LLVMValueRef err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope, &is_llvm_alloca);
6525 gen_panic(g, ir_llvm_value(g, instruction->msg), err_ret_trace_val, is_llvm_alloca);
6526 return nullptr;
6527 }
6528
ir_render_atomic_rmw(CodeGen * g,Stage1Air * executable,Stage1AirInstAtomicRmw * instruction)6529 static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, Stage1Air *executable,
6530 Stage1AirInstAtomicRmw *instruction)
6531 {
6532 bool is_signed;
6533 ZigType *operand_type = instruction->operand->value->type;
6534 bool is_float = operand_type->id == ZigTypeIdFloat;
6535 if (operand_type->id == ZigTypeIdInt) {
6536 is_signed = operand_type->data.integral.is_signed;
6537 } else {
6538 is_signed = false;
6539 }
6540 LLVMAtomicRMWBinOp op = to_LLVMAtomicRMWBinOp(instruction->op, is_signed, is_float);
6541 LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->ordering);
6542 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
6543 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
6544
6545 LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr,
6546 op == LLVMAtomicRMWBinOpXchg);
6547 if (actual_abi_type != nullptr) {
6548 // operand needs widening and truncating or bitcasting.
6549 LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr,
6550 LLVMPointerType(actual_abi_type, 0), "");
6551 LLVMValueRef casted_operand;
6552 if (is_float) {
6553 casted_operand = LLVMBuildBitCast(g->builder, operand, actual_abi_type, "");
6554 } else if (operand_type->data.integral.is_signed) {
6555 casted_operand = LLVMBuildSExt(g->builder, operand, actual_abi_type, "");
6556 } else {
6557 casted_operand = LLVMBuildZExt(g->builder, operand, actual_abi_type, "");
6558 }
6559 LLVMValueRef uncasted_result = LLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering,
6560 g->is_single_threaded);
6561 if (is_float) {
6562 return LLVMBuildBitCast(g->builder, uncasted_result, get_llvm_type(g, operand_type), "");
6563 } else {
6564 return LLVMBuildTrunc(g->builder, uncasted_result, get_llvm_type(g, operand_type), "");
6565 }
6566 }
6567
6568 if (get_codegen_ptr_type_bail(g, operand_type) == nullptr) {
6569 return LLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded);
6570 }
6571
6572 // it's a pointer but we need to treat it as an int
6573 LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr,
6574 LLVMPointerType(g->builtin_types.entry_usize->llvm_type, 0), "");
6575 LLVMValueRef casted_operand = LLVMBuildPtrToInt(g->builder, operand, g->builtin_types.entry_usize->llvm_type, "");
6576 LLVMValueRef uncasted_result = LLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering,
6577 g->is_single_threaded);
6578 return LLVMBuildIntToPtr(g->builder, uncasted_result, get_llvm_type(g, operand_type), "");
6579 }
6580
ir_render_atomic_load(CodeGen * g,Stage1Air * executable,Stage1AirInstAtomicLoad * instruction)6581 static LLVMValueRef ir_render_atomic_load(CodeGen *g, Stage1Air *executable,
6582 Stage1AirInstAtomicLoad *instruction)
6583 {
6584 LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->ordering);
6585 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
6586
6587 ZigType *operand_type = instruction->ptr->value->type->data.pointer.child_type;
6588 LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr, false);
6589 if (actual_abi_type != nullptr) {
6590 // operand needs widening and truncating
6591 ptr = LLVMBuildBitCast(g->builder, ptr,
6592 LLVMPointerType(actual_abi_type, 0), "");
6593 LLVMValueRef load_inst = gen_load(g, ptr, instruction->ptr->value->type, "");
6594 LLVMSetOrdering(load_inst, ordering);
6595 return LLVMBuildTrunc(g->builder, load_inst, get_llvm_type(g, operand_type), "");
6596 }
6597 LLVMValueRef load_inst = gen_load(g, ptr, instruction->ptr->value->type, "");
6598 LLVMSetOrdering(load_inst, ordering);
6599 return load_inst;
6600 }
6601
ir_render_atomic_store(CodeGen * g,Stage1Air * executable,Stage1AirInstAtomicStore * instruction)6602 static LLVMValueRef ir_render_atomic_store(CodeGen *g, Stage1Air *executable,
6603 Stage1AirInstAtomicStore *instruction)
6604 {
6605 LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->ordering);
6606 LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
6607 LLVMValueRef value = ir_llvm_value(g, instruction->value);
6608
6609 LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr, false);
6610 if (actual_abi_type != nullptr) {
6611 // operand needs widening
6612 ptr = LLVMBuildBitCast(g->builder, ptr,
6613 LLVMPointerType(actual_abi_type, 0), "");
6614 if (instruction->value->value->type->data.integral.is_signed) {
6615 value = LLVMBuildSExt(g->builder, value, actual_abi_type, "");
6616 } else {
6617 value = LLVMBuildZExt(g->builder, value, actual_abi_type, "");
6618 }
6619 }
6620 LLVMValueRef store_inst = gen_store(g, value, ptr, instruction->ptr->value->type);
6621 LLVMSetOrdering(store_inst, ordering);
6622 return nullptr;
6623 }
6624
ir_render_float_op(CodeGen * g,Stage1Air * executable,Stage1AirInstFloatOp * instruction)6625 static LLVMValueRef ir_render_float_op(CodeGen *g, Stage1Air *executable, Stage1AirInstFloatOp *instruction) {
6626 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
6627 LLVMValueRef fn_val = get_float_fn(g, instruction->base.value->type, ZigLLVMFnIdFloatOp, instruction->fn_id);
6628 return LLVMBuildCall(g->builder, fn_val, &operand, 1, "");
6629 }
6630
ir_render_mul_add(CodeGen * g,Stage1Air * executable,Stage1AirInstMulAdd * instruction)6631 static LLVMValueRef ir_render_mul_add(CodeGen *g, Stage1Air *executable, Stage1AirInstMulAdd *instruction) {
6632 LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);
6633 LLVMValueRef op2 = ir_llvm_value(g, instruction->op2);
6634 LLVMValueRef op3 = ir_llvm_value(g, instruction->op3);
6635 assert(instruction->base.value->type->id == ZigTypeIdFloat ||
6636 instruction->base.value->type->id == ZigTypeIdVector);
6637 LLVMValueRef fn_val = get_float_fn(g, instruction->base.value->type, ZigLLVMFnIdFMA, BuiltinFnIdMulAdd);
6638 LLVMValueRef args[3] = { op1, op2, op3 };
6639 return LLVMBuildCall(g->builder, fn_val, args, 3, "");
6640 }
6641
ir_render_bswap(CodeGen * g,Stage1Air * executable,Stage1AirInstBswap * instruction)6642 static LLVMValueRef ir_render_bswap(CodeGen *g, Stage1Air *executable, Stage1AirInstBswap *instruction) {
6643 LLVMValueRef op = ir_llvm_value(g, instruction->op);
6644 ZigType *expr_type = instruction->base.value->type;
6645 bool is_vector = expr_type->id == ZigTypeIdVector;
6646 ZigType *int_type = is_vector ? expr_type->data.vector.elem_type : expr_type;
6647 assert(int_type->id == ZigTypeIdInt);
6648 if (int_type->data.integral.bit_count % 16 == 0) {
6649 LLVMValueRef fn_val = get_int_builtin_fn(g, expr_type, BuiltinFnIdBswap);
6650 return LLVMBuildCall(g->builder, fn_val, &op, 1, "");
6651 }
6652 // Not an even number of bytes, so we zext 1 byte, then bswap, shift right 1 byte, truncate
6653 ZigType *extended_type = get_int_type(g, int_type->data.integral.is_signed,
6654 int_type->data.integral.bit_count + 8);
6655 LLVMValueRef shift_amt = LLVMConstInt(get_llvm_type(g, extended_type), 8, false);
6656 if (is_vector) {
6657 extended_type = get_vector_type(g, expr_type->data.vector.len, extended_type);
6658 LLVMValueRef *values = heap::c_allocator.allocate_nonzero<LLVMValueRef>(expr_type->data.vector.len);
6659 for (uint32_t i = 0; i < expr_type->data.vector.len; i += 1) {
6660 values[i] = shift_amt;
6661 }
6662 shift_amt = LLVMConstVector(values, expr_type->data.vector.len);
6663 heap::c_allocator.deallocate(values, expr_type->data.vector.len);
6664 }
6665 // aabbcc
6666 LLVMValueRef extended = LLVMBuildZExt(g->builder, op, get_llvm_type(g, extended_type), "");
6667 // 00aabbcc
6668 LLVMValueRef fn_val = get_int_builtin_fn(g, extended_type, BuiltinFnIdBswap);
6669 LLVMValueRef swapped = LLVMBuildCall(g->builder, fn_val, &extended, 1, "");
6670 // ccbbaa00
6671 LLVMValueRef shifted = ZigLLVMBuildLShrExact(g->builder, swapped, shift_amt, "");
6672 // 00ccbbaa
6673 return LLVMBuildTrunc(g->builder, shifted, get_llvm_type(g, expr_type), "");
6674 }
6675
ir_render_extern(CodeGen * g,Stage1Air * executable,Stage1AirInstExtern * instruction)6676 static LLVMValueRef ir_render_extern(CodeGen *g, Stage1Air *executable,
6677 Stage1AirInstExtern *instruction)
6678 {
6679 ZigType *expr_type = instruction->base.value->type;
6680 assert(get_src_ptr_type(expr_type));
6681
6682 const char *symbol_name = buf_ptr(instruction->name);
6683 const LLVMLinkage linkage = to_llvm_linkage(instruction->linkage, true);
6684
6685 LLVMValueRef global_value = LLVMGetNamedGlobal(g->module, symbol_name);
6686 if (global_value == nullptr) {
6687 global_value = LLVMAddGlobal(g->module, get_llvm_type(g, expr_type), symbol_name);
6688 LLVMSetLinkage(global_value, linkage);
6689 LLVMSetGlobalConstant(global_value, true);
6690 if (instruction->is_thread_local)
6691 LLVMSetThreadLocalMode(global_value, LLVMGeneralDynamicTLSModel);
6692 } else if (LLVMGetLinkage(global_value) != linkage) {
6693 // XXX: Handle this case better!
6694 zig_panic("duplicate extern symbol");
6695 }
6696
6697 return LLVMBuildBitCast(g->builder, global_value, get_llvm_type(g, expr_type), "");
6698 }
6699
ir_render_bit_reverse(CodeGen * g,Stage1Air * executable,Stage1AirInstBitReverse * instruction)6700 static LLVMValueRef ir_render_bit_reverse(CodeGen *g, Stage1Air *executable, Stage1AirInstBitReverse *instruction) {
6701 LLVMValueRef op = ir_llvm_value(g, instruction->op);
6702 ZigType *int_type = instruction->base.value->type;
6703 assert(int_type->id == ZigTypeIdInt);
6704 LLVMValueRef fn_val = get_int_builtin_fn(g, instruction->base.value->type, BuiltinFnIdBitReverse);
6705 return LLVMBuildCall(g->builder, fn_val, &op, 1, "");
6706 }
6707
ir_render_vector_to_array(CodeGen * g,Stage1Air * executable,Stage1AirInstVectorToArray * instruction)6708 static LLVMValueRef ir_render_vector_to_array(CodeGen *g, Stage1Air *executable,
6709 Stage1AirInstVectorToArray *instruction)
6710 {
6711 ZigType *array_type = instruction->base.value->type;
6712 assert(array_type->id == ZigTypeIdArray);
6713 assert(handle_is_ptr(g, array_type));
6714 LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
6715 LLVMValueRef vector = ir_llvm_value(g, instruction->vector);
6716
6717 ZigType *elem_type = array_type->data.array.child_type;
6718 bool bitcast_ok = elem_type->size_in_bits == elem_type->abi_size * 8;
6719 if (bitcast_ok) {
6720 LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, result_loc,
6721 LLVMPointerType(get_llvm_type(g, instruction->vector->value->type), 0), "");
6722 uint32_t alignment = get_ptr_align(g, instruction->result_loc->value->type);
6723 gen_store_untyped(g, vector, casted_ptr, alignment, false);
6724 } else {
6725 // If the ABI size of the element type is not evenly divisible by size_in_bits, a simple bitcast
6726 // will not work, and we fall back to extractelement.
6727 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
6728 LLVMTypeRef u32_type_ref = LLVMInt32Type();
6729 LLVMValueRef zero = LLVMConstInt(usize_type_ref, 0, false);
6730 for (uintptr_t i = 0; i < instruction->vector->value->type->data.vector.len; i++) {
6731 LLVMValueRef index_usize = LLVMConstInt(usize_type_ref, i, false);
6732 LLVMValueRef index_u32 = LLVMConstInt(u32_type_ref, i, false);
6733 LLVMValueRef indexes[] = { zero, index_usize };
6734 LLVMValueRef elem_ptr = LLVMBuildInBoundsGEP(g->builder, result_loc, indexes, 2, "");
6735 LLVMValueRef elem = LLVMBuildExtractElement(g->builder, vector, index_u32, "");
6736 LLVMBuildStore(g->builder, elem, elem_ptr);
6737 }
6738 }
6739 return result_loc;
6740 }
6741
ir_render_array_to_vector(CodeGen * g,Stage1Air * executable,Stage1AirInstArrayToVector * instruction)6742 static LLVMValueRef ir_render_array_to_vector(CodeGen *g, Stage1Air *executable,
6743 Stage1AirInstArrayToVector *instruction)
6744 {
6745 ZigType *vector_type = instruction->base.value->type;
6746 assert(vector_type->id == ZigTypeIdVector);
6747 assert(!handle_is_ptr(g, vector_type));
6748 LLVMValueRef array_ptr = ir_llvm_value(g, instruction->array);
6749 LLVMTypeRef vector_type_ref = get_llvm_type(g, vector_type);
6750
6751 ZigType *elem_type = vector_type->data.vector.elem_type;
6752 bool bitcast_ok = elem_type->size_in_bits == elem_type->abi_size * 8;
6753 if (bitcast_ok) {
6754 LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, array_ptr,
6755 LLVMPointerType(vector_type_ref, 0), "");
6756 ZigType *array_type = instruction->array->value->type;
6757 assert(array_type->id == ZigTypeIdArray);
6758 uint32_t alignment = get_abi_alignment(g, array_type->data.array.child_type);
6759 return gen_load_untyped(g, casted_ptr, alignment, false, "");
6760 } else {
6761 // If the ABI size of the element type is not evenly divisible by size_in_bits, a simple bitcast
6762 // will not work, and we fall back to insertelement.
6763 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
6764 LLVMTypeRef u32_type_ref = LLVMInt32Type();
6765 LLVMValueRef zero = LLVMConstInt(usize_type_ref, 0, false);
6766 LLVMValueRef vector = LLVMGetUndef(vector_type_ref);
6767 for (uintptr_t i = 0; i < instruction->base.value->type->data.vector.len; i++) {
6768 LLVMValueRef index_usize = LLVMConstInt(usize_type_ref, i, false);
6769 LLVMValueRef index_u32 = LLVMConstInt(u32_type_ref, i, false);
6770 LLVMValueRef indexes[] = { zero, index_usize };
6771 LLVMValueRef elem_ptr = LLVMBuildInBoundsGEP(g->builder, array_ptr, indexes, 2, "");
6772 LLVMValueRef elem = LLVMBuildLoad(g->builder, elem_ptr, "");
6773 vector = LLVMBuildInsertElement(g->builder, vector, elem, index_u32, "");
6774 }
6775 return vector;
6776 }
6777 }
6778
ir_render_assert_zero(CodeGen * g,Stage1Air * executable,Stage1AirInstAssertZero * instruction)6779 static LLVMValueRef ir_render_assert_zero(CodeGen *g, Stage1Air *executable,
6780 Stage1AirInstAssertZero *instruction)
6781 {
6782 LLVMValueRef target = ir_llvm_value(g, instruction->target);
6783 ZigType *int_type = instruction->target->value->type;
6784 if (ir_want_runtime_safety(g, &instruction->base)) {
6785 return gen_assert_zero(g, target, int_type);
6786 }
6787 return nullptr;
6788 }
6789
ir_render_assert_non_null(CodeGen * g,Stage1Air * executable,Stage1AirInstAssertNonNull * instruction)6790 static LLVMValueRef ir_render_assert_non_null(CodeGen *g, Stage1Air *executable,
6791 Stage1AirInstAssertNonNull *instruction)
6792 {
6793 LLVMValueRef target = ir_llvm_value(g, instruction->target);
6794 ZigType *target_type = instruction->target->value->type;
6795
6796 if (target_type->id == ZigTypeIdPointer) {
6797 assert(target_type->data.pointer.ptr_len == PtrLenC);
6798 LLVMValueRef non_null_bit = LLVMBuildICmp(g->builder, LLVMIntNE, target,
6799 LLVMConstNull(get_llvm_type(g, target_type)), "");
6800
6801 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "AssertNonNullFail");
6802 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "AssertNonNullOk");
6803 LLVMBuildCondBr(g->builder, non_null_bit, ok_block, fail_block);
6804
6805 LLVMPositionBuilderAtEnd(g->builder, fail_block);
6806 gen_assertion(g, PanicMsgIdUnwrapOptionalFail, &instruction->base);
6807
6808 LLVMPositionBuilderAtEnd(g->builder, ok_block);
6809 } else {
6810 zig_unreachable();
6811 }
6812 return nullptr;
6813 }
6814
ir_render_suspend_begin(CodeGen * g,Stage1Air * executable,Stage1AirInstSuspendBegin * instruction)6815 static LLVMValueRef ir_render_suspend_begin(CodeGen *g, Stage1Air *executable,
6816 Stage1AirInstSuspendBegin *instruction)
6817 {
6818 if (fn_is_async(g->cur_fn)) {
6819 instruction->resume_bb = gen_suspend_begin(g, "SuspendResume");
6820 }
6821 return nullptr;
6822 }
6823
ir_render_suspend_finish(CodeGen * g,Stage1Air * executable,Stage1AirInstSuspendFinish * instruction)6824 static LLVMValueRef ir_render_suspend_finish(CodeGen *g, Stage1Air *executable,
6825 Stage1AirInstSuspendFinish *instruction)
6826 {
6827 LLVMBuildRetVoid(g->builder);
6828
6829 LLVMPositionBuilderAtEnd(g->builder, instruction->begin->resume_bb);
6830 if (ir_want_runtime_safety(g, &instruction->base)) {
6831 LLVMBuildStore(g->builder, g->cur_bad_not_suspended_index, g->cur_async_resume_index_ptr);
6832 }
6833 render_async_var_decls(g, instruction->base.scope);
6834 return nullptr;
6835 }
6836
gen_await_early_return(CodeGen * g,Stage1AirInst * source_instr,LLVMValueRef target_frame_ptr,ZigType * result_type,ZigType * ptr_result_type,LLVMValueRef result_loc,bool non_async)6837 static LLVMValueRef gen_await_early_return(CodeGen *g, Stage1AirInst *source_instr,
6838 LLVMValueRef target_frame_ptr, ZigType *result_type, ZigType *ptr_result_type,
6839 LLVMValueRef result_loc, bool non_async)
6840 {
6841 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
6842 LLVMValueRef their_result_ptr = nullptr;
6843 if (type_has_bits(g, result_type) && (non_async || result_loc != nullptr)) {
6844 LLVMValueRef their_result_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_ret_start, "");
6845 their_result_ptr = LLVMBuildLoad(g->builder, their_result_ptr_ptr, "");
6846 if (result_loc != nullptr) {
6847 LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
6848 LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, result_loc, ptr_u8, "");
6849 LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, their_result_ptr, ptr_u8, "");
6850 bool is_volatile = false;
6851 uint32_t abi_align = get_abi_alignment(g, result_type);
6852 LLVMValueRef byte_count_val = LLVMConstInt(usize_type_ref, type_size(g, result_type), false);
6853 ZigLLVMBuildMemCpy(g->builder,
6854 dest_ptr_casted, abi_align,
6855 src_ptr_casted, abi_align, byte_count_val, is_volatile);
6856 }
6857 }
6858 if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) {
6859 LLVMValueRef their_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr,
6860 frame_index_trace_arg(g, result_type), "");
6861 LLVMValueRef src_trace_ptr = LLVMBuildLoad(g->builder, their_trace_ptr_ptr, "");
6862 bool is_llvm_alloca;
6863 LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, source_instr->scope, &is_llvm_alloca);
6864 LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr };
6865 ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2,
6866 get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_CallAttrAuto, "");
6867 }
6868 if (non_async && type_has_bits(g, result_type)) {
6869 LLVMValueRef result_ptr = (result_loc == nullptr) ? their_result_ptr : result_loc;
6870 return get_handle_value(g, result_ptr, result_type, ptr_result_type);
6871 } else {
6872 return nullptr;
6873 }
6874 }
6875
ir_render_await(CodeGen * g,Stage1Air * executable,Stage1AirInstAwait * instruction)6876 static LLVMValueRef ir_render_await(CodeGen *g, Stage1Air *executable, Stage1AirInstAwait *instruction) {
6877 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
6878 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
6879 LLVMValueRef target_frame_ptr = ir_llvm_value(g, instruction->frame);
6880 ZigType *result_type = instruction->base.value->type;
6881 ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true);
6882
6883 LLVMValueRef result_loc = (instruction->result_loc == nullptr) ?
6884 nullptr : ir_llvm_value(g, instruction->result_loc);
6885
6886 if (instruction->is_nosuspend ||
6887 (instruction->target_fn != nullptr && !fn_is_async(instruction->target_fn)))
6888 {
6889 return gen_await_early_return(g, &instruction->base, target_frame_ptr, result_type,
6890 ptr_result_type, result_loc, true);
6891 }
6892
6893 // Prepare to be suspended
6894 LLVMBasicBlockRef resume_bb = gen_suspend_begin(g, "AwaitResume");
6895 LLVMBasicBlockRef end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "AwaitEnd");
6896
6897 // At this point resuming the function will continue from resume_bb.
6898 // This code is as if it is running inside the suspend block.
6899
6900 // supply the awaiter return pointer
6901 if (type_has_bits(g, result_type)) {
6902 LLVMValueRef awaiter_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_ret_start + 1, "");
6903 if (result_loc == nullptr) {
6904 // no copy needed
6905 LLVMBuildStore(g->builder, LLVMConstNull(LLVMGetElementType(LLVMTypeOf(awaiter_ret_ptr_ptr))),
6906 awaiter_ret_ptr_ptr);
6907 } else {
6908 LLVMBuildStore(g->builder, result_loc, awaiter_ret_ptr_ptr);
6909 }
6910 }
6911
6912 // supply the error return trace pointer
6913 if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) {
6914 bool is_llvm_alloca;
6915 LLVMValueRef my_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope, &is_llvm_alloca);
6916 assert(my_err_ret_trace_val != nullptr);
6917 LLVMValueRef err_ret_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr,
6918 frame_index_trace_arg(g, result_type) + 1, "");
6919 LLVMBuildStore(g->builder, my_err_ret_trace_val, err_ret_trace_ptr_ptr);
6920 }
6921
6922 // caller's own frame pointer
6923 LLVMValueRef awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_frame_ptr, usize_type_ref, "");
6924 LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_awaiter_index, "");
6925 LLVMValueRef prev_val = gen_maybe_atomic_op(g, LLVMAtomicRMWBinOpXchg, awaiter_ptr, awaiter_init_val,
6926 LLVMAtomicOrderingRelease);
6927
6928 LLVMBasicBlockRef bad_await_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadAwait");
6929 LLVMBasicBlockRef complete_suspend_block = LLVMAppendBasicBlock(g->cur_fn_val, "CompleteSuspend");
6930 LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
6931
6932 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
6933 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, bad_await_block, 2);
6934
6935 LLVMAddCase(switch_instr, zero, complete_suspend_block);
6936 LLVMAddCase(switch_instr, all_ones, early_return_block);
6937
6938 // We discovered that another awaiter was already here.
6939 LLVMPositionBuilderAtEnd(g->builder, bad_await_block);
6940 gen_assertion(g, PanicMsgIdBadAwait, &instruction->base);
6941
6942 // Rely on the target to resume us from suspension.
6943 LLVMPositionBuilderAtEnd(g->builder, complete_suspend_block);
6944 LLVMBuildRetVoid(g->builder);
6945
6946 // Early return: The async function has already completed. We must copy the result and
6947 // the error return trace if applicable.
6948 LLVMPositionBuilderAtEnd(g->builder, early_return_block);
6949 gen_await_early_return(g, &instruction->base, target_frame_ptr, result_type, ptr_result_type,
6950 result_loc, false);
6951 LLVMBuildBr(g->builder, end_bb);
6952
6953 LLVMPositionBuilderAtEnd(g->builder, resume_bb);
6954 gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedAnAwaitingFn, nullptr);
6955 LLVMBuildBr(g->builder, end_bb);
6956
6957 LLVMPositionBuilderAtEnd(g->builder, end_bb);
6958 // Rely on the spill for the llvm_value to be populated.
6959 // See the implementation of ir_llvm_value.
6960 return nullptr;
6961 }
6962
ir_render_resume(CodeGen * g,Stage1Air * executable,Stage1AirInstResume * instruction)6963 static LLVMValueRef ir_render_resume(CodeGen *g, Stage1Air *executable, Stage1AirInstResume *instruction) {
6964 LLVMValueRef frame = ir_llvm_value(g, instruction->frame);
6965 ZigType *frame_type = instruction->frame->value->type;
6966 assert(frame_type->id == ZigTypeIdAnyFrame);
6967
6968 gen_resume(g, nullptr, frame, ResumeIdManual);
6969 return nullptr;
6970 }
6971
ir_render_frame_size(CodeGen * g,Stage1Air * executable,Stage1AirInstFrameSize * instruction)6972 static LLVMValueRef ir_render_frame_size(CodeGen *g, Stage1Air *executable,
6973 Stage1AirInstFrameSize *instruction)
6974 {
6975 LLVMValueRef fn_val = ir_llvm_value(g, instruction->fn);
6976 return gen_frame_size(g, fn_val);
6977 }
6978
ir_render_spill_begin(CodeGen * g,Stage1Air * executable,Stage1AirInstSpillBegin * instruction)6979 static LLVMValueRef ir_render_spill_begin(CodeGen *g, Stage1Air *executable,
6980 Stage1AirInstSpillBegin *instruction)
6981 {
6982 if (!fn_is_async(g->cur_fn))
6983 return nullptr;
6984
6985 switch (instruction->spill_id) {
6986 case SpillIdInvalid:
6987 zig_unreachable();
6988 case SpillIdRetErrCode: {
6989 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
6990 LLVMValueRef ptr = ir_llvm_value(g, g->cur_fn->err_code_spill);
6991 LLVMBuildStore(g->builder, operand, ptr);
6992 return nullptr;
6993 }
6994
6995 }
6996 zig_unreachable();
6997 }
6998
ir_render_spill_end(CodeGen * g,Stage1Air * executable,Stage1AirInstSpillEnd * instruction)6999 static LLVMValueRef ir_render_spill_end(CodeGen *g, Stage1Air *executable, Stage1AirInstSpillEnd *instruction) {
7000 if (!fn_is_async(g->cur_fn))
7001 return ir_llvm_value(g, instruction->begin->operand);
7002
7003 switch (instruction->begin->spill_id) {
7004 case SpillIdInvalid:
7005 zig_unreachable();
7006 case SpillIdRetErrCode: {
7007 LLVMValueRef ptr = ir_llvm_value(g, g->cur_fn->err_code_spill);
7008 return LLVMBuildLoad(g->builder, ptr, "");
7009 }
7010
7011 }
7012 zig_unreachable();
7013 }
7014
ir_render_vector_extract_elem(CodeGen * g,Stage1Air * executable,Stage1AirInstVectorExtractElem * instruction)7015 static LLVMValueRef ir_render_vector_extract_elem(CodeGen *g, Stage1Air *executable,
7016 Stage1AirInstVectorExtractElem *instruction)
7017 {
7018 LLVMValueRef vector = ir_llvm_value(g, instruction->vector);
7019 LLVMValueRef index = ir_llvm_value(g, instruction->index);
7020 return LLVMBuildExtractElement(g->builder, vector, index, "");
7021 }
7022
set_debug_location(CodeGen * g,Stage1AirInst * instruction)7023 static void set_debug_location(CodeGen *g, Stage1AirInst *instruction) {
7024 AstNode *source_node = instruction->source_node;
7025 Scope *scope = instruction->scope;
7026
7027 assert(source_node);
7028 assert(scope);
7029
7030 ZigLLVMSetCurrentDebugLocation(g->builder, node_line_onebased(source_node),
7031 node_column_onebased(source_node), get_di_scope(g, scope));
7032 }
7033
ir_render_instruction(CodeGen * g,Stage1Air * executable,Stage1AirInst * instruction)7034 static LLVMValueRef ir_render_instruction(CodeGen *g, Stage1Air *executable, Stage1AirInst *instruction) {
7035 switch (instruction->id) {
7036 case Stage1AirInstIdInvalid:
7037 case Stage1AirInstIdConst:
7038 case Stage1AirInstIdAlloca:
7039 zig_unreachable();
7040
7041 case Stage1AirInstIdDeclVar:
7042 return ir_render_decl_var(g, executable, (Stage1AirInstDeclVar *)instruction);
7043 case Stage1AirInstIdReturn:
7044 return ir_render_return(g, executable, (Stage1AirInstReturn *)instruction);
7045 case Stage1AirInstIdBinOp:
7046 return ir_render_bin_op(g, executable, (Stage1AirInstBinOp *)instruction);
7047 case Stage1AirInstIdCast:
7048 return ir_render_cast(g, executable, (Stage1AirInstCast *)instruction);
7049 case Stage1AirInstIdUnreachable:
7050 return ir_render_unreachable(g, executable, (Stage1AirInstUnreachable *)instruction);
7051 case Stage1AirInstIdCondBr:
7052 return ir_render_cond_br(g, executable, (Stage1AirInstCondBr *)instruction);
7053 case Stage1AirInstIdBr:
7054 return ir_render_br(g, executable, (Stage1AirInstBr *)instruction);
7055 case Stage1AirInstIdBinaryNot:
7056 return ir_render_binary_not(g, executable, (Stage1AirInstBinaryNot *)instruction);
7057 case Stage1AirInstIdNegation:
7058 return ir_render_negation(g, executable, (Stage1AirInstNegation *)instruction);
7059 case Stage1AirInstIdLoadPtr:
7060 return ir_render_load_ptr(g, executable, (Stage1AirInstLoadPtr *)instruction);
7061 case Stage1AirInstIdStorePtr:
7062 return ir_render_store_ptr(g, executable, (Stage1AirInstStorePtr *)instruction);
7063 case Stage1AirInstIdVectorStoreElem:
7064 return ir_render_vector_store_elem(g, executable, (Stage1AirInstVectorStoreElem *)instruction);
7065 case Stage1AirInstIdVarPtr:
7066 return ir_render_var_ptr(g, executable, (Stage1AirInstVarPtr *)instruction);
7067 case Stage1AirInstIdReturnPtr:
7068 return ir_render_return_ptr(g, executable, (Stage1AirInstReturnPtr *)instruction);
7069 case Stage1AirInstIdElemPtr:
7070 return ir_render_elem_ptr(g, executable, (Stage1AirInstElemPtr *)instruction);
7071 case Stage1AirInstIdCall:
7072 return ir_render_call(g, executable, (Stage1AirInstCall *)instruction);
7073 case Stage1AirInstIdStructFieldPtr:
7074 return ir_render_struct_field_ptr(g, executable, (Stage1AirInstStructFieldPtr *)instruction);
7075 case Stage1AirInstIdUnionFieldPtr:
7076 return ir_render_union_field_ptr(g, executable, (Stage1AirInstUnionFieldPtr *)instruction);
7077 case Stage1AirInstIdAsm:
7078 return ir_render_asm_gen(g, executable, (Stage1AirInstAsm *)instruction);
7079 case Stage1AirInstIdTestNonNull:
7080 return ir_render_test_non_null(g, executable, (Stage1AirInstTestNonNull *)instruction);
7081 case Stage1AirInstIdOptionalUnwrapPtr:
7082 return ir_render_optional_unwrap_ptr(g, executable, (Stage1AirInstOptionalUnwrapPtr *)instruction);
7083 case Stage1AirInstIdClz:
7084 return ir_render_clz(g, executable, (Stage1AirInstClz *)instruction);
7085 case Stage1AirInstIdCtz:
7086 return ir_render_ctz(g, executable, (Stage1AirInstCtz *)instruction);
7087 case Stage1AirInstIdPopCount:
7088 return ir_render_pop_count(g, executable, (Stage1AirInstPopCount *)instruction);
7089 case Stage1AirInstIdSwitchBr:
7090 return ir_render_switch_br(g, executable, (Stage1AirInstSwitchBr *)instruction);
7091 case Stage1AirInstIdBswap:
7092 return ir_render_bswap(g, executable, (Stage1AirInstBswap *)instruction);
7093 case Stage1AirInstIdBitReverse:
7094 return ir_render_bit_reverse(g, executable, (Stage1AirInstBitReverse *)instruction);
7095 case Stage1AirInstIdPhi:
7096 return ir_render_phi(g, executable, (Stage1AirInstPhi *)instruction);
7097 case Stage1AirInstIdRef:
7098 return ir_render_ref(g, executable, (Stage1AirInstRef *)instruction);
7099 case Stage1AirInstIdErrName:
7100 return ir_render_err_name(g, executable, (Stage1AirInstErrName *)instruction);
7101 case Stage1AirInstIdCmpxchg:
7102 return ir_render_cmpxchg(g, executable, (Stage1AirInstCmpxchg *)instruction);
7103 case Stage1AirInstIdFence:
7104 return ir_render_fence(g, executable, (Stage1AirInstFence *)instruction);
7105 case Stage1AirInstIdReduce:
7106 return ir_render_reduce(g, executable, (Stage1AirInstReduce *)instruction);
7107 case Stage1AirInstIdTruncate:
7108 return ir_render_truncate(g, executable, (Stage1AirInstTruncate *)instruction);
7109 case Stage1AirInstIdBoolNot:
7110 return ir_render_bool_not(g, executable, (Stage1AirInstBoolNot *)instruction);
7111 case Stage1AirInstIdMemset:
7112 return ir_render_memset(g, executable, (Stage1AirInstMemset *)instruction);
7113 case Stage1AirInstIdMemcpy:
7114 return ir_render_memcpy(g, executable, (Stage1AirInstMemcpy *)instruction);
7115 case Stage1AirInstIdSlice:
7116 return ir_render_slice(g, executable, (Stage1AirInstSlice *)instruction);
7117 case Stage1AirInstIdBreakpoint:
7118 return ir_render_breakpoint(g, executable, (Stage1AirInstBreakpoint *)instruction);
7119 case Stage1AirInstIdReturnAddress:
7120 return ir_render_return_address(g, executable, (Stage1AirInstReturnAddress *)instruction);
7121 case Stage1AirInstIdFrameAddress:
7122 return ir_render_frame_address(g, executable, (Stage1AirInstFrameAddress *)instruction);
7123 case Stage1AirInstIdFrameHandle:
7124 return ir_render_handle(g, executable, (Stage1AirInstFrameHandle *)instruction);
7125 case Stage1AirInstIdOverflowOp:
7126 return ir_render_overflow_op(g, executable, (Stage1AirInstOverflowOp *)instruction);
7127 case Stage1AirInstIdTestErr:
7128 return ir_render_test_err(g, executable, (Stage1AirInstTestErr *)instruction);
7129 case Stage1AirInstIdUnwrapErrCode:
7130 return ir_render_unwrap_err_code(g, executable, (Stage1AirInstUnwrapErrCode *)instruction);
7131 case Stage1AirInstIdUnwrapErrPayload:
7132 return ir_render_unwrap_err_payload(g, executable, (Stage1AirInstUnwrapErrPayload *)instruction);
7133 case Stage1AirInstIdOptionalWrap:
7134 return ir_render_optional_wrap(g, executable, (Stage1AirInstOptionalWrap *)instruction);
7135 case Stage1AirInstIdErrWrapCode:
7136 return ir_render_err_wrap_code(g, executable, (Stage1AirInstErrWrapCode *)instruction);
7137 case Stage1AirInstIdErrWrapPayload:
7138 return ir_render_err_wrap_payload(g, executable, (Stage1AirInstErrWrapPayload *)instruction);
7139 case Stage1AirInstIdUnionTag:
7140 return ir_render_union_tag(g, executable, (Stage1AirInstUnionTag *)instruction);
7141 case Stage1AirInstIdPtrCast:
7142 return ir_render_ptr_cast(g, executable, (Stage1AirInstPtrCast *)instruction);
7143 case Stage1AirInstIdBitCast:
7144 return ir_render_bit_cast(g, executable, (Stage1AirInstBitCast *)instruction);
7145 case Stage1AirInstIdWidenOrShorten:
7146 return ir_render_widen_or_shorten(g, executable, (Stage1AirInstWidenOrShorten *)instruction);
7147 case Stage1AirInstIdPtrToInt:
7148 return ir_render_ptr_to_int(g, executable, (Stage1AirInstPtrToInt *)instruction);
7149 case Stage1AirInstIdIntToPtr:
7150 return ir_render_int_to_ptr(g, executable, (Stage1AirInstIntToPtr *)instruction);
7151 case Stage1AirInstIdIntToEnum:
7152 return ir_render_int_to_enum(g, executable, (Stage1AirInstIntToEnum *)instruction);
7153 case Stage1AirInstIdIntToErr:
7154 return ir_render_int_to_err(g, executable, (Stage1AirInstIntToErr *)instruction);
7155 case Stage1AirInstIdErrToInt:
7156 return ir_render_err_to_int(g, executable, (Stage1AirInstErrToInt *)instruction);
7157 case Stage1AirInstIdPanic:
7158 return ir_render_panic(g, executable, (Stage1AirInstPanic *)instruction);
7159 case Stage1AirInstIdTagName:
7160 return ir_render_enum_tag_name(g, executable, (Stage1AirInstTagName *)instruction);
7161 case Stage1AirInstIdFieldParentPtr:
7162 return ir_render_field_parent_ptr(g, executable, (Stage1AirInstFieldParentPtr *)instruction);
7163 case Stage1AirInstIdAlignCast:
7164 return ir_render_align_cast(g, executable, (Stage1AirInstAlignCast *)instruction);
7165 case Stage1AirInstIdErrorReturnTrace:
7166 return ir_render_error_return_trace(g, executable, (Stage1AirInstErrorReturnTrace *)instruction);
7167 case Stage1AirInstIdAtomicRmw:
7168 return ir_render_atomic_rmw(g, executable, (Stage1AirInstAtomicRmw *)instruction);
7169 case Stage1AirInstIdAtomicLoad:
7170 return ir_render_atomic_load(g, executable, (Stage1AirInstAtomicLoad *)instruction);
7171 case Stage1AirInstIdAtomicStore:
7172 return ir_render_atomic_store(g, executable, (Stage1AirInstAtomicStore *)instruction);
7173 case Stage1AirInstIdSaveErrRetAddr:
7174 return ir_render_save_err_ret_addr(g, executable, (Stage1AirInstSaveErrRetAddr *)instruction);
7175 case Stage1AirInstIdFloatOp:
7176 return ir_render_float_op(g, executable, (Stage1AirInstFloatOp *)instruction);
7177 case Stage1AirInstIdMulAdd:
7178 return ir_render_mul_add(g, executable, (Stage1AirInstMulAdd *)instruction);
7179 case Stage1AirInstIdArrayToVector:
7180 return ir_render_array_to_vector(g, executable, (Stage1AirInstArrayToVector *)instruction);
7181 case Stage1AirInstIdVectorToArray:
7182 return ir_render_vector_to_array(g, executable, (Stage1AirInstVectorToArray *)instruction);
7183 case Stage1AirInstIdAssertZero:
7184 return ir_render_assert_zero(g, executable, (Stage1AirInstAssertZero *)instruction);
7185 case Stage1AirInstIdAssertNonNull:
7186 return ir_render_assert_non_null(g, executable, (Stage1AirInstAssertNonNull *)instruction);
7187 case Stage1AirInstIdPtrOfArrayToSlice:
7188 return ir_render_ptr_of_array_to_slice(g, executable, (Stage1AirInstPtrOfArrayToSlice *)instruction);
7189 case Stage1AirInstIdSuspendBegin:
7190 return ir_render_suspend_begin(g, executable, (Stage1AirInstSuspendBegin *)instruction);
7191 case Stage1AirInstIdSuspendFinish:
7192 return ir_render_suspend_finish(g, executable, (Stage1AirInstSuspendFinish *)instruction);
7193 case Stage1AirInstIdResume:
7194 return ir_render_resume(g, executable, (Stage1AirInstResume *)instruction);
7195 case Stage1AirInstIdFrameSize:
7196 return ir_render_frame_size(g, executable, (Stage1AirInstFrameSize *)instruction);
7197 case Stage1AirInstIdAwait:
7198 return ir_render_await(g, executable, (Stage1AirInstAwait *)instruction);
7199 case Stage1AirInstIdSpillBegin:
7200 return ir_render_spill_begin(g, executable, (Stage1AirInstSpillBegin *)instruction);
7201 case Stage1AirInstIdSpillEnd:
7202 return ir_render_spill_end(g, executable, (Stage1AirInstSpillEnd *)instruction);
7203 case Stage1AirInstIdShuffleVector:
7204 return ir_render_shuffle_vector(g, executable, (Stage1AirInstShuffleVector *) instruction);
7205 case Stage1AirInstIdSelect:
7206 return ir_render_select(g, executable, (Stage1AirInstSelect *) instruction);
7207 case Stage1AirInstIdSplat:
7208 return ir_render_splat(g, executable, (Stage1AirInstSplat *) instruction);
7209 case Stage1AirInstIdVectorExtractElem:
7210 return ir_render_vector_extract_elem(g, executable, (Stage1AirInstVectorExtractElem *) instruction);
7211 case Stage1AirInstIdWasmMemorySize:
7212 return ir_render_wasm_memory_size(g, executable, (Stage1AirInstWasmMemorySize *) instruction);
7213 case Stage1AirInstIdWasmMemoryGrow:
7214 return ir_render_wasm_memory_grow(g, executable, (Stage1AirInstWasmMemoryGrow *) instruction);
7215 case Stage1AirInstIdExtern:
7216 return ir_render_extern(g, executable, (Stage1AirInstExtern *) instruction);
7217 case Stage1AirInstIdPrefetch:
7218 return ir_render_prefetch(g, executable, (Stage1AirInstPrefetch *) instruction);
7219 }
7220 zig_unreachable();
7221 }
7222
ir_render(CodeGen * g,ZigFn * fn_entry)7223 static void ir_render(CodeGen *g, ZigFn *fn_entry) {
7224 assert(fn_entry);
7225
7226 Stage1Air *executable = &fn_entry->analyzed_executable;
7227 assert(executable->basic_block_list.length > 0);
7228
7229 for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) {
7230 Stage1AirBasicBlock *current_block = executable->basic_block_list.at(block_i);
7231 if (get_scope_typeof(current_block->scope) != nullptr) {
7232 LLVMBuildBr(g->builder, current_block->llvm_block);
7233 }
7234 assert(current_block->llvm_block);
7235 LLVMPositionBuilderAtEnd(g->builder, current_block->llvm_block);
7236 for (size_t instr_i = 0; instr_i < current_block->instruction_list.length; instr_i += 1) {
7237 Stage1AirInst *instruction = current_block->instruction_list.at(instr_i);
7238 if (instruction->ref_count == 0 && !ir_inst_gen_has_side_effects(instruction))
7239 continue;
7240 if (get_scope_typeof(instruction->scope) != nullptr)
7241 continue;
7242
7243 if (!g->strip_debug_symbols) {
7244 set_debug_location(g, instruction);
7245 }
7246 instruction->llvm_value = ir_render_instruction(g, executable, instruction);
7247 if (instruction->spill != nullptr && instruction->llvm_value != nullptr) {
7248 LLVMValueRef spill_ptr = ir_llvm_value(g, instruction->spill);
7249 gen_assign_raw(g, spill_ptr, instruction->spill->value->type, instruction->llvm_value);
7250 instruction->llvm_value = nullptr;
7251 }
7252 }
7253 current_block->llvm_exit_block = LLVMGetInsertBlock(g->builder);
7254 }
7255 }
7256
7257 static LLVMValueRef gen_const_ptr_struct_recursive(CodeGen *g, ZigValue *struct_const_val, size_t field_index);
7258 static LLVMValueRef gen_const_ptr_array_recursive(CodeGen *g, ZigValue *array_const_val, size_t index);
7259 static LLVMValueRef gen_const_ptr_union_recursive(CodeGen *g, ZigValue *union_const_val);
7260 static LLVMValueRef gen_const_ptr_err_union_code_recursive(CodeGen *g, ZigValue *err_union_const_val);
7261 static LLVMValueRef gen_const_ptr_err_union_payload_recursive(CodeGen *g, ZigValue *err_union_const_val);
7262 static LLVMValueRef gen_const_ptr_optional_payload_recursive(CodeGen *g, ZigValue *optional_const_val);
7263
gen_parent_ptr(CodeGen * g,ZigValue * val,ConstParent * parent)7264 static LLVMValueRef gen_parent_ptr(CodeGen *g, ZigValue *val, ConstParent *parent) {
7265 switch (parent->id) {
7266 case ConstParentIdNone:
7267 render_const_val(g, val, "");
7268 render_const_val_global(g, val, "");
7269 return val->llvm_global;
7270 case ConstParentIdStruct: {
7271 ZigValue *struct_val = parent->data.p_struct.struct_val;
7272 size_t src_field_index = parent->data.p_struct.field_index;
7273 size_t gen_field_index = struct_val->type->data.structure.fields[src_field_index]->gen_index;
7274 return gen_const_ptr_struct_recursive(g, struct_val, gen_field_index);
7275 }
7276 case ConstParentIdErrUnionCode:
7277 return gen_const_ptr_err_union_code_recursive(g, parent->data.p_err_union_code.err_union_val);
7278 case ConstParentIdErrUnionPayload:
7279 return gen_const_ptr_err_union_payload_recursive(g, parent->data.p_err_union_payload.err_union_val);
7280 case ConstParentIdOptionalPayload:
7281 return gen_const_ptr_optional_payload_recursive(g, parent->data.p_optional_payload.optional_val);
7282 case ConstParentIdArray:
7283 return gen_const_ptr_array_recursive(g, parent->data.p_array.array_val,
7284 parent->data.p_array.elem_index);
7285 case ConstParentIdUnion:
7286 return gen_const_ptr_union_recursive(g, parent->data.p_union.union_val);
7287 case ConstParentIdScalar:
7288 render_const_val(g, parent->data.p_scalar.scalar_val, "");
7289 render_const_val_global(g, parent->data.p_scalar.scalar_val, "");
7290 return parent->data.p_scalar.scalar_val->llvm_global;
7291 }
7292 zig_unreachable();
7293 }
7294
gen_const_ptr_array_recursive(CodeGen * g,ZigValue * array_const_val,size_t index)7295 static LLVMValueRef gen_const_ptr_array_recursive(CodeGen *g, ZigValue *array_const_val, size_t index) {
7296 expand_undef_array(g, array_const_val);
7297 ConstParent *parent = &array_const_val->parent;
7298 LLVMValueRef base_ptr = gen_parent_ptr(g, array_const_val, parent);
7299
7300 LLVMTypeKind el_type = LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(base_ptr)));
7301 if (el_type == LLVMArrayTypeKind) {
7302 ZigType *usize = g->builtin_types.entry_usize;
7303 LLVMValueRef indices[] = {
7304 LLVMConstNull(usize->llvm_type),
7305 LLVMConstInt(usize->llvm_type, index, false),
7306 };
7307 return LLVMConstInBoundsGEP(base_ptr, indices, 2);
7308 } else if (el_type == LLVMStructTypeKind) {
7309 ZigType *u32 = g->builtin_types.entry_u32;
7310 LLVMValueRef indices[] = {
7311 LLVMConstNull(get_llvm_type(g, u32)),
7312 LLVMConstInt(get_llvm_type(g, u32), index, false),
7313 };
7314 return LLVMConstInBoundsGEP(base_ptr, indices, 2);
7315 } else {
7316 return base_ptr;
7317 }
7318 }
7319
gen_const_ptr_struct_recursive(CodeGen * g,ZigValue * struct_const_val,size_t field_index)7320 static LLVMValueRef gen_const_ptr_struct_recursive(CodeGen *g, ZigValue *struct_const_val, size_t field_index) {
7321 ConstParent *parent = &struct_const_val->parent;
7322 LLVMValueRef base_ptr = gen_parent_ptr(g, struct_const_val, parent);
7323
7324 ZigType *u32 = g->builtin_types.entry_u32;
7325 LLVMValueRef indices[] = {
7326 LLVMConstNull(get_llvm_type(g, u32)),
7327 LLVMConstInt(get_llvm_type(g, u32), field_index, false),
7328 };
7329
7330 // The structure pointed by base_ptr may include trailing padding for
7331 // alignment purposes and have the following LLVM type: <{ %T, [N x i8] }>.
7332 // Add an extra bitcast as we're only interested in the %T part.
7333 assert(handle_is_ptr(g, struct_const_val->type));
7334 LLVMValueRef casted_base_ptr = LLVMConstBitCast(base_ptr,
7335 LLVMPointerType(get_llvm_type(g, struct_const_val->type), 0));
7336 return LLVMConstInBoundsGEP(casted_base_ptr, indices, 2);
7337 }
7338
gen_const_ptr_err_union_code_recursive(CodeGen * g,ZigValue * err_union_const_val)7339 static LLVMValueRef gen_const_ptr_err_union_code_recursive(CodeGen *g, ZigValue *err_union_const_val) {
7340 ConstParent *parent = &err_union_const_val->parent;
7341 LLVMValueRef base_ptr = gen_parent_ptr(g, err_union_const_val, parent);
7342
7343 ZigType *u32 = g->builtin_types.entry_u32;
7344 LLVMValueRef indices[] = {
7345 LLVMConstNull(get_llvm_type(g, u32)),
7346 LLVMConstInt(get_llvm_type(g, u32), err_union_err_index, false),
7347 };
7348 return LLVMConstInBoundsGEP(base_ptr, indices, 2);
7349 }
7350
gen_const_ptr_err_union_payload_recursive(CodeGen * g,ZigValue * err_union_const_val)7351 static LLVMValueRef gen_const_ptr_err_union_payload_recursive(CodeGen *g, ZigValue *err_union_const_val) {
7352 ConstParent *parent = &err_union_const_val->parent;
7353 LLVMValueRef base_ptr = gen_parent_ptr(g, err_union_const_val, parent);
7354
7355 ZigType *u32 = g->builtin_types.entry_u32;
7356 LLVMValueRef indices[] = {
7357 LLVMConstNull(get_llvm_type(g, u32)),
7358 LLVMConstInt(get_llvm_type(g, u32), err_union_payload_index, false),
7359 };
7360 return LLVMConstInBoundsGEP(base_ptr, indices, 2);
7361 }
7362
gen_const_ptr_optional_payload_recursive(CodeGen * g,ZigValue * optional_const_val)7363 static LLVMValueRef gen_const_ptr_optional_payload_recursive(CodeGen *g, ZigValue *optional_const_val) {
7364 ConstParent *parent = &optional_const_val->parent;
7365 LLVMValueRef base_ptr = gen_parent_ptr(g, optional_const_val, parent);
7366
7367 ZigType *u32 = g->builtin_types.entry_u32;
7368 LLVMValueRef indices[] = {
7369 LLVMConstNull(get_llvm_type(g, u32)),
7370 LLVMConstInt(get_llvm_type(g, u32), maybe_child_index, false),
7371 };
7372 return LLVMConstInBoundsGEP(base_ptr, indices, 2);
7373 }
7374
gen_const_ptr_union_recursive(CodeGen * g,ZigValue * union_const_val)7375 static LLVMValueRef gen_const_ptr_union_recursive(CodeGen *g, ZigValue *union_const_val) {
7376 ConstParent *parent = &union_const_val->parent;
7377 LLVMValueRef base_ptr = gen_parent_ptr(g, union_const_val, parent);
7378
7379 // Slot in the structure where the payload is stored, if equal to SIZE_MAX
7380 // the union has no tag and a single field and is collapsed into the field
7381 // itself
7382 size_t union_payload_index = union_const_val->type->data.unionation.gen_union_index;
7383
7384 ZigType *u32 = g->builtin_types.entry_u32;
7385 LLVMValueRef indices[] = {
7386 LLVMConstNull(get_llvm_type(g, u32)),
7387 LLVMConstInt(get_llvm_type(g, u32), union_payload_index, false),
7388 };
7389 return LLVMConstInBoundsGEP(base_ptr, indices, (union_payload_index != SIZE_MAX) ? 2 : 1);
7390 }
7391
pack_const_int(CodeGen * g,LLVMTypeRef big_int_type_ref,ZigValue * const_val)7392 static LLVMValueRef pack_const_int(CodeGen *g, LLVMTypeRef big_int_type_ref, ZigValue *const_val) {
7393 switch (const_val->special) {
7394 case ConstValSpecialLazy:
7395 case ConstValSpecialRuntime:
7396 zig_unreachable();
7397 case ConstValSpecialUndef:
7398 return LLVMConstInt(big_int_type_ref, 0, false);
7399 case ConstValSpecialStatic:
7400 break;
7401 }
7402
7403 ZigType *type_entry = const_val->type;
7404 assert(type_has_bits(g, type_entry));
7405 switch (type_entry->id) {
7406 case ZigTypeIdInvalid:
7407 case ZigTypeIdMetaType:
7408 case ZigTypeIdUnreachable:
7409 case ZigTypeIdComptimeFloat:
7410 case ZigTypeIdComptimeInt:
7411 case ZigTypeIdEnumLiteral:
7412 case ZigTypeIdUndefined:
7413 case ZigTypeIdNull:
7414 case ZigTypeIdErrorUnion:
7415 case ZigTypeIdErrorSet:
7416 case ZigTypeIdBoundFn:
7417 case ZigTypeIdVoid:
7418 case ZigTypeIdOpaque:
7419 zig_unreachable();
7420 case ZigTypeIdBool:
7421 return LLVMConstInt(big_int_type_ref, const_val->data.x_bool ? 1 : 0, false);
7422 case ZigTypeIdEnum:
7423 {
7424 assert(type_entry->data.enumeration.decl_node->data.container_decl.init_arg_expr != nullptr);
7425 LLVMValueRef int_val = gen_const_val(g, const_val, "");
7426 return LLVMConstZExt(int_val, big_int_type_ref);
7427 }
7428 case ZigTypeIdInt:
7429 {
7430 LLVMValueRef int_val = gen_const_val(g, const_val, "");
7431 return LLVMConstZExt(int_val, big_int_type_ref);
7432 }
7433 case ZigTypeIdFloat:
7434 {
7435 LLVMValueRef float_val = gen_const_val(g, const_val, "");
7436 LLVMValueRef int_val = LLVMConstFPToUI(float_val,
7437 LLVMIntType((unsigned)type_entry->data.floating.bit_count));
7438 return LLVMConstZExt(int_val, big_int_type_ref);
7439 }
7440 case ZigTypeIdPointer:
7441 case ZigTypeIdFn:
7442 case ZigTypeIdOptional:
7443 {
7444 LLVMValueRef ptr_val = gen_const_val(g, const_val, "");
7445 LLVMValueRef ptr_size_int_val = LLVMConstPtrToInt(ptr_val, g->builtin_types.entry_usize->llvm_type);
7446 return LLVMConstZExt(ptr_size_int_val, big_int_type_ref);
7447 }
7448 case ZigTypeIdArray: {
7449 LLVMValueRef val = LLVMConstInt(big_int_type_ref, 0, false);
7450 if (const_val->data.x_array.special == ConstArraySpecialUndef) {
7451 return val;
7452 }
7453 expand_undef_array(g, const_val);
7454 bool is_big_endian = g->is_big_endian; // TODO get endianness from struct type
7455 uint32_t packed_bits_size = type_size_bits(g, type_entry->data.array.child_type);
7456 size_t used_bits = 0;
7457 for (size_t i = 0; i < type_entry->data.array.len; i += 1) {
7458 ZigValue *elem_val = &const_val->data.x_array.data.s_none.elements[i];
7459 LLVMValueRef child_val = pack_const_int(g, big_int_type_ref, elem_val);
7460
7461 if (is_big_endian) {
7462 LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, packed_bits_size, false);
7463 val = LLVMConstShl(val, shift_amt);
7464 val = LLVMConstOr(val, child_val);
7465 } else {
7466 LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, used_bits, false);
7467 LLVMValueRef child_val_shifted = LLVMConstShl(child_val, shift_amt);
7468 val = LLVMConstOr(val, child_val_shifted);
7469 used_bits += packed_bits_size;
7470 }
7471 }
7472
7473 if (type_entry->data.array.sentinel != nullptr) {
7474 ZigValue *elem_val = type_entry->data.array.sentinel;
7475 LLVMValueRef child_val = pack_const_int(g, big_int_type_ref, elem_val);
7476
7477 if (is_big_endian) {
7478 LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, packed_bits_size, false);
7479 val = LLVMConstShl(val, shift_amt);
7480 val = LLVMConstOr(val, child_val);
7481 } else {
7482 LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, used_bits, false);
7483 LLVMValueRef child_val_shifted = LLVMConstShl(child_val, shift_amt);
7484 val = LLVMConstOr(val, child_val_shifted);
7485 used_bits += packed_bits_size;
7486 }
7487 }
7488 return val;
7489 }
7490 case ZigTypeIdVector:
7491 zig_panic("TODO bit pack a vector");
7492 case ZigTypeIdUnion:
7493 zig_panic("TODO bit pack a union");
7494 case ZigTypeIdStruct:
7495 {
7496 assert(type_entry->data.structure.layout == ContainerLayoutPacked);
7497 bool is_big_endian = g->is_big_endian; // TODO get endianness from struct type
7498
7499 LLVMValueRef val = LLVMConstInt(big_int_type_ref, 0, false);
7500 size_t used_bits = 0;
7501 for (size_t i = 0; i < type_entry->data.structure.src_field_count; i += 1) {
7502 TypeStructField *field = type_entry->data.structure.fields[i];
7503 if (field->gen_index == SIZE_MAX || field->is_comptime) {
7504 continue;
7505 }
7506 LLVMValueRef child_val = pack_const_int(g, big_int_type_ref, const_val->data.x_struct.fields[i]);
7507 uint32_t packed_bits_size = type_size_bits(g, field->type_entry);
7508 if (is_big_endian) {
7509 LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, packed_bits_size, false);
7510 val = LLVMConstShl(val, shift_amt);
7511 val = LLVMConstOr(val, child_val);
7512 } else {
7513 LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, used_bits, false);
7514 LLVMValueRef child_val_shifted = LLVMConstShl(child_val, shift_amt);
7515 val = LLVMConstOr(val, child_val_shifted);
7516 used_bits += packed_bits_size;
7517 }
7518 }
7519 return val;
7520 }
7521 case ZigTypeIdFnFrame:
7522 zig_panic("TODO bit pack an async function frame");
7523 case ZigTypeIdAnyFrame:
7524 zig_panic("TODO bit pack an anyframe");
7525 }
7526 zig_unreachable();
7527 }
7528
7529 // We have this because union constants can't be represented by the official union type,
7530 // and this property bubbles up in whatever aggregate type contains a union constant
is_llvm_value_unnamed_type(CodeGen * g,ZigType * type_entry,LLVMValueRef val)7531 static bool is_llvm_value_unnamed_type(CodeGen *g, ZigType *type_entry, LLVMValueRef val) {
7532 return LLVMTypeOf(val) != get_llvm_type(g, type_entry);
7533 }
7534
gen_const_val_ptr(CodeGen * g,ZigValue * const_val,const char * name)7535 static LLVMValueRef gen_const_val_ptr(CodeGen *g, ZigValue *const_val, const char *name) {
7536 switch (const_val->data.x_ptr.special) {
7537 case ConstPtrSpecialInvalid:
7538 case ConstPtrSpecialDiscard:
7539 zig_unreachable();
7540 case ConstPtrSpecialRef:
7541 {
7542 ZigValue *pointee = const_val->data.x_ptr.data.ref.pointee;
7543 render_const_val(g, pointee, "");
7544 render_const_val_global(g, pointee, "");
7545 const_val->llvm_value = LLVMConstBitCast(pointee->llvm_global,
7546 get_llvm_type(g, const_val->type));
7547 return const_val->llvm_value;
7548 }
7549 case ConstPtrSpecialBaseArray:
7550 case ConstPtrSpecialSubArray:
7551 {
7552 ZigValue *array_const_val = const_val->data.x_ptr.data.base_array.array_val;
7553 assert(array_const_val->type->id == ZigTypeIdArray);
7554 if (!type_has_bits(g, array_const_val->type)) {
7555 // make this a null pointer
7556 ZigType *usize = g->builtin_types.entry_usize;
7557 const_val->llvm_value = LLVMConstIntToPtr(LLVMConstNull(usize->llvm_type),
7558 get_llvm_type(g, const_val->type));
7559 return const_val->llvm_value;
7560 }
7561 size_t elem_index = const_val->data.x_ptr.data.base_array.elem_index;
7562 LLVMValueRef uncasted_ptr_val = gen_const_ptr_array_recursive(g, array_const_val, elem_index);
7563 LLVMValueRef ptr_val = LLVMConstBitCast(uncasted_ptr_val, get_llvm_type(g, const_val->type));
7564 const_val->llvm_value = ptr_val;
7565 return ptr_val;
7566 }
7567 case ConstPtrSpecialBaseStruct:
7568 {
7569 ZigValue *struct_const_val = const_val->data.x_ptr.data.base_struct.struct_val;
7570 assert(struct_const_val->type->id == ZigTypeIdStruct);
7571 if (!type_has_bits(g, struct_const_val->type)) {
7572 // make this a null pointer
7573 ZigType *usize = g->builtin_types.entry_usize;
7574 const_val->llvm_value = LLVMConstIntToPtr(LLVMConstNull(usize->llvm_type),
7575 get_llvm_type(g, const_val->type));
7576 return const_val->llvm_value;
7577 }
7578 size_t src_field_index = const_val->data.x_ptr.data.base_struct.field_index;
7579 size_t gen_field_index = struct_const_val->type->data.structure.fields[src_field_index]->gen_index;
7580 LLVMValueRef uncasted_ptr_val = gen_const_ptr_struct_recursive(g, struct_const_val,
7581 gen_field_index);
7582 LLVMValueRef ptr_val = LLVMConstBitCast(uncasted_ptr_val, get_llvm_type(g, const_val->type));
7583 const_val->llvm_value = ptr_val;
7584 return ptr_val;
7585 }
7586 case ConstPtrSpecialBaseErrorUnionCode:
7587 {
7588 ZigValue *err_union_const_val = const_val->data.x_ptr.data.base_err_union_code.err_union_val;
7589 assert(err_union_const_val->type->id == ZigTypeIdErrorUnion);
7590 if (!type_has_bits(g, err_union_const_val->type)) {
7591 // make this a null pointer
7592 ZigType *usize = g->builtin_types.entry_usize;
7593 const_val->llvm_value = LLVMConstIntToPtr(LLVMConstNull(usize->llvm_type),
7594 get_llvm_type(g, const_val->type));
7595 return const_val->llvm_value;
7596 }
7597 LLVMValueRef uncasted_ptr_val = gen_const_ptr_err_union_code_recursive(g, err_union_const_val);
7598 LLVMValueRef ptr_val = LLVMConstBitCast(uncasted_ptr_val, get_llvm_type(g, const_val->type));
7599 const_val->llvm_value = ptr_val;
7600 return ptr_val;
7601 }
7602 case ConstPtrSpecialBaseErrorUnionPayload:
7603 {
7604 ZigValue *err_union_const_val = const_val->data.x_ptr.data.base_err_union_payload.err_union_val;
7605 assert(err_union_const_val->type->id == ZigTypeIdErrorUnion);
7606 if (!type_has_bits(g, err_union_const_val->type)) {
7607 // make this a null pointer
7608 ZigType *usize = g->builtin_types.entry_usize;
7609 const_val->llvm_value = LLVMConstIntToPtr(LLVMConstNull(usize->llvm_type),
7610 get_llvm_type(g, const_val->type));
7611 return const_val->llvm_value;
7612 }
7613 LLVMValueRef uncasted_ptr_val = gen_const_ptr_err_union_payload_recursive(g, err_union_const_val);
7614 LLVMValueRef ptr_val = LLVMConstBitCast(uncasted_ptr_val, get_llvm_type(g, const_val->type));
7615 const_val->llvm_value = ptr_val;
7616 return ptr_val;
7617 }
7618 case ConstPtrSpecialBaseOptionalPayload:
7619 {
7620 ZigValue *optional_const_val = const_val->data.x_ptr.data.base_optional_payload.optional_val;
7621 assert(optional_const_val->type->id == ZigTypeIdOptional);
7622 if (!type_has_bits(g, optional_const_val->type)) {
7623 // make this a null pointer
7624 ZigType *usize = g->builtin_types.entry_usize;
7625 const_val->llvm_value = LLVMConstIntToPtr(LLVMConstNull(usize->llvm_type),
7626 get_llvm_type(g, const_val->type));
7627 return const_val->llvm_value;
7628 }
7629 LLVMValueRef uncasted_ptr_val = gen_const_ptr_optional_payload_recursive(g, optional_const_val);
7630 LLVMValueRef ptr_val = LLVMConstBitCast(uncasted_ptr_val, get_llvm_type(g, const_val->type));
7631 const_val->llvm_value = ptr_val;
7632 return ptr_val;
7633 }
7634 case ConstPtrSpecialHardCodedAddr:
7635 {
7636 uint64_t addr_value = const_val->data.x_ptr.data.hard_coded_addr.addr;
7637 ZigType *usize = g->builtin_types.entry_usize;
7638 const_val->llvm_value = LLVMConstIntToPtr(
7639 LLVMConstInt(usize->llvm_type, addr_value, false), get_llvm_type(g, const_val->type));
7640 return const_val->llvm_value;
7641 }
7642 case ConstPtrSpecialFunction:
7643 return LLVMConstBitCast(fn_llvm_value(g, const_val->data.x_ptr.data.fn.fn_entry),
7644 get_llvm_type(g, const_val->type));
7645 case ConstPtrSpecialNull:
7646 return LLVMConstNull(get_llvm_type(g, const_val->type));
7647 }
7648 zig_unreachable();
7649 }
7650
gen_const_val_err_set(CodeGen * g,ZigValue * const_val,const char * name)7651 static LLVMValueRef gen_const_val_err_set(CodeGen *g, ZigValue *const_val, const char *name) {
7652 uint64_t value = (const_val->data.x_err_set == nullptr) ? 0 : const_val->data.x_err_set->value;
7653 return LLVMConstInt(get_llvm_type(g, g->builtin_types.entry_global_error_set), value, false);
7654 }
7655
gen_const_val(CodeGen * g,ZigValue * const_val,const char * name)7656 static LLVMValueRef gen_const_val(CodeGen *g, ZigValue *const_val, const char *name) {
7657 Error err;
7658
7659 ZigType *type_entry = const_val->type;
7660 assert(type_has_bits(g, type_entry));
7661
7662 if (const_val->special == ConstValSpecialLazy &&
7663 (err = ir_resolve_lazy(g, nullptr, const_val)))
7664 codegen_report_errors_and_exit(g);
7665
7666 switch (const_val->special) {
7667 case ConstValSpecialLazy:
7668 case ConstValSpecialRuntime:
7669 zig_unreachable();
7670 case ConstValSpecialUndef:
7671 return LLVMGetUndef(get_llvm_type(g, type_entry));
7672 case ConstValSpecialStatic:
7673 break;
7674 }
7675
7676 if ((err = type_resolve(g, type_entry, ResolveStatusLLVMFull)))
7677 zig_unreachable();
7678
7679 switch (type_entry->id) {
7680 case ZigTypeIdInt:
7681 return bigint_to_llvm_const(get_llvm_type(g, type_entry), &const_val->data.x_bigint);
7682 case ZigTypeIdErrorSet:
7683 return gen_const_val_err_set(g, const_val, name);
7684 case ZigTypeIdFloat:
7685 switch (type_entry->data.floating.bit_count) {
7686 case 16:
7687 {
7688 LLVMValueRef as_int = LLVMConstInt(LLVMInt16Type(), const_val->data.x_f16.v, false);
7689 return LLVMConstBitCast(as_int, get_llvm_type(g, type_entry));
7690 }
7691 case 32:
7692 return LLVMConstReal(get_llvm_type(g, type_entry), const_val->data.x_f32);
7693 case 64:
7694 return LLVMConstReal(get_llvm_type(g, type_entry), const_val->data.x_f64);
7695 case 128:
7696 {
7697 uint64_t buf[2];
7698
7699 // LLVM seems to require that the lower half of the f128 be placed first in the buffer.
7700 #if defined(ZIG_BYTE_ORDER) && ZIG_BYTE_ORDER == ZIG_LITTLE_ENDIAN
7701 buf[0] = const_val->data.x_f128.v[0];
7702 buf[1] = const_val->data.x_f128.v[1];
7703 #elif defined(ZIG_BYTE_ORDER) && ZIG_BYTE_ORDER == ZIG_BIG_ENDIAN
7704 buf[0] = const_val->data.x_f128.v[1];
7705 buf[1] = const_val->data.x_f128.v[0];
7706 #else
7707 #error Unsupported endian
7708 #endif
7709
7710 LLVMValueRef as_int = LLVMConstIntOfArbitraryPrecision(LLVMInt128Type(), 2, buf);
7711 return LLVMConstBitCast(as_int, get_llvm_type(g, type_entry));
7712 }
7713 default:
7714 zig_unreachable();
7715 }
7716 case ZigTypeIdBool:
7717 if (const_val->data.x_bool) {
7718 return LLVMConstAllOnes(LLVMInt1Type());
7719 } else {
7720 return LLVMConstNull(LLVMInt1Type());
7721 }
7722 case ZigTypeIdOptional:
7723 {
7724 ZigType *child_type = type_entry->data.maybe.child_type;
7725
7726 if (get_src_ptr_type(type_entry) != nullptr) {
7727 bool has_bits;
7728 if ((err = type_has_bits2(g, child_type, &has_bits)))
7729 codegen_report_errors_and_exit(g);
7730
7731 if (has_bits)
7732 return gen_const_val_ptr(g, const_val, name);
7733
7734 // No bits, treat this value as a boolean
7735 const unsigned bool_val = optional_value_is_null(const_val) ? 0 : 1;
7736 return LLVMConstInt(LLVMInt1Type(), bool_val, false);
7737 } else if (child_type->id == ZigTypeIdErrorSet) {
7738 return gen_const_val_err_set(g, const_val, name);
7739 } else if (!type_has_bits(g, child_type)) {
7740 return LLVMConstInt(LLVMInt1Type(), const_val->data.x_optional ? 1 : 0, false);
7741 } else {
7742 LLVMValueRef child_val;
7743 LLVMValueRef maybe_val;
7744 bool make_unnamed_struct;
7745 if (const_val->data.x_optional) {
7746 child_val = gen_const_val(g, const_val->data.x_optional, "");
7747 maybe_val = LLVMConstAllOnes(LLVMInt1Type());
7748
7749 make_unnamed_struct = is_llvm_value_unnamed_type(g, const_val->type, child_val);
7750 } else {
7751 child_val = LLVMGetUndef(get_llvm_type(g, child_type));
7752 maybe_val = LLVMConstNull(LLVMInt1Type());
7753
7754 make_unnamed_struct = false;
7755 }
7756
7757 LLVMValueRef fields[] = {
7758 child_val,
7759 maybe_val,
7760 nullptr,
7761 };
7762 if (make_unnamed_struct) {
7763 LLVMValueRef result = LLVMConstStruct(fields, 2, false);
7764 uint64_t last_field_offset = LLVMOffsetOfElement(g->target_data_ref, LLVMTypeOf(result), 1);
7765 uint64_t end_offset = last_field_offset +
7766 LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(fields[1]));
7767 uint64_t expected_sz = LLVMABISizeOfType(g->target_data_ref, get_llvm_type(g, type_entry));
7768 unsigned pad_sz = expected_sz - end_offset;
7769 if (pad_sz != 0) {
7770 fields[2] = LLVMGetUndef(LLVMArrayType(LLVMInt8Type(), pad_sz));
7771 result = LLVMConstStruct(fields, 3, false);
7772 }
7773 uint64_t actual_sz = LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(result));
7774 assert(actual_sz == expected_sz);
7775 return result;
7776 } else {
7777 return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, 2);
7778 }
7779 }
7780 }
7781 case ZigTypeIdStruct:
7782 {
7783 LLVMValueRef *fields = heap::c_allocator.allocate<LLVMValueRef>(type_entry->data.structure.gen_field_count);
7784 size_t src_field_count = type_entry->data.structure.src_field_count;
7785 bool make_unnamed_struct = false;
7786 assert(type_entry->data.structure.resolve_status == ResolveStatusLLVMFull);
7787 if (type_entry->data.structure.layout == ContainerLayoutPacked) {
7788 size_t src_field_index = 0;
7789 while (src_field_index < src_field_count) {
7790 TypeStructField *type_struct_field = type_entry->data.structure.fields[src_field_index];
7791 if (type_struct_field->gen_index == SIZE_MAX || type_struct_field->is_comptime) {
7792 src_field_index += 1;
7793 continue;
7794 }
7795
7796 size_t src_field_index_end = src_field_index + 1;
7797 for (; src_field_index_end < src_field_count; src_field_index_end += 1) {
7798 TypeStructField *it_field = type_entry->data.structure.fields[src_field_index_end];
7799 if (it_field->gen_index != type_struct_field->gen_index)
7800 break;
7801 }
7802
7803 if (src_field_index + 1 == src_field_index_end) {
7804 ZigValue *field_val = const_val->data.x_struct.fields[src_field_index];
7805 LLVMValueRef val = gen_const_val(g, field_val, "");
7806 fields[type_struct_field->gen_index] = val;
7807 make_unnamed_struct = make_unnamed_struct || is_llvm_value_unnamed_type(g, field_val->type, val);
7808 } else {
7809 bool is_big_endian = g->is_big_endian; // TODO get endianness from struct type
7810 LLVMTypeRef field_ty = LLVMStructGetTypeAtIndex(get_llvm_type(g, type_entry),
7811 (unsigned)type_struct_field->gen_index);
7812 const size_t size_in_bytes = LLVMStoreSizeOfType(g->target_data_ref, field_ty);
7813 const size_t size_in_bits = size_in_bytes * 8;
7814 LLVMTypeRef big_int_type_ref = LLVMIntType(size_in_bits);
7815 LLVMValueRef val = LLVMConstInt(big_int_type_ref, 0, false);
7816 size_t used_bits = 0;
7817 for (size_t i = src_field_index; i < src_field_index_end; i += 1) {
7818 TypeStructField *it_field = type_entry->data.structure.fields[i];
7819 if (it_field->gen_index == SIZE_MAX) {
7820 continue;
7821 }
7822 LLVMValueRef child_val = pack_const_int(g, big_int_type_ref,
7823 const_val->data.x_struct.fields[i]);
7824 uint32_t packed_bits_size = type_size_bits(g, it_field->type_entry);
7825 if (is_big_endian) {
7826 LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref,
7827 size_in_bits - used_bits - packed_bits_size, false);
7828 LLVMValueRef child_val_shifted = LLVMConstShl(child_val, shift_amt);
7829 val = LLVMConstOr(val, child_val_shifted);
7830 } else {
7831 LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, used_bits, false);
7832 LLVMValueRef child_val_shifted = LLVMConstShl(child_val, shift_amt);
7833 val = LLVMConstOr(val, child_val_shifted);
7834 }
7835 used_bits += packed_bits_size;
7836 }
7837 assert(size_in_bits >= used_bits);
7838 if (LLVMGetTypeKind(field_ty) != LLVMArrayTypeKind) {
7839 assert(LLVMGetTypeKind(field_ty) == LLVMIntegerTypeKind);
7840 fields[type_struct_field->gen_index] = val;
7841 } else {
7842 const LLVMValueRef AMT = LLVMConstInt(LLVMTypeOf(val), 8, false);
7843
7844 LLVMValueRef *values = heap::c_allocator.allocate<LLVMValueRef>(size_in_bytes);
7845 for (size_t i = 0; i < size_in_bytes; i++) {
7846 const size_t idx = is_big_endian ? size_in_bytes - 1 - i : i;
7847 values[idx] = LLVMConstTruncOrBitCast(val, LLVMInt8Type());
7848 val = LLVMConstLShr(val, AMT);
7849 }
7850
7851 fields[type_struct_field->gen_index] = LLVMConstArray(LLVMInt8Type(), values, size_in_bytes);
7852 }
7853 }
7854
7855 src_field_index = src_field_index_end;
7856 }
7857 } else {
7858 for (uint32_t i = 0; i < src_field_count; i += 1) {
7859 TypeStructField *type_struct_field = type_entry->data.structure.fields[i];
7860 if (type_struct_field->gen_index == SIZE_MAX || type_struct_field->is_comptime) {
7861 continue;
7862 }
7863 ZigValue *field_val = const_val->data.x_struct.fields[i];
7864 if (field_val == nullptr) {
7865 add_node_error(g, type_struct_field->decl_node,
7866 buf_sprintf("compiler bug: generating const value for struct field '%s'",
7867 buf_ptr(type_struct_field->name)));
7868 codegen_report_errors_and_exit(g);
7869 }
7870 ZigType *field_type = field_val->type;
7871 assert(field_type != nullptr);
7872 if ((err = ensure_const_val_repr(nullptr, g, nullptr, field_val, field_type))) {
7873 zig_unreachable();
7874 }
7875
7876 LLVMValueRef val = gen_const_val(g, field_val, "");
7877 make_unnamed_struct = make_unnamed_struct || is_llvm_value_unnamed_type(g, field_type, val);
7878
7879 // Find the next runtime field
7880 size_t next_rt_gen_index = type_entry->data.structure.gen_field_count;
7881 size_t next_offset = type_entry->abi_size;
7882 for (size_t j = i + 1; j < src_field_count; j++) {
7883 const size_t index = type_entry->data.structure.fields[j]->gen_index;
7884 const size_t offset = type_entry->data.structure.fields[j]->offset;
7885
7886 if (index != SIZE_MAX) {
7887 next_rt_gen_index = index;
7888 next_offset = offset;
7889 break;
7890 }
7891 }
7892
7893 // How much padding is needed to reach the next field
7894 const size_t pad_bytes = next_offset -
7895 (type_struct_field->offset + LLVMABISizeOfType(g->target_data_ref, LLVMTypeOf(val)));
7896 // Catch underflow
7897 assert((ssize_t)pad_bytes >= 0);
7898
7899 if (type_struct_field->gen_index + 1 != next_rt_gen_index) {
7900 // If there's a hole between this field and the next
7901 // we have an alignment gap to fill
7902 fields[type_struct_field->gen_index] = val;
7903 fields[type_struct_field->gen_index + 1] = LLVMGetUndef(LLVMArrayType(LLVMInt8Type(), pad_bytes));
7904 } else if (pad_bytes != 0) {
7905 LLVMValueRef padded_val[] = {
7906 val,
7907 LLVMGetUndef(LLVMArrayType(LLVMInt8Type(), pad_bytes)),
7908 };
7909 fields[type_struct_field->gen_index] = LLVMConstStruct(padded_val, 2, true);
7910 make_unnamed_struct = true;
7911 } else {
7912 fields[type_struct_field->gen_index] = val;
7913 }
7914 }
7915 }
7916 if (make_unnamed_struct) {
7917 LLVMValueRef unnamed_struct = LLVMConstStruct(fields, type_entry->data.structure.gen_field_count,
7918 type_entry->data.structure.layout == ContainerLayoutPacked);
7919 heap::c_allocator.deallocate(fields, type_entry->data.structure.gen_field_count);
7920 return unnamed_struct;
7921 } else {
7922 LLVMValueRef named_struct = LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, type_entry->data.structure.gen_field_count);
7923 heap::c_allocator.deallocate(fields, type_entry->data.structure.gen_field_count);
7924 return named_struct;
7925 }
7926 }
7927 case ZigTypeIdArray:
7928 {
7929 uint64_t len = type_entry->data.array.len;
7930 switch (const_val->data.x_array.special) {
7931 case ConstArraySpecialUndef:
7932 return LLVMGetUndef(get_llvm_type(g, type_entry));
7933 case ConstArraySpecialNone: {
7934 uint64_t extra_len_from_sentinel = (type_entry->data.array.sentinel != nullptr) ? 1 : 0;
7935 uint64_t full_len = len + extra_len_from_sentinel;
7936 LLVMValueRef *values = heap::c_allocator.allocate<LLVMValueRef>(full_len);
7937 LLVMTypeRef element_type_ref = get_llvm_type(g, type_entry->data.array.child_type);
7938 bool make_unnamed_struct = false;
7939 for (uint64_t i = 0; i < len; i += 1) {
7940 ZigValue *elem_value = &const_val->data.x_array.data.s_none.elements[i];
7941 LLVMValueRef val = gen_const_val(g, elem_value, "");
7942 values[i] = val;
7943 make_unnamed_struct = make_unnamed_struct || is_llvm_value_unnamed_type(g, elem_value->type, val);
7944 }
7945 if (type_entry->data.array.sentinel != nullptr) {
7946 values[len] = gen_const_val(g, type_entry->data.array.sentinel, "");
7947 }
7948 if (make_unnamed_struct) {
7949 LLVMValueRef unnamed_struct = LLVMConstStruct(values, full_len, true);
7950 heap::c_allocator.deallocate(values, full_len);
7951 return unnamed_struct;
7952 } else {
7953 LLVMValueRef array = LLVMConstArray(element_type_ref, values, (unsigned)full_len);
7954 heap::c_allocator.deallocate(values, full_len);
7955 return array;
7956 }
7957 }
7958 case ConstArraySpecialBuf: {
7959 Buf *buf = const_val->data.x_array.data.s_buf;
7960 return LLVMConstString(buf_ptr(buf), (unsigned)buf_len(buf),
7961 type_entry->data.array.sentinel == nullptr);
7962 }
7963 }
7964 zig_unreachable();
7965 }
7966 case ZigTypeIdVector: {
7967 uint32_t len = type_entry->data.vector.len;
7968 switch (const_val->data.x_array.special) {
7969 case ConstArraySpecialUndef:
7970 return LLVMGetUndef(get_llvm_type(g, type_entry));
7971 case ConstArraySpecialNone: {
7972 LLVMValueRef *values = heap::c_allocator.allocate<LLVMValueRef>(len);
7973 for (uint64_t i = 0; i < len; i += 1) {
7974 ZigValue *elem_value = &const_val->data.x_array.data.s_none.elements[i];
7975 values[i] = gen_const_val(g, elem_value, "");
7976 }
7977 LLVMValueRef vector = LLVMConstVector(values, len);
7978 heap::c_allocator.deallocate(values, len);
7979 return vector;
7980 }
7981 case ConstArraySpecialBuf: {
7982 Buf *buf = const_val->data.x_array.data.s_buf;
7983 assert(buf_len(buf) == len);
7984 LLVMValueRef *values = heap::c_allocator.allocate<LLVMValueRef>(len);
7985 for (uint64_t i = 0; i < len; i += 1) {
7986 values[i] = LLVMConstInt(g->builtin_types.entry_u8->llvm_type, buf_ptr(buf)[i], false);
7987 }
7988 LLVMValueRef vector = LLVMConstVector(values, len);
7989 heap::c_allocator.deallocate(values, len);
7990 return vector;
7991 }
7992 }
7993 zig_unreachable();
7994 }
7995 case ZigTypeIdUnion:
7996 {
7997 // Force type_entry->data.unionation.union_llvm_type to get resolved
7998 (void)get_llvm_type(g, type_entry);
7999
8000 if (type_entry->data.unionation.gen_field_count == 0) {
8001 if (type_entry->data.unionation.tag_type == nullptr) {
8002 return nullptr;
8003 } else {
8004 return bigint_to_llvm_const(get_llvm_type(g, type_entry->data.unionation.tag_type),
8005 &const_val->data.x_union.tag);
8006 }
8007 }
8008
8009 LLVMTypeRef union_type_ref = type_entry->data.unionation.union_llvm_type;
8010 assert(union_type_ref != nullptr);
8011
8012 LLVMValueRef union_value_ref;
8013 bool make_unnamed_struct;
8014 ZigValue *payload_value = const_val->data.x_union.payload;
8015 if (payload_value == nullptr || !type_has_bits(g, payload_value->type)) {
8016 if (type_entry->data.unionation.gen_tag_index == SIZE_MAX)
8017 return LLVMGetUndef(get_llvm_type(g, type_entry));
8018
8019 union_value_ref = LLVMGetUndef(union_type_ref);
8020 make_unnamed_struct = false;
8021 } else {
8022 uint64_t field_type_bytes = LLVMABISizeOfType(g->target_data_ref,
8023 get_llvm_type(g, payload_value->type));
8024 uint64_t pad_bytes = type_entry->data.unionation.union_abi_size - field_type_bytes;
8025 LLVMValueRef correctly_typed_value = gen_const_val(g, payload_value, "");
8026 make_unnamed_struct = is_llvm_value_unnamed_type(g, payload_value->type, correctly_typed_value) ||
8027 payload_value->type != type_entry->data.unionation.most_aligned_union_member->type_entry;
8028
8029 {
8030 if (pad_bytes == 0) {
8031 union_value_ref = correctly_typed_value;
8032 } else {
8033 LLVMValueRef fields[2];
8034 fields[0] = correctly_typed_value;
8035 fields[1] = LLVMGetUndef(LLVMArrayType(LLVMInt8Type(), (unsigned)pad_bytes));
8036 if (make_unnamed_struct || type_entry->data.unionation.gen_tag_index != SIZE_MAX) {
8037 union_value_ref = LLVMConstStruct(fields, 2, false);
8038 } else {
8039 union_value_ref = LLVMConstNamedStruct(union_type_ref, fields, 2);
8040 }
8041 }
8042 }
8043
8044 if (type_entry->data.unionation.gen_tag_index == SIZE_MAX) {
8045 return union_value_ref;
8046 }
8047 }
8048
8049 LLVMValueRef tag_value = bigint_to_llvm_const(
8050 get_llvm_type(g, type_entry->data.unionation.tag_type),
8051 &const_val->data.x_union.tag);
8052
8053 LLVMValueRef fields[3];
8054 fields[type_entry->data.unionation.gen_union_index] = union_value_ref;
8055 fields[type_entry->data.unionation.gen_tag_index] = tag_value;
8056
8057 if (make_unnamed_struct) {
8058 LLVMValueRef result = LLVMConstStruct(fields, 2, false);
8059 uint64_t last_field_offset = LLVMOffsetOfElement(g->target_data_ref, LLVMTypeOf(result), 1);
8060 uint64_t end_offset = last_field_offset +
8061 LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(fields[1]));
8062 uint64_t expected_sz = LLVMABISizeOfType(g->target_data_ref, get_llvm_type(g, type_entry));
8063 unsigned pad_sz = expected_sz - end_offset;
8064 if (pad_sz != 0) {
8065 fields[2] = LLVMGetUndef(LLVMArrayType(LLVMInt8Type(), pad_sz));
8066 result = LLVMConstStruct(fields, 3, false);
8067 }
8068 uint64_t actual_sz = LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(result));
8069 assert(actual_sz == expected_sz);
8070 return result;
8071 } else {
8072 return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, 2);
8073 }
8074
8075 }
8076
8077 case ZigTypeIdEnum:
8078 return bigint_to_llvm_const(get_llvm_type(g, type_entry), &const_val->data.x_enum_tag);
8079 case ZigTypeIdFn:
8080 if (const_val->data.x_ptr.special == ConstPtrSpecialFunction &&
8081 const_val->data.x_ptr.mut != ConstPtrMutComptimeConst) {
8082 zig_unreachable();
8083 }
8084 // Treat it the same as we do for pointers
8085 return gen_const_val_ptr(g, const_val, name);
8086 case ZigTypeIdPointer:
8087 return gen_const_val_ptr(g, const_val, name);
8088 case ZigTypeIdErrorUnion:
8089 {
8090 ZigType *payload_type = type_entry->data.error_union.payload_type;
8091 ZigType *err_set_type = type_entry->data.error_union.err_set_type;
8092 if (!type_has_bits(g, payload_type)) {
8093 assert(type_has_bits(g, err_set_type));
8094 ErrorTableEntry *err_set = const_val->data.x_err_union.error_set->data.x_err_set;
8095 uint64_t value = (err_set == nullptr) ? 0 : err_set->value;
8096 return LLVMConstInt(get_llvm_type(g, g->err_tag_type), value, false);
8097 } else if (!type_has_bits(g, err_set_type)) {
8098 assert(type_has_bits(g, payload_type));
8099 return gen_const_val(g, const_val->data.x_err_union.payload, "");
8100 } else {
8101 LLVMValueRef err_tag_value;
8102 LLVMValueRef err_payload_value;
8103 bool make_unnamed_struct;
8104 ErrorTableEntry *err_set = const_val->data.x_err_union.error_set->data.x_err_set;
8105 if (err_set != nullptr) {
8106 err_tag_value = LLVMConstInt(get_llvm_type(g, g->err_tag_type), err_set->value, false);
8107 err_payload_value = LLVMConstNull(get_llvm_type(g, payload_type));
8108 make_unnamed_struct = false;
8109 } else {
8110 err_tag_value = LLVMConstNull(get_llvm_type(g, g->err_tag_type));
8111 ZigValue *payload_val = const_val->data.x_err_union.payload;
8112 err_payload_value = gen_const_val(g, payload_val, "");
8113 make_unnamed_struct = is_llvm_value_unnamed_type(g, payload_val->type, err_payload_value);
8114 }
8115 LLVMValueRef fields[3];
8116 fields[err_union_err_index] = err_tag_value;
8117 fields[err_union_payload_index] = err_payload_value;
8118 size_t field_count = 2;
8119 if (type_entry->data.error_union.pad_llvm_type != nullptr) {
8120 fields[2] = LLVMGetUndef(type_entry->data.error_union.pad_llvm_type);
8121 field_count = 3;
8122 }
8123 if (make_unnamed_struct) {
8124 return LLVMConstStruct(fields, field_count, false);
8125 } else {
8126 return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, field_count);
8127 }
8128 }
8129 }
8130 case ZigTypeIdVoid:
8131 return nullptr;
8132 case ZigTypeIdInvalid:
8133 case ZigTypeIdMetaType:
8134 case ZigTypeIdUnreachable:
8135 case ZigTypeIdComptimeFloat:
8136 case ZigTypeIdComptimeInt:
8137 case ZigTypeIdEnumLiteral:
8138 case ZigTypeIdUndefined:
8139 case ZigTypeIdNull:
8140 case ZigTypeIdBoundFn:
8141 case ZigTypeIdOpaque:
8142 zig_unreachable();
8143 case ZigTypeIdFnFrame:
8144 zig_panic("TODO: gen_const_val ZigTypeIdFnFrame");
8145 case ZigTypeIdAnyFrame:
8146 zig_panic("TODO: gen_const_val ZigTypeIdAnyFrame");
8147 }
8148 zig_unreachable();
8149 }
8150
render_const_val(CodeGen * g,ZigValue * const_val,const char * name)8151 static void render_const_val(CodeGen *g, ZigValue *const_val, const char *name) {
8152 if (!const_val->llvm_value)
8153 const_val->llvm_value = gen_const_val(g, const_val, name);
8154
8155 if (const_val->llvm_global)
8156 LLVMSetInitializer(const_val->llvm_global, const_val->llvm_value);
8157 }
8158
render_const_val_global(CodeGen * g,ZigValue * const_val,const char * name)8159 static void render_const_val_global(CodeGen *g, ZigValue *const_val, const char *name) {
8160 if (!const_val->llvm_global) {
8161 LLVMTypeRef type_ref = const_val->llvm_value ?
8162 LLVMTypeOf(const_val->llvm_value) : get_llvm_type(g, const_val->type);
8163 LLVMValueRef global_value = LLVMAddGlobal(g->module, type_ref, name);
8164 LLVMSetLinkage(global_value, (name == nullptr) ? LLVMPrivateLinkage : LLVMInternalLinkage);
8165 LLVMSetGlobalConstant(global_value, true);
8166 LLVMSetUnnamedAddr(global_value, true);
8167 LLVMSetAlignment(global_value, (const_val->llvm_align == 0) ?
8168 get_abi_alignment(g, const_val->type) : const_val->llvm_align);
8169
8170 const_val->llvm_global = global_value;
8171 }
8172
8173 if (const_val->llvm_value)
8174 LLVMSetInitializer(const_val->llvm_global, const_val->llvm_value);
8175 }
8176
generate_error_name_table(CodeGen * g)8177 static void generate_error_name_table(CodeGen *g) {
8178 if (g->err_name_table != nullptr || !g->generate_error_name_table) {
8179 return;
8180 }
8181
8182 assert(g->errors_by_index.length > 0);
8183
8184 ZigType *u8_ptr_type = get_pointer_to_type_extra(g, g->builtin_types.entry_u8, true, false,
8185 PtrLenUnknown, get_abi_alignment(g, g->builtin_types.entry_u8), 0, 0, false);
8186 ZigType *str_type = get_slice_type(g, u8_ptr_type);
8187
8188 LLVMValueRef *values = heap::c_allocator.allocate<LLVMValueRef>(g->errors_by_index.length);
8189 values[0] = LLVMGetUndef(get_llvm_type(g, str_type));
8190 for (size_t i = 1; i < g->errors_by_index.length; i += 1) {
8191 ErrorTableEntry *err_entry = g->errors_by_index.at(i);
8192 Buf *name = &err_entry->name;
8193
8194 g->largest_err_name_len = max(g->largest_err_name_len, buf_len(name));
8195
8196 LLVMValueRef str_init = LLVMConstString(buf_ptr(name), (unsigned)buf_len(name), true);
8197 LLVMValueRef str_global = LLVMAddGlobal(g->module, LLVMTypeOf(str_init), "");
8198 LLVMSetInitializer(str_global, str_init);
8199 LLVMSetLinkage(str_global, LLVMPrivateLinkage);
8200 LLVMSetGlobalConstant(str_global, true);
8201 LLVMSetUnnamedAddr(str_global, true);
8202 LLVMSetAlignment(str_global, LLVMABIAlignmentOfType(g->target_data_ref, LLVMTypeOf(str_init)));
8203
8204 LLVMValueRef fields[] = {
8205 LLVMConstBitCast(str_global, get_llvm_type(g, u8_ptr_type)),
8206 LLVMConstInt(g->builtin_types.entry_usize->llvm_type, buf_len(name), false),
8207 };
8208 values[i] = LLVMConstNamedStruct(get_llvm_type(g, str_type), fields, 2);
8209 }
8210
8211 LLVMValueRef err_name_table_init = LLVMConstArray(get_llvm_type(g, str_type), values, (unsigned)g->errors_by_index.length);
8212 heap::c_allocator.deallocate(values, g->errors_by_index.length);
8213
8214 g->err_name_table = LLVMAddGlobal(g->module, LLVMTypeOf(err_name_table_init),
8215 get_mangled_name(g, buf_ptr(buf_create_from_str("__zig_err_name_table"))));
8216 LLVMSetInitializer(g->err_name_table, err_name_table_init);
8217 LLVMSetLinkage(g->err_name_table, LLVMPrivateLinkage);
8218 LLVMSetGlobalConstant(g->err_name_table, true);
8219 LLVMSetUnnamedAddr(g->err_name_table, true);
8220 LLVMSetAlignment(g->err_name_table, LLVMABIAlignmentOfType(g->target_data_ref, LLVMTypeOf(err_name_table_init)));
8221 }
8222
build_all_basic_blocks(CodeGen * g,ZigFn * fn)8223 static void build_all_basic_blocks(CodeGen *g, ZigFn *fn) {
8224 Stage1Air *executable = &fn->analyzed_executable;
8225 assert(executable->basic_block_list.length > 0);
8226 LLVMValueRef fn_val = fn_llvm_value(g, fn);
8227 LLVMBasicBlockRef first_bb = nullptr;
8228 if (fn_is_async(fn)) {
8229 first_bb = LLVMAppendBasicBlock(fn_val, "AsyncSwitch");
8230 g->cur_preamble_llvm_block = first_bb;
8231 }
8232 for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) {
8233 Stage1AirBasicBlock *bb = executable->basic_block_list.at(block_i);
8234 bb->llvm_block = LLVMAppendBasicBlock(fn_val, bb->name_hint);
8235 }
8236 if (first_bb == nullptr) {
8237 first_bb = executable->basic_block_list.at(0)->llvm_block;
8238 }
8239 LLVMPositionBuilderAtEnd(g->builder, first_bb);
8240 }
8241
gen_global_var(CodeGen * g,ZigVar * var,LLVMValueRef init_val,ZigType * type_entry)8242 static void gen_global_var(CodeGen *g, ZigVar *var, LLVMValueRef init_val,
8243 ZigType *type_entry)
8244 {
8245 if (g->strip_debug_symbols) {
8246 return;
8247 }
8248
8249 assert(var->gen_is_const);
8250 assert(type_entry);
8251
8252 ZigType *import = get_scope_import(var->parent_scope);
8253 assert(import);
8254
8255 bool is_local_to_unit = true;
8256 ZigLLVMCreateGlobalVariable(g->dbuilder, get_di_scope(g, var->parent_scope), var->name,
8257 var->name, import->data.structure.root_struct->di_file,
8258 node_line_onebased(var->decl_node),
8259 get_llvm_di_type(g, type_entry), is_local_to_unit);
8260
8261 // TODO ^^ make an actual global variable
8262 }
8263
set_global_tls(CodeGen * g,ZigVar * var,LLVMValueRef global_value)8264 static void set_global_tls(CodeGen *g, ZigVar *var, LLVMValueRef global_value) {
8265 bool is_extern = var->decl_node->data.variable_declaration.is_extern;
8266 bool is_export = var->decl_node->data.variable_declaration.is_export;
8267 bool is_internal_linkage = !is_extern && !is_export;
8268 if (var->is_thread_local && (!g->is_single_threaded || !is_internal_linkage)) {
8269 LLVMSetThreadLocalMode(global_value, LLVMGeneralDynamicTLSModel);
8270 }
8271 }
8272
do_code_gen(CodeGen * g)8273 static void do_code_gen(CodeGen *g) {
8274 Error err;
8275 assert(!g->errors.length);
8276
8277 generate_error_name_table(g);
8278
8279 // Generate module level variables
8280 for (size_t i = 0; i < g->global_vars.length; i += 1) {
8281 TldVar *tld_var = g->global_vars.at(i);
8282 ZigVar *var = tld_var->var;
8283
8284 if (var->var_type->id == ZigTypeIdComptimeFloat) {
8285 // Generate debug info for it but that's it.
8286 ZigValue *const_val = var->const_value;
8287 assert(const_val->special != ConstValSpecialRuntime);
8288 if ((err = ir_resolve_lazy(g, var->decl_node, const_val)))
8289 zig_unreachable();
8290 if (const_val->type != var->var_type) {
8291 zig_panic("TODO debug info for var with ptr casted value");
8292 }
8293 ZigType *var_type = g->builtin_types.entry_f128;
8294 ZigValue coerced_value = {};
8295 coerced_value.special = ConstValSpecialStatic;
8296 coerced_value.type = var_type;
8297 coerced_value.data.x_f128 = bigfloat_to_f128(&const_val->data.x_bigfloat);
8298 LLVMValueRef init_val = gen_const_val(g, &coerced_value, "");
8299 gen_global_var(g, var, init_val, var_type);
8300 continue;
8301 }
8302
8303 if (var->var_type->id == ZigTypeIdComptimeInt) {
8304 // Generate debug info for it but that's it.
8305 ZigValue *const_val = var->const_value;
8306 assert(const_val->special != ConstValSpecialRuntime);
8307 if ((err = ir_resolve_lazy(g, var->decl_node, const_val)))
8308 zig_unreachable();
8309 if (const_val->type != var->var_type) {
8310 zig_panic("TODO debug info for var with ptr casted value");
8311 }
8312 size_t bits_needed = bigint_bits_needed(&const_val->data.x_bigint);
8313 if (bits_needed < 8) {
8314 bits_needed = 8;
8315 }
8316 ZigType *var_type = get_int_type(g, const_val->data.x_bigint.is_negative, bits_needed);
8317 LLVMValueRef init_val = bigint_to_llvm_const(get_llvm_type(g, var_type), &const_val->data.x_bigint);
8318 gen_global_var(g, var, init_val, var_type);
8319 continue;
8320 }
8321
8322 if (!type_has_bits(g, var->var_type))
8323 continue;
8324
8325 assert(var->decl_node);
8326
8327 GlobalLinkageId linkage;
8328 const char *unmangled_name = var->name;
8329 const char *symbol_name;
8330 if (var->export_list.length == 0) {
8331 if (var->decl_node->data.variable_declaration.is_extern) {
8332 symbol_name = unmangled_name;
8333 linkage = GlobalLinkageIdStrong;
8334 } else {
8335 symbol_name = get_mangled_name(g, unmangled_name);
8336 linkage = GlobalLinkageIdInternal;
8337 }
8338 } else {
8339 GlobalExport *global_export = &var->export_list.items[0];
8340 symbol_name = buf_ptr(&global_export->name);
8341 linkage = global_export->linkage;
8342 }
8343
8344 LLVMValueRef global_value;
8345 bool externally_initialized = var->decl_node->data.variable_declaration.expr == nullptr;
8346 if (externally_initialized) {
8347 LLVMValueRef existing_llvm_var = LLVMGetNamedGlobal(g->module, symbol_name);
8348 if (existing_llvm_var) {
8349 global_value = LLVMConstBitCast(existing_llvm_var,
8350 LLVMPointerType(get_llvm_type(g, var->var_type), 0));
8351 } else {
8352 global_value = LLVMAddGlobal(g->module, get_llvm_type(g, var->var_type), symbol_name);
8353 // TODO debug info for the extern variable
8354
8355 LLVMSetLinkage(global_value, to_llvm_linkage(linkage, true));
8356 maybe_import_dll(g, global_value, GlobalLinkageIdStrong);
8357 LLVMSetAlignment(global_value, var->align_bytes);
8358 LLVMSetGlobalConstant(global_value, var->gen_is_const);
8359 set_global_tls(g, var, global_value);
8360 }
8361 } else {
8362 bool exported = (linkage != GlobalLinkageIdInternal);
8363 render_const_val(g, var->const_value, symbol_name);
8364 render_const_val_global(g, var->const_value, symbol_name);
8365 global_value = var->const_value->llvm_global;
8366
8367 if (exported) {
8368 LLVMSetLinkage(global_value, to_llvm_linkage(linkage, false));
8369 maybe_export_dll(g, global_value, GlobalLinkageIdStrong);
8370 }
8371 if (var->section_name) {
8372 LLVMSetSection(global_value, buf_ptr(var->section_name));
8373 }
8374 LLVMSetAlignment(global_value, var->align_bytes);
8375
8376 // TODO debug info for function pointers
8377 // Here we use const_value->type because that's the type of the llvm global,
8378 // which we const ptr cast upon use to whatever it needs to be.
8379 if (var->gen_is_const && var->const_value->type->id != ZigTypeIdFn) {
8380 gen_global_var(g, var, var->const_value->llvm_value, var->const_value->type);
8381 }
8382
8383 LLVMSetGlobalConstant(global_value, var->gen_is_const);
8384 set_global_tls(g, var, global_value);
8385 }
8386
8387 var->value_ref = global_value;
8388
8389 for (size_t export_i = 1; export_i < var->export_list.length; export_i += 1) {
8390 GlobalExport *global_export = &var->export_list.items[export_i];
8391 LLVMAddAlias(g->module, LLVMTypeOf(var->value_ref), var->value_ref, buf_ptr(&global_export->name));
8392 }
8393 }
8394
8395 // Generate function definitions.
8396 stage2_progress_update_node(g->sub_progress_node, 0, g->fn_defs.length);
8397 for (size_t fn_i = 0; fn_i < g->fn_defs.length; fn_i += 1) {
8398 ZigFn *fn_table_entry = g->fn_defs.at(fn_i);
8399 Stage2ProgressNode *fn_prog_node = stage2_progress_start(g->sub_progress_node,
8400 buf_ptr(&fn_table_entry->symbol_name), buf_len(&fn_table_entry->symbol_name), 0);
8401
8402 FnTypeId *fn_type_id = &fn_table_entry->type_entry->data.fn.fn_type_id;
8403 CallingConvention cc = fn_type_id->cc;
8404 bool is_c_abi = !calling_convention_allows_zig_types(cc);
8405 bool want_sret = want_first_arg_sret(g, fn_type_id);
8406
8407 LLVMValueRef fn = fn_llvm_value(g, fn_table_entry);
8408 g->cur_fn = fn_table_entry;
8409 g->cur_fn_val = fn;
8410
8411 build_all_basic_blocks(g, fn_table_entry);
8412 clear_debug_source_node(g);
8413
8414 bool is_async = fn_is_async(fn_table_entry);
8415
8416 if (is_async) {
8417 g->cur_frame_ptr = LLVMGetParam(fn, 0);
8418 } else {
8419 if (want_sret) {
8420 g->cur_ret_ptr = LLVMGetParam(fn, 0);
8421 } else if (type_has_bits(g, fn_type_id->return_type)) {
8422 g->cur_ret_ptr = build_alloca(g, fn_type_id->return_type, "result", 0);
8423 // TODO add debug info variable for this
8424 } else {
8425 g->cur_ret_ptr = nullptr;
8426 }
8427 }
8428
8429 uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn_table_entry);
8430 bool have_err_ret_trace_arg = err_ret_trace_arg_index != UINT32_MAX;
8431 if (have_err_ret_trace_arg) {
8432 g->cur_err_ret_trace_val_arg = LLVMGetParam(fn, err_ret_trace_arg_index);
8433 } else {
8434 g->cur_err_ret_trace_val_arg = nullptr;
8435 }
8436
8437 // error return tracing setup
8438 bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn &&
8439 !is_async && !have_err_ret_trace_arg;
8440 LLVMValueRef err_ret_array_val = nullptr;
8441 if (have_err_ret_trace_stack) {
8442 ZigType *array_type = get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count, nullptr);
8443 err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type));
8444
8445 (void)get_llvm_type(g, get_stack_trace_type(g));
8446 g->cur_err_ret_trace_val_stack = build_alloca(g, get_stack_trace_type(g), "error_return_trace",
8447 get_abi_alignment(g, g->stack_trace_type));
8448 } else {
8449 g->cur_err_ret_trace_val_stack = nullptr;
8450 }
8451
8452 if (fn_returns_c_abi_small_struct(fn_type_id)) {
8453 LLVMTypeRef abi_type = get_llvm_c_abi_type(g, fn_type_id->return_type);
8454 fn_table_entry->abi_return_value = LLVMBuildAlloca(g->builder, abi_type, "");
8455 }
8456
8457 if (!is_async) {
8458 // allocate async frames for nosuspend calls & awaits to async functions
8459 ZigType *largest_call_frame_type = nullptr;
8460 Stage1AirInst *all_calls_alloca = ir_create_alloca(g, &fn_table_entry->fndef_scope->base,
8461 fn_table_entry->body_node, fn_table_entry, g->builtin_types.entry_void, "@async_call_frame");
8462 for (size_t i = 0; i < fn_table_entry->call_list.length; i += 1) {
8463 Stage1AirInstCall *call = fn_table_entry->call_list.at(i);
8464 if (call->fn_entry == nullptr)
8465 continue;
8466 if (!fn_is_async(call->fn_entry))
8467 continue;
8468 if (call->modifier != CallModifierNoSuspend)
8469 continue;
8470 if (call->frame_result_loc != nullptr)
8471 continue;
8472 ZigType *callee_frame_type = get_fn_frame_type(g, call->fn_entry);
8473 if (largest_call_frame_type == nullptr ||
8474 callee_frame_type->abi_size > largest_call_frame_type->abi_size)
8475 {
8476 largest_call_frame_type = callee_frame_type;
8477 }
8478 call->frame_result_loc = all_calls_alloca;
8479 }
8480 if (largest_call_frame_type != nullptr) {
8481 all_calls_alloca->value->type = get_pointer_to_type(g, largest_call_frame_type, false);
8482 }
8483 // allocate temporary stack data
8484 for (size_t alloca_i = 0; alloca_i < fn_table_entry->alloca_gen_list.length; alloca_i += 1) {
8485 Stage1AirInstAlloca *instruction = fn_table_entry->alloca_gen_list.at(alloca_i);
8486 ZigType *ptr_type = instruction->base.value->type;
8487 assert(ptr_type->id == ZigTypeIdPointer);
8488 ZigType *child_type = ptr_type->data.pointer.child_type;
8489 if (type_resolve(g, child_type, ResolveStatusSizeKnown))
8490 zig_unreachable();
8491 if (!type_has_bits(g, child_type))
8492 continue;
8493 if (instruction->base.ref_count == 0)
8494 continue;
8495 if (instruction->base.value->special != ConstValSpecialRuntime) {
8496 if (const_ptr_pointee(nullptr, g, instruction->base.value, nullptr)->special !=
8497 ConstValSpecialRuntime)
8498 {
8499 continue;
8500 }
8501 }
8502 if (type_resolve(g, child_type, ResolveStatusLLVMFull))
8503 zig_unreachable();
8504 instruction->base.llvm_value = build_alloca(g, child_type, instruction->name_hint,
8505 get_ptr_align(g, ptr_type));
8506 }
8507 }
8508
8509 ZigType *import = get_scope_import(&fn_table_entry->fndef_scope->base);
8510 unsigned gen_i_init = want_sret ? 1 : 0;
8511
8512 // create debug variable declarations for variables and allocate all local variables
8513 FnWalk fn_walk_var = {};
8514 fn_walk_var.id = FnWalkIdVars;
8515 fn_walk_var.data.vars.import = import;
8516 fn_walk_var.data.vars.fn = fn_table_entry;
8517 fn_walk_var.data.vars.llvm_fn = fn;
8518 fn_walk_var.data.vars.gen_i = gen_i_init;
8519 for (size_t var_i = 0; var_i < fn_table_entry->variable_list.length; var_i += 1) {
8520 ZigVar *var = fn_table_entry->variable_list.at(var_i);
8521
8522 if (!type_has_bits(g, var->var_type)) {
8523 continue;
8524 }
8525 if (ir_get_var_is_comptime(var))
8526 continue;
8527 switch (type_requires_comptime(g, var->var_type)) {
8528 case ReqCompTimeInvalid:
8529 zig_unreachable();
8530 case ReqCompTimeYes:
8531 continue;
8532 case ReqCompTimeNo:
8533 break;
8534 }
8535
8536 if (var->src_arg_index == SIZE_MAX) {
8537 var->di_loc_var = ZigLLVMCreateAutoVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
8538 var->name, import->data.structure.root_struct->di_file,
8539 node_line_onebased(var->decl_node),
8540 get_llvm_di_type(g, var->var_type), !g->strip_debug_symbols, 0);
8541
8542 } else if (is_c_abi) {
8543 fn_walk_var.data.vars.var = var;
8544 iter_function_params_c_abi(g, fn_table_entry->type_entry, &fn_walk_var, var->src_arg_index);
8545 } else if (!is_async) {
8546 ZigType *gen_type;
8547 FnGenParamInfo *gen_info = &fn_table_entry->type_entry->data.fn.gen_param_info[var->src_arg_index];
8548 assert(gen_info->gen_index != SIZE_MAX);
8549
8550 if (handle_is_ptr(g, var->var_type)) {
8551 if (gen_info->is_byval) {
8552 gen_type = var->var_type;
8553 } else {
8554 gen_type = gen_info->type;
8555 }
8556 var->value_ref = LLVMGetParam(fn, gen_info->gen_index);
8557 } else {
8558 gen_type = var->var_type;
8559 var->value_ref = build_alloca(g, var->var_type, var->name, var->align_bytes);
8560 }
8561 if (var->decl_node) {
8562 var->di_loc_var = ZigLLVMCreateParameterVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
8563 var->name, import->data.structure.root_struct->di_file,
8564 node_line_onebased(var->decl_node),
8565 get_llvm_di_type(g, gen_type), !g->strip_debug_symbols, 0, (unsigned)(gen_info->gen_index+1));
8566 }
8567
8568 }
8569 }
8570
8571 // finishing error return trace setup. we have to do this after all the allocas.
8572 if (have_err_ret_trace_stack) {
8573 ZigType *usize = g->builtin_types.entry_usize;
8574 size_t index_field_index = g->stack_trace_type->data.structure.fields[0]->gen_index;
8575 LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, "");
8576 gen_store_untyped(g, LLVMConstNull(usize->llvm_type), index_field_ptr, 0, false);
8577
8578 size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1]->gen_index;
8579 LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)addresses_field_index, "");
8580
8581 ZigType *slice_type = g->stack_trace_type->data.structure.fields[1]->type_entry;
8582 size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index]->gen_index;
8583 LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, "");
8584 LLVMValueRef zero = LLVMConstNull(usize->llvm_type);
8585 LLVMValueRef indices[] = {zero, zero};
8586 LLVMValueRef err_ret_array_val_elem0_ptr = LLVMBuildInBoundsGEP(g->builder, err_ret_array_val,
8587 indices, 2, "");
8588 ZigType *ptr_ptr_usize_type = get_pointer_to_type(g, get_pointer_to_type(g, usize, false), false);
8589 gen_store(g, err_ret_array_val_elem0_ptr, ptr_field_ptr, ptr_ptr_usize_type);
8590
8591 size_t len_field_index = slice_type->data.structure.fields[slice_len_index]->gen_index;
8592 LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, "");
8593 gen_store(g, LLVMConstInt(usize->llvm_type, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false));
8594 }
8595
8596 if (is_async) {
8597 (void)get_llvm_type(g, fn_table_entry->frame_type);
8598 g->cur_resume_block_count = 0;
8599
8600 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
8601 LLVMValueRef size_val = LLVMConstInt(usize_type_ref, fn_table_entry->frame_type->abi_size, false);
8602 if (g->need_frame_size_prefix_data) {
8603 ZigLLVMFunctionSetPrefixData(fn_table_entry->llvm_value, size_val);
8604 }
8605
8606 if (!g->strip_debug_symbols) {
8607 AstNode *source_node = fn_table_entry->proto_node;
8608 ZigLLVMSetCurrentDebugLocation(g->builder,
8609 node_line_onebased(source_node), node_column_onebased(source_node),
8610 get_di_scope(g, fn_table_entry->child_scope));
8611 }
8612 Stage1Air *executable = &fn_table_entry->analyzed_executable;
8613 LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
8614 LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
8615 gen_assertion_scope(g, PanicMsgIdBadResume, fn_table_entry->child_scope);
8616
8617 LLVMPositionBuilderAtEnd(g->builder, g->cur_preamble_llvm_block);
8618 render_async_spills(g);
8619 g->cur_async_awaiter_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_awaiter_index, "");
8620 LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_resume_index, "");
8621 g->cur_async_resume_index_ptr = resume_index_ptr;
8622
8623 if (type_has_bits(g, fn_type_id->return_type)) {
8624 LLVMValueRef cur_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_ret_start, "");
8625 g->cur_ret_ptr = LLVMBuildLoad(g->builder, cur_ret_ptr_ptr, "");
8626 }
8627 uint32_t trace_field_index_stack = UINT32_MAX;
8628 if (codegen_fn_has_err_ret_tracing_stack(g, fn_table_entry, true)) {
8629 trace_field_index_stack = frame_index_trace_stack(g, fn_table_entry);
8630 g->cur_err_ret_trace_val_stack = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
8631 trace_field_index_stack, "");
8632 }
8633
8634 LLVMValueRef resume_index = LLVMBuildLoad(g->builder, resume_index_ptr, "");
8635 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, resume_index, bad_resume_block, 4);
8636 g->cur_async_switch_instr = switch_instr;
8637
8638 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
8639 Stage1AirBasicBlock *entry_block = executable->basic_block_list.at(0);
8640 LLVMAddCase(switch_instr, zero, entry_block->llvm_block);
8641 g->cur_resume_block_count += 1;
8642
8643 {
8644 LLVMBasicBlockRef bad_not_suspended_bb = LLVMAppendBasicBlock(g->cur_fn_val, "NotSuspended");
8645 size_t new_block_index = g->cur_resume_block_count;
8646 g->cur_resume_block_count += 1;
8647 g->cur_bad_not_suspended_index = LLVMConstInt(usize_type_ref, new_block_index, false);
8648 LLVMAddCase(g->cur_async_switch_instr, g->cur_bad_not_suspended_index, bad_not_suspended_bb);
8649
8650 LLVMPositionBuilderAtEnd(g->builder, bad_not_suspended_bb);
8651 gen_assertion_scope(g, PanicMsgIdResumeNotSuspendedFn, fn_table_entry->child_scope);
8652 }
8653
8654 LLVMPositionBuilderAtEnd(g->builder, entry_block->llvm_block);
8655 LLVMBuildStore(g->builder, g->cur_bad_not_suspended_index, g->cur_async_resume_index_ptr);
8656 if (trace_field_index_stack != UINT32_MAX) {
8657 if (codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type)) {
8658 LLVMValueRef trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
8659 frame_index_trace_arg(g, fn_type_id->return_type), "");
8660 LLVMValueRef zero_ptr = LLVMConstNull(LLVMGetElementType(LLVMTypeOf(trace_ptr_ptr)));
8661 LLVMBuildStore(g->builder, zero_ptr, trace_ptr_ptr);
8662 }
8663
8664 LLVMValueRef trace_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
8665 trace_field_index_stack, "");
8666 LLVMValueRef addrs_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
8667 trace_field_index_stack + 1, "");
8668
8669 gen_init_stack_trace(g, trace_field_ptr, addrs_field_ptr);
8670 }
8671 render_async_var_decls(g, entry_block->instruction_list.at(0)->scope);
8672 } else {
8673 // create debug variable declarations for parameters
8674 // rely on the first variables in the variable_list being parameters.
8675 FnWalk fn_walk_init = {};
8676 fn_walk_init.id = FnWalkIdInits;
8677 fn_walk_init.data.inits.fn = fn_table_entry;
8678 fn_walk_init.data.inits.llvm_fn = fn;
8679 fn_walk_init.data.inits.gen_i = gen_i_init;
8680 walk_function_params(g, fn_table_entry->type_entry, &fn_walk_init);
8681 }
8682
8683 ir_render(g, fn_table_entry);
8684
8685 stage2_progress_end(fn_prog_node);
8686 }
8687
8688 assert(!g->errors.length);
8689
8690 if (buf_len(&g->global_asm) != 0) {
8691 LLVMSetModuleInlineAsm2(g->module, buf_ptr(&g->global_asm), buf_len(&g->global_asm));
8692 }
8693
8694 while (g->type_resolve_stack.length != 0) {
8695 ZigType *ty = g->type_resolve_stack.last();
8696 if (type_resolve(g, ty, ResolveStatusLLVMFull))
8697 zig_unreachable();
8698 }
8699
8700 ZigLLVMDIBuilderFinalize(g->dbuilder);
8701
8702 if (g->verbose_llvm_ir) {
8703 fflush(stderr);
8704 LLVMDumpModule(g->module);
8705 }
8706
8707 char *error = nullptr;
8708 if (LLVMVerifyModule(g->module, LLVMReturnStatusAction, &error)) {
8709 zig_panic("broken LLVM module found: %s\nThis is a bug in the Zig compiler.", error);
8710 }
8711 }
8712
zig_llvm_emit_output(CodeGen * g)8713 static void zig_llvm_emit_output(CodeGen *g) {
8714 g->pass1_arena->destruct(&heap::c_allocator);
8715 g->pass1_arena = nullptr;
8716
8717 bool is_small = g->build_mode == BuildModeSmallRelease;
8718
8719 char *err_msg = nullptr;
8720 const char *asm_filename = nullptr;
8721 const char *bin_filename = nullptr;
8722 const char *llvm_ir_filename = nullptr;
8723 const char *bitcode_filename = nullptr;
8724
8725 if (buf_len(&g->o_file_output_path) != 0) bin_filename = buf_ptr(&g->o_file_output_path);
8726 if (buf_len(&g->asm_file_output_path) != 0) asm_filename = buf_ptr(&g->asm_file_output_path);
8727 if (buf_len(&g->llvm_ir_file_output_path) != 0) llvm_ir_filename = buf_ptr(&g->llvm_ir_file_output_path);
8728 if (buf_len(&g->bitcode_file_output_path) != 0) bitcode_filename = buf_ptr(&g->bitcode_file_output_path);
8729
8730 // Unfortunately, LLVM shits the bed when we ask for both binary and assembly.
8731 // So we call the entire pipeline multiple times if this is requested.
8732 if (asm_filename != nullptr && bin_filename != nullptr) {
8733 if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, &err_msg,
8734 g->build_mode == BuildModeDebug, is_small, g->enable_time_report, g->tsan_enabled,
8735 g->have_lto, nullptr, bin_filename, llvm_ir_filename, nullptr))
8736 {
8737 fprintf(stderr, "LLVM failed to emit bin=%s, ir=%s: %s\n",
8738 bin_filename, llvm_ir_filename, err_msg);
8739 exit(1);
8740 }
8741 bin_filename = nullptr;
8742 llvm_ir_filename = nullptr;
8743 }
8744
8745 if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, &err_msg,
8746 g->build_mode == BuildModeDebug, is_small, g->enable_time_report, g->tsan_enabled,
8747 g->have_lto, asm_filename, bin_filename, llvm_ir_filename, bitcode_filename))
8748 {
8749 fprintf(stderr, "LLVM failed to emit asm=%s, bin=%s, ir=%s, bc=%s: %s\n",
8750 asm_filename, bin_filename, llvm_ir_filename, bitcode_filename,
8751 err_msg);
8752 exit(1);
8753 }
8754
8755 LLVMDisposeModule(g->module);
8756 g->module = nullptr;
8757 LLVMDisposeTargetData(g->target_data_ref);
8758 g->target_data_ref = nullptr;
8759 LLVMDisposeTargetMachine(g->target_machine);
8760 g->target_machine = nullptr;
8761 }
8762
8763 struct CIntTypeInfo {
8764 CIntType id;
8765 const char *name;
8766 bool is_signed;
8767 };
8768
8769 static const CIntTypeInfo c_int_type_infos[] = {
8770 {CIntTypeShort, "c_short", true},
8771 {CIntTypeUShort, "c_ushort", false},
8772 {CIntTypeInt, "c_int", true},
8773 {CIntTypeUInt, "c_uint", false},
8774 {CIntTypeLong, "c_long", true},
8775 {CIntTypeULong, "c_ulong", false},
8776 {CIntTypeLongLong, "c_longlong", true},
8777 {CIntTypeULongLong, "c_ulonglong", false},
8778 };
8779
8780 static const bool is_signed_list[] = { false, true, };
8781
8782 struct GlobalLinkageValue {
8783 GlobalLinkageId id;
8784 const char *name;
8785 };
8786
add_fp_entry(CodeGen * g,const char * name,uint32_t bit_count,LLVMTypeRef type_ref,ZigType ** field)8787 static void add_fp_entry(CodeGen *g, const char *name, uint32_t bit_count, LLVMTypeRef type_ref,
8788 ZigType **field)
8789 {
8790 ZigType *entry = new_type_table_entry(ZigTypeIdFloat);
8791 entry->llvm_type = type_ref;
8792 entry->size_in_bits = 8*LLVMStoreSizeOfType(g->target_data_ref, entry->llvm_type);
8793 entry->abi_size = LLVMABISizeOfType(g->target_data_ref, entry->llvm_type);
8794 entry->abi_align = LLVMABIAlignmentOfType(g->target_data_ref, entry->llvm_type);
8795 buf_init_from_str(&entry->name, name);
8796 entry->data.floating.bit_count = bit_count;
8797
8798 entry->llvm_di_type = ZigLLVMCreateDebugBasicType(g->dbuilder, buf_ptr(&entry->name),
8799 entry->size_in_bits, ZigLLVMEncoding_DW_ATE_float());
8800 *field = entry;
8801 g->primitive_type_table.put(&entry->name, entry);
8802 }
8803
define_builtin_types(CodeGen * g)8804 static void define_builtin_types(CodeGen *g) {
8805 {
8806 // if this type is anywhere in the AST, we should never hit codegen.
8807 ZigType *entry = new_type_table_entry(ZigTypeIdInvalid);
8808 buf_init_from_str(&entry->name, "(invalid)");
8809 g->builtin_types.entry_invalid = entry;
8810 }
8811 {
8812 ZigType *entry = new_type_table_entry(ZigTypeIdComptimeFloat);
8813 buf_init_from_str(&entry->name, "comptime_float");
8814 g->builtin_types.entry_num_lit_float = entry;
8815 g->primitive_type_table.put(&entry->name, entry);
8816 }
8817 {
8818 ZigType *entry = new_type_table_entry(ZigTypeIdComptimeInt);
8819 buf_init_from_str(&entry->name, "comptime_int");
8820 g->builtin_types.entry_num_lit_int = entry;
8821 g->primitive_type_table.put(&entry->name, entry);
8822 }
8823 {
8824 ZigType *entry = new_type_table_entry(ZigTypeIdEnumLiteral);
8825 buf_init_from_str(&entry->name, "@Type(.EnumLiteral)");
8826 g->builtin_types.entry_enum_literal = entry;
8827 }
8828 {
8829 ZigType *entry = new_type_table_entry(ZigTypeIdUndefined);
8830 buf_init_from_str(&entry->name, "@Type(.Undefined)");
8831 g->builtin_types.entry_undef = entry;
8832 }
8833 {
8834 ZigType *entry = new_type_table_entry(ZigTypeIdNull);
8835 buf_init_from_str(&entry->name, "@Type(.Null)");
8836 g->builtin_types.entry_null = entry;
8837 }
8838 {
8839 ZigType *entry = new_type_table_entry(ZigTypeIdOpaque);
8840 buf_init_from_str(&entry->name, "(anytype)");
8841 g->builtin_types.entry_anytype = entry;
8842 }
8843
8844 for (size_t i = 0; i < array_length(c_int_type_infos); i += 1) {
8845 const CIntTypeInfo *info = &c_int_type_infos[i];
8846 uint32_t size_in_bits = target_c_type_size_in_bits(g->zig_target, info->id);
8847 bool is_signed = info->is_signed;
8848
8849 ZigType *entry = new_type_table_entry(ZigTypeIdInt);
8850 entry->llvm_type = LLVMIntType(size_in_bits);
8851 entry->size_in_bits = size_in_bits;
8852 entry->abi_size = LLVMABISizeOfType(g->target_data_ref, entry->llvm_type);
8853 entry->abi_align = LLVMABIAlignmentOfType(g->target_data_ref, entry->llvm_type);
8854
8855 buf_init_from_str(&entry->name, info->name);
8856
8857 entry->llvm_di_type = ZigLLVMCreateDebugBasicType(g->dbuilder, buf_ptr(&entry->name),
8858 8*LLVMStoreSizeOfType(g->target_data_ref, entry->llvm_type),
8859 is_signed ? ZigLLVMEncoding_DW_ATE_signed() : ZigLLVMEncoding_DW_ATE_unsigned());
8860 entry->data.integral.is_signed = is_signed;
8861 entry->data.integral.bit_count = size_in_bits;
8862 g->primitive_type_table.put(&entry->name, entry);
8863
8864 get_c_int_type_ptr(g, info->id)[0] = entry;
8865 }
8866
8867 {
8868 ZigType *entry = new_type_table_entry(ZigTypeIdBool);
8869 entry->llvm_type = LLVMInt1Type();
8870 entry->size_in_bits = 1;
8871 entry->abi_size = LLVMABISizeOfType(g->target_data_ref, entry->llvm_type);
8872 entry->abi_align = LLVMABIAlignmentOfType(g->target_data_ref, entry->llvm_type);
8873 buf_init_from_str(&entry->name, "bool");
8874 entry->llvm_di_type = ZigLLVMCreateDebugBasicType(g->dbuilder, buf_ptr(&entry->name),
8875 8*LLVMStoreSizeOfType(g->target_data_ref, entry->llvm_type),
8876 ZigLLVMEncoding_DW_ATE_boolean());
8877 g->builtin_types.entry_bool = entry;
8878 g->primitive_type_table.put(&entry->name, entry);
8879 }
8880
8881 for (size_t sign_i = 0; sign_i < array_length(is_signed_list); sign_i += 1) {
8882 bool is_signed = is_signed_list[sign_i];
8883
8884 ZigType *entry = new_type_table_entry(ZigTypeIdInt);
8885 entry->llvm_type = LLVMIntType(g->pointer_size_bytes * 8);
8886 entry->size_in_bits = g->pointer_size_bytes * 8;
8887 entry->abi_size = LLVMABISizeOfType(g->target_data_ref, entry->llvm_type);
8888 entry->abi_align = LLVMABIAlignmentOfType(g->target_data_ref, entry->llvm_type);
8889
8890 const char u_or_i = is_signed ? 'i' : 'u';
8891 buf_resize(&entry->name, 0);
8892 buf_appendf(&entry->name, "%csize", u_or_i);
8893
8894 entry->data.integral.is_signed = is_signed;
8895 entry->data.integral.bit_count = g->pointer_size_bytes * 8;
8896
8897 entry->llvm_di_type = ZigLLVMCreateDebugBasicType(g->dbuilder, buf_ptr(&entry->name),
8898 8*LLVMStoreSizeOfType(g->target_data_ref, entry->llvm_type),
8899 is_signed ? ZigLLVMEncoding_DW_ATE_signed() : ZigLLVMEncoding_DW_ATE_unsigned());
8900 g->primitive_type_table.put(&entry->name, entry);
8901
8902 if (is_signed) {
8903 g->builtin_types.entry_isize = entry;
8904 } else {
8905 g->builtin_types.entry_usize = entry;
8906 }
8907 }
8908
8909 add_fp_entry(g, "f16", 16, LLVMHalfType(), &g->builtin_types.entry_f16);
8910 add_fp_entry(g, "f32", 32, LLVMFloatType(), &g->builtin_types.entry_f32);
8911 add_fp_entry(g, "f64", 64, LLVMDoubleType(), &g->builtin_types.entry_f64);
8912 add_fp_entry(g, "f128", 128, LLVMFP128Type(), &g->builtin_types.entry_f128);
8913
8914 switch (g->zig_target->arch) {
8915 case ZigLLVM_x86:
8916 case ZigLLVM_x86_64:
8917 if (g->zig_target->abi != ZigLLVM_MSVC)
8918 add_fp_entry(g, "c_longdouble", 80, LLVMX86FP80Type(), &g->builtin_types.entry_c_longdouble);
8919 else
8920 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8921 break;
8922 case ZigLLVM_arm:
8923 case ZigLLVM_armeb:
8924 case ZigLLVM_thumb:
8925 case ZigLLVM_thumbeb:
8926 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8927 break;
8928 case ZigLLVM_aarch64:
8929 case ZigLLVM_aarch64_be:
8930 if (g->zig_target->os == OsWindows || target_os_is_darwin(g->zig_target->os))
8931 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8932 else
8933 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8934 break;
8935 case ZigLLVM_riscv32:
8936 case ZigLLVM_riscv64:
8937 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8938 break;
8939 case ZigLLVM_wasm32:
8940 case ZigLLVM_wasm64:
8941 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8942 break;
8943 case ZigLLVM_mips:
8944 case ZigLLVM_mipsel:
8945 // Assume o32 ABI
8946 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8947 break;
8948 case ZigLLVM_mips64:
8949 case ZigLLVM_mips64el:
8950 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8951 break;
8952 case ZigLLVM_ppc:
8953 case ZigLLVM_ppc64:
8954 case ZigLLVM_ppc64le:
8955 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8956 break;
8957 case ZigLLVM_sparcv9:
8958 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8959 break;
8960 case ZigLLVM_systemz:
8961 add_fp_entry(g, "c_longdouble", 128, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8962 break;
8963 case ZigLLVM_avr:
8964 // It's either a float or a double, depending on a toolchain switch
8965 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8966 break;
8967 case ZigLLVM_msp430:
8968 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8969 break;
8970 case ZigLLVM_bpfel:
8971 case ZigLLVM_bpfeb:
8972 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8973 break;
8974 case ZigLLVM_nvptx:
8975 case ZigLLVM_nvptx64:
8976 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8977 break;
8978 default:
8979 zig_panic("TODO implement mapping for c_longdouble");
8980 }
8981
8982 {
8983 ZigType *entry = new_type_table_entry(ZigTypeIdVoid);
8984 entry->llvm_type = LLVMVoidType();
8985 buf_init_from_str(&entry->name, "void");
8986 entry->llvm_di_type = ZigLLVMCreateDebugBasicType(g->dbuilder, buf_ptr(&entry->name),
8987 0,
8988 ZigLLVMEncoding_DW_ATE_signed());
8989 g->builtin_types.entry_void = entry;
8990 g->primitive_type_table.put(&entry->name, entry);
8991 }
8992 {
8993 ZigType *entry = new_type_table_entry(ZigTypeIdUnreachable);
8994 entry->llvm_type = LLVMVoidType();
8995 buf_init_from_str(&entry->name, "noreturn");
8996 entry->llvm_di_type = g->builtin_types.entry_void->llvm_di_type;
8997 g->builtin_types.entry_unreachable = entry;
8998 g->primitive_type_table.put(&entry->name, entry);
8999 }
9000 {
9001 ZigType *entry = new_type_table_entry(ZigTypeIdMetaType);
9002 buf_init_from_str(&entry->name, "type");
9003 g->builtin_types.entry_type = entry;
9004 g->primitive_type_table.put(&entry->name, entry);
9005 }
9006
9007 g->builtin_types.entry_u8 = get_int_type(g, false, 8);
9008 g->builtin_types.entry_u16 = get_int_type(g, false, 16);
9009 g->builtin_types.entry_u29 = get_int_type(g, false, 29);
9010 g->builtin_types.entry_u32 = get_int_type(g, false, 32);
9011 g->builtin_types.entry_u64 = get_int_type(g, false, 64);
9012 g->builtin_types.entry_i8 = get_int_type(g, true, 8);
9013 g->builtin_types.entry_i32 = get_int_type(g, true, 32);
9014 g->builtin_types.entry_i64 = get_int_type(g, true, 64);
9015
9016 {
9017 g->builtin_types.entry_anyopaque = get_opaque_type(g, nullptr, nullptr, "anyopaque",
9018 buf_create_from_str("anyopaque"));
9019 g->primitive_type_table.put(&g->builtin_types.entry_anyopaque->name, g->builtin_types.entry_anyopaque);
9020 }
9021
9022 {
9023 ZigType *entry = new_type_table_entry(ZigTypeIdErrorSet);
9024 buf_init_from_str(&entry->name, "anyerror");
9025 entry->data.error_set.err_count = UINT32_MAX;
9026
9027 // TODO https://github.com/ziglang/zig/issues/786
9028 g->err_tag_type = g->builtin_types.entry_u16;
9029
9030 entry->size_in_bits = g->err_tag_type->size_in_bits;
9031 entry->abi_align = g->err_tag_type->abi_align;
9032 entry->abi_size = g->err_tag_type->abi_size;
9033
9034 g->builtin_types.entry_global_error_set = entry;
9035
9036 g->errors_by_index.append(nullptr);
9037
9038 g->primitive_type_table.put(&entry->name, entry);
9039 }
9040 }
9041
define_intern_values(CodeGen * g)9042 static void define_intern_values(CodeGen *g) {
9043 {
9044 auto& value = g->intern.x_undefined;
9045 value.type = g->builtin_types.entry_undef;
9046 value.special = ConstValSpecialStatic;
9047 }
9048 {
9049 auto& value = g->intern.x_void;
9050 value.type = g->builtin_types.entry_void;
9051 value.special = ConstValSpecialStatic;
9052 }
9053 {
9054 auto& value = g->intern.x_null;
9055 value.type = g->builtin_types.entry_null;
9056 value.special = ConstValSpecialStatic;
9057 }
9058 {
9059 auto& value = g->intern.x_unreachable;
9060 value.type = g->builtin_types.entry_unreachable;
9061 value.special = ConstValSpecialStatic;
9062 }
9063 {
9064 auto& value = g->intern.zero_byte;
9065 value.type = g->builtin_types.entry_u8;
9066 value.special = ConstValSpecialStatic;
9067 bigint_init_unsigned(&value.data.x_bigint, 0);
9068 }
9069 }
9070
create_builtin_fn(CodeGen * g,BuiltinFnId id,const char * name,size_t count)9071 static BuiltinFnEntry *create_builtin_fn(CodeGen *g, BuiltinFnId id, const char *name, size_t count) {
9072 BuiltinFnEntry *builtin_fn = heap::c_allocator.create<BuiltinFnEntry>();
9073 buf_init_from_str(&builtin_fn->name, name);
9074 builtin_fn->id = id;
9075 builtin_fn->param_count = count;
9076 g->builtin_fn_table.put(&builtin_fn->name, builtin_fn);
9077 return builtin_fn;
9078 }
9079
define_builtin_fns(CodeGen * g)9080 static void define_builtin_fns(CodeGen *g) {
9081 create_builtin_fn(g, BuiltinFnIdBreakpoint, "breakpoint", 0);
9082 create_builtin_fn(g, BuiltinFnIdReturnAddress, "returnAddress", 0);
9083 create_builtin_fn(g, BuiltinFnIdMemcpy, "memcpy", 3);
9084 create_builtin_fn(g, BuiltinFnIdMemset, "memset", 3);
9085 create_builtin_fn(g, BuiltinFnIdSizeof, "sizeOf", 1);
9086 create_builtin_fn(g, BuiltinFnIdAlignOf, "alignOf", 1);
9087 create_builtin_fn(g, BuiltinFnIdField, "field", 2);
9088 create_builtin_fn(g, BuiltinFnIdTypeInfo, "typeInfo", 1);
9089 create_builtin_fn(g, BuiltinFnIdType, "Type", 1);
9090 create_builtin_fn(g, BuiltinFnIdHasField, "hasField", 2);
9091 create_builtin_fn(g, BuiltinFnIdTypeof, "TypeOf", SIZE_MAX);
9092 create_builtin_fn(g, BuiltinFnIdAddWithOverflow, "addWithOverflow", 4);
9093 create_builtin_fn(g, BuiltinFnIdSubWithOverflow, "subWithOverflow", 4);
9094 create_builtin_fn(g, BuiltinFnIdMulWithOverflow, "mulWithOverflow", 4);
9095 create_builtin_fn(g, BuiltinFnIdShlWithOverflow, "shlWithOverflow", 4);
9096 create_builtin_fn(g, BuiltinFnIdCInclude, "cInclude", 1);
9097 create_builtin_fn(g, BuiltinFnIdCDefine, "cDefine", 2);
9098 create_builtin_fn(g, BuiltinFnIdCUndef, "cUndef", 1);
9099 create_builtin_fn(g, BuiltinFnIdCtz, "ctz", 2);
9100 create_builtin_fn(g, BuiltinFnIdClz, "clz", 2);
9101 create_builtin_fn(g, BuiltinFnIdPopCount, "popCount", 2);
9102 create_builtin_fn(g, BuiltinFnIdBswap, "byteSwap", 2);
9103 create_builtin_fn(g, BuiltinFnIdBitReverse, "bitReverse", 2);
9104 create_builtin_fn(g, BuiltinFnIdImport, "import", 1);
9105 create_builtin_fn(g, BuiltinFnIdCImport, "cImport", 1);
9106 create_builtin_fn(g, BuiltinFnIdErrName, "errorName", 1);
9107 create_builtin_fn(g, BuiltinFnIdTypeName, "typeName", 1);
9108 create_builtin_fn(g, BuiltinFnIdEmbedFile, "embedFile", 1);
9109 create_builtin_fn(g, BuiltinFnIdCmpxchgWeak, "cmpxchgWeak", 6);
9110 create_builtin_fn(g, BuiltinFnIdCmpxchgStrong, "cmpxchgStrong", 6);
9111 create_builtin_fn(g, BuiltinFnIdFence, "fence", 1);
9112 create_builtin_fn(g, BuiltinFnIdTruncate, "truncate", 2);
9113 create_builtin_fn(g, BuiltinFnIdIntCast, "intCast", 2);
9114 create_builtin_fn(g, BuiltinFnIdFloatCast, "floatCast", 2);
9115 create_builtin_fn(g, BuiltinFnIdIntToFloat, "intToFloat", 2);
9116 create_builtin_fn(g, BuiltinFnIdFloatToInt, "floatToInt", 2);
9117 create_builtin_fn(g, BuiltinFnIdBoolToInt, "boolToInt", 1);
9118 create_builtin_fn(g, BuiltinFnIdErrToInt, "errorToInt", 1);
9119 create_builtin_fn(g, BuiltinFnIdIntToErr, "intToError", 1);
9120 create_builtin_fn(g, BuiltinFnIdEnumToInt, "enumToInt", 1);
9121 create_builtin_fn(g, BuiltinFnIdIntToEnum, "intToEnum", 2);
9122 create_builtin_fn(g, BuiltinFnIdCompileErr, "compileError", 1);
9123 create_builtin_fn(g, BuiltinFnIdCompileLog, "compileLog", SIZE_MAX);
9124 create_builtin_fn(g, BuiltinFnIdVectorType, "Vector", 2);
9125 create_builtin_fn(g, BuiltinFnIdShuffle, "shuffle", 4);
9126 create_builtin_fn(g, BuiltinFnIdSelect, "select", 4);
9127 create_builtin_fn(g, BuiltinFnIdSplat, "splat", 2);
9128 create_builtin_fn(g, BuiltinFnIdSetCold, "setCold", 1);
9129 create_builtin_fn(g, BuiltinFnIdSetRuntimeSafety, "setRuntimeSafety", 1);
9130 create_builtin_fn(g, BuiltinFnIdSetFloatMode, "setFloatMode", 1);
9131 create_builtin_fn(g, BuiltinFnIdPanic, "panic", 1);
9132 create_builtin_fn(g, BuiltinFnIdPtrCast, "ptrCast", 2);
9133 create_builtin_fn(g, BuiltinFnIdBitCast, "bitCast", 2);
9134 create_builtin_fn(g, BuiltinFnIdIntToPtr, "intToPtr", 2);
9135 create_builtin_fn(g, BuiltinFnIdPtrToInt, "ptrToInt", 1);
9136 create_builtin_fn(g, BuiltinFnIdTagName, "tagName", 1);
9137 create_builtin_fn(g, BuiltinFnIdFieldParentPtr, "fieldParentPtr", 3);
9138 create_builtin_fn(g, BuiltinFnIdOffsetOf, "offsetOf", 2);
9139 create_builtin_fn(g, BuiltinFnIdBitOffsetOf, "bitOffsetOf", 2);
9140 create_builtin_fn(g, BuiltinFnIdDivExact, "divExact", 2);
9141 create_builtin_fn(g, BuiltinFnIdDivTrunc, "divTrunc", 2);
9142 create_builtin_fn(g, BuiltinFnIdDivFloor, "divFloor", 2);
9143 create_builtin_fn(g, BuiltinFnIdRem, "rem", 2);
9144 create_builtin_fn(g, BuiltinFnIdMod, "mod", 2);
9145 create_builtin_fn(g, BuiltinFnIdSqrt, "sqrt", 1);
9146 create_builtin_fn(g, BuiltinFnIdSin, "sin", 1);
9147 create_builtin_fn(g, BuiltinFnIdCos, "cos", 1);
9148 create_builtin_fn(g, BuiltinFnIdExp, "exp", 1);
9149 create_builtin_fn(g, BuiltinFnIdExp2, "exp2", 1);
9150 create_builtin_fn(g, BuiltinFnIdLog, "log", 1);
9151 create_builtin_fn(g, BuiltinFnIdLog2, "log2", 1);
9152 create_builtin_fn(g, BuiltinFnIdLog10, "log10", 1);
9153 create_builtin_fn(g, BuiltinFnIdFabs, "fabs", 1);
9154 create_builtin_fn(g, BuiltinFnIdFloor, "floor", 1);
9155 create_builtin_fn(g, BuiltinFnIdCeil, "ceil", 1);
9156 create_builtin_fn(g, BuiltinFnIdTrunc, "trunc", 1);
9157 create_builtin_fn(g, BuiltinFnIdNearbyInt, "nearbyInt", 1);
9158 create_builtin_fn(g, BuiltinFnIdRound, "round", 1);
9159 create_builtin_fn(g, BuiltinFnIdMulAdd, "mulAdd", 4);
9160 create_builtin_fn(g, BuiltinFnIdAsyncCall, "asyncCall", SIZE_MAX);
9161 create_builtin_fn(g, BuiltinFnIdShlExact, "shlExact", 2);
9162 create_builtin_fn(g, BuiltinFnIdShrExact, "shrExact", 2);
9163 create_builtin_fn(g, BuiltinFnIdSetEvalBranchQuota, "setEvalBranchQuota", 1);
9164 create_builtin_fn(g, BuiltinFnIdAlignCast, "alignCast", 2);
9165 create_builtin_fn(g, BuiltinFnIdSetAlignStack, "setAlignStack", 1);
9166 create_builtin_fn(g, BuiltinFnIdExport, "export", 2);
9167 create_builtin_fn(g, BuiltinFnIdExtern, "extern", 2);
9168 create_builtin_fn(g, BuiltinFnIdErrorReturnTrace, "errorReturnTrace", 0);
9169 create_builtin_fn(g, BuiltinFnIdAtomicRmw, "atomicRmw", 5);
9170 create_builtin_fn(g, BuiltinFnIdAtomicLoad, "atomicLoad", 3);
9171 create_builtin_fn(g, BuiltinFnIdAtomicStore, "atomicStore", 4);
9172 create_builtin_fn(g, BuiltinFnIdErrSetCast, "errSetCast", 2);
9173 create_builtin_fn(g, BuiltinFnIdThis, "This", 0);
9174 create_builtin_fn(g, BuiltinFnIdHasDecl, "hasDecl", 2);
9175 create_builtin_fn(g, BuiltinFnIdUnionInit, "unionInit", 3);
9176 create_builtin_fn(g, BuiltinFnIdFrameHandle, "frame", 0);
9177 create_builtin_fn(g, BuiltinFnIdFrameType, "Frame", 1);
9178 create_builtin_fn(g, BuiltinFnIdFrameAddress, "frameAddress", 0);
9179 create_builtin_fn(g, BuiltinFnIdFrameSize, "frameSize", 1);
9180 create_builtin_fn(g, BuiltinFnIdAs, "as", 2);
9181 create_builtin_fn(g, BuiltinFnIdCall, "call", 3);
9182 create_builtin_fn(g, BuiltinFnIdBitSizeof, "bitSizeOf", 1);
9183 create_builtin_fn(g, BuiltinFnIdWasmMemorySize, "wasmMemorySize", 1);
9184 create_builtin_fn(g, BuiltinFnIdWasmMemoryGrow, "wasmMemoryGrow", 2);
9185 create_builtin_fn(g, BuiltinFnIdSrc, "src", 0);
9186 create_builtin_fn(g, BuiltinFnIdReduce, "reduce", 2);
9187 create_builtin_fn(g, BuiltinFnIdMaximum, "maximum", 2);
9188 create_builtin_fn(g, BuiltinFnIdMinimum, "minimum", 2);
9189 create_builtin_fn(g, BuiltinFnIdPrefetch, "prefetch", 2);
9190 }
9191
bool_to_str(bool b)9192 static const char *bool_to_str(bool b) {
9193 return b ? "true" : "false";
9194 }
9195
build_mode_to_str(BuildMode build_mode)9196 static const char *build_mode_to_str(BuildMode build_mode) {
9197 switch (build_mode) {
9198 case BuildModeDebug: return "Debug";
9199 case BuildModeSafeRelease: return "ReleaseSafe";
9200 case BuildModeFastRelease: return "ReleaseFast";
9201 case BuildModeSmallRelease: return "ReleaseSmall";
9202 }
9203 zig_unreachable();
9204 }
9205
subsystem_to_str(TargetSubsystem subsystem)9206 static const char *subsystem_to_str(TargetSubsystem subsystem) {
9207 switch (subsystem) {
9208 case TargetSubsystemConsole: return "Console";
9209 case TargetSubsystemWindows: return "Windows";
9210 case TargetSubsystemPosix: return "Posix";
9211 case TargetSubsystemNative: return "Native";
9212 case TargetSubsystemEfiApplication: return "EfiApplication";
9213 case TargetSubsystemEfiBootServiceDriver: return "EfiBootServiceDriver";
9214 case TargetSubsystemEfiRom: return "EfiRom";
9215 case TargetSubsystemEfiRuntimeDriver: return "EfiRuntimeDriver";
9216 case TargetSubsystemAuto: zig_unreachable();
9217 }
9218 zig_unreachable();
9219 }
9220
9221 // Returns TargetSubsystemAuto to mean "no subsystem"
detect_subsystem(CodeGen * g)9222 TargetSubsystem detect_subsystem(CodeGen *g) {
9223 if (g->subsystem != TargetSubsystemAuto)
9224 return g->subsystem;
9225 if (g->zig_target->os == OsWindows) {
9226 if (g->stage1.have_dllmain_crt_startup)
9227 return TargetSubsystemAuto;
9228 if (g->stage1.have_c_main || g->is_test_build || g->stage1.have_winmain_crt_startup || g->stage1.have_wwinmain_crt_startup)
9229 return TargetSubsystemConsole;
9230 if (g->stage1.have_winmain || g->stage1.have_wwinmain)
9231 return TargetSubsystemWindows;
9232 } else if (g->zig_target->os == OsUefi) {
9233 return TargetSubsystemEfiApplication;
9234 }
9235 return TargetSubsystemAuto;
9236 }
9237
detect_err_ret_tracing(CodeGen * g)9238 static bool detect_err_ret_tracing(CodeGen *g) {
9239 return !g->strip_debug_symbols &&
9240 g->build_mode != BuildModeFastRelease &&
9241 g->build_mode != BuildModeSmallRelease;
9242 }
9243
to_llvm_code_model(CodeGen * g)9244 static LLVMCodeModel to_llvm_code_model(CodeGen *g) {
9245 switch (g->code_model) {
9246 case CodeModelDefault:
9247 return LLVMCodeModelDefault;
9248 case CodeModelTiny:
9249 return LLVMCodeModelTiny;
9250 case CodeModelSmall:
9251 return LLVMCodeModelSmall;
9252 case CodeModelKernel:
9253 return LLVMCodeModelKernel;
9254 case CodeModelMedium:
9255 return LLVMCodeModelMedium;
9256 case CodeModelLarge:
9257 return LLVMCodeModelLarge;
9258 }
9259
9260 zig_unreachable();
9261 }
9262
codegen_generate_builtin_source(CodeGen * g)9263 Buf *codegen_generate_builtin_source(CodeGen *g) {
9264 // Note that this only runs when zig0 is building the self-hosted zig compiler code,
9265 // so it makes a few assumption that are always true for that case. Once we have
9266 // built the stage2 zig components then zig is in charge of generating the builtin.zig
9267 // file.
9268
9269 g->have_err_ret_tracing = detect_err_ret_tracing(g);
9270
9271 Buf *contents = buf_alloc();
9272 buf_appendf(contents,
9273 "const std = @import(\"std\");\n"
9274 );
9275
9276 const char *cur_os = nullptr;
9277 {
9278 uint32_t field_count = (uint32_t)target_os_count();
9279 for (uint32_t i = 0; i < field_count; i += 1) {
9280 Os os_type = target_os_enum(i);
9281 const char *name = target_os_name(os_type);
9282
9283 if (os_type == g->zig_target->os) {
9284 cur_os = name;
9285 }
9286 }
9287 }
9288 assert(cur_os != nullptr);
9289
9290 const char *cur_arch = nullptr;
9291 {
9292 uint32_t field_count = (uint32_t)target_arch_count();
9293 for (uint32_t arch_i = 0; arch_i < field_count; arch_i += 1) {
9294 ZigLLVM_ArchType arch = target_arch_enum(arch_i);
9295 const char *arch_name = target_arch_name(arch);
9296 if (arch == g->zig_target->arch) {
9297 cur_arch = arch_name;
9298 }
9299 }
9300 }
9301 assert(cur_arch != nullptr);
9302
9303 const char *cur_abi = nullptr;
9304 {
9305 uint32_t field_count = (uint32_t)target_abi_count();
9306 for (uint32_t i = 0; i < field_count; i += 1) {
9307 ZigLLVM_EnvironmentType abi = target_abi_enum(i);
9308 const char *name = target_abi_name(abi);
9309
9310 if (abi == g->zig_target->abi) {
9311 cur_abi = name;
9312 }
9313 }
9314 }
9315 assert(cur_abi != nullptr);
9316
9317 const char *cur_obj_fmt = nullptr;
9318 {
9319 uint32_t field_count = (uint32_t)target_oformat_count();
9320 for (uint32_t i = 0; i < field_count; i += 1) {
9321 ZigLLVM_ObjectFormatType oformat = target_oformat_enum(i);
9322 const char *name = target_oformat_name(oformat);
9323
9324 ZigLLVM_ObjectFormatType target_oformat = target_object_format(g->zig_target);
9325 if (oformat == target_oformat) {
9326 cur_obj_fmt = name;
9327 }
9328 }
9329
9330 }
9331 assert(cur_obj_fmt != nullptr);
9332
9333 // If any of these asserts trip then you need to either fix the internal compiler enum
9334 // or the corresponding one in std.Target or std.builtin.
9335 static_assert(ContainerLayoutAuto == 0, "");
9336 static_assert(ContainerLayoutExtern == 1, "");
9337 static_assert(ContainerLayoutPacked == 2, "");
9338
9339 static_assert(CallingConventionUnspecified == 0, "");
9340 static_assert(CallingConventionC == 1, "");
9341 static_assert(CallingConventionNaked == 2, "");
9342 static_assert(CallingConventionAsync == 3, "");
9343 static_assert(CallingConventionInline == 4, "");
9344 static_assert(CallingConventionInterrupt == 5, "");
9345 static_assert(CallingConventionSignal == 6, "");
9346 static_assert(CallingConventionStdcall == 7, "");
9347 static_assert(CallingConventionFastcall == 8, "");
9348 static_assert(CallingConventionVectorcall == 9, "");
9349 static_assert(CallingConventionThiscall == 10, "");
9350 static_assert(CallingConventionAPCS == 11, "");
9351 static_assert(CallingConventionAAPCS == 12, "");
9352 static_assert(CallingConventionAAPCSVFP == 13, "");
9353 static_assert(CallingConventionSysV == 14, "");
9354
9355 static_assert(BuiltinPtrSizeOne == 0, "");
9356 static_assert(BuiltinPtrSizeMany == 1, "");
9357 static_assert(BuiltinPtrSizeSlice == 2, "");
9358 static_assert(BuiltinPtrSizeC == 3, "");
9359
9360 static_assert(TargetSubsystemConsole == 0, "");
9361 static_assert(TargetSubsystemWindows == 1, "");
9362 static_assert(TargetSubsystemPosix == 2, "");
9363 static_assert(TargetSubsystemNative == 3, "");
9364 static_assert(TargetSubsystemEfiApplication == 4, "");
9365 static_assert(TargetSubsystemEfiBootServiceDriver == 5, "");
9366 static_assert(TargetSubsystemEfiRom == 6, "");
9367 static_assert(TargetSubsystemEfiRuntimeDriver == 7, "");
9368
9369 buf_appendf(contents, "pub const output_mode = std.builtin.OutputMode.Obj;\n");
9370 buf_appendf(contents, "pub const link_mode = std.builtin.LinkMode.%s;\n", ZIG_QUOTE(ZIG_LINK_MODE));
9371 buf_appendf(contents, "pub const is_test = false;\n");
9372 buf_appendf(contents, "pub const single_threaded = %s;\n", bool_to_str(g->is_single_threaded));
9373 buf_appendf(contents, "pub const abi = std.Target.Abi.%s;\n", cur_abi);
9374 buf_appendf(contents, "pub const cpu = std.Target.Cpu.baseline(.%s);\n", cur_arch);
9375 buf_appendf(contents, "pub const stage2_arch: std.Target.Cpu.Arch = .%s;\n", cur_arch);
9376 buf_appendf(contents, "pub const os = std.Target.Os.Tag.defaultVersionRange(.%s, .%s);\n", cur_os, cur_arch);
9377 buf_appendf(contents,
9378 "pub const target = std.Target{\n"
9379 " .cpu = cpu,\n"
9380 " .os = os,\n"
9381 " .abi = abi,\n"
9382 "};\n"
9383 );
9384
9385 buf_appendf(contents, "pub const object_format = std.Target.ObjectFormat.%s;\n", cur_obj_fmt);
9386 buf_appendf(contents, "pub const mode = std.builtin.Mode.%s;\n", build_mode_to_str(g->build_mode));
9387 buf_appendf(contents, "pub const link_libc = %s;\n", bool_to_str(g->link_libc));
9388 buf_appendf(contents, "pub const link_libcpp = %s;\n", bool_to_str(g->link_libcpp));
9389 buf_appendf(contents, "pub const have_error_return_tracing = %s;\n", bool_to_str(g->have_err_ret_tracing));
9390 buf_appendf(contents, "pub const valgrind_support = false;\n");
9391 buf_appendf(contents, "pub const position_independent_code = %s;\n", bool_to_str(g->have_pic));
9392 buf_appendf(contents, "pub const position_independent_executable = %s;\n", bool_to_str(g->have_pie));
9393 buf_appendf(contents, "pub const strip_debug_info = %s;\n", bool_to_str(g->strip_debug_symbols));
9394 buf_appendf(contents, "pub const code_model = std.builtin.CodeModel.default;\n");
9395 buf_appendf(contents, "pub const zig_is_stage2 = false;\n");
9396
9397 {
9398 TargetSubsystem detected_subsystem = detect_subsystem(g);
9399 if (detected_subsystem != TargetSubsystemAuto) {
9400 buf_appendf(contents, "pub const explicit_subsystem = std.builtin.SubSystem.%s;\n", subsystem_to_str(detected_subsystem));
9401 }
9402 }
9403
9404 return contents;
9405 }
9406
create_test_runner_pkg(CodeGen * g)9407 static ZigPackage *create_test_runner_pkg(CodeGen *g) {
9408 return codegen_create_package(g, buf_ptr(g->zig_std_special_dir), "test_runner.zig", "std.special");
9409 }
9410
define_builtin_compile_vars(CodeGen * g)9411 static Error define_builtin_compile_vars(CodeGen *g) {
9412 Error err;
9413
9414 if (g->std_package == nullptr)
9415 return ErrorNone;
9416
9417 assert(g->main_pkg);
9418
9419 const char *builtin_zig_basename = "builtin.zig";
9420
9421 Buf *contents;
9422 if (g->builtin_zig_path == nullptr) {
9423 // Then this is zig0 building stage2. We can make many assumptions about the compilation.
9424 Buf *out_dir = buf_alloc();
9425 os_path_split(&g->o_file_output_path, out_dir, nullptr);
9426 g->builtin_zig_path = buf_alloc();
9427 os_path_join(out_dir, buf_create_from_str(builtin_zig_basename), g->builtin_zig_path);
9428
9429 Buf *resolve_paths[] = { g->builtin_zig_path, };
9430 *g->builtin_zig_path = os_path_resolve(resolve_paths, 1);
9431
9432 contents = codegen_generate_builtin_source(g);
9433 if ((err = os_write_file(g->builtin_zig_path, contents))) {
9434 fprintf(stderr, "Unable to write file '%s': %s\n", buf_ptr(g->builtin_zig_path), err_str(err));
9435 exit(1);
9436 }
9437
9438 g->compile_var_package = new_package(buf_ptr(out_dir), builtin_zig_basename, "builtin");
9439 } else {
9440 Buf *resolve_paths[] = { g->builtin_zig_path, };
9441 *g->builtin_zig_path = os_path_resolve(resolve_paths, 1);
9442
9443 contents = buf_alloc();
9444 if ((err = os_fetch_file_path(g->builtin_zig_path, contents))) {
9445 fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(g->builtin_zig_path), err_str(err));
9446 exit(1);
9447 }
9448 Buf builtin_dirname = BUF_INIT;
9449 os_path_dirname(g->builtin_zig_path, &builtin_dirname);
9450 g->compile_var_package = new_package(buf_ptr(&builtin_dirname), builtin_zig_basename, "builtin");
9451 }
9452
9453 if (g->is_test_build) {
9454 if (g->test_runner_package == nullptr) {
9455 g->test_runner_package = create_test_runner_pkg(g);
9456 }
9457 g->root_pkg = g->test_runner_package;
9458 } else {
9459 g->root_pkg = g->main_pkg;
9460 }
9461 g->compile_var_package->package_table.put(buf_create_from_str("std"), g->std_package);
9462 g->main_pkg->package_table.put(buf_create_from_str("builtin"), g->compile_var_package);
9463 g->main_pkg->package_table.put(buf_create_from_str("root"), g->root_pkg);
9464 g->std_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package);
9465 g->std_package->package_table.put(buf_create_from_str("std"), g->std_package);
9466 g->std_package->package_table.put(buf_create_from_str("root"), g->root_pkg);
9467 g->compile_var_import = add_source_file(g, g->compile_var_package, g->builtin_zig_path, contents,
9468 SourceKindPkgMain);
9469
9470 return ErrorNone;
9471 }
9472
init(CodeGen * g)9473 static void init(CodeGen *g) {
9474 if (g->module)
9475 return;
9476
9477 codegen_add_time_event(g, "Initialize");
9478 {
9479 const char *progress_name = "Initialize";
9480 codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node,
9481 progress_name, strlen(progress_name), 0));
9482 }
9483
9484 g->have_err_ret_tracing = detect_err_ret_tracing(g);
9485
9486 assert(g->root_out_name);
9487 g->module = LLVMModuleCreateWithName(buf_ptr(g->root_out_name));
9488
9489 LLVMSetTarget(g->module, buf_ptr(&g->llvm_triple_str));
9490
9491 if (target_object_format(g->zig_target) == ZigLLVM_COFF) {
9492 ZigLLVMAddModuleCodeViewFlag(g->module);
9493 } else {
9494 ZigLLVMAddModuleDebugInfoFlag(g->module);
9495 }
9496
9497 LLVMTargetRef target_ref;
9498 char *err_msg = nullptr;
9499 if (LLVMGetTargetFromTriple(buf_ptr(&g->llvm_triple_str), &target_ref, &err_msg)) {
9500 fprintf(stderr,
9501 "Zig is expecting LLVM to understand this target: '%s'\n"
9502 "However LLVM responded with: \"%s\"\n"
9503 "Zig is unable to continue. This is a bug in Zig:\n"
9504 "https://github.com/ziglang/zig/issues/438\n"
9505 , buf_ptr(&g->llvm_triple_str), err_msg);
9506 exit(1);
9507 }
9508
9509 bool is_optimized = g->build_mode != BuildModeDebug;
9510 LLVMCodeGenOptLevel opt_level = is_optimized ? LLVMCodeGenLevelAggressive : LLVMCodeGenLevelNone;
9511
9512 LLVMRelocMode reloc_mode;
9513 if (g->have_pic) {
9514 reloc_mode = LLVMRelocPIC;
9515 } else if (g->link_mode_dynamic) {
9516 reloc_mode = LLVMRelocDynamicNoPic;
9517 } else {
9518 reloc_mode = LLVMRelocStatic;
9519 }
9520
9521 if (g->have_pic) {
9522 ZigLLVMSetModulePICLevel(g->module);
9523 }
9524
9525 if (g->have_pie) {
9526 ZigLLVMSetModulePIELevel(g->module);
9527 }
9528
9529 if (g->code_model != CodeModelDefault) {
9530 ZigLLVMSetModuleCodeModel(g->module, to_llvm_code_model(g));
9531 }
9532
9533 const char *target_specific_cpu_args = "";
9534 const char *target_specific_features = "";
9535
9536 if (g->zig_target->is_native_cpu) {
9537 target_specific_cpu_args = ZigLLVMGetHostCPUName();
9538 target_specific_features = ZigLLVMGetNativeFeatures();
9539 }
9540
9541 // Override CPU and features if defined by user.
9542 if (g->zig_target->llvm_cpu_name != nullptr) {
9543 target_specific_cpu_args = g->zig_target->llvm_cpu_name;
9544 }
9545 if (g->zig_target->llvm_cpu_features != nullptr) {
9546 target_specific_features = g->zig_target->llvm_cpu_features;
9547 }
9548 if (g->verbose_llvm_cpu_features) {
9549 fprintf(stderr, "name=%s triple=%s\n", buf_ptr(g->root_out_name), buf_ptr(&g->llvm_triple_str));
9550 fprintf(stderr, "name=%s target_specific_cpu_args=%s\n", buf_ptr(g->root_out_name), target_specific_cpu_args);
9551 fprintf(stderr, "name=%s target_specific_features=%s\n", buf_ptr(g->root_out_name), target_specific_features);
9552 }
9553
9554 // TODO handle float ABI better- it should depend on the ABI portion of std.Target
9555 ZigLLVMABIType float_abi = ZigLLVMABITypeDefault;
9556
9557 const char *abi_name = g->zig_target->llvm_target_abi;
9558 if (abi_name == nullptr && target_is_riscv(g->zig_target)) {
9559 // RISC-V Linux defaults to ilp32d/lp64d
9560 if (g->zig_target->os == OsLinux) {
9561 abi_name = (g->zig_target->arch == ZigLLVM_riscv32) ? "ilp32d" : "lp64d";
9562 } else {
9563 abi_name = (g->zig_target->arch == ZigLLVM_riscv32) ? "ilp32" : "lp64";
9564 }
9565 }
9566
9567 g->target_machine = ZigLLVMCreateTargetMachine(target_ref, buf_ptr(&g->llvm_triple_str),
9568 target_specific_cpu_args, target_specific_features, opt_level, reloc_mode,
9569 to_llvm_code_model(g), g->function_sections, float_abi, abi_name);
9570
9571 g->target_data_ref = LLVMCreateTargetDataLayout(g->target_machine);
9572
9573 char *layout_str = LLVMCopyStringRepOfTargetData(g->target_data_ref);
9574 LLVMSetDataLayout(g->module, layout_str);
9575
9576 assert(g->pointer_size_bytes == LLVMPointerSize(g->target_data_ref));
9577 g->is_big_endian = (LLVMByteOrder(g->target_data_ref) == LLVMBigEndian);
9578
9579 g->builder = LLVMCreateBuilder();
9580 g->dbuilder = ZigLLVMCreateDIBuilder(g->module, true);
9581
9582 // Don't use the version string here, llvm misparses it when it includes the git revision.
9583 Stage2SemVer semver = stage2_version();
9584 Buf *producer = buf_sprintf("zig %d.%d.%d", semver.major, semver.minor, semver.patch);
9585 const char *flags = "";
9586 unsigned runtime_version = 0;
9587
9588 // For macOS stack traces, we want to avoid having to parse the compilation unit debug
9589 // info. As long as each debug info file has a path independent of the compilation unit
9590 // directory (DW_AT_comp_dir), then we never have to look at the compilation unit debug
9591 // info. If we provide an absolute path to LLVM here for the compilation unit debug info,
9592 // LLVM will emit DWARF info that depends on DW_AT_comp_dir. To avoid this, we pass "."
9593 // for the compilation unit directory. This forces each debug file to have a directory
9594 // rather than be relative to DW_AT_comp_dir. According to DWARF 5, debug files will
9595 // no longer reference DW_AT_comp_dir, for the purpose of being able to support the
9596 // common practice of stripping all but the line number sections from an executable.
9597 const char *compile_unit_dir = target_os_is_darwin(g->zig_target->os) ? "." :
9598 buf_ptr(&g->main_pkg->root_src_dir);
9599
9600 ZigLLVMDIFile *compile_unit_file = ZigLLVMCreateFile(g->dbuilder, buf_ptr(g->root_out_name),
9601 compile_unit_dir);
9602 g->compile_unit = ZigLLVMCreateCompileUnit(g->dbuilder, ZigLLVMLang_DW_LANG_C99(),
9603 compile_unit_file, buf_ptr(producer), is_optimized, flags, runtime_version,
9604 "", 0, !g->strip_debug_symbols);
9605
9606 // This is for debug stuff that doesn't have a real file.
9607 g->dummy_di_file = nullptr;
9608
9609 define_builtin_types(g);
9610 define_intern_values(g);
9611
9612 Stage1AirInst *sentinel_instructions = heap::c_allocator.allocate<Stage1AirInst>(2);
9613 g->invalid_inst_gen = &sentinel_instructions[0];
9614 g->invalid_inst_gen->value = g->pass1_arena->create<ZigValue>();
9615 g->invalid_inst_gen->value->type = g->builtin_types.entry_invalid;
9616
9617 g->unreach_instruction = &sentinel_instructions[1];
9618 g->unreach_instruction->value = g->pass1_arena->create<ZigValue>();
9619 g->unreach_instruction->value->type = g->builtin_types.entry_unreachable;
9620
9621 g->invalid_inst_src = heap::c_allocator.create<Stage1ZirInst>();
9622
9623 define_builtin_fns(g);
9624 Error err;
9625 if ((err = define_builtin_compile_vars(g))) {
9626 fprintf(stderr, "Unable to create builtin.zig: %s\n", err_str(err));
9627 exit(1);
9628 }
9629 }
9630
update_test_functions_builtin_decl(CodeGen * g)9631 static void update_test_functions_builtin_decl(CodeGen *g) {
9632 Error err;
9633
9634 assert(g->is_test_build);
9635
9636 if (g->test_fns.length == 0) {
9637 fprintf(stderr, "No tests to run.\n");
9638 exit(0);
9639 }
9640
9641 ZigType *fn_type = get_test_fn_type(g);
9642
9643 ZigValue *test_fn_type_val = get_builtin_value(g, "TestFn");
9644 assert(test_fn_type_val->type->id == ZigTypeIdMetaType);
9645 ZigType *struct_type = test_fn_type_val->data.x_type;
9646 if ((err = type_resolve(g, struct_type, ResolveStatusSizeKnown)))
9647 zig_unreachable();
9648
9649 ZigValue *test_fn_array = g->pass1_arena->create<ZigValue>();
9650 test_fn_array->type = get_array_type(g, struct_type, g->test_fns.length, nullptr);
9651 test_fn_array->special = ConstValSpecialStatic;
9652 test_fn_array->data.x_array.data.s_none.elements = g->pass1_arena->allocate<ZigValue>(g->test_fns.length);
9653
9654 for (size_t i = 0; i < g->test_fns.length; i += 1) {
9655 ZigFn *test_fn_entry = g->test_fns.at(i);
9656
9657 ZigValue *this_val = &test_fn_array->data.x_array.data.s_none.elements[i];
9658 this_val->special = ConstValSpecialStatic;
9659 this_val->type = struct_type;
9660 this_val->parent.id = ConstParentIdArray;
9661 this_val->parent.data.p_array.array_val = test_fn_array;
9662 this_val->parent.data.p_array.elem_index = i;
9663 this_val->data.x_struct.fields = alloc_const_vals_ptrs(g, 3);
9664
9665 ZigValue *name_field = this_val->data.x_struct.fields[0];
9666 ZigValue *name_array_val = create_const_str_lit(g, &test_fn_entry->symbol_name)->data.x_ptr.data.ref.pointee;
9667 init_const_slice(g, name_field, name_array_val, 0, buf_len(&test_fn_entry->symbol_name), true, nullptr);
9668
9669 ZigValue *fn_field = this_val->data.x_struct.fields[1];
9670 fn_field->type = fn_type;
9671 fn_field->special = ConstValSpecialStatic;
9672 fn_field->data.x_ptr.special = ConstPtrSpecialFunction;
9673 fn_field->data.x_ptr.mut = ConstPtrMutComptimeConst;
9674 fn_field->data.x_ptr.data.fn.fn_entry = test_fn_entry;
9675
9676 ZigValue *frame_size_field = this_val->data.x_struct.fields[2];
9677 frame_size_field->type = get_optional_type(g, g->builtin_types.entry_usize);
9678 frame_size_field->special = ConstValSpecialStatic;
9679 frame_size_field->data.x_optional = nullptr;
9680
9681 if (fn_is_async(test_fn_entry)) {
9682 frame_size_field->data.x_optional = g->pass1_arena->create<ZigValue>();
9683 frame_size_field->data.x_optional->special = ConstValSpecialStatic;
9684 frame_size_field->data.x_optional->type = g->builtin_types.entry_usize;
9685 bigint_init_unsigned(&frame_size_field->data.x_optional->data.x_bigint,
9686 test_fn_entry->frame_type->abi_size);
9687 }
9688 }
9689 report_errors_and_maybe_exit(g);
9690
9691 ZigValue *test_fn_slice = create_const_slice(g, test_fn_array, 0, g->test_fns.length, true, nullptr);
9692
9693 update_compile_var(g, buf_create_from_str("test_functions"), test_fn_slice);
9694 assert(g->test_runner_package != nullptr);
9695 }
9696
get_resolved_root_src_path(CodeGen * g)9697 static Buf *get_resolved_root_src_path(CodeGen *g) {
9698 // TODO memoize
9699 if (buf_len(&g->main_pkg->root_src_path) == 0)
9700 return nullptr;
9701
9702 Buf rel_full_path = BUF_INIT;
9703 os_path_join(&g->main_pkg->root_src_dir, &g->main_pkg->root_src_path, &rel_full_path);
9704
9705 Buf *resolved_path = buf_alloc();
9706 Buf *resolve_paths[] = {&rel_full_path};
9707 *resolved_path = os_path_resolve(resolve_paths, 1);
9708
9709 return resolved_path;
9710 }
9711
gen_root_source(CodeGen * g)9712 static void gen_root_source(CodeGen *g) {
9713 Buf *resolved_path = get_resolved_root_src_path(g);
9714 if (resolved_path == nullptr)
9715 return;
9716
9717 Buf *source_code = buf_alloc();
9718 Error err;
9719 // No need for using the caching system for this file fetch because it is handled
9720 // separately.
9721 if ((err = os_fetch_file_path(resolved_path, source_code))) {
9722 fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(resolved_path), err_str(err));
9723 exit(1);
9724 }
9725
9726 ZigType *root_import_alias = add_source_file(g, g->main_pkg, resolved_path, source_code, SourceKindRoot);
9727 assert(root_import_alias == g->root_import);
9728
9729 assert(g->root_out_name);
9730
9731 // Zig has lazy top level definitions. Here we semantically analyze the panic function.
9732 Buf *import_target_path;
9733 Buf full_path = BUF_INIT;
9734 ZigType *std_import;
9735 if ((err = analyze_import(g, g->root_import, buf_create_from_str("std"), &std_import,
9736 &import_target_path, &full_path)))
9737 {
9738 if (err == ErrorFileNotFound) {
9739 fprintf(stderr, "unable to find '%s'", buf_ptr(import_target_path));
9740 } else {
9741 fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(&full_path), err_str(err));
9742 }
9743 exit(1);
9744 }
9745
9746 Tld *builtin_tld = find_decl(g, &get_container_scope(std_import)->base,
9747 buf_create_from_str("builtin"));
9748 assert(builtin_tld != nullptr);
9749 resolve_top_level_decl(g, builtin_tld, nullptr, false);
9750 report_errors_and_maybe_exit(g);
9751 assert(builtin_tld->id == TldIdVar);
9752 TldVar *builtin_tld_var = (TldVar*)builtin_tld;
9753 ZigValue *builtin_val = builtin_tld_var->var->const_value;
9754 assert(builtin_val->type->id == ZigTypeIdMetaType);
9755 g->std_builtin_import = builtin_val->data.x_type;
9756
9757 Tld *panic_tld = find_decl(g, &get_container_scope(g->std_builtin_import)->base,
9758 buf_create_from_str("panic"));
9759 assert(panic_tld != nullptr);
9760 resolve_top_level_decl(g, panic_tld, nullptr, false);
9761 report_errors_and_maybe_exit(g);
9762 assert(panic_tld->id == TldIdVar);
9763 TldVar *panic_tld_var = (TldVar*)panic_tld;
9764 ZigValue *panic_fn_val = panic_tld_var->var->const_value;
9765 assert(panic_fn_val->type->id == ZigTypeIdFn);
9766 assert(panic_fn_val->data.x_ptr.special == ConstPtrSpecialFunction);
9767 g->panic_fn = panic_fn_val->data.x_ptr.data.fn.fn_entry;
9768 assert(g->panic_fn != nullptr);
9769
9770 if (g->include_compiler_rt) {
9771 Buf *import_target_path;
9772 Buf full_path = BUF_INIT;
9773 ZigType *compiler_rt_import;
9774 if ((err = analyze_import(g, std_import, buf_create_from_str("./special/compiler_rt.zig"),
9775 &compiler_rt_import, &import_target_path, &full_path)))
9776 {
9777 if (err == ErrorFileNotFound) {
9778 fprintf(stderr, "unable to find '%s'", buf_ptr(import_target_path));
9779 } else {
9780 fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(&full_path), err_str(err));
9781 }
9782 exit(1);
9783 }
9784 }
9785
9786 if (!g->error_during_imports) {
9787 semantic_analyze(g);
9788 }
9789 report_errors_and_maybe_exit(g);
9790
9791 if (g->is_test_build) {
9792 update_test_functions_builtin_decl(g);
9793 if (!g->error_during_imports) {
9794 semantic_analyze(g);
9795 }
9796 }
9797
9798 report_errors_and_maybe_exit(g);
9799
9800 }
9801
codegen_print_timing_report(CodeGen * g,FILE * f)9802 void codegen_print_timing_report(CodeGen *g, FILE *f) {
9803 double start_time = g->timing_events.at(0).time;
9804 double end_time = g->timing_events.last().time;
9805 double total = end_time - start_time;
9806 fprintf(f, "%20s%12s%12s%12s%12s\n", "Name", "Start", "End", "Duration", "Percent");
9807 for (size_t i = 0; i < g->timing_events.length - 1; i += 1) {
9808 TimeEvent *te = &g->timing_events.at(i);
9809 TimeEvent *next_te = &g->timing_events.at(i + 1);
9810 fprintf(f, "%20s%12.4f%12.4f%12.4f%12.4f\n", te->name,
9811 te->time - start_time,
9812 next_te->time - start_time,
9813 next_te->time - te->time,
9814 (next_te->time - te->time) / total);
9815 }
9816 fprintf(f, "%20s%12.4f%12.4f%12.4f%12.4f\n", "Total", 0.0, total, total, 1.0);
9817 }
9818
codegen_add_time_event(CodeGen * g,const char * name)9819 void codegen_add_time_event(CodeGen *g, const char *name) {
9820 OsTimeStamp timestamp = os_timestamp_monotonic();
9821 double seconds = (double)timestamp.sec;
9822 seconds += ((double)timestamp.nsec) / 1000000000.0;
9823 g->timing_events.append({seconds, name});
9824 }
9825
codegen_build_object(CodeGen * g)9826 void codegen_build_object(CodeGen *g) {
9827 g->have_err_ret_tracing = detect_err_ret_tracing(g);
9828
9829 init(g);
9830
9831 codegen_add_time_event(g, "Semantic Analysis");
9832 const char *progress_name = "Semantic Analysis";
9833 codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node,
9834 progress_name, strlen(progress_name), 0));
9835
9836 gen_root_source(g);
9837
9838 if (buf_len(&g->analysis_json_output_path) != 0) {
9839 const char *analysis_json_filename = buf_ptr(&g->analysis_json_output_path);
9840 FILE *f = fopen(analysis_json_filename, "wb");
9841 if (f == nullptr) {
9842 fprintf(stderr, "Unable to open '%s': %s\n", analysis_json_filename, strerror(errno));
9843 exit(1);
9844 }
9845 zig_print_analysis_dump(g, f, " ", "\n");
9846 if (fclose(f) != 0) {
9847 fprintf(stderr, "Unable to write '%s': %s\n", analysis_json_filename, strerror(errno));
9848 exit(1);
9849 }
9850 }
9851 if (buf_len(&g->docs_output_path) != 0) {
9852 Error err;
9853 Buf *doc_dir_path = &g->docs_output_path;
9854 if ((err = os_make_path(doc_dir_path))) {
9855 fprintf(stderr, "Unable to create directory %s: %s\n", buf_ptr(doc_dir_path), err_str(err));
9856 exit(1);
9857 }
9858 Buf *index_html_src_path = buf_sprintf("%s" OS_SEP "special" OS_SEP "docs" OS_SEP "index.html",
9859 buf_ptr(g->zig_std_dir));
9860 Buf *index_html_dest_path = buf_sprintf("%s" OS_SEP "index.html", buf_ptr(doc_dir_path));
9861 Buf *main_js_src_path = buf_sprintf("%s" OS_SEP "special" OS_SEP "docs" OS_SEP "main.js",
9862 buf_ptr(g->zig_std_dir));
9863 Buf *main_js_dest_path = buf_sprintf("%s" OS_SEP "main.js", buf_ptr(doc_dir_path));
9864
9865 if ((err = os_copy_file(index_html_src_path, index_html_dest_path))) {
9866 fprintf(stderr, "Unable to copy %s to %s: %s\n", buf_ptr(index_html_src_path),
9867 buf_ptr(index_html_dest_path), err_str(err));
9868 exit(1);
9869 }
9870 if ((err = os_copy_file(main_js_src_path, main_js_dest_path))) {
9871 fprintf(stderr, "Unable to copy %s to %s: %s\n", buf_ptr(main_js_src_path),
9872 buf_ptr(main_js_dest_path), err_str(err));
9873 exit(1);
9874 }
9875 const char *data_js_filename = buf_ptr(buf_sprintf("%s" OS_SEP "data.js", buf_ptr(doc_dir_path)));
9876 FILE *f = fopen(data_js_filename, "wb");
9877 if (f == nullptr) {
9878 fprintf(stderr, "Unable to open '%s': %s\n", data_js_filename, strerror(errno));
9879 exit(1);
9880 }
9881 fprintf(f, "zigAnalysis=");
9882 zig_print_analysis_dump(g, f, "", "");
9883 fprintf(f, ";");
9884 if (fclose(f) != 0) {
9885 fprintf(stderr, "Unable to write '%s': %s\n", data_js_filename, strerror(errno));
9886 exit(1);
9887 }
9888 }
9889
9890 codegen_add_time_event(g, "Code Generation");
9891 {
9892 const char *progress_name = "Code Generation";
9893 codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node,
9894 progress_name, strlen(progress_name), 0));
9895 }
9896
9897 do_code_gen(g);
9898 codegen_add_time_event(g, "LLVM Emit Object");
9899 {
9900 const char *progress_name = "LLVM Emit Object";
9901 codegen_switch_sub_prog_node(g, stage2_progress_start(g->main_progress_node,
9902 progress_name, strlen(progress_name), 0));
9903 }
9904 zig_llvm_emit_output(g);
9905
9906 codegen_add_time_event(g, "Done");
9907 codegen_switch_sub_prog_node(g, nullptr);
9908 }
9909
codegen_create_package(CodeGen * g,const char * root_src_dir,const char * root_src_path,const char * pkg_path)9910 ZigPackage *codegen_create_package(CodeGen *g, const char *root_src_dir, const char *root_src_path,
9911 const char *pkg_path)
9912 {
9913 init(g);
9914 ZigPackage *pkg = new_package(root_src_dir, root_src_path, pkg_path);
9915 if (g->std_package != nullptr) {
9916 assert(g->compile_var_package != nullptr);
9917 pkg->package_table.put(buf_create_from_str("std"), g->std_package);
9918
9919 pkg->package_table.put(buf_create_from_str("root"), g->root_pkg);
9920
9921 pkg->package_table.put(buf_create_from_str("builtin"), g->compile_var_package);
9922 }
9923 return pkg;
9924 }
9925
codegen_destroy(CodeGen * g)9926 void codegen_destroy(CodeGen *g) {
9927 if (g->pass1_arena != nullptr) {
9928 g->pass1_arena->destruct(&heap::c_allocator);
9929 g->pass1_arena = nullptr;
9930 }
9931 heap::c_allocator.destroy<CodeGen>(g);
9932 }
9933
codegen_create(Buf * main_pkg_path,Buf * root_src_path,const ZigTarget * target,BuildMode build_mode,Buf * override_lib_dir,bool is_test_build)9934 CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget *target,
9935 BuildMode build_mode, Buf *override_lib_dir,
9936 bool is_test_build)
9937 {
9938 CodeGen *g = heap::c_allocator.create<CodeGen>();
9939 g->pass1_arena = heap::ArenaAllocator::construct(&heap::c_allocator, &heap::c_allocator, "pass1");
9940
9941 g->subsystem = TargetSubsystemAuto;
9942 g->zig_target = target;
9943
9944 assert(override_lib_dir != nullptr);
9945 g->zig_lib_dir = override_lib_dir;
9946
9947 g->zig_std_dir = buf_alloc();
9948 os_path_join(g->zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir);
9949
9950 g->build_mode = build_mode;
9951 g->import_table.init(32);
9952 g->builtin_fn_table.init(32);
9953 g->primitive_type_table.init(32);
9954 g->type_table.init(32);
9955 g->fn_type_table.init(32);
9956 g->error_table.init(16);
9957 g->generic_table.init(16);
9958 g->llvm_fn_table.init(16);
9959 g->memoized_fn_eval_table.init(16);
9960 g->exported_symbol_names.init(8);
9961 g->external_symbol_names.init(8);
9962 g->string_literals_table.init(16);
9963 g->type_info_cache.init(32);
9964 g->one_possible_values.init(32);
9965 g->is_test_build = is_test_build;
9966 g->is_single_threaded = false;
9967 g->code_model = CodeModelDefault;
9968 buf_resize(&g->global_asm, 0);
9969
9970 for (size_t i = 0; i < array_length(symbols_that_llvm_depends_on); i += 1) {
9971 g->external_symbol_names.put(buf_create_from_str(symbols_that_llvm_depends_on[i]), nullptr);
9972 }
9973
9974 if (root_src_path) {
9975 Buf *root_pkg_path;
9976 Buf *rel_root_src_path;
9977 if (main_pkg_path == nullptr) {
9978 Buf *src_basename = buf_alloc();
9979 Buf *src_dir = buf_alloc();
9980 os_path_split(root_src_path, src_dir, src_basename);
9981
9982 if (buf_len(src_basename) == 0) {
9983 fprintf(stderr, "Invalid root source path: %s\n", buf_ptr(root_src_path));
9984 exit(1);
9985 }
9986 root_pkg_path = src_dir;
9987 rel_root_src_path = src_basename;
9988 } else {
9989 Buf resolved_root_src_path = os_path_resolve(&root_src_path, 1);
9990 Buf resolved_main_pkg_path = os_path_resolve(&main_pkg_path, 1);
9991
9992 if (!buf_starts_with_buf(&resolved_root_src_path, &resolved_main_pkg_path)) {
9993 fprintf(stderr, "Root source path '%s' outside main package path '%s'\n",
9994 buf_ptr(root_src_path), buf_ptr(main_pkg_path));
9995 exit(1);
9996 }
9997 root_pkg_path = main_pkg_path;
9998 rel_root_src_path = buf_create_from_mem(
9999 buf_ptr(&resolved_root_src_path) + buf_len(&resolved_main_pkg_path) + 1,
10000 buf_len(&resolved_root_src_path) - buf_len(&resolved_main_pkg_path) - 1);
10001 }
10002
10003 g->main_pkg = new_package(buf_ptr(root_pkg_path), buf_ptr(rel_root_src_path), "");
10004 g->std_package = new_package(buf_ptr(g->zig_std_dir), "std.zig", "std");
10005 g->main_pkg->package_table.put(buf_create_from_str("std"), g->std_package);
10006 } else {
10007 g->main_pkg = new_package(".", "", "");
10008 }
10009
10010 g->zig_std_special_dir = buf_alloc();
10011 os_path_join(g->zig_std_dir, buf_sprintf("special"), g->zig_std_special_dir);
10012
10013 target_triple_llvm(&g->llvm_triple_str, g->zig_target);
10014 g->pointer_size_bytes = target_arch_pointer_bit_width(g->zig_target->arch) / 8;
10015
10016 if (!target_has_debug_info(g->zig_target)) {
10017 g->strip_debug_symbols = true;
10018 }
10019
10020 return g;
10021 }
10022
codegen_fn_has_err_ret_tracing_arg(CodeGen * g,ZigType * return_type)10023 bool codegen_fn_has_err_ret_tracing_arg(CodeGen *g, ZigType *return_type) {
10024 return g->have_err_ret_tracing &&
10025 (return_type->id == ZigTypeIdErrorUnion ||
10026 return_type->id == ZigTypeIdErrorSet);
10027 }
10028
codegen_fn_has_err_ret_tracing_stack(CodeGen * g,ZigFn * fn,bool is_async)10029 bool codegen_fn_has_err_ret_tracing_stack(CodeGen *g, ZigFn *fn, bool is_async) {
10030 if (is_async) {
10031 return g->have_err_ret_tracing && (fn->calls_or_awaits_errorable_fn ||
10032 codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type));
10033 } else {
10034 return g->have_err_ret_tracing && fn->calls_or_awaits_errorable_fn &&
10035 !codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type);
10036 }
10037 }
10038
codegen_switch_sub_prog_node(CodeGen * g,Stage2ProgressNode * node)10039 void codegen_switch_sub_prog_node(CodeGen *g, Stage2ProgressNode *node) {
10040 if (g->sub_progress_node != nullptr) {
10041 stage2_progress_end(g->sub_progress_node);
10042 }
10043 g->sub_progress_node = node;
10044 }
10045
for_undefined()10046 ZigValue *CodeGen::Intern::for_undefined() {
10047 return &this->x_undefined;
10048 }
10049
for_void()10050 ZigValue *CodeGen::Intern::for_void() {
10051 return &this->x_void;
10052 }
10053
for_null()10054 ZigValue *CodeGen::Intern::for_null() {
10055 return &this->x_null;
10056 }
10057
for_unreachable()10058 ZigValue *CodeGen::Intern::for_unreachable() {
10059 return &this->x_unreachable;
10060 }
10061
for_zero_byte()10062 ZigValue *CodeGen::Intern::for_zero_byte() {
10063 return &this->zero_byte;
10064 }
10065