1//! This file contains the functionality for lowering AArch64 MIR into 2//! machine code 3 4const Emit = @This(); 5const std = @import("std"); 6const math = std.math; 7const Mir = @import("Mir.zig"); 8const bits = @import("bits.zig"); 9const link = @import("../../link.zig"); 10const Module = @import("../../Module.zig"); 11const ErrorMsg = Module.ErrorMsg; 12const assert = std.debug.assert; 13const DW = std.dwarf; 14const leb128 = std.leb; 15const Instruction = bits.Instruction; 16const Register = bits.Register; 17const log = std.log.scoped(.aarch64_emit); 18const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput; 19 20mir: Mir, 21bin_file: *link.File, 22debug_output: DebugInfoOutput, 23target: *const std.Target, 24err_msg: ?*ErrorMsg = null, 25src_loc: Module.SrcLoc, 26code: *std.ArrayList(u8), 27 28prev_di_line: u32, 29prev_di_column: u32, 30/// Relative to the beginning of `code`. 31prev_di_pc: usize, 32 33/// The branch type of every branch 34branch_types: std.AutoHashMapUnmanaged(Mir.Inst.Index, BranchType) = .{}, 35/// For every forward branch, maps the target instruction to a list of 36/// branches which branch to this target instruction 37branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUnmanaged(Mir.Inst.Index)) = .{}, 38/// For backward branches: stores the code offset of the target 39/// instruction 40/// 41/// For forward branches: stores the code offset of the branch 42/// instruction 43code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{}, 44 45stack_size: u32, 46 47const InnerError = error{ 48 OutOfMemory, 49 EmitFail, 50}; 51 52const BranchType = enum { 53 b_cond, 54 unconditional_branch_immediate, 55 56 fn default(tag: Mir.Inst.Tag) BranchType { 57 return switch (tag) { 58 .b, .bl => .unconditional_branch_immediate, 59 .b_cond => .b_cond, 60 else => unreachable, 61 }; 62 } 63}; 64 65pub fn emitMir( 66 emit: *Emit, 67) !void { 68 const mir_tags = emit.mir.instructions.items(.tag); 69 70 // Find smallest lowerings for branch instructions 71 try emit.lowerBranches(); 72 73 // Emit machine code 74 for (mir_tags) |tag, index| { 75 const inst = @intCast(u32, index); 76 switch (tag) { 77 .add_immediate => try emit.mirAddSubtractImmediate(inst), 78 .cmp_immediate => try emit.mirAddSubtractImmediate(inst), 79 .sub_immediate => try emit.mirAddSubtractImmediate(inst), 80 81 .b_cond => try emit.mirConditionalBranchImmediate(inst), 82 83 .b => try emit.mirBranch(inst), 84 .bl => try emit.mirBranch(inst), 85 86 .blr => try emit.mirUnconditionalBranchRegister(inst), 87 .ret => try emit.mirUnconditionalBranchRegister(inst), 88 89 .brk => try emit.mirExceptionGeneration(inst), 90 .svc => try emit.mirExceptionGeneration(inst), 91 92 .call_extern => try emit.mirCallExtern(inst), 93 94 .cmp_shifted_register => try emit.mirAddSubtractShiftedRegister(inst), 95 96 .cset => try emit.mirConditionalSelect(inst), 97 98 .dbg_line => try emit.mirDbgLine(inst), 99 100 .dbg_prologue_end => try emit.mirDebugPrologueEnd(), 101 .dbg_epilogue_begin => try emit.mirDebugEpilogueBegin(), 102 103 .load_memory => try emit.mirLoadMemory(inst), 104 105 .ldp => try emit.mirLoadStoreRegisterPair(inst), 106 .stp => try emit.mirLoadStoreRegisterPair(inst), 107 108 .ldr_stack => try emit.mirLoadStoreStack(inst), 109 .ldrb_stack => try emit.mirLoadStoreStack(inst), 110 .ldrh_stack => try emit.mirLoadStoreStack(inst), 111 .str_stack => try emit.mirLoadStoreStack(inst), 112 .strb_stack => try emit.mirLoadStoreStack(inst), 113 .strh_stack => try emit.mirLoadStoreStack(inst), 114 115 .ldr => try emit.mirLoadStoreRegister(inst), 116 .ldrb => try emit.mirLoadStoreRegister(inst), 117 .ldrh => try emit.mirLoadStoreRegister(inst), 118 .str => try emit.mirLoadStoreRegister(inst), 119 .strb => try emit.mirLoadStoreRegister(inst), 120 .strh => try emit.mirLoadStoreRegister(inst), 121 122 .mov_register => try emit.mirMoveRegister(inst), 123 .mov_to_from_sp => try emit.mirMoveRegister(inst), 124 125 .movk => try emit.mirMoveWideImmediate(inst), 126 .movz => try emit.mirMoveWideImmediate(inst), 127 128 .nop => try emit.mirNop(), 129 130 .push_regs => try emit.mirPushPopRegs(inst), 131 .pop_regs => try emit.mirPushPopRegs(inst), 132 } 133 } 134} 135 136pub fn deinit(emit: *Emit) void { 137 var iter = emit.branch_forward_origins.valueIterator(); 138 while (iter.next()) |origin_list| { 139 origin_list.deinit(emit.bin_file.allocator); 140 } 141 142 emit.branch_types.deinit(emit.bin_file.allocator); 143 emit.branch_forward_origins.deinit(emit.bin_file.allocator); 144 emit.code_offset_mapping.deinit(emit.bin_file.allocator); 145 emit.* = undefined; 146} 147 148fn optimalBranchType(emit: *Emit, tag: Mir.Inst.Tag, offset: i64) !BranchType { 149 assert(offset & 0b11 == 0); 150 151 switch (tag) { 152 .b, .bl => { 153 if (std.math.cast(i26, offset >> 2)) |_| { 154 return BranchType.unconditional_branch_immediate; 155 } else |_| { 156 return emit.fail("TODO support branches larger than +-128 MiB", .{}); 157 } 158 }, 159 .b_cond => { 160 if (std.math.cast(i19, offset >> 2)) |_| { 161 return BranchType.b_cond; 162 } else |_| { 163 return emit.fail("TODO support conditional branches larger than +-1 MiB", .{}); 164 } 165 }, 166 else => unreachable, 167 } 168} 169 170fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize { 171 const tag = emit.mir.instructions.items(.tag)[inst]; 172 173 if (isBranch(tag)) { 174 switch (emit.branch_types.get(inst).?) { 175 .unconditional_branch_immediate => return 4, 176 .b_cond => return 4, 177 } 178 } 179 180 switch (tag) { 181 .load_memory => { 182 if (emit.bin_file.options.pie) { 183 // adrp, ldr 184 return 2 * 4; 185 } else { 186 const payload = emit.mir.instructions.items(.data)[inst].payload; 187 const load_memory = emit.mir.extraData(Mir.LoadMemory, payload).data; 188 const addr = load_memory.addr; 189 190 // movz, [movk, ...], ldr 191 if (addr <= math.maxInt(u16)) return 2 * 4; 192 if (addr <= math.maxInt(u32)) return 3 * 4; 193 if (addr <= math.maxInt(u48)) return 4 * 4; 194 return 5 * 4; 195 } 196 }, 197 .call_extern => return 4, 198 .dbg_line, 199 .dbg_epilogue_begin, 200 .dbg_prologue_end, 201 => return 0, 202 else => return 4, 203 } 204} 205 206fn isBranch(tag: Mir.Inst.Tag) bool { 207 return switch (tag) { 208 .b, .bl, .b_cond => true, 209 else => false, 210 }; 211} 212 213fn branchTarget(emit: *Emit, inst: Mir.Inst.Index) Mir.Inst.Index { 214 const tag = emit.mir.instructions.items(.tag)[inst]; 215 216 switch (tag) { 217 .b, .bl => return emit.mir.instructions.items(.data)[inst].inst, 218 .b_cond => return emit.mir.instructions.items(.data)[inst].inst_cond.inst, 219 else => unreachable, 220 } 221} 222 223fn lowerBranches(emit: *Emit) !void { 224 const mir_tags = emit.mir.instructions.items(.tag); 225 const allocator = emit.bin_file.allocator; 226 227 // First pass: Note down all branches and their target 228 // instructions, i.e. populate branch_types, 229 // branch_forward_origins, and code_offset_mapping 230 // 231 // TODO optimization opportunity: do this in codegen while 232 // generating MIR 233 for (mir_tags) |tag, index| { 234 const inst = @intCast(u32, index); 235 if (isBranch(tag)) { 236 const target_inst = emit.branchTarget(inst); 237 238 // Remember this branch instruction 239 try emit.branch_types.put(allocator, inst, BranchType.default(tag)); 240 241 // Forward branches require some extra stuff: We only 242 // know their offset once we arrive at the target 243 // instruction. Therefore, we need to be able to 244 // access the branch instruction when we visit the 245 // target instruction in order to manipulate its type 246 // etc. 247 if (target_inst > inst) { 248 // Remember the branch instruction index 249 try emit.code_offset_mapping.put(allocator, inst, 0); 250 251 if (emit.branch_forward_origins.getPtr(target_inst)) |origin_list| { 252 try origin_list.append(allocator, inst); 253 } else { 254 var origin_list: std.ArrayListUnmanaged(Mir.Inst.Index) = .{}; 255 try origin_list.append(allocator, inst); 256 try emit.branch_forward_origins.put(allocator, target_inst, origin_list); 257 } 258 } 259 260 // Remember the target instruction index so that we 261 // update the real code offset in all future passes 262 // 263 // putNoClobber may not be used as the put operation 264 // may clobber the entry when multiple branches branch 265 // to the same target instruction 266 try emit.code_offset_mapping.put(allocator, target_inst, 0); 267 } 268 } 269 270 // Further passes: Until all branches are lowered, interate 271 // through all instructions and calculate new offsets and 272 // potentially new branch types 273 var all_branches_lowered = false; 274 while (!all_branches_lowered) { 275 all_branches_lowered = true; 276 var current_code_offset: usize = 0; 277 278 for (mir_tags) |tag, index| { 279 const inst = @intCast(u32, index); 280 281 // If this instruction contained in the code offset 282 // mapping (when it is a target of a branch or if it is a 283 // forward branch), update the code offset 284 if (emit.code_offset_mapping.getPtr(inst)) |offset| { 285 offset.* = current_code_offset; 286 } 287 288 // If this instruction is a backward branch, calculate the 289 // offset, which may potentially update the branch type 290 if (isBranch(tag)) { 291 const target_inst = emit.branchTarget(inst); 292 if (target_inst < inst) { 293 const target_offset = emit.code_offset_mapping.get(target_inst).?; 294 const offset = @intCast(i64, target_offset) - @intCast(i64, current_code_offset); 295 const branch_type = emit.branch_types.getPtr(inst).?; 296 const optimal_branch_type = try emit.optimalBranchType(tag, offset); 297 if (branch_type.* != optimal_branch_type) { 298 branch_type.* = optimal_branch_type; 299 all_branches_lowered = false; 300 } 301 302 log.debug("lowerBranches: branch {} has offset {}", .{ inst, offset }); 303 } 304 } 305 306 // If this instruction is the target of one or more 307 // forward branches, calculate the offset, which may 308 // potentially update the branch type 309 if (emit.branch_forward_origins.get(inst)) |origin_list| { 310 for (origin_list.items) |forward_branch_inst| { 311 const branch_tag = emit.mir.instructions.items(.tag)[forward_branch_inst]; 312 const forward_branch_inst_offset = emit.code_offset_mapping.get(forward_branch_inst).?; 313 const offset = @intCast(i64, current_code_offset) - @intCast(i64, forward_branch_inst_offset); 314 const branch_type = emit.branch_types.getPtr(forward_branch_inst).?; 315 const optimal_branch_type = try emit.optimalBranchType(branch_tag, offset); 316 if (branch_type.* != optimal_branch_type) { 317 branch_type.* = optimal_branch_type; 318 all_branches_lowered = false; 319 } 320 321 log.debug("lowerBranches: branch {} has offset {}", .{ forward_branch_inst, offset }); 322 } 323 } 324 325 // Increment code offset 326 current_code_offset += emit.instructionSize(inst); 327 } 328 } 329} 330 331fn writeInstruction(emit: *Emit, instruction: Instruction) !void { 332 const endian = emit.target.cpu.arch.endian(); 333 std.mem.writeInt(u32, try emit.code.addManyAsArray(4), instruction.toU32(), endian); 334} 335 336fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError { 337 @setCold(true); 338 assert(emit.err_msg == null); 339 emit.err_msg = try ErrorMsg.create(emit.bin_file.allocator, emit.src_loc, format, args); 340 return error.EmitFail; 341} 342 343fn moveImmediate(emit: *Emit, reg: Register, imm64: u64) !void { 344 try emit.writeInstruction(Instruction.movz(reg, @truncate(u16, imm64), 0)); 345 346 if (imm64 > math.maxInt(u16)) { 347 try emit.writeInstruction(Instruction.movk(reg, @truncate(u16, imm64 >> 16), 16)); 348 } 349 if (imm64 > math.maxInt(u32)) { 350 try emit.writeInstruction(Instruction.movk(reg, @truncate(u16, imm64 >> 32), 32)); 351 } 352 if (imm64 > math.maxInt(u48)) { 353 try emit.writeInstruction(Instruction.movk(reg, @truncate(u16, imm64 >> 48), 48)); 354 } 355} 356 357fn dbgAdvancePCAndLine(self: *Emit, line: u32, column: u32) !void { 358 const delta_line = @intCast(i32, line) - @intCast(i32, self.prev_di_line); 359 const delta_pc: usize = self.code.items.len - self.prev_di_pc; 360 switch (self.debug_output) { 361 .dwarf => |dbg_out| { 362 // TODO Look into using the DWARF special opcodes to compress this data. 363 // It lets you emit single-byte opcodes that add different numbers to 364 // both the PC and the line number at the same time. 365 try dbg_out.dbg_line.ensureUnusedCapacity(11); 366 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc); 367 leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable; 368 if (delta_line != 0) { 369 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line); 370 leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable; 371 } 372 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy); 373 self.prev_di_pc = self.code.items.len; 374 self.prev_di_line = line; 375 self.prev_di_column = column; 376 self.prev_di_pc = self.code.items.len; 377 }, 378 .plan9 => |dbg_out| { 379 if (delta_pc <= 0) return; // only do this when the pc changes 380 // we have already checked the target in the linker to make sure it is compatable 381 const quant = @import("../../link/Plan9/aout.zig").getPCQuant(self.target.cpu.arch) catch unreachable; 382 383 // increasing the line number 384 try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line); 385 // increasing the pc 386 const d_pc_p9 = @intCast(i64, delta_pc) - quant; 387 if (d_pc_p9 > 0) { 388 // minus one because if its the last one, we want to leave space to change the line which is one quanta 389 try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant); 390 if (dbg_out.pcop_change_index.*) |pci| 391 dbg_out.dbg_line.items[pci] += 1; 392 dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1); 393 } else if (d_pc_p9 == 0) { 394 // we don't need to do anything, because adding the quant does it for us 395 } else unreachable; 396 if (dbg_out.start_line.* == null) 397 dbg_out.start_line.* = self.prev_di_line; 398 dbg_out.end_line.* = line; 399 // only do this if the pc changed 400 self.prev_di_line = line; 401 self.prev_di_column = column; 402 self.prev_di_pc = self.code.items.len; 403 }, 404 .none => {}, 405 } 406} 407 408fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void { 409 const tag = emit.mir.instructions.items(.tag)[inst]; 410 const rr_imm12_sh = emit.mir.instructions.items(.data)[inst].rr_imm12_sh; 411 412 switch (tag) { 413 .add_immediate => try emit.writeInstruction(Instruction.add( 414 rr_imm12_sh.rd, 415 rr_imm12_sh.rn, 416 rr_imm12_sh.imm12, 417 rr_imm12_sh.sh == 1, 418 )), 419 .cmp_immediate => try emit.writeInstruction(Instruction.subs( 420 rr_imm12_sh.rd, 421 rr_imm12_sh.rn, 422 rr_imm12_sh.imm12, 423 rr_imm12_sh.sh == 1, 424 )), 425 .sub_immediate => try emit.writeInstruction(Instruction.sub( 426 rr_imm12_sh.rd, 427 rr_imm12_sh.rn, 428 rr_imm12_sh.imm12, 429 rr_imm12_sh.sh == 1, 430 )), 431 else => unreachable, 432 } 433} 434 435fn mirConditionalBranchImmediate(emit: *Emit, inst: Mir.Inst.Index) !void { 436 const tag = emit.mir.instructions.items(.tag)[inst]; 437 const inst_cond = emit.mir.instructions.items(.data)[inst].inst_cond; 438 439 const offset = @intCast(i64, emit.code_offset_mapping.get(inst_cond.inst).?) - @intCast(i64, emit.code.items.len); 440 const branch_type = emit.branch_types.get(inst).?; 441 log.debug("mirConditionalBranchImmediate: {} offset={}", .{ inst, offset }); 442 443 switch (branch_type) { 444 .b_cond => switch (tag) { 445 .b_cond => try emit.writeInstruction(Instruction.bCond(inst_cond.cond, @intCast(i21, offset))), 446 else => unreachable, 447 }, 448 else => unreachable, 449 } 450} 451 452fn mirBranch(emit: *Emit, inst: Mir.Inst.Index) !void { 453 const tag = emit.mir.instructions.items(.tag)[inst]; 454 const target_inst = emit.mir.instructions.items(.data)[inst].inst; 455 456 log.debug("branch {}(tag: {}) -> {}(tag: {})", .{ 457 inst, 458 tag, 459 target_inst, 460 emit.mir.instructions.items(.tag)[target_inst], 461 }); 462 463 const offset = @intCast(i64, emit.code_offset_mapping.get(target_inst).?) - @intCast(i64, emit.code.items.len); 464 const branch_type = emit.branch_types.get(inst).?; 465 log.debug("mirBranch: {} offset={}", .{ inst, offset }); 466 467 switch (branch_type) { 468 .unconditional_branch_immediate => switch (tag) { 469 .b => try emit.writeInstruction(Instruction.b(@intCast(i28, offset))), 470 .bl => try emit.writeInstruction(Instruction.bl(@intCast(i28, offset))), 471 else => unreachable, 472 }, 473 else => unreachable, 474 } 475} 476 477fn mirUnconditionalBranchRegister(emit: *Emit, inst: Mir.Inst.Index) !void { 478 const tag = emit.mir.instructions.items(.tag)[inst]; 479 const reg = emit.mir.instructions.items(.data)[inst].reg; 480 481 switch (tag) { 482 .blr => try emit.writeInstruction(Instruction.blr(reg)), 483 .ret => try emit.writeInstruction(Instruction.ret(reg)), 484 else => unreachable, 485 } 486} 487 488fn mirExceptionGeneration(emit: *Emit, inst: Mir.Inst.Index) !void { 489 const tag = emit.mir.instructions.items(.tag)[inst]; 490 const imm16 = emit.mir.instructions.items(.data)[inst].imm16; 491 492 switch (tag) { 493 .brk => try emit.writeInstruction(Instruction.brk(imm16)), 494 .svc => try emit.writeInstruction(Instruction.svc(imm16)), 495 else => unreachable, 496 } 497} 498 499fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) !void { 500 const tag = emit.mir.instructions.items(.tag)[inst]; 501 const dbg_line_column = emit.mir.instructions.items(.data)[inst].dbg_line_column; 502 503 switch (tag) { 504 .dbg_line => try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column), 505 else => unreachable, 506 } 507} 508 509fn mirDebugPrologueEnd(self: *Emit) !void { 510 switch (self.debug_output) { 511 .dwarf => |dbg_out| { 512 try dbg_out.dbg_line.append(DW.LNS.set_prologue_end); 513 try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column); 514 }, 515 .plan9 => {}, 516 .none => {}, 517 } 518} 519 520fn mirDebugEpilogueBegin(self: *Emit) !void { 521 switch (self.debug_output) { 522 .dwarf => |dbg_out| { 523 try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin); 524 try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column); 525 }, 526 .plan9 => {}, 527 .none => {}, 528 } 529} 530 531fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) !void { 532 assert(emit.mir.instructions.items(.tag)[inst] == .call_extern); 533 const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn; 534 535 if (emit.bin_file.cast(link.File.MachO)) |macho_file| { 536 const offset = blk: { 537 const offset = @intCast(u32, emit.code.items.len); 538 // bl 539 try emit.writeInstruction(Instruction.bl(0)); 540 break :blk offset; 541 }; 542 // Add relocation to the decl. 543 try macho_file.active_decl.?.link.macho.relocs.append(emit.bin_file.allocator, .{ 544 .offset = offset, 545 .target = .{ .global = n_strx }, 546 .addend = 0, 547 .subtractor = null, 548 .pcrel = true, 549 .length = 2, 550 .@"type" = @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_BRANCH26), 551 }); 552 } else { 553 return emit.fail("Implement call_extern for linking backends != MachO", .{}); 554 } 555} 556 557fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void { 558 const tag = emit.mir.instructions.items(.tag)[inst]; 559 const rrr_imm6_shift = emit.mir.instructions.items(.data)[inst].rrr_imm6_shift; 560 561 switch (tag) { 562 .cmp_shifted_register => try emit.writeInstruction(Instruction.subsShiftedRegister( 563 rrr_imm6_shift.rd, 564 rrr_imm6_shift.rn, 565 rrr_imm6_shift.rm, 566 rrr_imm6_shift.shift, 567 rrr_imm6_shift.imm6, 568 )), 569 else => unreachable, 570 } 571} 572 573fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void { 574 const tag = emit.mir.instructions.items(.tag)[inst]; 575 const rrr_cond = emit.mir.instructions.items(.data)[inst].rrr_cond; 576 577 switch (tag) { 578 .cset => try emit.writeInstruction(Instruction.csinc( 579 rrr_cond.rd, 580 rrr_cond.rn, 581 rrr_cond.rm, 582 rrr_cond.cond, 583 )), 584 else => unreachable, 585 } 586} 587 588fn mirLoadMemory(emit: *Emit, inst: Mir.Inst.Index) !void { 589 assert(emit.mir.instructions.items(.tag)[inst] == .load_memory); 590 const payload = emit.mir.instructions.items(.data)[inst].payload; 591 const load_memory = emit.mir.extraData(Mir.LoadMemory, payload).data; 592 const reg = @intToEnum(Register, load_memory.register); 593 const addr = load_memory.addr; 594 595 if (emit.bin_file.options.pie) { 596 // PC-relative displacement to the entry in the GOT table. 597 // adrp 598 const offset = @intCast(u32, emit.code.items.len); 599 try emit.writeInstruction(Instruction.adrp(reg, 0)); 600 601 // ldr reg, reg, offset 602 try emit.writeInstruction(Instruction.ldr( 603 reg, 604 reg, 605 Instruction.LoadStoreOffset.imm(0), 606 )); 607 608 if (emit.bin_file.cast(link.File.MachO)) |macho_file| { 609 // TODO I think the reloc might be in the wrong place. 610 const decl = macho_file.active_decl.?; 611 // Page reloc for adrp instruction. 612 try decl.link.macho.relocs.append(emit.bin_file.allocator, .{ 613 .offset = offset, 614 .target = .{ .local = addr }, 615 .addend = 0, 616 .subtractor = null, 617 .pcrel = true, 618 .length = 2, 619 .@"type" = @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_GOT_LOAD_PAGE21), 620 }); 621 // Pageoff reloc for adrp instruction. 622 try decl.link.macho.relocs.append(emit.bin_file.allocator, .{ 623 .offset = offset + 4, 624 .target = .{ .local = addr }, 625 .addend = 0, 626 .subtractor = null, 627 .pcrel = false, 628 .length = 2, 629 .@"type" = @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_GOT_LOAD_PAGEOFF12), 630 }); 631 } else { 632 return emit.fail("TODO implement load_memory for PIE GOT indirection on this platform", .{}); 633 } 634 } else { 635 // The value is in memory at a hard-coded address. 636 // If the type is a pointer, it means the pointer address is at this memory location. 637 try emit.moveImmediate(reg, addr); 638 try emit.writeInstruction(Instruction.ldr( 639 reg, 640 reg, 641 Instruction.LoadStoreOffset.none, 642 )); 643 } 644} 645 646fn mirLoadStoreRegisterPair(emit: *Emit, inst: Mir.Inst.Index) !void { 647 const tag = emit.mir.instructions.items(.tag)[inst]; 648 const load_store_register_pair = emit.mir.instructions.items(.data)[inst].load_store_register_pair; 649 650 switch (tag) { 651 .stp => try emit.writeInstruction(Instruction.stp( 652 load_store_register_pair.rt, 653 load_store_register_pair.rt2, 654 load_store_register_pair.rn, 655 load_store_register_pair.offset, 656 )), 657 .ldp => try emit.writeInstruction(Instruction.ldp( 658 load_store_register_pair.rt, 659 load_store_register_pair.rt2, 660 load_store_register_pair.rn, 661 load_store_register_pair.offset, 662 )), 663 else => unreachable, 664 } 665} 666 667fn mirLoadStoreStack(emit: *Emit, inst: Mir.Inst.Index) !void { 668 const tag = emit.mir.instructions.items(.tag)[inst]; 669 const load_store_stack = emit.mir.instructions.items(.data)[inst].load_store_stack; 670 671 const raw_offset = emit.stack_size - load_store_stack.offset; 672 const offset = switch (tag) { 673 .ldrb_stack, .strb_stack => blk: { 674 if (math.cast(u12, raw_offset)) |imm| { 675 break :blk Instruction.LoadStoreOffset.imm(imm); 676 } else |_| { 677 return emit.fail("TODO load/store stack byte with larger offset", .{}); 678 } 679 }, 680 .ldrh_stack, .strh_stack => blk: { 681 assert(std.mem.isAlignedGeneric(u32, raw_offset, 2)); // misaligned stack entry 682 if (math.cast(u12, @divExact(raw_offset, 2))) |imm| { 683 break :blk Instruction.LoadStoreOffset.imm(imm); 684 } else |_| { 685 return emit.fail("TODO load/store stack halfword with larger offset", .{}); 686 } 687 }, 688 .ldr_stack, .str_stack => blk: { 689 const alignment: u32 = switch (load_store_stack.rt.size()) { 690 32 => 4, 691 64 => 8, 692 else => unreachable, 693 }; 694 695 assert(std.mem.isAlignedGeneric(u32, raw_offset, alignment)); // misaligned stack entry 696 if (math.cast(u12, @divExact(raw_offset, alignment))) |imm| { 697 break :blk Instruction.LoadStoreOffset.imm(imm); 698 } else |_| { 699 return emit.fail("TODO load/store stack with larger offset", .{}); 700 } 701 }, 702 else => unreachable, 703 }; 704 705 switch (tag) { 706 .ldr_stack => try emit.writeInstruction(Instruction.ldr( 707 load_store_stack.rt, 708 Register.sp, 709 offset, 710 )), 711 .ldrb_stack => try emit.writeInstruction(Instruction.ldrb( 712 load_store_stack.rt, 713 Register.sp, 714 offset, 715 )), 716 .ldrh_stack => try emit.writeInstruction(Instruction.ldrh( 717 load_store_stack.rt, 718 Register.sp, 719 offset, 720 )), 721 .str_stack => try emit.writeInstruction(Instruction.str( 722 load_store_stack.rt, 723 Register.sp, 724 offset, 725 )), 726 .strb_stack => try emit.writeInstruction(Instruction.strb( 727 load_store_stack.rt, 728 Register.sp, 729 offset, 730 )), 731 .strh_stack => try emit.writeInstruction(Instruction.strh( 732 load_store_stack.rt, 733 Register.sp, 734 offset, 735 )), 736 else => unreachable, 737 } 738} 739 740fn mirLoadStoreRegister(emit: *Emit, inst: Mir.Inst.Index) !void { 741 const tag = emit.mir.instructions.items(.tag)[inst]; 742 const load_store_register = emit.mir.instructions.items(.data)[inst].load_store_register; 743 744 switch (tag) { 745 .ldr => try emit.writeInstruction(Instruction.ldr( 746 load_store_register.rt, 747 load_store_register.rn, 748 load_store_register.offset, 749 )), 750 .ldrb => try emit.writeInstruction(Instruction.ldrb( 751 load_store_register.rt, 752 load_store_register.rn, 753 load_store_register.offset, 754 )), 755 .ldrh => try emit.writeInstruction(Instruction.ldrh( 756 load_store_register.rt, 757 load_store_register.rn, 758 load_store_register.offset, 759 )), 760 .str => try emit.writeInstruction(Instruction.str( 761 load_store_register.rt, 762 load_store_register.rn, 763 load_store_register.offset, 764 )), 765 .strb => try emit.writeInstruction(Instruction.strb( 766 load_store_register.rt, 767 load_store_register.rn, 768 load_store_register.offset, 769 )), 770 .strh => try emit.writeInstruction(Instruction.strh( 771 load_store_register.rt, 772 load_store_register.rn, 773 load_store_register.offset, 774 )), 775 else => unreachable, 776 } 777} 778 779fn mirMoveRegister(emit: *Emit, inst: Mir.Inst.Index) !void { 780 const tag = emit.mir.instructions.items(.tag)[inst]; 781 const rr = emit.mir.instructions.items(.data)[inst].rr; 782 783 switch (tag) { 784 .mov_register => try emit.writeInstruction(Instruction.orr(rr.rd, .xzr, rr.rn, Instruction.Shift.none)), 785 .mov_to_from_sp => try emit.writeInstruction(Instruction.add(rr.rd, rr.rn, 0, false)), 786 else => unreachable, 787 } 788} 789 790fn mirMoveWideImmediate(emit: *Emit, inst: Mir.Inst.Index) !void { 791 const tag = emit.mir.instructions.items(.tag)[inst]; 792 const r_imm16_sh = emit.mir.instructions.items(.data)[inst].r_imm16_sh; 793 794 switch (tag) { 795 .movz => try emit.writeInstruction(Instruction.movz(r_imm16_sh.rd, r_imm16_sh.imm16, @as(u6, r_imm16_sh.hw) << 4)), 796 .movk => try emit.writeInstruction(Instruction.movk(r_imm16_sh.rd, r_imm16_sh.imm16, @as(u6, r_imm16_sh.hw) << 4)), 797 else => unreachable, 798 } 799} 800 801fn mirNop(emit: *Emit) !void { 802 try emit.writeInstruction(Instruction.nop()); 803} 804 805fn mirPushPopRegs(emit: *Emit, inst: Mir.Inst.Index) !void { 806 const tag = emit.mir.instructions.items(.tag)[inst]; 807 const reg_list = emit.mir.instructions.items(.data)[inst].reg_list; 808 809 if (reg_list & @as(u32, 1) << 31 != 0) return emit.fail("xzr is not a valid register for {}", .{tag}); 810 811 // sp must be aligned at all times, so we only use stp and ldp 812 // instructions for minimal instruction count. However, if we do 813 // not have an even number of registers, we use str and ldr 814 const number_of_regs = @popCount(u32, reg_list); 815 816 switch (tag) { 817 .pop_regs => { 818 var i: u6 = 32; 819 var count: u6 = 0; 820 var other_reg: Register = undefined; 821 while (i > 0) : (i -= 1) { 822 const reg = @intToEnum(Register, i - 1); 823 if (reg_list & @as(u32, 1) << reg.id() != 0) { 824 if (count % 2 == 0) { 825 if (count == number_of_regs - 1) { 826 try emit.writeInstruction(Instruction.ldr( 827 reg, 828 Register.sp, 829 Instruction.LoadStoreOffset.imm_post_index(16), 830 )); 831 } else { 832 other_reg = reg; 833 } 834 } else { 835 try emit.writeInstruction(Instruction.ldp( 836 reg, 837 other_reg, 838 Register.sp, 839 Instruction.LoadStorePairOffset.post_index(16), 840 )); 841 } 842 count += 1; 843 } 844 } 845 assert(count == number_of_regs); 846 }, 847 .push_regs => { 848 var i: u6 = 0; 849 var count: u6 = 0; 850 var other_reg: Register = undefined; 851 while (i < 32) : (i += 1) { 852 const reg = @intToEnum(Register, i); 853 if (reg_list & @as(u32, 1) << reg.id() != 0) { 854 if (count % 2 == 0) { 855 if (count == number_of_regs - 1) { 856 try emit.writeInstruction(Instruction.str( 857 reg, 858 Register.sp, 859 Instruction.LoadStoreOffset.imm_pre_index(-16), 860 )); 861 } else { 862 other_reg = reg; 863 } 864 } else { 865 try emit.writeInstruction(Instruction.stp( 866 other_reg, 867 reg, 868 Register.sp, 869 Instruction.LoadStorePairOffset.pre_index(-16), 870 )); 871 } 872 count += 1; 873 } 874 } 875 assert(count == number_of_regs); 876 }, 877 else => unreachable, 878 } 879} 880