1const std = @import("std"); 2const os = std.os; 3const mem = std.mem; 4const elf = std.elf; 5const math = std.math; 6const assert = std.debug.assert; 7const native_arch = @import("builtin").cpu.arch; 8 9// This file implements the two TLS variants [1] used by ELF-based systems. 10// 11// The variant I has the following layout in memory: 12// ------------------------------------------------------- 13// | DTV | Zig | DTV | Alignment | TLS | 14// | storage | thread data | pointer | | block | 15// ------------------------^------------------------------ 16// `-- The thread pointer register points here 17// 18// In this case we allocate additional space for our control structure that's 19// placed _before_ the DTV pointer together with the DTV. 20// 21// NOTE: Some systems such as power64 or mips use this variant with a twist: the 22// alignment is not present and the tp and DTV addresses are offset by a 23// constant. 24// 25// On the other hand the variant II has the following layout in memory: 26// --------------------------------------- 27// | TLS | TCB | Zig | DTV | 28// | block | | thread data | storage | 29// --------^------------------------------ 30// `-- The thread pointer register points here 31// 32// The structure of the TCB is not defined by the ABI so we reserve enough space 33// for a single pointer as some architectures such as i386 and x86_64 need a 34// pointer to the TCB block itself at the address pointed by the tp. 35// 36// In this case the control structure and DTV are placed one after another right 37// after the TLS block data. 38// 39// At the moment the DTV is very simple since we only support static TLS, all we 40// need is a two word vector to hold the number of entries (1) and the address 41// of the first TLS block. 42// 43// [1] https://www.akkadia.org/drepper/tls.pdf 44 45const TLSVariant = enum { 46 VariantI, 47 VariantII, 48}; 49 50const tls_variant = switch (native_arch) { 51 .arm, .armeb, .thumb, .aarch64, .aarch64_be, .riscv32, .riscv64, .mips, .mipsel, .powerpc, .powerpc64, .powerpc64le => TLSVariant.VariantI, 52 .x86_64, .i386, .sparcv9 => TLSVariant.VariantII, 53 else => @compileError("undefined tls_variant for this architecture"), 54}; 55 56// Controls how many bytes are reserved for the Thread Control Block 57const tls_tcb_size = switch (native_arch) { 58 // ARM EABI mandates enough space for two pointers: the first one points to 59 // the DTV while the second one is unspecified but reserved 60 .arm, .armeb, .thumb, .aarch64, .aarch64_be => 2 * @sizeOf(usize), 61 // One pointer-sized word that points either to the DTV or the TCB itself 62 else => @sizeOf(usize), 63}; 64 65// Controls if the TP points to the end of the TCB instead of its beginning 66const tls_tp_points_past_tcb = switch (native_arch) { 67 .riscv32, .riscv64, .mips, .mipsel, .powerpc, .powerpc64, .powerpc64le => true, 68 else => false, 69}; 70 71// Some architectures add some offset to the tp and dtv addresses in order to 72// make the generated code more efficient 73 74const tls_tp_offset = switch (native_arch) { 75 .mips, .mipsel, .powerpc, .powerpc64, .powerpc64le => 0x7000, 76 else => 0, 77}; 78 79const tls_dtv_offset = switch (native_arch) { 80 .mips, .mipsel, .powerpc, .powerpc64, .powerpc64le => 0x8000, 81 .riscv32, .riscv64 => 0x800, 82 else => 0, 83}; 84 85// Per-thread storage for Zig's use 86const CustomData = struct { 87 dummy: usize, 88}; 89 90// Dynamic Thread Vector 91const DTV = extern struct { 92 entries: usize, 93 tls_block: [1][*]u8, 94}; 95 96// Holds all the information about the process TLS image 97const TLSImage = struct { 98 init_data: []const u8, 99 alloc_size: usize, 100 alloc_align: usize, 101 tcb_offset: usize, 102 dtv_offset: usize, 103 data_offset: usize, 104 data_size: usize, 105 // Only used on the i386 architecture 106 gdt_entry_number: usize, 107}; 108 109pub var tls_image: TLSImage = undefined; 110 111pub fn setThreadPointer(addr: usize) void { 112 switch (native_arch) { 113 .i386 => { 114 var user_desc = std.os.linux.user_desc{ 115 .entry_number = tls_image.gdt_entry_number, 116 .base_addr = addr, 117 .limit = 0xfffff, 118 .seg_32bit = 1, 119 .contents = 0, // Data 120 .read_exec_only = 0, 121 .limit_in_pages = 1, 122 .seg_not_present = 0, 123 .useable = 1, 124 }; 125 const rc = std.os.linux.syscall1(.set_thread_area, @ptrToInt(&user_desc)); 126 assert(rc == 0); 127 128 const gdt_entry_number = user_desc.entry_number; 129 // We have to keep track of our slot as it's also needed for clone() 130 tls_image.gdt_entry_number = gdt_entry_number; 131 // Update the %gs selector 132 asm volatile ("movl %[gs_val], %%gs" 133 : 134 : [gs_val] "r" (gdt_entry_number << 3 | 3), 135 ); 136 }, 137 .x86_64 => { 138 const rc = std.os.linux.syscall2(.arch_prctl, std.os.linux.ARCH.SET_FS, addr); 139 assert(rc == 0); 140 }, 141 .aarch64 => { 142 asm volatile ( 143 \\ msr tpidr_el0, %[addr] 144 : 145 : [addr] "r" (addr), 146 ); 147 }, 148 .arm, .thumb => { 149 const rc = std.os.linux.syscall1(.set_tls, addr); 150 assert(rc == 0); 151 }, 152 .riscv64 => { 153 asm volatile ( 154 \\ mv tp, %[addr] 155 : 156 : [addr] "r" (addr), 157 ); 158 }, 159 .mips, .mipsel => { 160 const rc = std.os.linux.syscall1(.set_thread_area, addr); 161 assert(rc == 0); 162 }, 163 .powerpc => { 164 asm volatile ( 165 \\ mr 2, %[addr] 166 : 167 : [addr] "r" (addr), 168 ); 169 }, 170 .powerpc64, .powerpc64le => { 171 asm volatile ( 172 \\ mr 13, %[addr] 173 : 174 : [addr] "r" (addr), 175 ); 176 }, 177 .sparcv9 => { 178 asm volatile ( 179 \\ mov %[addr], %%g7 180 : 181 : [addr] "r" (addr), 182 ); 183 }, 184 else => @compileError("Unsupported architecture"), 185 } 186} 187 188fn initTLS(phdrs: []elf.Phdr) void { 189 var tls_phdr: ?*elf.Phdr = null; 190 var img_base: usize = 0; 191 192 for (phdrs) |*phdr| { 193 switch (phdr.p_type) { 194 elf.PT_PHDR => img_base = @ptrToInt(phdrs.ptr) - phdr.p_vaddr, 195 elf.PT_TLS => tls_phdr = phdr, 196 else => {}, 197 } 198 } 199 200 var tls_align_factor: usize = undefined; 201 var tls_data: []const u8 = undefined; 202 var tls_data_alloc_size: usize = undefined; 203 if (tls_phdr) |phdr| { 204 // The effective size in memory is represented by p_memsz, the length of 205 // the data stored in the PT_TLS segment is p_filesz and may be less 206 // than the former 207 tls_align_factor = phdr.p_align; 208 tls_data = @intToPtr([*]u8, img_base + phdr.p_vaddr)[0..phdr.p_filesz]; 209 tls_data_alloc_size = phdr.p_memsz; 210 } else { 211 tls_align_factor = @alignOf(usize); 212 tls_data = &[_]u8{}; 213 tls_data_alloc_size = 0; 214 } 215 216 // Offsets into the allocated TLS area 217 var tcb_offset: usize = undefined; 218 var dtv_offset: usize = undefined; 219 var data_offset: usize = undefined; 220 // Compute the total size of the ABI-specific data plus our own control 221 // structures. All the offset calculated here assume a well-aligned base 222 // address. 223 const alloc_size = switch (tls_variant) { 224 .VariantI => blk: { 225 var l: usize = 0; 226 dtv_offset = l; 227 l += @sizeOf(DTV); 228 // Add some padding here so that the thread pointer (tcb_offset) is 229 // aligned to p_align and the CustomData structure can be found by 230 // simply subtracting its @sizeOf from the tp value 231 const delta = (l + @sizeOf(CustomData)) & (tls_align_factor - 1); 232 if (delta > 0) 233 l += tls_align_factor - delta; 234 l += @sizeOf(CustomData); 235 tcb_offset = l; 236 l += mem.alignForward(tls_tcb_size, tls_align_factor); 237 data_offset = l; 238 l += tls_data_alloc_size; 239 break :blk l; 240 }, 241 .VariantII => blk: { 242 var l: usize = 0; 243 data_offset = l; 244 l += mem.alignForward(tls_data_alloc_size, tls_align_factor); 245 // The thread pointer is aligned to p_align 246 tcb_offset = l; 247 l += tls_tcb_size; 248 // The CustomData structure is right after the TCB with no padding 249 // in between so it can be easily found 250 l += @sizeOf(CustomData); 251 l = mem.alignForward(l, @alignOf(DTV)); 252 dtv_offset = l; 253 l += @sizeOf(DTV); 254 break :blk l; 255 }, 256 }; 257 258 tls_image = TLSImage{ 259 .init_data = tls_data, 260 .alloc_size = alloc_size, 261 .alloc_align = tls_align_factor, 262 .tcb_offset = tcb_offset, 263 .dtv_offset = dtv_offset, 264 .data_offset = data_offset, 265 .data_size = tls_data_alloc_size, 266 .gdt_entry_number = @bitCast(usize, @as(isize, -1)), 267 }; 268} 269 270inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T { 271 return @ptrCast(*T, @alignCast(@alignOf(T), ptr)); 272} 273 274/// Initializes all the fields of the static TLS area and returns the computed 275/// architecture-specific value of the thread-pointer register 276pub fn prepareTLS(area: []u8) usize { 277 // Clear the area we're going to use, just to be safe 278 mem.set(u8, area, 0); 279 // Prepare the DTV 280 const dtv = alignPtrCast(DTV, area.ptr + tls_image.dtv_offset); 281 dtv.entries = 1; 282 dtv.tls_block[0] = area.ptr + tls_dtv_offset + tls_image.data_offset; 283 // Prepare the TCB 284 const tcb_ptr = alignPtrCast([*]u8, area.ptr + tls_image.tcb_offset); 285 tcb_ptr.* = switch (tls_variant) { 286 .VariantI => area.ptr + tls_image.dtv_offset, 287 .VariantII => area.ptr + tls_image.tcb_offset, 288 }; 289 // Copy the data 290 mem.copy(u8, area[tls_image.data_offset..], tls_image.init_data); 291 292 // Return the corrected value (if needed) for the tp register. 293 // Overflow here is not a problem, the pointer arithmetic involving the tp 294 // is done with wrapping semantics. 295 return @ptrToInt(area.ptr) +% tls_tp_offset +% 296 if (tls_tp_points_past_tcb) tls_image.data_offset else tls_image.tcb_offset; 297} 298 299// The main motivation for the size chosen here is this is how much ends up being 300// requested for the thread local variables of the std.crypto.random implementation. 301// I'm not sure why it ends up being so much; the struct itself is only 64 bytes. 302// I think it has to do with being page aligned and LLVM or LLD is not smart enough 303// to lay out the TLS data in a space conserving way. Anyway I think it's fine 304// because it's less than 3 pages of memory, and putting it in the ELF like this 305// is equivalent to moving the mmap call below into the kernel, avoiding syscall 306// overhead. 307var main_thread_tls_buffer: [0x2100]u8 align(mem.page_size) = undefined; 308 309pub fn initStaticTLS(phdrs: []elf.Phdr) void { 310 initTLS(phdrs); 311 312 const tls_area = blk: { 313 // Fast path for the common case where the TLS data is really small, 314 // avoid an allocation and use our local buffer. 315 if (tls_image.alloc_align <= mem.page_size and 316 tls_image.alloc_size <= main_thread_tls_buffer.len) 317 { 318 break :blk main_thread_tls_buffer[0..tls_image.alloc_size]; 319 } 320 321 const alloc_tls_area = os.mmap( 322 null, 323 tls_image.alloc_size + tls_image.alloc_align - 1, 324 os.PROT.READ | os.PROT.WRITE, 325 os.MAP.PRIVATE | os.MAP.ANONYMOUS, 326 -1, 327 0, 328 ) catch os.abort(); 329 330 // Make sure the slice is correctly aligned. 331 const begin_addr = @ptrToInt(alloc_tls_area.ptr); 332 const begin_aligned_addr = mem.alignForward(begin_addr, tls_image.alloc_align); 333 const start = begin_aligned_addr - begin_addr; 334 break :blk alloc_tls_area[start .. start + tls_image.alloc_size]; 335 }; 336 337 const tp_value = prepareTLS(tls_area); 338 setThreadPointer(tp_value); 339} 340