1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "jit/ProcessExecutableMemory.h"
8
9 #include "mozilla/Array.h"
10 #include "mozilla/Atomics.h"
11 #include "mozilla/DebugOnly.h"
12 #include "mozilla/Maybe.h"
13 #include "mozilla/TaggedAnonymousMemory.h"
14 #include "mozilla/XorShift128PlusRNG.h"
15
16 #include <errno.h>
17
18 #include "jsfriendapi.h"
19 #include "jsmath.h"
20
21 #include "gc/Memory.h"
22 #ifdef JS_CODEGEN_ARM64
23 # include "jit/arm64/vixl/Cpu-vixl.h"
24 #endif
25 #include "jit/AtomicOperations.h"
26 #include "jit/FlushICache.h" // js::jit::FlushICache
27 #include "threading/LockGuard.h"
28 #include "threading/Mutex.h"
29 #include "util/Memory.h"
30 #include "util/Poison.h"
31 #include "util/Windows.h"
32 #include "vm/MutexIDs.h"
33
34 #ifdef XP_WIN
35 # include "mozilla/StackWalk_windows.h"
36 # include "mozilla/WindowsVersion.h"
37 #elif defined(__wasi__)
38 // Nothing.
39 #else
40 # include <sys/mman.h>
41 # include <unistd.h>
42 #endif
43
44 #ifdef MOZ_VALGRIND
45 # include <valgrind/valgrind.h>
46 #endif
47
48 using namespace js;
49 using namespace js::jit;
50
51 #ifdef XP_WIN
52 # if defined(HAVE_64BIT_BUILD)
53 # define NEED_JIT_UNWIND_HANDLING
54 # endif
55
ComputeRandomAllocationAddress()56 static void* ComputeRandomAllocationAddress() {
57 /*
58 * Inspiration is V8's OS::Allocate in platform-win32.cc.
59 *
60 * VirtualAlloc takes 64K chunks out of the virtual address space, so we
61 * keep 16b alignment.
62 *
63 * x86: V8 comments say that keeping addresses in the [64MiB, 1GiB) range
64 * tries to avoid system default DLL mapping space. In the end, we get 13
65 * bits of randomness in our selection.
66 * x64: [2GiB, 4TiB), with 25 bits of randomness.
67 */
68 # ifdef HAVE_64BIT_BUILD
69 static const uintptr_t base = 0x0000000080000000;
70 static const uintptr_t mask = 0x000003ffffff0000;
71 # elif defined(_M_IX86) || defined(__i386__)
72 static const uintptr_t base = 0x04000000;
73 static const uintptr_t mask = 0x3fff0000;
74 # else
75 # error "Unsupported architecture"
76 # endif
77
78 uint64_t rand = js::GenerateRandomSeed();
79 return (void*)(base | (rand & mask));
80 }
81
82 # ifdef NEED_JIT_UNWIND_HANDLING
83 static js::JitExceptionHandler sJitExceptionHandler;
84 # endif
85
SetJitExceptionHandler(JitExceptionHandler handler)86 JS_PUBLIC_API void js::SetJitExceptionHandler(JitExceptionHandler handler) {
87 # ifdef NEED_JIT_UNWIND_HANDLING
88 MOZ_ASSERT(!sJitExceptionHandler);
89 sJitExceptionHandler = handler;
90 # else
91 // Just do nothing if unwind handling is disabled.
92 # endif
93 }
94
95 # ifdef NEED_JIT_UNWIND_HANDLING
96 # if defined(_M_ARM64)
97 // See the ".xdata records" section of
98 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
99 // These records can have various fields present or absent depending on the
100 // bits set in the header. Our struct will use one 32-bit slot for unwind codes,
101 // and no slots for epilog scopes.
102 struct UnwindInfo {
103 uint32_t functionLength : 18;
104 uint32_t version : 2;
105 uint32_t hasExceptionHandler : 1;
106 uint32_t packedEpilog : 1;
107 uint32_t epilogCount : 5;
108 uint32_t codeWords : 5;
109 uint8_t unwindCodes[4];
110 uint32_t exceptionHandler;
111 };
112 static const unsigned ThunkLength = 20;
113 # else
114 // From documentation for UNWIND_INFO on
115 // http://msdn.microsoft.com/en-us/library/ddssxxy8.aspx
116 struct UnwindInfo {
117 uint8_t version : 3;
118 uint8_t flags : 5;
119 uint8_t sizeOfPrologue;
120 uint8_t countOfUnwindCodes;
121 uint8_t frameRegister : 4;
122 uint8_t frameOffset : 4;
123 ULONG exceptionHandler;
124 };
125 static const unsigned ThunkLength = 12;
126 # endif
127
128 struct ExceptionHandlerRecord {
129 RUNTIME_FUNCTION runtimeFunction;
130 UnwindInfo unwindInfo;
131 uint8_t thunk[ThunkLength];
132 };
133
134 // This function must match the function pointer type PEXCEPTION_HANDLER
135 // mentioned in:
136 // http://msdn.microsoft.com/en-us/library/ssa62fwe.aspx.
137 // This type is rather elusive in documentation; Wine is the best I've found:
138 // http://source.winehq.org/source/include/winnt.h
ExceptionHandler(PEXCEPTION_RECORD exceptionRecord,_EXCEPTION_REGISTRATION_RECORD *,PCONTEXT context,_EXCEPTION_REGISTRATION_RECORD **)139 static DWORD ExceptionHandler(PEXCEPTION_RECORD exceptionRecord,
140 _EXCEPTION_REGISTRATION_RECORD*, PCONTEXT context,
141 _EXCEPTION_REGISTRATION_RECORD**) {
142 return sJitExceptionHandler(exceptionRecord, context);
143 }
144
145 PRUNTIME_FUNCTION RuntimeFunctionCallback(DWORD64 ControlPc, PVOID Context);
146
147 // For an explanation of the problem being solved here, see
148 // SetJitExceptionFilter in jsfriendapi.h.
RegisterExecutableMemory(void * p,size_t bytes,size_t pageSize)149 static bool RegisterExecutableMemory(void* p, size_t bytes, size_t pageSize) {
150 if (!VirtualAlloc(p, pageSize, MEM_COMMIT, PAGE_READWRITE)) {
151 MOZ_CRASH();
152 }
153
154 ExceptionHandlerRecord* r = reinterpret_cast<ExceptionHandlerRecord*>(p);
155 void* handler = JS_FUNC_TO_DATA_PTR(void*, ExceptionHandler);
156
157 // Because the .xdata format on ARM64 can only encode sizes up to 1M (much
158 // too small for our JIT code regions), we register a function table callback
159 // to provide RUNTIME_FUNCTIONs at runtime. Windows doesn't seem to care about
160 // the size fields on RUNTIME_FUNCTIONs that are created in this way, so the
161 // same RUNTIME_FUNCTION can work for any address in the region. We'll set up
162 // a generic one now and the callback can just return a pointer to it.
163
164 // All these fields are specified to be offsets from the base of the
165 // executable code (which is 'p'), even if they have 'Address' in their
166 // names. In particular, exceptionHandler is a ULONG offset which is a
167 // 32-bit integer. Since 'p' can be farther than INT32_MAX away from
168 // sJitExceptionHandler, we must generate a little thunk inside the
169 // record. The record is put on its own page so that we can take away write
170 // access to protect against accidental clobbering.
171
172 # if defined(_M_ARM64)
173 r->runtimeFunction.BeginAddress = pageSize;
174 r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindInfo);
175 static_assert(offsetof(ExceptionHandlerRecord, unwindInfo) % 4 == 0,
176 "The ARM64 .pdata format requires that exception information "
177 "RVAs be 4-byte aligned.");
178
179 memset(&r->unwindInfo, 0, sizeof(r->unwindInfo));
180 r->unwindInfo.hasExceptionHandler = true;
181 r->unwindInfo.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);
182
183 // Use a fake unwind code to make the Windows unwinder do _something_. If the
184 // PC and SP both stay unchanged, we'll fail the unwinder's sanity checks and
185 // it won't call our exception handler.
186 r->unwindInfo.codeWords = 1; // one 32-bit word gives us up to 4 codes
187 r->unwindInfo.unwindCodes[0] =
188 0b00000001; // alloc_s small stack of size 1*16
189 r->unwindInfo.unwindCodes[1] = 0b11100100; // end
190
191 uint32_t* thunk = (uint32_t*)r->thunk;
192 uint16_t* addr = (uint16_t*)&handler;
193
194 // xip0/r16 should be safe to clobber: Windows just used it to call our thunk.
195 const uint8_t reg = 16;
196
197 // Say `handler` is 0x4444333322221111, then:
198 thunk[0] = 0xd2800000 | addr[0] << 5 | reg; // mov xip0, 1111
199 thunk[1] = 0xf2a00000 | addr[1] << 5 | reg; // movk xip0, 2222 lsl #0x10
200 thunk[2] = 0xf2c00000 | addr[2] << 5 | reg; // movk xip0, 3333 lsl #0x20
201 thunk[3] = 0xf2e00000 | addr[3] << 5 | reg; // movk xip0, 4444 lsl #0x30
202 thunk[4] = 0xd61f0000 | reg << 5; // br xip0
203 # else
204 r->runtimeFunction.BeginAddress = pageSize;
205 r->runtimeFunction.EndAddress = (DWORD)bytes;
206 r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindInfo);
207
208 r->unwindInfo.version = 1;
209 r->unwindInfo.flags = UNW_FLAG_EHANDLER;
210 r->unwindInfo.sizeOfPrologue = 0;
211 r->unwindInfo.countOfUnwindCodes = 0;
212 r->unwindInfo.frameRegister = 0;
213 r->unwindInfo.frameOffset = 0;
214 r->unwindInfo.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);
215
216 // mov imm64, rax
217 r->thunk[0] = 0x48;
218 r->thunk[1] = 0xb8;
219 memcpy(&r->thunk[2], &handler, 8);
220
221 // jmp rax
222 r->thunk[10] = 0xff;
223 r->thunk[11] = 0xe0;
224 # endif
225
226 DWORD oldProtect;
227 if (!VirtualProtect(p, pageSize, PAGE_EXECUTE_READ, &oldProtect)) {
228 MOZ_CRASH();
229 }
230
231 // XXX NB: The profiler believes this function is only called from the main
232 // thread. If that ever becomes untrue, the profiler must be updated
233 // immediately.
234 AutoSuppressStackWalking suppress;
235 return RtlInstallFunctionTableCallback((DWORD64)p | 0x3, (DWORD64)p, bytes,
236 RuntimeFunctionCallback, NULL, NULL);
237 }
238
UnregisterExecutableMemory(void * p,size_t bytes,size_t pageSize)239 static void UnregisterExecutableMemory(void* p, size_t bytes, size_t pageSize) {
240 // There's no such thing as RtlUninstallFunctionTableCallback, so there's
241 // nothing to do here.
242 }
243 # endif
244
ReserveProcessExecutableMemory(size_t bytes)245 static void* ReserveProcessExecutableMemory(size_t bytes) {
246 # ifdef NEED_JIT_UNWIND_HANDLING
247 size_t pageSize = gc::SystemPageSize();
248 if (sJitExceptionHandler) {
249 bytes += pageSize;
250 }
251 # endif
252
253 void* p = nullptr;
254 for (size_t i = 0; i < 10; i++) {
255 void* randomAddr = ComputeRandomAllocationAddress();
256 p = VirtualAlloc(randomAddr, bytes, MEM_RESERVE, PAGE_NOACCESS);
257 if (p) {
258 break;
259 }
260 }
261
262 if (!p) {
263 // Try again without randomization.
264 p = VirtualAlloc(nullptr, bytes, MEM_RESERVE, PAGE_NOACCESS);
265 if (!p) {
266 return nullptr;
267 }
268 }
269
270 # ifdef NEED_JIT_UNWIND_HANDLING
271 if (sJitExceptionHandler) {
272 if (!RegisterExecutableMemory(p, bytes, pageSize)) {
273 VirtualFree(p, 0, MEM_RELEASE);
274 return nullptr;
275 }
276
277 p = (uint8_t*)p + pageSize;
278 bytes -= pageSize;
279 }
280
281 RegisterJitCodeRegion((uint8_t*)p, bytes);
282 # endif
283
284 return p;
285 }
286
DeallocateProcessExecutableMemory(void * addr,size_t bytes)287 static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
288 # ifdef NEED_JIT_UNWIND_HANDLING
289 UnregisterJitCodeRegion((uint8_t*)addr, bytes);
290
291 if (sJitExceptionHandler) {
292 size_t pageSize = gc::SystemPageSize();
293 addr = (uint8_t*)addr - pageSize;
294 UnregisterExecutableMemory(addr, bytes, pageSize);
295 }
296 # endif
297
298 VirtualFree(addr, 0, MEM_RELEASE);
299 }
300
ProtectionSettingToFlags(ProtectionSetting protection)301 static DWORD ProtectionSettingToFlags(ProtectionSetting protection) {
302 switch (protection) {
303 case ProtectionSetting::Protected:
304 return PAGE_NOACCESS;
305 case ProtectionSetting::Writable:
306 return PAGE_READWRITE;
307 case ProtectionSetting::Executable:
308 return PAGE_EXECUTE_READ;
309 }
310 MOZ_CRASH();
311 }
312
CommitPages(void * addr,size_t bytes,ProtectionSetting protection)313 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
314 ProtectionSetting protection) {
315 void* p = VirtualAlloc(addr, bytes, MEM_COMMIT,
316 ProtectionSettingToFlags(protection));
317 if (!p) {
318 return false;
319 }
320 MOZ_RELEASE_ASSERT(p == addr);
321 return true;
322 }
323
DecommitPages(void * addr,size_t bytes)324 static void DecommitPages(void* addr, size_t bytes) {
325 if (!VirtualFree(addr, bytes, MEM_DECOMMIT)) {
326 MOZ_CRASH("DecommitPages failed");
327 }
328 }
329 #elif defined(__wasi__)
ReserveProcessExecutableMemory(size_t bytes)330 static void* ReserveProcessExecutableMemory(size_t bytes) {
331 MOZ_CRASH("NYI for WASI.");
332 return nullptr;
333 }
CommitPages(void * addr,size_t bytes,ProtectionSetting protection)334 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
335 ProtectionSetting protection) {
336 MOZ_CRASH("NYI for WASI.");
337 return false;
338 }
DecommitPages(void * addr,size_t bytes)339 static void DecommitPages(void* addr, size_t bytes) {
340 MOZ_CRASH("NYI for WASI.");
341 }
342 #else // !XP_WIN && !__wasi__
343 # ifndef MAP_NORESERVE
344 # define MAP_NORESERVE 0
345 # endif
346
ComputeRandomAllocationAddress()347 static void* ComputeRandomAllocationAddress() {
348 # ifdef __OpenBSD__
349 // OpenBSD already has random mmap and the idea that all x64 cpus
350 // have 48-bit address space is not correct. Returning nullptr
351 // allows OpenBSD do to the right thing.
352 return nullptr;
353 # else
354 uint64_t rand = js::GenerateRandomSeed();
355
356 # ifdef HAVE_64BIT_BUILD
357 // x64 CPUs have a 48-bit address space and on some platforms the OS will
358 // give us access to 47 bits, so to be safe we right shift by 18 to leave
359 // 46 bits.
360 rand >>= 18;
361 # else
362 // On 32-bit, right shift by 34 to leave 30 bits, range [0, 1GiB). Then add
363 // 512MiB to get range [512MiB, 1.5GiB), or [0x20000000, 0x60000000). This
364 // is based on V8 comments in platform-posix.cc saying this range is
365 // relatively unpopulated across a variety of kernels.
366 rand >>= 34;
367 rand += 512 * 1024 * 1024;
368 # endif
369
370 // Ensure page alignment.
371 uintptr_t mask = ~uintptr_t(gc::SystemPageSize() - 1);
372 return (void*)uintptr_t(rand & mask);
373 # endif
374 }
375
ReserveProcessExecutableMemory(size_t bytes)376 static void* ReserveProcessExecutableMemory(size_t bytes) {
377 // Note that randomAddr is just a hint: if the address is not available
378 // mmap will pick a different address.
379 void* randomAddr = ComputeRandomAllocationAddress();
380 void* p = MozTaggedAnonymousMmap(randomAddr, bytes, PROT_NONE,
381 MAP_NORESERVE | MAP_PRIVATE | MAP_ANON, -1,
382 0, "js-executable-memory");
383 if (p == MAP_FAILED) {
384 return nullptr;
385 }
386 return p;
387 }
388
DeallocateProcessExecutableMemory(void * addr,size_t bytes)389 static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
390 mozilla::DebugOnly<int> result = munmap(addr, bytes);
391 MOZ_ASSERT(!result || errno == ENOMEM);
392 }
393
ProtectionSettingToFlags(ProtectionSetting protection)394 static unsigned ProtectionSettingToFlags(ProtectionSetting protection) {
395 # ifdef MOZ_VALGRIND
396 // If we're configured for Valgrind and running on it, use a slacker
397 // scheme that doesn't change execute permissions, since doing so causes
398 // Valgrind a lot of extra overhead re-JITting code that loses and later
399 // regains execute permission. See bug 1338179.
400 if (RUNNING_ON_VALGRIND) {
401 switch (protection) {
402 case ProtectionSetting::Protected:
403 return PROT_NONE;
404 case ProtectionSetting::Writable:
405 return PROT_READ | PROT_WRITE | PROT_EXEC;
406 case ProtectionSetting::Executable:
407 return PROT_READ | PROT_EXEC;
408 }
409 MOZ_CRASH();
410 }
411 // If we get here, we're configured for Valgrind but not running on
412 // it, so use the standard scheme.
413 # endif
414 switch (protection) {
415 case ProtectionSetting::Protected:
416 return PROT_NONE;
417 case ProtectionSetting::Writable:
418 return PROT_READ | PROT_WRITE;
419 case ProtectionSetting::Executable:
420 return PROT_READ | PROT_EXEC;
421 }
422 MOZ_CRASH();
423 }
424
CommitPages(void * addr,size_t bytes,ProtectionSetting protection)425 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
426 ProtectionSetting protection) {
427 void* p = MozTaggedAnonymousMmap(
428 addr, bytes, ProtectionSettingToFlags(protection),
429 MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0, "js-executable-memory");
430 if (p == MAP_FAILED) {
431 return false;
432 }
433 MOZ_RELEASE_ASSERT(p == addr);
434 return true;
435 }
436
DecommitPages(void * addr,size_t bytes)437 static void DecommitPages(void* addr, size_t bytes) {
438 // Use mmap with MAP_FIXED and PROT_NONE. Inspired by jemalloc's
439 // pages_decommit.
440 void* p = MozTaggedAnonymousMmap(addr, bytes, PROT_NONE,
441 MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0,
442 "js-executable-memory");
443 MOZ_RELEASE_ASSERT(addr == p);
444 }
445 #endif
446
447 template <size_t NumBits>
448 class PageBitSet {
449 using WordType = uint32_t;
450 static const size_t BitsPerWord = sizeof(WordType) * 8;
451
452 static_assert((NumBits % BitsPerWord) == 0,
453 "NumBits must be a multiple of BitsPerWord");
454 static const size_t NumWords = NumBits / BitsPerWord;
455
456 mozilla::Array<WordType, NumWords> words_;
457
indexToWord(uint32_t index) const458 uint32_t indexToWord(uint32_t index) const {
459 MOZ_ASSERT(index < NumBits);
460 return index / BitsPerWord;
461 }
indexToBit(uint32_t index) const462 WordType indexToBit(uint32_t index) const {
463 MOZ_ASSERT(index < NumBits);
464 return WordType(1) << (index % BitsPerWord);
465 }
466
467 public:
init()468 void init() { mozilla::PodArrayZero(words_); }
contains(size_t index) const469 bool contains(size_t index) const {
470 uint32_t word = indexToWord(index);
471 return words_[word] & indexToBit(index);
472 }
insert(size_t index)473 void insert(size_t index) {
474 MOZ_ASSERT(!contains(index));
475 uint32_t word = indexToWord(index);
476 words_[word] |= indexToBit(index);
477 }
remove(size_t index)478 void remove(size_t index) {
479 MOZ_ASSERT(contains(index));
480 uint32_t word = indexToWord(index);
481 words_[word] &= ~indexToBit(index);
482 }
483
484 #ifdef DEBUG
empty() const485 bool empty() const {
486 for (size_t i = 0; i < NumWords; i++) {
487 if (words_[i] != 0) {
488 return false;
489 }
490 }
491 return true;
492 }
493 #endif
494 };
495
496 // Per-process executable memory allocator. It reserves a block of memory of
497 // MaxCodeBytesPerProcess bytes, then allocates/deallocates pages from that.
498 //
499 // This has a number of benefits compared to raw mmap/VirtualAlloc:
500 //
501 // * More resillient against certain attacks.
502 //
503 // * Behaves more consistently across platforms: it avoids the 64K granularity
504 // issues on Windows, for instance.
505 //
506 // * On x64, near jumps can be used for jumps to other JIT pages.
507 //
508 // * On Win64, we have to register the exception handler only once (at process
509 // startup). This saves some memory and avoids RtlAddFunctionTable profiler
510 // deadlocks.
511 class ProcessExecutableMemory {
512 static_assert(
513 (MaxCodeBytesPerProcess % ExecutableCodePageSize) == 0,
514 "MaxCodeBytesPerProcess must be a multiple of ExecutableCodePageSize");
515 static const size_t MaxCodePages =
516 MaxCodeBytesPerProcess / ExecutableCodePageSize;
517
518 // Start of the MaxCodeBytesPerProcess memory block or nullptr if
519 // uninitialized. Note that this is NOT guaranteed to be aligned to
520 // ExecutableCodePageSize.
521 uint8_t* base_;
522
523 // The fields below should only be accessed while we hold the lock.
524 Mutex lock_;
525
526 // pagesAllocated_ is an Atomic so that bytesAllocated does not have to
527 // take the lock.
528 mozilla::Atomic<size_t, mozilla::ReleaseAcquire> pagesAllocated_;
529
530 // Page where we should try to allocate next.
531 size_t cursor_;
532
533 mozilla::Maybe<mozilla::non_crypto::XorShift128PlusRNG> rng_;
534 PageBitSet<MaxCodePages> pages_;
535
536 public:
ProcessExecutableMemory()537 ProcessExecutableMemory()
538 : base_(nullptr),
539 lock_(mutexid::ProcessExecutableRegion),
540 pagesAllocated_(0),
541 cursor_(0),
542 rng_(),
543 pages_() {}
544
init()545 [[nodiscard]] bool init() {
546 pages_.init();
547
548 MOZ_RELEASE_ASSERT(!initialized());
549 MOZ_RELEASE_ASSERT(gc::SystemPageSize() <= ExecutableCodePageSize);
550
551 void* p = ReserveProcessExecutableMemory(MaxCodeBytesPerProcess);
552 if (!p) {
553 return false;
554 }
555
556 base_ = static_cast<uint8_t*>(p);
557
558 mozilla::Array<uint64_t, 2> seed;
559 GenerateXorShift128PlusSeed(seed);
560 rng_.emplace(seed[0], seed[1]);
561 return true;
562 }
563
base() const564 uint8_t* base() const { return base_; }
565
initialized() const566 bool initialized() const { return base_ != nullptr; }
567
bytesAllocated() const568 size_t bytesAllocated() const {
569 MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
570 return pagesAllocated_ * ExecutableCodePageSize;
571 }
572
release()573 void release() {
574 #if defined(__wasi__)
575 MOZ_ASSERT(!initialized());
576 #else
577 MOZ_ASSERT(initialized());
578 MOZ_ASSERT(pages_.empty());
579 MOZ_ASSERT(pagesAllocated_ == 0);
580 DeallocateProcessExecutableMemory(base_, MaxCodeBytesPerProcess);
581 base_ = nullptr;
582 rng_.reset();
583 MOZ_ASSERT(!initialized());
584 #endif
585 }
586
assertValidAddress(void * p,size_t bytes) const587 void assertValidAddress(void* p, size_t bytes) const {
588 MOZ_RELEASE_ASSERT(p >= base_ &&
589 uintptr_t(p) + bytes <=
590 uintptr_t(base_) + MaxCodeBytesPerProcess);
591 }
592
containsAddress(const void * p) const593 bool containsAddress(const void* p) const {
594 return p >= base_ &&
595 uintptr_t(p) < uintptr_t(base_) + MaxCodeBytesPerProcess;
596 }
597
598 void* allocate(size_t bytes, ProtectionSetting protection,
599 MemCheckKind checkKind);
600 void deallocate(void* addr, size_t bytes, bool decommit);
601 };
602
allocate(size_t bytes,ProtectionSetting protection,MemCheckKind checkKind)603 void* ProcessExecutableMemory::allocate(size_t bytes,
604 ProtectionSetting protection,
605 MemCheckKind checkKind) {
606 MOZ_ASSERT(initialized());
607 MOZ_ASSERT(bytes > 0);
608 MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);
609
610 size_t numPages = bytes / ExecutableCodePageSize;
611
612 // Take the lock and try to allocate.
613 void* p = nullptr;
614 {
615 LockGuard<Mutex> guard(lock_);
616 MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
617
618 // Check if we have enough pages available.
619 if (pagesAllocated_ + numPages >= MaxCodePages) {
620 return nullptr;
621 }
622
623 MOZ_ASSERT(bytes <= MaxCodeBytesPerProcess);
624
625 // Maybe skip a page to make allocations less predictable.
626 size_t page = cursor_ + (rng_.ref().next() % 2);
627
628 for (size_t i = 0; i < MaxCodePages; i++) {
629 // Make sure page + numPages - 1 is a valid index.
630 if (page + numPages > MaxCodePages) {
631 page = 0;
632 }
633
634 bool available = true;
635 for (size_t j = 0; j < numPages; j++) {
636 if (pages_.contains(page + j)) {
637 available = false;
638 break;
639 }
640 }
641 if (!available) {
642 page++;
643 continue;
644 }
645
646 // Mark the pages as unavailable.
647 for (size_t j = 0; j < numPages; j++) {
648 pages_.insert(page + j);
649 }
650
651 pagesAllocated_ += numPages;
652 MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
653
654 // If we allocated a small number of pages, move cursor_ to the
655 // next page. We don't do this for larger allocations to avoid
656 // skipping a large number of small holes.
657 if (numPages <= 2) {
658 cursor_ = page + numPages;
659 }
660
661 p = base_ + page * ExecutableCodePageSize;
662 break;
663 }
664 if (!p) {
665 return nullptr;
666 }
667 }
668
669 // Commit the pages after releasing the lock.
670 if (!CommitPages(p, bytes, protection)) {
671 deallocate(p, bytes, /* decommit = */ false);
672 return nullptr;
673 }
674
675 SetMemCheckKind(p, bytes, checkKind);
676
677 return p;
678 }
679
deallocate(void * addr,size_t bytes,bool decommit)680 void ProcessExecutableMemory::deallocate(void* addr, size_t bytes,
681 bool decommit) {
682 MOZ_ASSERT(initialized());
683 MOZ_ASSERT(addr);
684 MOZ_ASSERT((uintptr_t(addr) % gc::SystemPageSize()) == 0);
685 MOZ_ASSERT(bytes > 0);
686 MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);
687
688 assertValidAddress(addr, bytes);
689
690 size_t firstPage =
691 (static_cast<uint8_t*>(addr) - base_) / ExecutableCodePageSize;
692 size_t numPages = bytes / ExecutableCodePageSize;
693
694 // Decommit before taking the lock.
695 MOZ_MAKE_MEM_NOACCESS(addr, bytes);
696 if (decommit) {
697 DecommitPages(addr, bytes);
698 }
699
700 LockGuard<Mutex> guard(lock_);
701 MOZ_ASSERT(numPages <= pagesAllocated_);
702 pagesAllocated_ -= numPages;
703
704 for (size_t i = 0; i < numPages; i++) {
705 pages_.remove(firstPage + i);
706 }
707
708 // Move the cursor back so we can reuse pages instead of fragmenting the
709 // whole region.
710 if (firstPage < cursor_) {
711 cursor_ = firstPage;
712 }
713 }
714
715 static ProcessExecutableMemory execMemory;
716
AllocateExecutableMemory(size_t bytes,ProtectionSetting protection,MemCheckKind checkKind)717 void* js::jit::AllocateExecutableMemory(size_t bytes,
718 ProtectionSetting protection,
719 MemCheckKind checkKind) {
720 return execMemory.allocate(bytes, protection, checkKind);
721 }
722
DeallocateExecutableMemory(void * addr,size_t bytes)723 void js::jit::DeallocateExecutableMemory(void* addr, size_t bytes) {
724 execMemory.deallocate(addr, bytes, /* decommit = */ true);
725 }
726
InitProcessExecutableMemory()727 bool js::jit::InitProcessExecutableMemory() {
728 #ifdef JS_CODEGEN_ARM64
729 // Initialize instruction cache flushing.
730 vixl::CPU::SetUp();
731 #elif defined(__wasi__)
732 return true;
733 #endif
734 return execMemory.init();
735 }
736
ReleaseProcessExecutableMemory()737 void js::jit::ReleaseProcessExecutableMemory() { execMemory.release(); }
738
LikelyAvailableExecutableMemory()739 size_t js::jit::LikelyAvailableExecutableMemory() {
740 // Round down available memory to the closest MB.
741 return MaxCodeBytesPerProcess -
742 AlignBytes(execMemory.bytesAllocated(), 0x100000U);
743 }
744
CanLikelyAllocateMoreExecutableMemory()745 bool js::jit::CanLikelyAllocateMoreExecutableMemory() {
746 // Use a 8 MB buffer.
747 static const size_t BufferSize = 8 * 1024 * 1024;
748
749 MOZ_ASSERT(execMemory.bytesAllocated() <= MaxCodeBytesPerProcess);
750
751 return execMemory.bytesAllocated() + BufferSize <= MaxCodeBytesPerProcess;
752 }
753
AddressIsInExecutableMemory(const void * p)754 bool js::jit::AddressIsInExecutableMemory(const void* p) {
755 return execMemory.containsAddress(p);
756 }
757
ReprotectRegion(void * start,size_t size,ProtectionSetting protection,MustFlushICache flushICache)758 bool js::jit::ReprotectRegion(void* start, size_t size,
759 ProtectionSetting protection,
760 MustFlushICache flushICache) {
761 // Flush ICache when making code executable, before we modify |size|.
762 if (flushICache == MustFlushICache::LocalThreadOnly ||
763 flushICache == MustFlushICache::AllThreads) {
764 MOZ_ASSERT(protection == ProtectionSetting::Executable);
765 bool codeIsThreadLocal = flushICache == MustFlushICache::LocalThreadOnly;
766 jit::FlushICache(start, size, codeIsThreadLocal);
767 }
768
769 // Calculate the start of the page containing this region,
770 // and account for this extra memory within size.
771 size_t pageSize = gc::SystemPageSize();
772 intptr_t startPtr = reinterpret_cast<intptr_t>(start);
773 intptr_t pageStartPtr = startPtr & ~(pageSize - 1);
774 void* pageStart = reinterpret_cast<void*>(pageStartPtr);
775 size += (startPtr - pageStartPtr);
776
777 // Round size up
778 size += (pageSize - 1);
779 size &= ~(pageSize - 1);
780
781 MOZ_ASSERT((uintptr_t(pageStart) % pageSize) == 0);
782
783 execMemory.assertValidAddress(pageStart, size);
784
785 // On weak memory systems, make sure new code is visible on all cores before
786 // addresses of the code are made public. Now is the latest moment in time
787 // when we can do that, and we're assuming that every other thread that has
788 // written into the memory that is being reprotected here has synchronized
789 // with this thread in such a way that the memory writes have become visible
790 // and we therefore only need to execute the fence once here. See bug 1529933
791 // for a longer discussion of why this is both necessary and sufficient.
792 //
793 // We use the C++ fence here -- and not AtomicOperations::fenceSeqCst() --
794 // primarily because ReprotectRegion will be called while we construct our own
795 // jitted atomics. But the C++ fence is sufficient and correct, too.
796 #ifdef __wasi__
797 MOZ_CRASH("NYI FOR WASI.");
798 #else
799 std::atomic_thread_fence(std::memory_order_seq_cst);
800
801 # ifdef XP_WIN
802 DWORD oldProtect;
803 DWORD flags = ProtectionSettingToFlags(protection);
804 if (!VirtualProtect(pageStart, size, flags, &oldProtect)) {
805 return false;
806 }
807 # else
808 unsigned flags = ProtectionSettingToFlags(protection);
809 if (mprotect(pageStart, size, flags)) {
810 return false;
811 }
812 # endif
813 #endif // __wasi__
814
815 execMemory.assertValidAddress(pageStart, size);
816 return true;
817 }
818
819 #if defined(XP_WIN) && defined(NEED_JIT_UNWIND_HANDLING)
RuntimeFunctionCallback(DWORD64 ControlPc,PVOID Context)820 static PRUNTIME_FUNCTION RuntimeFunctionCallback(DWORD64 ControlPc,
821 PVOID Context) {
822 MOZ_ASSERT(sJitExceptionHandler);
823
824 // RegisterExecutableMemory already set up the runtime function in the
825 // exception-data page preceding the allocation.
826 uint8_t* p = execMemory.base();
827 if (!p) {
828 return nullptr;
829 }
830 return (PRUNTIME_FUNCTION)(p - gc::SystemPageSize());
831 }
832 #endif
833