src/jit/ProcessExecutableMemory.cpp

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
 * vim: set ts=8 sts=4 et sw=4 tw=99:
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "jit/ProcessExecutableMemory.h"

#include "mozilla/Array.h"
#include "mozilla/Atomics.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/Maybe.h"
#include "mozilla/TaggedAnonymousMemory.h"
#include "mozilla/XorShift128PlusRNG.h"

#include "jsfriendapi.h"
#include "jslock.h"
#include "jsmath.h"
#include "jsutil.h"
#include "jswin.h"

#include <errno.h>

#include "gc/Memory.h"

#ifdef XP_WIN
# include "mozilla/WindowsVersion.h"
#else
# include <sys/mman.h>
# include <unistd.h>
#endif

using namespace js;
using namespace js::jit;

#ifdef XP_WIN
static void*
ComputeRandomAllocationAddress()
{
    /*
     * Inspiration is V8's OS::Allocate in platform-win32.cc.
     *
     * VirtualAlloc takes 64K chunks out of the virtual address space, so we
     * keep 16b alignment.
     *
     * x86: V8 comments say that keeping addresses in the [64MiB, 1GiB) range
     * tries to avoid system default DLL mapping space. In the end, we get 13
     * bits of randomness in our selection.
     * x64: [2GiB, 4TiB), with 25 bits of randomness.
     */
# ifdef HAVE_64BIT_BUILD
    static const uintptr_t base = 0x0000000080000000;
    static const uintptr_t mask = 0x000003ffffff0000;
# elif defined(_M_IX86) || defined(__i386__)
    static const uintptr_t base = 0x04000000;
    static const uintptr_t mask = 0x3fff0000;
# else
#  error "Unsupported architecture"
# endif

    uint64_t rand = js::GenerateRandomSeed();
    return (void*) (base | (rand & mask));
}

# ifdef HAVE_64BIT_BUILD
static js::JitExceptionHandler sJitExceptionHandler;

JS_FRIEND_API(void)
js::SetJitExceptionHandler(JitExceptionHandler handler)
{
    MOZ_ASSERT(!sJitExceptionHandler);
    sJitExceptionHandler = handler;
}

// From documentation for UNWIND_INFO on
// http://msdn.microsoft.com/en-us/library/ddssxxy8.aspx
struct UnwindInfo
{
    uint8_t version : 3;
    uint8_t flags : 5;
    uint8_t sizeOfPrologue;
    uint8_t countOfUnwindCodes;
    uint8_t frameRegister : 4;
    uint8_t frameOffset : 4;
    ULONG exceptionHandler;
};

static const unsigned ThunkLength = 12;

struct ExceptionHandlerRecord
{
    RUNTIME_FUNCTION runtimeFunction;
    UnwindInfo unwindInfo;
    uint8_t thunk[ThunkLength];
};

// This function must match the function pointer type PEXCEPTION_HANDLER
// mentioned in:
//   http://msdn.microsoft.com/en-us/library/ssa62fwe.aspx.
// This type is rather elusive in documentation; Wine is the best I've found:
//   http://source.winehq.org/source/include/winnt.h
static DWORD
ExceptionHandler(PEXCEPTION_RECORD exceptionRecord, _EXCEPTION_REGISTRATION_RECORD*,
                 PCONTEXT context, _EXCEPTION_REGISTRATION_RECORD**)
{
    return sJitExceptionHandler(exceptionRecord, context);
}

// For an explanation of the problem being solved here, see
// SetJitExceptionFilter in jsfriendapi.h.
static bool
RegisterExecutableMemory(void* p, size_t bytes, size_t pageSize)
{
    if (!VirtualAlloc(p, pageSize, MEM_COMMIT, PAGE_READWRITE))
        MOZ_CRASH();

    ExceptionHandlerRecord* r = reinterpret_cast<ExceptionHandlerRecord*>(p);

    // All these fields are specified to be offsets from the base of the
    // executable code (which is 'p'), even if they have 'Address' in their
    // names. In particular, exceptionHandler is a ULONG offset which is a
    // 32-bit integer. Since 'p' can be farther than INT32_MAX away from
    // sJitExceptionHandler, we must generate a little thunk inside the
    // record. The record is put on its own page so that we can take away write
    // access to protect against accidental clobbering.

    r->runtimeFunction.BeginAddress = pageSize;
    r->runtimeFunction.EndAddress = (DWORD)bytes;
    r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindInfo);

    r->unwindInfo.version = 1;
    r->unwindInfo.flags = UNW_FLAG_EHANDLER;
    r->unwindInfo.sizeOfPrologue = 0;
    r->unwindInfo.countOfUnwindCodes = 0;
    r->unwindInfo.frameRegister = 0;
    r->unwindInfo.frameOffset = 0;
    r->unwindInfo.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);

    // mov imm64, rax
    r->thunk[0]  = 0x48;
    r->thunk[1]  = 0xb8;
    void* handler = JS_FUNC_TO_DATA_PTR(void*, ExceptionHandler);
    memcpy(&r->thunk[2], &handler, 8);

    // jmp rax
    r->thunk[10] = 0xff;
    r->thunk[11] = 0xe0;

    DWORD oldProtect;
    if (!VirtualProtect(p, pageSize, PAGE_EXECUTE_READ, &oldProtect))
        MOZ_CRASH();

    bool success = RtlAddFunctionTable(&r->runtimeFunction, 1, reinterpret_cast<DWORD64>(p));

    return success;
}

static void
UnregisterExecutableMemory(void* p, size_t bytes, size_t pageSize)
{
    ExceptionHandlerRecord* r = reinterpret_cast<ExceptionHandlerRecord*>(p);

    RtlDeleteFunctionTable(&r->runtimeFunction);
}
# endif

static void*
ReserveProcessExecutableMemory(size_t bytes)
{
# ifdef HAVE_64BIT_BUILD
    size_t pageSize = gc::SystemPageSize();
    if (sJitExceptionHandler)
        bytes += pageSize;
# endif

    void* p = nullptr;
    for (size_t i = 0; i < 10; i++) {
        void* randomAddr = ComputeRandomAllocationAddress();
        p = VirtualAlloc(randomAddr, bytes, MEM_RESERVE, PAGE_NOACCESS);
        if (p)
            break;
    }

    if (!p) {
        // Try again without randomization.
        p = VirtualAlloc(nullptr, bytes, MEM_RESERVE, PAGE_NOACCESS);
        if (!p)
            return nullptr;
    }

# ifdef HAVE_64BIT_BUILD
    if (sJitExceptionHandler) {
        if (!RegisterExecutableMemory(p, bytes, pageSize)) {
            VirtualFree(p, 0, MEM_RELEASE);
            return nullptr;
        }

        p = (uint8_t*)p + pageSize;
    }
# endif

    return p;
}

static void
DeallocateProcessExecutableMemory(void* addr, size_t bytes)
{
# ifdef HAVE_64BIT_BUILD
    if (sJitExceptionHandler) {
        size_t pageSize = gc::SystemPageSize();
        addr = (uint8_t*)addr - pageSize;
        UnregisterExecutableMemory(addr, bytes, pageSize);
    }
# endif

    VirtualFree(addr, 0, MEM_RELEASE);
}

static DWORD
ProtectionSettingToFlags(ProtectionSetting protection)
{
    return PAGE_EXECUTE_READWRITE;
}

static void
CommitPages(void* addr, size_t bytes, ProtectionSetting protection)
{
    if (!VirtualAlloc(addr, bytes, MEM_COMMIT, ProtectionSettingToFlags(protection)))
        MOZ_CRASH("CommitPages failed");
}

static void
DecommitPages(void* addr, size_t bytes)
{
    if (!VirtualFree(addr, bytes, MEM_DECOMMIT))
        MOZ_CRASH("DecommitPages failed");
}
#else // !XP_WIN
static void*
ComputeRandomAllocationAddress()
{
    uint64_t rand = js::GenerateRandomSeed();

# ifdef HAVE_64BIT_BUILD
    // x64 CPUs have a 48-bit address space and on some platforms the OS will
    // give us access to 47 bits, so to be safe we right shift by 18 to leave
    // 46 bits.
    rand >>= 18;
# else
    // On 32-bit, right shift by 34 to leave 30 bits, range [0, 1GiB). Then add
    // 512MiB to get range [512MiB, 1.5GiB), or [0x20000000, 0x60000000). This
    // is based on V8 comments in platform-posix.cc saying this range is
    // relatively unpopulated across a variety of kernels.
    rand >>= 34;
    rand += 512 * 1024 * 1024;
# endif

    // Ensure page alignment.
    uintptr_t mask = ~uintptr_t(gc::SystemPageSize() - 1);
    return (void*) uintptr_t(rand & mask);
}

static void*
ReserveProcessExecutableMemory(size_t bytes)
{
    // Note that randomAddr is just a hint: if the address is not available
    // mmap will pick a different address.
    void* randomAddr = ComputeRandomAllocationAddress();
#ifdef SOLARIS
    // Do not reserve swap space for the entire allocation. Regardless of what mmap(2) says,
    // Solaris reserves swap space even though this allocation is not marked as writable.
    void* p = MozTaggedAnonymousMmap(randomAddr, bytes, PROT_NONE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE,
                                     -1, 0, "js-executable-memory");
#else
    void* p = MozTaggedAnonymousMmap(randomAddr, bytes, PROT_NONE, MAP_PRIVATE | MAP_ANON,
                                     -1, 0, "js-executable-memory");
#endif
    if (p == MAP_FAILED)
        return nullptr;
    return p;
}

static void
DeallocateProcessExecutableMemory(void* addr, size_t bytes)
{
    mozilla::DebugOnly<int> result = munmap(addr, bytes);
    MOZ_ASSERT(!result || errno == ENOMEM);
}

static unsigned
ProtectionSettingToFlags(ProtectionSetting protection)
{
    return PROT_READ | PROT_WRITE | PROT_EXEC;
}

static void
CommitPages(void* addr, size_t bytes, ProtectionSetting protection)
{
    void* p = MozTaggedAnonymousMmap(addr, bytes, ProtectionSettingToFlags(protection),
                                     MAP_FIXED | MAP_PRIVATE | MAP_ANON,
                                     -1, 0, "js-executable-memory");
    MOZ_RELEASE_ASSERT(addr == p);
}

static void
DecommitPages(void* addr, size_t bytes)
{
    // Use mmap with MAP_FIXED and PROT_NONE. Inspired by jemalloc's
    // pages_decommit.
    void* p = MozTaggedAnonymousMmap(addr, bytes, PROT_NONE,
                                     MAP_FIXED | MAP_PRIVATE | MAP_ANON,
                                     -1, 0, "js-executable-memory");
    MOZ_RELEASE_ASSERT(addr == p);
}
#endif

template <size_t NumBits>
class PageBitSet
{
    using WordType = uint32_t;
    static const size_t BitsPerWord = sizeof(WordType) * 8;

    static_assert((NumBits % BitsPerWord) == 0,
                  "NumBits must be a multiple of BitsPerWord");
    static const size_t NumWords = NumBits / BitsPerWord;

    mozilla::Array<WordType, NumWords> words_;

    uint32_t indexToWord(uint32_t index) const {
        MOZ_ASSERT(index < NumBits);
        return index / BitsPerWord;
    }
    WordType indexToBit(uint32_t index) const {
        MOZ_ASSERT(index < NumBits);
        return WordType(1) << (index % BitsPerWord);
    }

  public:
    void init() {
        mozilla::PodArrayZero(words_);
    }
    bool contains(size_t index) const {
        uint32_t word = indexToWord(index);
        return words_[word] & indexToBit(index);
    }
    void insert(size_t index) {
        MOZ_ASSERT(!contains(index));
        uint32_t word = indexToWord(index);
        words_[word] |= indexToBit(index);
    }
    void remove(size_t index) {
        MOZ_ASSERT(contains(index));
        uint32_t word = indexToWord(index);
        words_[word] &= ~indexToBit(index);
    }

#ifdef DEBUG
    bool empty() const {
        for (size_t i = 0; i < NumWords; i++) {
            if (words_[i] != 0)
                return false;
        }
        return true;
    }
#endif
};

// Limit on the number of bytes of executable memory to prevent JIT spraying
// attacks.
#if JS_BITS_PER_WORD == 32
static const size_t MaxCodeBytesPerProcess = 128 * 1024 * 1024;
#else
static const size_t MaxCodeBytesPerProcess = 640 * 1024 * 1024;
#endif

// Per-process executable memory allocator. It reserves a block of memory of
// MaxCodeBytesPerProcess bytes, then allocates/deallocates pages from that.
//
// This has a number of benefits compared to raw mmap/VirtualAlloc:
//
// * More resillient against certain attacks.
//
// * Behaves more consistently across platforms: it avoids the 64K granularity
//   issues on Windows, for instance.
//
// * On x64, near jumps can be used for jumps to other JIT pages.
//
// * On Win64, we have to register the exception handler only once (at process
//   startup). This saves some memory and avoids RtlAddFunctionTable profiler
//   deadlocks.
class ProcessExecutableMemory
{
    static_assert((MaxCodeBytesPerProcess % ExecutableCodePageSize) == 0,
                  "MaxCodeBytesPerProcess must be a multiple of ExecutableCodePageSize");
    static const size_t MaxCodePages = MaxCodeBytesPerProcess / ExecutableCodePageSize;

    // Start of the MaxCodeBytesPerProcess memory block or nullptr if
    // uninitialized. Note that this is NOT guaranteed to be aligned to
    // ExecutableCodePageSize.
    uint8_t* base_;

    // The fields below should only be accessed while we hold the lock.
    PRLock* lock_;

    // pagesAllocated_ is an Atomic so that bytesAllocated does not have to
    // take the lock.
    mozilla::Atomic<size_t, mozilla::ReleaseAcquire> pagesAllocated_;

    // Page where we should try to allocate next.
    size_t cursor_;

    mozilla::Maybe<mozilla::non_crypto::XorShift128PlusRNG> rng_;
    PageBitSet<MaxCodePages> pages_;

  public:
    ProcessExecutableMemory()
      : base_(nullptr),
        lock_(nullptr),
        pagesAllocated_(0),
        cursor_(0),
        rng_(),
        pages_()
    {}

    MOZ_MUST_USE bool init() {
        pages_.init();

        MOZ_RELEASE_ASSERT(!initialized());
        MOZ_RELEASE_ASSERT(gc::SystemPageSize() <= ExecutableCodePageSize);

        lock_ = PR_NewLock();
        if (!lock_)
            return false;

        void* p = ReserveProcessExecutableMemory(MaxCodeBytesPerProcess);
        if (!p)
            return false;

        base_ = static_cast<uint8_t*>(p);

        mozilla::Array<uint64_t, 2> seed;
        GenerateXorShift128PlusSeed(seed);
        rng_.emplace(seed[0], seed[1]);
        return true;
    }

    bool initialized() const {
        return base_ != nullptr;
    }

    size_t bytesAllocated() const {
        MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
        return pagesAllocated_ * ExecutableCodePageSize;
    }

    void release() {
        MOZ_ASSERT(initialized());
        MOZ_ASSERT(pages_.empty());
        MOZ_ASSERT(pagesAllocated_ == 0);
        DeallocateProcessExecutableMemory(base_, MaxCodeBytesPerProcess);
        base_ = nullptr;
        rng_.reset();
        MOZ_ASSERT(lock_);
        PR_DestroyLock(lock_);
        lock_ = nullptr;
        MOZ_ASSERT(!initialized());
    }

    void assertValidAddress(void* p, size_t bytes) const {
        MOZ_RELEASE_ASSERT(p >= base_ &&
                           uintptr_t(p) + bytes <= uintptr_t(base_) + MaxCodeBytesPerProcess);
    }

    void* allocate(size_t bytes, ProtectionSetting protection);
    void deallocate(void* addr, size_t bytes);
};

void*
ProcessExecutableMemory::allocate(size_t bytes, ProtectionSetting protection)
{
    MOZ_ASSERT(initialized());
    MOZ_ASSERT(bytes > 0);
    MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);

    size_t numPages = bytes / ExecutableCodePageSize;

    // Take the lock and try to allocate.
    void* p = nullptr;
    {
        PR_Lock(lock_);
        MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);

        // Check if we have enough pages available.
        if (pagesAllocated_ + numPages >= MaxCodePages) {
            PR_Unlock(lock_);
            return nullptr;
        }

        MOZ_ASSERT(bytes <= MaxCodeBytesPerProcess);

        // Maybe skip a page to make allocations less predictable.
        size_t page = cursor_ + (rng_.ref().next() % 2);

        for (size_t i = 0; i < MaxCodePages; i++) {
            // Make sure page + numPages - 1 is a valid index.
            if (page + numPages > MaxCodePages)
                page = 0;

            bool available = true;
            for (size_t j = 0; j < numPages; j++) {
                if (pages_.contains(page + j)) {
                    available = false;
                    break;
                }
            }
            if (!available) {
                page++;
                continue;
            }

            // Mark the pages as unavailable.
            for (size_t j = 0; j < numPages; j++)
                pages_.insert(page + j);

            pagesAllocated_ += numPages;
            MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);

            // If we allocated a small number of pages, move cursor_ to the
            // next page. We don't do this for larger allocations to avoid
            // skipping a large number of small holes.
            if (numPages <= 2)
                cursor_ = page + numPages;

            p = base_ + page * ExecutableCodePageSize;
            break;
        }
        PR_Unlock(lock_);
        if (!p)
            return nullptr;
    }

    // Commit the pages after releasing the lock.
    CommitPages(p, bytes, protection);
    return p;
}

void
ProcessExecutableMemory::deallocate(void* addr, size_t bytes)
{
    MOZ_ASSERT(initialized());
    MOZ_ASSERT(addr);
    MOZ_ASSERT((uintptr_t(addr) % gc::SystemPageSize()) == 0);
    MOZ_ASSERT(bytes > 0);
    MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);

    assertValidAddress(addr, bytes);

    size_t firstPage = (static_cast<uint8_t*>(addr) - base_) / ExecutableCodePageSize;
    size_t numPages = bytes / ExecutableCodePageSize;

    // Decommit before taking the lock.
    DecommitPages(addr, bytes);

    PR_Lock(lock_);
    MOZ_ASSERT(numPages <= pagesAllocated_);
    pagesAllocated_ -= numPages;

    for (size_t i = 0; i < numPages; i++)
        pages_.remove(firstPage + i);

    // Move the cursor back so we can reuse pages instead of fragmenting the
    // whole region.
    if (firstPage < cursor_)
        cursor_ = firstPage;

    PR_Unlock(lock_);
}

static ProcessExecutableMemory execMemory;

void*
js::jit::AllocateExecutableMemory(size_t bytes, ProtectionSetting protection)
{
    return execMemory.allocate(bytes, protection);
}

void
js::jit::DeallocateExecutableMemory(void* addr, size_t bytes)
{
    execMemory.deallocate(addr, bytes);
}

bool
js::jit::InitProcessExecutableMemory()
{
    return execMemory.init();
}

void
js::jit::ReleaseProcessExecutableMemory()
{
    execMemory.release();
}

bool
js::jit::CanLikelyAllocateMoreExecutableMemory()
{
    // Use a 16 MB buffer.
    static const size_t BufferSize = 16 * 1024 * 1024;

    MOZ_ASSERT(execMemory.bytesAllocated() <= MaxCodeBytesPerProcess);

    return execMemory.bytesAllocated() + BufferSize <= MaxCodeBytesPerProcess;
}