diff --git a/Makefile b/Makefile index 9c35cb8..afb1ece 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,6 @@ BIN_DIR := bin ZIALLOC_MAIN := allocators/zialloc/alloc.cpp ifeq ($(ALLOCATOR),$(ZIALLOC_MAIN)) ALLOC_CPP_SRCS := allocators/zialloc/alloc.cpp \ - allocators/zialloc/free.cpp \ allocators/zialloc/os.cpp \ allocators/zialloc/segments.cpp ALLOC_OBJ := $(patsubst %.cpp,$(BUILD_DIR)/%.o,$(ALLOC_CPP_SRCS)) diff --git a/allocators/zialloc/free.cpp b/allocators/zialloc/free.cpp deleted file mode 100644 index 9045269..0000000 --- a/allocators/zialloc/free.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include "zialloc_memory.hpp" - -namespace zialloc::memory { - -void free_dispatch(void *ptr) { - (void)free_dispatch_with_size(ptr, nullptr); -} - -} // namespace zialloc::memory diff --git a/allocators/zialloc/mem.h b/allocators/zialloc/mem.h index dc0cb62..a98c51f 100644 --- a/allocators/zialloc/mem.h +++ b/allocators/zialloc/mem.h @@ -86,12 +86,6 @@ constexpr size_t page_kind_size(int kind) { SEGMENT_SIZE; } -typedef enum segment_kind_e { - SEGMENT_NORM, // most allocations - SEGMENT_HUGE, // if page kind is "huge" - SEGMENT_GUARD -} segment_kind_t; - typedef enum page_status_e { FULL, ACTIVE, EMPTY } page_status_t; // all power aligned (i think), we can fill unused space w/ guard chunks. @@ -107,46 +101,6 @@ typedef enum chunk_max_e { CHUNK_XL // whatever it wants to be } chunk_max_t; -// Memory can reside in arena's, direct OS allocated, or statically allocated. -// The memid keeps track of this. -typedef enum memkind_e { - MEM_NONE, // not allocated - MEM_EXTERNAL, // not owned by mimalloc but provided externally (via - // `mi_manage_os_memory` for example) - MEM_STATIC, // allocated in a static area and should not be freed (for arena - // meta data for example) - MEM_OS, // allocated from the OS - MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical - // memory) - MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`) - MEM_ARENA // allocated from an arena (the usual case) -} memkind_t; - -constexpr static inline bool memkind_is_os(memkind_t memkind) { - return (memkind >= MEM_OS && memkind <= MEM_OS_REMAP); -} - -typedef struct memid_os_info { - void *base; // actual base address of the block (used for offset aligned - // allocations) - size_t size; // full allocation size -} memid_os_info_t; -typedef struct memid_arena_info { - size_t block_index; // index in the arena - uint8_t id; // arena id (>= 1) - bool is_exclusive; // this arena can only be used for specific arena - // allocations -} memid_arena_info_t; -typedef struct memid_s { - union { - memid_os_info_t os; // MI_MEM_OS - memid_arena_info_t arena; // MI_MEM_ARENA - } mem; - bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory - bool initially_committed; - bool initially_zero; -} memid_t; - static inline pid_t current_tid() { return syscall(SYS_gettid); } static inline uint64_t generate_canary() { diff --git a/allocators/zialloc/segments.cpp b/allocators/zialloc/segments.cpp index 5b07ddc..1681341 100644 --- a/allocators/zialloc/segments.cpp +++ b/allocators/zialloc/segments.cpp @@ -415,8 +415,6 @@ class Page { page_status_t get_status() const { return status; } size_t get_chunk_usable() const { return chunk_usable; } pid_t get_owner_tid() const { return owner_tid; } - uintptr_t get_base_addr() const { return reinterpret_cast(base); } - size_t get_span_size() const { return page_span; } size_t get_segment_index() const { return owner_segment_idx; } Segment *get_owner_segment() const { return owner_segment; } }; @@ -512,8 +510,7 @@ class Segment { void *out = nullptr; page_status_t before = EMPTY; page_status_t after = EMPTY; - if (mt) { - std::lock_guard lk(page_lock_for(&page)); + auto alloc_from_page = [&]() -> void * { const size_t target_req = (fixed_chunk_set.load(std::memory_order_relaxed)) ? fixed_chunk_usable.load(std::memory_order_relaxed) @@ -521,36 +518,24 @@ class Segment { if (!page.is_initialized()) { void *page_base = static_cast(static_cast(base) + idx * page_size); if (!page.init(page_base, size_class, target_req)) - continue; + return nullptr; if (!fixed_chunk_set.load(std::memory_order_relaxed)) { fixed_chunk_usable.store(page.get_chunk_usable(), std::memory_order_relaxed); fixed_chunk_set.store(true, std::memory_order_relaxed); } } if (!page.can_hold(req)) { - continue; + return nullptr; } - out = page.allocate(req, &before, &after); // dispatch to page lvl alloc - } else { - const size_t target_req = - (fixed_chunk_set.load(std::memory_order_relaxed)) - ? fixed_chunk_usable.load(std::memory_order_relaxed) - : req; - if (!page.is_initialized()) { - void *page_base = static_cast(static_cast(base) + idx * page_size); - if (!page.init(page_base, size_class, target_req)) - continue; - if (!fixed_chunk_set.load(std::memory_order_relaxed)) { - fixed_chunk_usable.store(page.get_chunk_usable(), std::memory_order_relaxed); - fixed_chunk_set.store(true, std::memory_order_relaxed); - } - } - if (!page.can_hold(req)) { - continue; - } + return page.allocate(req, &before, &after); + }; - out = page.allocate(req, &before, &after); + if (mt) { + std::lock_guard lk(page_lock_for(&page)); + out = alloc_from_page(); + } else { + out = alloc_from_page(); } if (!out) @@ -592,30 +577,19 @@ class Segment { } }; -typedef struct tc_page_s { - Page *loc; - page_kind_t kind; - size_t size; - void *freelist; -} tc_page_t; - class ThreadCache { private: static std::atomic live_threads; pid_t tid; bool is_active; - std::vector pages; Page *cached_pages[3]; - uintptr_t cached_page_bases[3]; - uintptr_t cached_page_ends[3]; size_t preferred_seg_idx[3]; bool preferred_seg_valid[3]; public: ThreadCache() - : tid(current_tid()), is_active(true), pages(), - cached_pages{nullptr, nullptr, nullptr}, cached_page_bases{0, 0, 0}, - cached_page_ends{0, 0, 0}, preferred_seg_idx{0, 0, 0}, + : tid(current_tid()), is_active(true), cached_pages{nullptr, nullptr, nullptr}, + preferred_seg_idx{0, 0, 0}, preferred_seg_valid{false, false, false} { live_threads.fetch_add(1, std::memory_order_relaxed); g_live_threads.fetch_add(1, std::memory_order_relaxed); @@ -644,13 +618,11 @@ class ThreadCache { return cached_pages[class_index_for_kind(kind)]; } - void cache_page(page_kind_t kind, Page *page, uintptr_t page_base, size_t page_size) { + void cache_page(page_kind_t kind, Page *page) { if (kind > PAGE_LG) return; const size_t idx = class_index_for_kind(kind); cached_pages[idx] = page; - cached_page_bases[idx] = page_base; - cached_page_ends[idx] = page_base + page_size; } void clear_cached_page(page_kind_t kind, Page *page) { @@ -659,8 +631,6 @@ class ThreadCache { const size_t idx = class_index_for_kind(kind); if (cached_pages[idx] == page) { cached_pages[idx] = nullptr; - cached_page_bases[idx] = 0; - cached_page_ends[idx] = 0; } } @@ -695,15 +665,11 @@ struct ClassShard { class HeapState { private: - memid_t memid; void *base; size_t reserved_size; uint32_t num_segments; std::vector> layout; - std::vector seg_kind; std::vector seg_bases; - std::vector seg_page_kind; - memkind_t mem_kind; uint64_t canary; size_t reserved_cursor; std::mutex heap_mu; @@ -727,7 +693,7 @@ class HeapState { shard.non_full_segments.push_back(seg_idx); } - bool add_segment_nolock(void *segment_base, segment_kind_t kind, page_kind_t page_kind) { + bool add_segment_nolock(void *segment_base, page_kind_t page_kind) { if (!segment_base) return false; @@ -737,9 +703,7 @@ class HeapState { return false; layout.push_back(std::move(seg)); - seg_kind.push_back(kind); seg_bases.push_back(segment_base); - seg_page_kind.push_back(page_kind); num_segments = static_cast(layout.size()); ClassShard &shard = shard_for(page_kind); @@ -755,7 +719,7 @@ class HeapState { return true; } - bool add_segment_from_reserved_nolock(segment_kind_t kind, page_kind_t page_kind) { + bool add_segment_from_reserved_nolock(page_kind_t page_kind) { if (!base || reserved_size == 0) return false; if (reserved_cursor + SEGMENT_SIZE > reserved_size) @@ -766,7 +730,7 @@ class HeapState { return false; reserved_cursor += SEGMENT_SIZE; - return add_segment_nolock(seg_base, kind, page_kind); + return add_segment_nolock(seg_base, page_kind); } void *alloc_xl(size_t size) { @@ -819,8 +783,8 @@ class HeapState { public: HeapState() - : memid(), base(nullptr), reserved_size(0), num_segments(0), layout(), - seg_kind(), seg_bases(), seg_page_kind(), mem_kind(MEM_NONE), canary(0), + : base(nullptr), reserved_size(0), num_segments(0), layout(), + seg_bases(), canary(0), reserved_cursor(0), heap_mu(), class_shards() {} static HeapState &instance() { @@ -837,13 +801,10 @@ class HeapState { reserved_size = size; reserved_cursor = 0; canary = generate_canary(); - mem_kind = MEM_OS; const size_t cap = size / SEGMENT_SIZE; layout.reserve(cap); - seg_kind.reserve(cap); seg_bases.reserve(cap); - seg_page_kind.reserve(cap); for (ClassShard &shard : class_shards) { std::lock_guard shard_lk(shard.mu); @@ -854,14 +815,9 @@ class HeapState { return true; } - bool add_segment(void *segment_base, segment_kind_t kind, page_kind_t page_kind) { + bool add_segment_from_reserved(page_kind_t page_kind) { std::lock_guard lk(heap_mu); - return add_segment_nolock(segment_base, kind, page_kind); - } - - bool add_segment_from_reserved(segment_kind_t kind, page_kind_t page_kind) { - std::lock_guard lk(heap_mu); - return add_segment_from_reserved_nolock(kind, page_kind); + return add_segment_from_reserved_nolock(page_kind); } void *allocate(size_t size) { @@ -930,7 +886,7 @@ class HeapState { if (tc->get_active()) { tc->set_preferred_segment(kind, seg_idx); if (page) - tc->cache_page(kind, page, page->get_base_addr(), page->get_span_size()); + tc->cache_page(kind, page); } if (page) g_last_alloc_usable = page->get_chunk_usable(); @@ -992,7 +948,7 @@ class HeapState { // grow from reserved heap (ideal) instead of mmaping more mem to expand. { std::lock_guard lk(heap_mu); - if (add_segment_from_reserved_nolock(SEGMENT_NORM, kind)) { + if (add_segment_from_reserved_nolock(kind)) { return try_segment(layout.size() - 1); } } @@ -1003,7 +959,7 @@ class HeapState { { std::lock_guard lk(heap_mu); - if (!add_segment_nolock(seg_mem, SEGMENT_NORM, kind)) { + if (!add_segment_nolock(seg_mem, kind)) { free_segment(seg_mem, SEGMENT_SIZE); return nullptr; } @@ -1050,7 +1006,7 @@ class HeapState { } if (tc->get_active()) { - tc->cache_page(kind, page, page->get_base_addr(), page->get_span_size()); + tc->cache_page(kind, page); if (page->get_status() == EMPTY) tc->clear_cached_page(kind, page); } @@ -1076,7 +1032,6 @@ class HeapState { return usable_xl(ptr); } - std::vector get_segment_kinds() { return seg_kind; } uint32_t get_num_segments() { return num_segments; } bool is_corrupted() { @@ -1111,9 +1066,7 @@ class HeapState { } layout.clear(); - seg_kind.clear(); seg_bases.clear(); - seg_page_kind.clear(); for (ClassShard &shard : class_shards) { std::lock_guard shard_lk(shard.mu); @@ -1125,29 +1078,20 @@ class HeapState { reserved_size = 0; num_segments = 0; canary = 0; - mem_kind = MEM_NONE; reserved_cursor = 0; } }; } // namespace -bool heap_register_segment(void *segment_base) { - return HeapState::instance().add_segment(segment_base, SEGMENT_NORM, PAGE_SM); -} - void heap_clear_metadata() { HeapState::instance().clear_metadata(); } bool heap_init_reserved(void *reserved_base, size_t size) { return HeapState::instance().init_reserved(reserved_base, size); } -bool heap_add_segment_from_reserved(segment_kind_t kind) { - return HeapState::instance().add_segment_from_reserved(kind, PAGE_SM); -} - bool heap_add_segment_for_class(page_kind_t kind) { - return HeapState::instance().add_segment_from_reserved(SEGMENT_NORM, kind); + return HeapState::instance().add_segment_from_reserved(kind); } void *heap_alloc(size_t size) { return HeapState::instance().allocate(size); } diff --git a/allocators/zialloc/zialloc_memory.hpp b/allocators/zialloc/zialloc_memory.hpp index 1d3e4cb..659ad6f 100644 --- a/allocators/zialloc/zialloc_memory.hpp +++ b/allocators/zialloc/zialloc_memory.hpp @@ -18,8 +18,6 @@ bool setup_guard(void* ptr, size_t size); void lock_page(void* ptr, size_t size); void unlock_page(void* ptr, size_t size); -// free-path entry -void free_dispatch(void* ptr); bool free_dispatch_with_size(void* ptr, size_t* usable_size); void set_zero_on_free_enabled(bool enabled); void set_uaf_check_enabled(bool enabled); @@ -31,11 +29,8 @@ size_t heap_usable_size(void* ptr); bool heap_validate(); bool heap_add_segment_for_class(page_kind_t kind); -// heap metadata registration -bool heap_register_segment(void* segment_base); void heap_clear_metadata(); bool heap_init_reserved(void* reserved_base, size_t size); -bool heap_add_segment_from_reserved(segment_kind_t kind); class Chunk; class Page; @@ -43,12 +38,3 @@ class Segment; class Heap; } // namespace zialloc::memory - -namespace zialloc { - -// allocator hooks -void allocator_free_internal(void* ptr); -size_t allocator_usable_size_internal(void* ptr); -bool allocator_validate_internal(); - -} // namespace zialloc