luagc: from-scratch incremental GC with slab allocator
Replace WoW's stop-the-world Lua 5.0 GC with a fully custom incremental mark+sweep. The module owns both allocation (slab allocator) and collection. Key components: - Slab allocator (luaalloc.zig): copied from weirdperformance, stripped of generational age bitmaps. Hooks memory_pool_allocate + lua_gc_step. - Incremental mark: tri-color with gray stack, write barriers on table writes. Correctly handles weak table strong/weak dimensions (condmarkobject). - From-scratch sweep: cursor-based rootgc, rootudata, chunked string buckets. Sweep order matches native (rootudata -> strings -> rootgc). - Weak table detection: scans metatable for __mode, sets KEYWEAK/VALUEWEAK. - Weak clearing: removekey semantics (nil value + TNONE dead keys). - Birth mark scoped to string sweep only (rootgc objects prepend to head). - Stack nilling: traverseThread nils slots above L->top matching native. - Hooks lua_setgcthreshold to prevent external threshold stomping. Native functions retained: lua_gc_free_object (type dispatch), luaCallUserDataGC (__gc finalizers), luaC_separateudata (dead udata separation).
This commit is contained in:
@@ -0,0 +1,326 @@
|
||||
//! Hardcoded addresses and struct offsets for WoW 1.12.1's Lua 5.0.
|
||||
//! All addresses are client-absolute (build 5875).
|
||||
//!
|
||||
//! All struct offsets below are verified via fresh Ghidra disassembly
|
||||
//! of the mark_table (0x6F7590), mark_closure (0x6F77B0), mark_thread
|
||||
//! (0x6F7860), and mark_proto (0x6F7710) functions. Each offset is
|
||||
//! annotated with the exact instruction that proves it.
|
||||
//!
|
||||
//! DO NOT "correct" offsets based on the Lua 5.0 source without also
|
||||
//! checking the disasm: WoW's layout diverges in a few places (TValue
|
||||
//! is 16 bytes including padding; globals table TValue is at thread+0x40
|
||||
//! rather than +0x38; etc.).
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// GC function addresses (all confirmed from cross-reference)
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const luaC_collectgarbage: usize = 0x6F7340;
|
||||
pub const lua_gc_full_collection: usize = 0x6F73E0;
|
||||
pub const lua_gc_remove_objects: usize = 0x6F7210;
|
||||
pub const lua_gc_sweep_all_lists: usize = 0x6F72F0;
|
||||
pub const lua_gc_shrink_memory: usize = 0x6F7370;
|
||||
pub const luaCallUserDataGC: usize = 0x6F7080;
|
||||
pub const lua_gc_free_object: usize = 0x6F7260;
|
||||
|
||||
/// Walks rootudata list. Moves dead udata with __gc metamethods to tmudata.
|
||||
/// __fastcall: ECX = global_State*.
|
||||
pub const luaC_separateudata: usize = 0x6F6FF0;
|
||||
|
||||
/// Re-marks tmudata list so __gc-pending userdata survive the sweep.
|
||||
/// __fastcall: ECX = global_State*.
|
||||
pub const marktmu: usize = 0x6F7470;
|
||||
|
||||
/// Clears weak-value table values in an atomic post-mark pass.
|
||||
pub const cleartablevalues: usize = 0x6F7A20;
|
||||
|
||||
/// Clears weak-key table keys in an atomic post-mark pass.
|
||||
pub const cleartablekeys: usize = 0x6F7970;
|
||||
|
||||
/// Resize string hash table. __fastcall(ECX=L, EDX=new_size).
|
||||
pub const lua_string_hash_resize: usize = 0x6F9C50;
|
||||
|
||||
/// lua_setgcthreshold: __fastcall(ECX=L, EDX=newthreshold).
|
||||
/// Writes (newthreshold << 10) to g->gcthreshold, then calls luaC_checkGC.
|
||||
/// WoW calls this on screen transitions with threshold=256 (=262144 bytes).
|
||||
pub const lua_setgcthreshold: usize = 0x6F4400;
|
||||
|
||||
/// Mark functions (for reference; zluagen reimplements these to support
|
||||
/// incremental chunking).
|
||||
pub const markroot: usize = 0x6F7AB0;
|
||||
pub const propagatemarks: usize = 0x6F7510;
|
||||
pub const mark_object_gray: usize = 0x6F74A0;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// Write barrier hook addresses
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const lua_table_set_value: usize = 0x6FA840;
|
||||
pub const lua_table_set_int_key: usize = 0x6FAD80;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// Birth mark patch addresses
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
/// `MOV byte [EDX+0x05], 0x00` inside luaC_link. Patch immediate to 0x01
|
||||
/// to make new allocations born marked-alive during chunked sweep.
|
||||
pub const birth_mark_luaC_link: usize = 0x6F7B37;
|
||||
|
||||
/// `MOV byte [EBX+0x05], 0x00` inside lua_create_string_object. Same idea.
|
||||
pub const birth_mark_string: usize = 0x6F9DC1;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// global_State struct offsets
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
/// stringtable struct starts at g+0x04 (after WoW's pool_ptrs at g+0x00).
|
||||
/// Layout: {hash: GCObject**, nuse: int, size: int} = 12 bytes.
|
||||
/// Verified: strt+12 = 0x04+12 = 0x10 = rootgc.
|
||||
pub const GS_strt_hash: u32 = 0x04; // GCObject** bucket array
|
||||
pub const GS_strt_nuse: u32 = 0x08; // int: number of strings
|
||||
pub const GS_strt_size: u32 = 0x0C; // int: number of buckets
|
||||
pub const GS_rootgc: u32 = 0x10; // main GC list head
|
||||
pub const GS_rootudata: u32 = 0x14; // userdata list head
|
||||
pub const GS_tmudata: u32 = 0x18; // finalizer-pending udata list
|
||||
pub const GS_gcthreshold: u32 = 0x24; // bytes-alive threshold for next GC
|
||||
pub const GS_totalbytes: u32 = 0x28; // currently allocated bytes
|
||||
pub const GS_registry: u32 = 0x38; // registry table pointer
|
||||
pub const GS_defaultmeta: u32 = 0x48; // default metatable pointer
|
||||
pub const GS_mainthread: u32 = 0x50; // main lua_State*
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// GCObject common header (all GC types start with this prefix)
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const OBJ_next: u32 = 0x00; // linked list next pointer
|
||||
pub const OBJ_tt: u32 = 0x04; // type tag (u8)
|
||||
|
||||
/// GC mark byte. Layout:
|
||||
/// bit 0: mark flag (1 = reachable / traversed)
|
||||
/// bit 1: KEYWEAK (tables with __mode 'k')
|
||||
/// bit 2: VALUEWEAK (tables with __mode 'v')
|
||||
/// bit 3: FINALIZED (userdata __gc already run)
|
||||
/// bit 4: propagation flag (used with bit 0 via TEST 0x11)
|
||||
/// bits 5-7: UNUSED — do NOT use for age tracking!
|
||||
///
|
||||
/// CRITICAL: upvalues in Lua closures use the WHOLE BYTE as a "processed"
|
||||
/// flag. Any non-zero value causes mark_closure to skip the upvalue entirely
|
||||
/// (see TEST AL, AL at 0x6F7828). Don't write any bits into an upvalue's
|
||||
/// marked byte except 0 or 1.
|
||||
pub const OBJ_marked: u32 = 0x05;
|
||||
|
||||
/// Bits 0 + 4 used by TEST 0x11 at mark_table child references.
|
||||
pub const MARK_MASK: u8 = 0x11;
|
||||
|
||||
/// AND mask to clear mark bit 0 while preserving other bits.
|
||||
pub const CLEAR_MARK: u8 = 0xFE;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// Type tags (from propagate dispatch at 0x6F7510 + standard Lua 5.0)
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const LUA_TNONE: u8 = 0xFF; // -1 as u8, marks dead keys in hash nodes
|
||||
pub const LUA_TNIL: u8 = 0;
|
||||
pub const LUA_TBOOLEAN: u8 = 1;
|
||||
pub const LUA_TLIGHTUSERDATA: u8 = 2;
|
||||
pub const LUA_TNUMBER: u8 = 3;
|
||||
pub const LUA_TSTRING: u8 = 4;
|
||||
pub const LUA_TTABLE: u8 = 5;
|
||||
pub const LUA_TFUNCTION: u8 = 6;
|
||||
pub const LUA_TUSERDATA: u8 = 7;
|
||||
pub const LUA_TTHREAD: u8 = 8;
|
||||
pub const LUA_TPROTO: u8 = 9;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// TValue (16 bytes in WoW Lua 5.0)
|
||||
//
|
||||
// Verified from mark_table array loop:
|
||||
// 0x6F766A: SHL ESI, 0x4 ; count * 16 = TValue stride
|
||||
// 0x6F7673: SUB ESI, 0x10 ; decrement by 16 per iteration
|
||||
// 0x6F7680: MOV EAX, [EAX+0x8] ; gcptr at +0x08 (earlier CMP at +0x00 for tt)
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const TVALUE_size: u32 = 16;
|
||||
pub const TVALUE_tt: u32 = 0x00;
|
||||
pub const TVALUE_gcptr: u32 = 0x08;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// Table struct
|
||||
//
|
||||
// Verified from mark_table 0x6F7590:
|
||||
// 0x6F759D: MOV EDX, [EDI+0x08] ; metatable
|
||||
// 0x6F7661: MOV EAX, [EDI+0x1C] ; sizearray (used as int count)
|
||||
// 0x6F7670: MOV EAX, [EDI+0x0C] ; array pointer (dereferenced)
|
||||
// 0x6F7697: MOV CL, [EDI+0x07] ; lsizenode (u8)
|
||||
// 0x6F769A: MOV EAX, 1
|
||||
// 0x6F769F: SHL EAX, CL ; sizenode = 1 << lsizenode
|
||||
// 0x6F76B0: MOV ESI, [EDI+0x10] ; node pointer
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const TABLE_flags: u32 = 0x06; // TM cache / weak flags (also in marked byte)
|
||||
pub const TABLE_lsizenode: u32 = 0x07; // u8: log2 of sizenode
|
||||
pub const TABLE_metatable: u32 = 0x08; // Table* metatable
|
||||
pub const TABLE_array: u32 = 0x0C; // TValue* array part
|
||||
pub const TABLE_node: u32 = 0x10; // Node* hash part
|
||||
pub const TABLE_sizearray: u32 = 0x1C; // int size of array part
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// Node (hash bucket, 40 bytes)
|
||||
//
|
||||
// Verified from mark_table node loop (0x6F76B0-0x6F76FE):
|
||||
// 0x6F76A5: LEA EBX, [EAX+EAX*4] ; count*5
|
||||
// 0x6F76A8: SHL EBX, 0x3 ; count*40 = Node size * count
|
||||
// 0x6F76B6: MOV EAX, [ESI+EBX+0x10] ; value.tt at node+0x10
|
||||
// 0x6F76BC: TEST EAX, EAX ; skip dead nodes (nil value)
|
||||
// 0x6F76C0: CMP [ESI], 0x4 ; key.tt at node+0x00
|
||||
// 0x6F76C5: MOV EDX, [ESI+0x08] ; key.gcptr at node+0x08
|
||||
// 0x6F76E3: MOV EDX, [ESI+0x18] ; value.gcptr at node+0x18
|
||||
//
|
||||
// Iteration goes count-1 down to 0 (full 0-indexed range), NOT starting at 1.
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const NODE_size: u32 = 40;
|
||||
pub const NODE_key_tt: u32 = 0x00;
|
||||
pub const NODE_key_gcptr: u32 = 0x08;
|
||||
pub const NODE_value_tt: u32 = 0x10;
|
||||
pub const NODE_value_gcptr: u32 = 0x18;
|
||||
pub const NODE_next: u32 = 0x20;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// Closure (from mark_closure 0x6F77B0)
|
||||
//
|
||||
// Verified:
|
||||
// 0x6F77B9: MOV AL, [EDI+0x06] ; isC flag
|
||||
// 0x6F77C0: MOV AL, [EDI+0x07] ; nupvalues (u8)
|
||||
//
|
||||
// C closure branch (LEA ESI, [EDI+0x18] then [ESI-8]/[ESI] with stride 16):
|
||||
// elements start at cl+0x10, each 16 bytes (TValue)
|
||||
// tt at element+0x00, gcptr at element+0x08
|
||||
//
|
||||
// Lua closure branch (0x6F77F6-0x6F7854):
|
||||
// 0x6F77F6: MOV EDX, [EDI+0x18] ; mark this pointer
|
||||
// 0x6F7804: MOV EDX, [EDI+0x0C] ; mark this pointer
|
||||
// 0x6F7820: LEA EBX, [EDI+0x20] ; upval pointer array start
|
||||
// 0x6F7823: MOV ESI, [EBX] ; ESI = upvals[i] (the UpVal* itself)
|
||||
// upval ptr array stride = 4 bytes per entry
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const CLOSURE_isC: u32 = 0x06;
|
||||
pub const CLOSURE_nupvalues: u32 = 0x07;
|
||||
|
||||
/// First pointer marked for Lua closures (see disasm above).
|
||||
pub const CLOSURE_lua_mark1: u32 = 0x18;
|
||||
/// Second pointer marked for Lua closures.
|
||||
pub const CLOSURE_lua_mark2: u32 = 0x0C;
|
||||
|
||||
/// C closure: upvalue array (inline TValues, each 16 bytes) starts here.
|
||||
pub const CLOSURE_c_upvals: u32 = 0x10;
|
||||
|
||||
/// Lua closure: upvalue POINTER array (each 4 bytes = UpVal*) starts here.
|
||||
pub const CLOSURE_lua_upval_ptrs: u32 = 0x20;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// UpVal (verified from mark_closure's Lua upvalue loop 0x6F7820-0x6F7840)
|
||||
//
|
||||
// 0x6F7823: MOV ESI, [EBX] ; ESI = UpVal* (upvals[i])
|
||||
// 0x6F7825: MOV AL, [ESI+0x05] ; marked byte
|
||||
// 0x6F7828: TEST AL, AL ; any non-zero = skip (whole byte)
|
||||
// 0x6F782C: CMP [ESI+0x10], 0x4 ; value.tt at upv+0x10
|
||||
// 0x6F7832: MOV EDX, [ESI+0x18] ; value.gcptr at upv+0x18
|
||||
// 0x6F7840: MOV byte [ESI+0x05], 0x01 ; mark as processed
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const UPVAL_marked: u32 = 0x05;
|
||||
pub const UPVAL_value_tt: u32 = 0x10;
|
||||
pub const UPVAL_value_gcptr: u32 = 0x18;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// Thread / lua_State (from mark_thread 0x6F7860)
|
||||
//
|
||||
// Verified:
|
||||
// 0x6F7869: CMP [EDI+0x40], 0x4 ; gt.tt at thread+0x40 (TValue)
|
||||
// 0x6F7872: MOV EDX, [EDI+0x48] ; gt.gcptr at thread+0x48
|
||||
// 0x6F7880: MOV EAX, [EDI+0x28] ; base_ci pointer (CallInfo*)
|
||||
// 0x6F7883: MOV ESI, [EDI+0x14] ; ci pointer (CallInfo*)
|
||||
// 0x6F7888: MOV ECX, [EDI+0x08] ; top (StkId)
|
||||
// 0x6F78A6: MOV ESI, [EDI+0x1C] ; stack base (StkId)
|
||||
//
|
||||
// Stack iteration (0x6F78B0-0x6F78CE):
|
||||
// stride 16 bytes per slot (TValue)
|
||||
// tt at slot+0x00, gcptr at slot+0x08
|
||||
// loop: from [thread+0x1C] to [thread+0x08]
|
||||
//
|
||||
// CallInfo iteration stride: 24 bytes (0x18) — used to find max stack top.
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const THREAD_top: u32 = 0x08; // StkId top
|
||||
pub const THREAD_ci: u32 = 0x14; // CallInfo* current
|
||||
pub const THREAD_stack: u32 = 0x1C; // StkId base (first slot)
|
||||
pub const THREAD_base_ci: u32 = 0x28; // CallInfo* base
|
||||
pub const THREAD_gt_tt: u32 = 0x40; // gt.tt (TValue holding globals)
|
||||
pub const THREAD_gt_gcptr: u32 = 0x48; // gt.gcptr
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// Proto (from mark_proto 0x6F7710)
|
||||
//
|
||||
// Verified:
|
||||
// 0x6F7714: MOV EAX, [ESI+0x20] ; source (TString*)
|
||||
// 0x6F7727: MOV EAX, [ESI+0x08] ; k (constants array TValue*)
|
||||
// 0x6F772A: tt read from EAX + offset (start of TValue)
|
||||
// 0x6F772F: CMP EBX, 0x4 ; check tt == LUA_TSTRING (stringmark only)
|
||||
// 0x6F7734: MOV EAX, [EAX+0x08] ; gcptr at TValue+0x08
|
||||
// 0x6F773F: ADD EDX, 0x10 ; stride 16 (TValue)
|
||||
// 0x6F773B: MOV EAX, [ESI+0x28] ; sizek at proto+0x28
|
||||
// 0x6F7746: MOV EDX, [ESI+0x24] ; sizeupvalues at proto+0x24
|
||||
// 0x6F7750: MOV EDX, [ESI+0x1C] ; upvalues array (TString*) at proto+0x1C
|
||||
// 0x6F7762: MOV EAX, [ESI+0x34] ; sizep at proto+0x34
|
||||
// 0x6F7770: MOV EAX, [ESI+0x10] ; p (nested Proto** array) at proto+0x10
|
||||
// 0x6F7789: MOV EAX, [ESI+0x38] ; sizelocvars at proto+0x38
|
||||
// 0x6F7794: MOV EAX, [ESI+0x18] ; locvars array at proto+0x18
|
||||
// 0x6F77A2: ADD EDX, 0x0C ; LocVar stride 12
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const PROTO_k: u32 = 0x08; // TValue* constants array
|
||||
pub const PROTO_p: u32 = 0x10; // Proto** nested protos array
|
||||
pub const PROTO_locvars: u32 = 0x18; // LocVar* (12 bytes each)
|
||||
pub const PROTO_upvalues: u32 = 0x1C; // TString** upvalue names
|
||||
pub const PROTO_source: u32 = 0x20; // TString* source
|
||||
pub const PROTO_sizeupvalues: u32 = 0x24; // int
|
||||
pub const PROTO_sizek: u32 = 0x28; // int
|
||||
pub const PROTO_sizep: u32 = 0x34; // int
|
||||
pub const PROTO_sizelocvars: u32 = 0x38; // int
|
||||
|
||||
pub const LOCVAR_size: u32 = 12;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// lua_State non-thread fields (not all threads, just the active L pointer)
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
/// Pointer to global_State (at L + 0x10).
|
||||
pub const L_global: u32 = 0x10;
|
||||
|
||||
/// Check if a lua_State is active. When the state is being destroyed,
|
||||
/// the game stores 0 in L+0x60.
|
||||
pub const L_active_check: u32 = 0x60;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// Userdata (from reallymarkobject LUA_TUSERDATA case)
|
||||
//
|
||||
// Udata layout (Lua 5.0):
|
||||
// CommonHeader (next+0x00, tt+0x04, marked+0x05)
|
||||
// metatable at +0x08 (same offset as Table.metatable)
|
||||
// len at +0x0C
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
pub const UDATA_metatable: u32 = 0x08;
|
||||
|
||||
/// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
/// ---------------------------------------------------------------------------
|
||||
|
||||
/// Number of objects to process per incremental step.
|
||||
/// ~200k objects in ~80ms = ~400ns/object → 5000 ≈ 2ms/step.
|
||||
pub const CHUNK_SIZE: u32 = 5000;
|
||||
|
||||
/// Headroom added to totalbytes when setting gcthreshold between steps.
|
||||
pub const BATCH_HEADROOM: u32 = 128 * 1024;
|
||||
@@ -0,0 +1,268 @@
|
||||
//! Lua slab allocator for the incremental GC module.
|
||||
//!
|
||||
//! Copied from weirdperformance/luaalloc.zig and stripped of generational
|
||||
//! age-bitmap infrastructure (not needed for incremental GC).
|
||||
//!
|
||||
//! Hooks memory_pool_allocate (0x6FAE90) to replace WoW's 6-class slab
|
||||
//! allocator with a faster Zig slab. Also hooks lua_gc_step (0x6FAE00)
|
||||
//! which normally reads pool metadata to calculate GC thresholds.
|
||||
//!
|
||||
//! LuaMemoryRealloc (the accounting wrapper at 0x6FC980) is left intact
|
||||
//! so totalbytes tracking works unmodified.
|
||||
|
||||
const hook = @import("zhook");
|
||||
|
||||
// ============================================================================
|
||||
// Slab allocator -- segment-based lookup
|
||||
//
|
||||
// Each 64KB page is dedicated to one size class. A 64KB segment table
|
||||
// maps (ptr >> 16) -> class index, giving O(1) lookup with zero
|
||||
// per-allocation overhead. Free slots store a next pointer in the
|
||||
// slot body (minimum slot size is 4 bytes, smallest class is 16).
|
||||
//
|
||||
// Pages are allocated via VirtualAlloc which guarantees 64KB alignment
|
||||
// on Windows (allocation granularity). Large allocations (>4096 bytes)
|
||||
// also use VirtualAlloc with a size header.
|
||||
// ============================================================================
|
||||
|
||||
const MEM_COMMIT = 0x1000;
|
||||
const MEM_RESERVE = 0x2000;
|
||||
const MEM_RELEASE = 0x8000;
|
||||
const PAGE_READWRITE = 0x04;
|
||||
extern "kernel32" fn VirtualAlloc(lpAddress: ?*anyopaque, dwSize: u32, flAllocationType: u32, flProtect: u32) callconv(hook.cc.stdcall) ?[*]u8;
|
||||
extern "kernel32" fn VirtualFree(lpAddress: *anyopaque, dwSize: u32, dwFreeType: u32) callconv(hook.cc.stdcall) i32;
|
||||
|
||||
const SEGMENT_SHIFT = 16; // 64KB pages
|
||||
const PAGE_SIZE = 1 << SEGMENT_SHIFT; // 65536
|
||||
|
||||
// Class index values: 1-15 = slab classes, LARGE_CLASS = large alloc, 0 = unowned
|
||||
const LARGE_CLASS = 0xFF;
|
||||
|
||||
// Segment table: one byte per 64KB of 32-bit address space = 64KB table.
|
||||
var segment_table: [65536]u8 = .{0} ** 65536;
|
||||
|
||||
// Slot sizes tuned from runtime profiling of 6M Lua allocations.
|
||||
const class_sizes = [_]u32{ 16, 24, 32, 48, 64, 80, 96, 128, 160, 192, 256, 320, 384, 512, 640, 768, 1024, 1280, 2048, 2560, 4096 };
|
||||
const NUM_CLASSES = class_sizes.len;
|
||||
|
||||
// Precomputed lookup table: size_class_lut[n] = class index for size n.
|
||||
const LUT_SIZE = 4097; // covers sizes 0-4096
|
||||
const size_class_lut: [LUT_SIZE]u8 = blk: {
|
||||
@setEvalBranchQuota(100000);
|
||||
var lut: [LUT_SIZE]u8 = .{0xFF} ** LUT_SIZE;
|
||||
lut[0] = 0; // size 0 -> class 0
|
||||
for (1..LUT_SIZE) |sz| {
|
||||
for (class_sizes, 0..) |cs, ci| {
|
||||
if (sz <= cs) {
|
||||
lut[sz] = @intCast(ci);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
break :blk lut;
|
||||
};
|
||||
|
||||
/// Find the smallest size class index that fits `size` bytes.
|
||||
inline fn sizeClassIndex(size: u32) ?usize {
|
||||
if (size >= LUT_SIZE) return null;
|
||||
const cls = size_class_lut[size];
|
||||
if (cls == 0xFF) return null;
|
||||
return cls;
|
||||
}
|
||||
|
||||
/// Look up class index from pointer via segment table.
|
||||
inline fn classFromPtr(ptr: u32) u8 {
|
||||
return segment_table[ptr >> SEGMENT_SHIFT];
|
||||
}
|
||||
|
||||
var free_lists: [NUM_CLASSES]u32 = .{0} ** NUM_CLASSES;
|
||||
|
||||
/// Allocate a new 64KB page for the given class, register in segment table,
|
||||
/// and link all slots into the free list.
|
||||
fn refillClass(class_idx: usize) bool {
|
||||
const slot_size = class_sizes[class_idx];
|
||||
const page = VirtualAlloc(null, PAGE_SIZE, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE) orelse return false;
|
||||
const base = @intFromPtr(page);
|
||||
|
||||
// Register this page in the segment table (class indices are 0-based,
|
||||
// store as idx+1 so 0 remains "unowned")
|
||||
const seg = base >> SEGMENT_SHIFT;
|
||||
segment_table[seg] = @intCast(class_idx + 1);
|
||||
|
||||
// Carve page into slots, chain from last to first
|
||||
const slots_per_page = PAGE_SIZE / slot_size;
|
||||
var i: u32 = slots_per_page;
|
||||
while (i > 0) {
|
||||
i -= 1;
|
||||
const slot_addr = base + i * slot_size;
|
||||
const next_ptr: *u32 = @ptrFromInt(slot_addr);
|
||||
next_ptr.* = free_lists[class_idx];
|
||||
free_lists[class_idx] = slot_addr;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
fn slabAlloc(size: u32) ?[*]u8 {
|
||||
const class_idx = sizeClassIndex(size) orelse return largeMalloc(size);
|
||||
|
||||
if (free_lists[class_idx] == 0) {
|
||||
if (!refillClass(class_idx)) return null;
|
||||
}
|
||||
const slot_addr = free_lists[class_idx];
|
||||
const next_ptr: *const u32 = @ptrFromInt(slot_addr);
|
||||
free_lists[class_idx] = next_ptr.*;
|
||||
|
||||
return @ptrFromInt(slot_addr);
|
||||
}
|
||||
|
||||
fn slabFree(ptr: u32) void {
|
||||
const seg_val = classFromPtr(ptr);
|
||||
if (seg_val == LARGE_CLASS) {
|
||||
largeFree(ptr);
|
||||
return;
|
||||
}
|
||||
if (seg_val == 0) return; // not ours (shouldn't happen)
|
||||
const class_idx: usize = seg_val - 1;
|
||||
|
||||
const next_ptr: *u32 = @ptrFromInt(ptr);
|
||||
next_ptr.* = free_lists[class_idx];
|
||||
free_lists[class_idx] = ptr;
|
||||
}
|
||||
|
||||
fn slabRealloc(ptr: u32, new_size: u32) ?[*]u8 {
|
||||
const seg_val = classFromPtr(ptr);
|
||||
if (seg_val == LARGE_CLASS) return largeRealloc(ptr, new_size);
|
||||
if (seg_val == 0) return null; // not ours (shouldn't happen)
|
||||
|
||||
const class_idx: usize = seg_val - 1;
|
||||
const old_slot_size = class_sizes[class_idx];
|
||||
|
||||
// If new size fits in current slot, return same pointer
|
||||
if (new_size <= old_slot_size) return @ptrFromInt(ptr);
|
||||
|
||||
// Allocate new, copy, free old
|
||||
const new_ptr = slabAlloc(new_size) orelse return null;
|
||||
const copy_len = @min(old_slot_size, new_size);
|
||||
const dst: [*]u8 = new_ptr;
|
||||
const src: [*]const u8 = @ptrFromInt(ptr);
|
||||
@memcpy(dst[0..copy_len], src[0..copy_len]);
|
||||
slabFree(ptr);
|
||||
return new_ptr;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Large allocations (>4096 bytes): VirtualAlloc with 8-byte header.
|
||||
// ============================================================================
|
||||
|
||||
const LARGE_HEADER = 8;
|
||||
const LARGE_MAGIC: u32 = 0x4C554121; // "LUA!"
|
||||
|
||||
fn largeMalloc(size: u32) ?[*]u8 {
|
||||
const total = size + LARGE_HEADER;
|
||||
const alloc_size = (total + 0xFFF) & ~@as(u32, 0xFFF);
|
||||
const mem = VirtualAlloc(null, alloc_size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE) orelse return null;
|
||||
const base = @intFromPtr(mem);
|
||||
|
||||
var seg = base >> SEGMENT_SHIFT;
|
||||
const end_seg = (base + alloc_size - 1) >> SEGMENT_SHIFT;
|
||||
while (seg <= end_seg) : (seg += 1) {
|
||||
segment_table[seg] = LARGE_CLASS;
|
||||
}
|
||||
|
||||
const hdr_size: *u32 = @ptrFromInt(base);
|
||||
const hdr_magic: *u32 = @ptrFromInt(base + 4);
|
||||
hdr_size.* = size;
|
||||
hdr_magic.* = LARGE_MAGIC;
|
||||
return @ptrFromInt(base + LARGE_HEADER);
|
||||
}
|
||||
|
||||
fn largeFree(ptr: u32) void {
|
||||
const base = ptr - LARGE_HEADER;
|
||||
const hdr_magic: *const u32 = @ptrFromInt(base + 4);
|
||||
if (hdr_magic.* != LARGE_MAGIC) return;
|
||||
const hdr_size: *const u32 = @ptrFromInt(base);
|
||||
const total = hdr_size.* + LARGE_HEADER;
|
||||
const alloc_size = (total + 0xFFF) & ~@as(u32, 0xFFF);
|
||||
|
||||
var seg = base >> SEGMENT_SHIFT;
|
||||
const end_seg = (base + alloc_size - 1) >> SEGMENT_SHIFT;
|
||||
while (seg <= end_seg) : (seg += 1) {
|
||||
segment_table[seg] = 0;
|
||||
}
|
||||
|
||||
_ = VirtualFree(@ptrFromInt(base), 0, MEM_RELEASE);
|
||||
}
|
||||
|
||||
fn largeRealloc(ptr: u32, new_size: u32) ?[*]u8 {
|
||||
const base = ptr - LARGE_HEADER;
|
||||
const hdr_magic: *const u32 = @ptrFromInt(base + 4);
|
||||
if (hdr_magic.* != LARGE_MAGIC) return null;
|
||||
const hdr_size: *const u32 = @ptrFromInt(base);
|
||||
const old_size = hdr_size.*;
|
||||
|
||||
if (new_size <= old_size) return @ptrFromInt(ptr);
|
||||
|
||||
const new_ptr = slabAlloc(new_size) orelse return null;
|
||||
const copy_len = @min(old_size, new_size);
|
||||
const dst: [*]u8 = new_ptr;
|
||||
const src: [*]const u8 = @ptrFromInt(ptr);
|
||||
@memcpy(dst[0..copy_len], src[0..copy_len]);
|
||||
largeFree(ptr);
|
||||
return new_ptr;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// memory_pool_allocate hook
|
||||
//
|
||||
// Original at 0x6FAE90:
|
||||
// __fastcall(ECX=pool_context, EDX=old_ptr, stack: new_size)
|
||||
// RET 0x4 (cleans 1 stack param)
|
||||
// ============================================================================
|
||||
|
||||
const PoolAllocFn = fn (u32, u32, u32) callconv(hook.cc.fastcall) ?[*]u8;
|
||||
|
||||
var pool_alloc_hook: hook.Detour(PoolAllocFn) = .{};
|
||||
|
||||
fn poolAllocDetour(_: u32, old_ptr_raw: u32, new_size: u32) callconv(hook.cc.fastcall) ?[*]u8 {
|
||||
if (new_size == 0) {
|
||||
if (old_ptr_raw != 0) slabFree(old_ptr_raw);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (old_ptr_raw != 0) {
|
||||
return slabRealloc(old_ptr_raw, new_size);
|
||||
}
|
||||
|
||||
return slabAlloc(new_size);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// lua_gc_step hook
|
||||
//
|
||||
// Original at 0x6FAE00:
|
||||
// __fastcall(ECX=lua_State*), plain RET (no stack cleanup).
|
||||
// Original reads pool descriptor array (g+0x00) which is NULL with our
|
||||
// allocator, causing a crash. We intercept and no-op: our GC module
|
||||
// manages the threshold directly via setFinalThreshold.
|
||||
// ============================================================================
|
||||
|
||||
const GcStepFn = fn (u32) callconv(hook.cc.fastcall) void;
|
||||
|
||||
var gc_step_hook: hook.Detour(GcStepFn) = .{};
|
||||
|
||||
fn gcStepDetour(_: u32) callconv(hook.cc.fastcall) void {
|
||||
// No-op. Threshold is managed by luagc.zig's setFinalThreshold.
|
||||
// Must still intercept to prevent crash from NULL pool descriptors.
|
||||
}
|
||||
|
||||
pub fn install() u32 {
|
||||
var installed: u32 = 0;
|
||||
if (pool_alloc_hook.attach(0x6FAE90, &poolAllocDetour) == .ok) installed += 1;
|
||||
if (gc_step_hook.attach(0x6FAE00, &gcStepDetour) == .ok) installed += 1;
|
||||
return installed;
|
||||
}
|
||||
|
||||
pub fn remove() void {
|
||||
gc_step_hook.detach();
|
||||
pool_alloc_hook.detach();
|
||||
}
|
||||
+1331
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user