diff --git a/src/weirdperformance/luaalloc.zig b/src/weirdperformance/luaalloc.zig index c0de8ad..3a4073f 100644 --- a/src/weirdperformance/luaalloc.zig +++ b/src/weirdperformance/luaalloc.zig @@ -98,6 +98,62 @@ inline fn classFromPtr(ptr: u32) u8 { var free_lists: [NUM_CLASSES]u32 = .{0} ** NUM_CLASSES; +// ============================================================================ +// Age bitmaps for generational GC +// +// One bit per slot: 0 = young, 1 = old. +// Indexed by segment number (ptr >> 16). Each slab page gets a 512-byte +// bitmap (enough for 4096 slots at the smallest 16-byte class). +// Bitmaps are allocated alongside pages via VirtualAlloc. +// ============================================================================ + +const BITMAP_SIZE = 512; // 4096 bits = covers max slots per page +var age_bitmaps: [65536]?[*]u8 = .{null} ** 65536; + +fn slotIndex(ptr: u32, class_idx: usize) u32 { + const page_base = ptr & ~@as(u32, PAGE_SIZE - 1); + return (ptr - page_base) / class_sizes[class_idx]; +} + +/// Check if an object is old. Returns false for non-slab pointers. +pub fn isOld(ptr: u32) bool { + const seg = ptr >> SEGMENT_SHIFT; + const seg_val = segment_table[seg]; + if (seg_val == 0 or seg_val == LARGE_CLASS) return false; + const bitmap = age_bitmaps[seg] orelse return false; + const idx = slotIndex(ptr, seg_val - 1); + return (bitmap[idx >> 3] & (@as(u8, 1) << @intCast(idx & 7))) != 0; +} + +/// Mark an object as old. +pub fn setOld(ptr: u32) void { + const seg = ptr >> SEGMENT_SHIFT; + const seg_val = segment_table[seg]; + if (seg_val == 0 or seg_val == LARGE_CLASS) return; + const bitmap = age_bitmaps[seg] orelse return; + const idx = slotIndex(ptr, seg_val - 1); + bitmap[idx >> 3] |= @as(u8, 1) << @intCast(idx & 7); +} + +/// Mark an object as young (clear old bit). +pub fn setYoung(ptr: u32) void { + const seg = ptr >> SEGMENT_SHIFT; + const seg_val = segment_table[seg]; + if (seg_val == 0 or seg_val == LARGE_CLASS) return; + const bitmap = age_bitmaps[seg] orelse return; + const idx = slotIndex(ptr, seg_val - 1); + bitmap[idx >> 3] &= ~(@as(u8, 1) << @intCast(idx & 7)); +} + +/// Clear all age bits for all pages (used before major collection). +pub fn clearAllAges() void { + for (age_bitmaps) |bm_opt| { + if (bm_opt) |bm| { + @memset(bm[0..BITMAP_SIZE], 0); + } + } +} + /// Allocate a new 64KB page for the given class, register in segment table, /// and link all slots into the free list. fn refillClass(class_idx: usize) bool { @@ -108,7 +164,15 @@ fn refillClass(class_idx: usize) bool { // Register this page in the segment table (class indices are 0-based, // store as idx+1 so 0 remains "unowned") - segment_table[base >> SEGMENT_SHIFT] = @intCast(class_idx + 1); + const seg = base >> SEGMENT_SHIFT; + segment_table[seg] = @intCast(class_idx + 1); + + // Allocate age bitmap for this page (all zeros = all young) + if (age_bitmaps[seg] == null) { + if (VirtualAlloc(null, BITMAP_SIZE, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE)) |bm| { + age_bitmaps[seg] = bm; + } + } // Carve page into slots, chain from last to first const slots_per_page = PAGE_SIZE / slot_size; diff --git a/src/weirdperformance/luagc.zig b/src/weirdperformance/luagc.zig index 3801ba6..26d7876 100644 --- a/src/weirdperformance/luagc.zig +++ b/src/weirdperformance/luagc.zig @@ -1,19 +1,30 @@ -//! Incremental GC for Lua 5.0. +//! Generational incremental GC for Lua 5.0. //! //! The original stop-the-world GC freezes the game for up to 5 seconds. -//! This module splits the rootgc sweep across multiple GC triggers. +//! This module layers two optimizations on top: //! -//! Strategy: -//! 1. Mark + udata sweep + string sweep run atomically (mark is fast) -//! 2. Rootgc is split into chunks. Each chunk is swept by the original -//! lua_gc_remove_objects. After sweeping, surviving objects are moved -//! to a separate "swept" list so they can't be re-swept. -//! 3. After all chunks are processed, the swept list is reconnected. +//! 1. Incremental rootgc sweep: the rootgc list is swept in chunks across +//! multiple GC triggers. The 5s pause becomes ~9ms chunks. +//! +//! 2. Generational tracking: surviving rootgc objects are marked "old" in +//! an external age bitmap (managed by luaalloc). Writes to old tables +//! are captured by a write barrier (tableSetBarrier) and added to a +//! touched set, which will be re-traversed by future minor mark +//! implementations. +//! +//! Current minor/major distinction is only in the machinery: both cycles +//! still run the full lua_gc_full_collection (full mark). The minor path +//! preserves old objects in the age bitmap; the major path clears the +//! bitmap and touched set so the full cycle starts from scratch. +//! +//! A custom minor mark phase that avoids traversing untouched old objects +//! is the next optimization layer on top of this machinery. //! //! Birth-mark: binary patch in luaC_link makes new objects born marked=1 //! during sweep so they survive until the next GC cycle. const hook = @import("zhook"); +const luaalloc = @import("luaalloc.zig"); const GS_ROOTGC = 0x10; const GS_ROOTUDATA = 0x14; @@ -33,6 +44,86 @@ const lua_gc_remove_objects: *const fn (u32, u32, u32) callconv(hook.cc.fastcall var sweeping: bool = false; var in_gc: bool = false; +// ============================================================================ +// Generational cycle tracking +// ============================================================================ +// +// is_major: true if the current collection is a major (full) cycle. Set at +// the start of a collection based on cycle_count / touched overflow. +// cycle_count: number of minor cycles since the last major. Reset to 0 at +// the end of a major. +// MINORS_PER_MAJOR: force a major every N minor cycles to bound the amount +// of dead-old garbage that accumulates. +// first_collection: flag to force the very first collection to be a major +// so we can populate the age bitmap from a clean slate. +// +var is_major: bool = false; +var cycle_count: u32 = 0; +const MINORS_PER_MAJOR: u32 = 32; +var first_collection: bool = true; + +/// Decide minor vs major for this cycle. Called once at the start. +fn selectCycleMode() void { + if (first_collection or touched_overflow or cycle_count >= MINORS_PER_MAJOR) { + is_major = true; + } else { + is_major = false; + } +} + +/// Called at the start of a collection: clear the touched set (writes from +/// the previous cycle). For a major cycle, also clear the entire age bitmap +/// so survivors can be re-aged from zero. +fn cycleStart() void { + if (is_major) { + luaalloc.clearAllAges(); + } + touched_count = 0; + touched_overflow = false; +} + +/// Called at the end of a completed collection. Repopulates the age bitmap +/// (all survivors are now old) and advances cycle_count / first_collection. +fn cycleFinish(g: u32) void { + markAllOld(g); + if (is_major) { + cycle_count = 0; + } else { + cycle_count += 1; + } + first_collection = false; +} + +// ============================================================================ +// Write barrier for generational GC +// Hooks lua_table_set_value to detect writes to old tables. +// __fastcall(ECX=L, EDX=table, stack=key_TValue_ptr) -> u32 (value slot ptr), RET 0x4 +// ============================================================================ + +const MAX_TOUCHED = 4096; +var touched_set: [MAX_TOUCHED]u32 = .{0} ** MAX_TOUCHED; +var touched_count: u32 = 0; +var touched_overflow: bool = false; + +fn addTouched(table: u32) void { + if (touched_count >= MAX_TOUCHED) { + touched_overflow = true; + return; + } + touched_set[touched_count] = table; + touched_count += 1; +} + +const TableSetFn = fn (u32, u32, u32) callconv(hook.cc.fastcall) u32; +var table_set_hook: hook.Detour(TableSetFn) = .{}; + +fn tableSetBarrier(L: u32, table: u32, key: u32) callconv(hook.cc.fastcall) u32 { + if (luaalloc.isOld(table)) { + addTouched(table); + } + return table_set_hook.callOriginal(.{ L, table, key }); +} + // The "already swept" list: objects that survived sweep, detached from rootgc. // swept_head -> first swept survivor, swept_tail -> last (for O(1) append). var swept_head: u32 = 0; @@ -85,6 +176,15 @@ fn getGlobalState(L: u32) u32 { return @as(*const u32, @ptrFromInt(L + 0x10)).*; } +/// Walk rootgc and mark all objects as old in the external age bitmap. +fn markAllOld(g: u32) void { + var obj = readU32(g + GS_ROOTGC); + while (obj != 0) { + luaalloc.setOld(obj); + obj = readU32(obj + OBJ_NEXT); + } +} + /// Walk the list from a head pointer, find the Nth object. /// Returns (obj_addr, count_walked). obj_addr=0 if list shorter than N. fn findNth(head: u32, n: u32) struct { obj: u32, count: u32 } { @@ -137,6 +237,10 @@ fn collectGarbageDetour(L: u32) callconv(hook.cc.fastcall) void { const g = getGlobalState(L); if (!sweeping) { + // === Start of a new cycle: decide minor vs major === + selectCycleMode(); + cycleStart(); + // === Atomic: mark + udata sweep + string sweep === const t0 = rdtsc(); lua_gc_full_collection(L); @@ -159,6 +263,7 @@ fn collectGarbageDetour(L: u32) callconv(hook.cc.fastcall) void { _ = lua_gc_remove_objects(L, g + GS_ROOTGC, 0); lua_gc_shrink_memory(L); luaCallUserDataGC(L); + cycleFinish(g); return; } @@ -211,6 +316,7 @@ fn collectGarbageDetour(L: u32) callconv(hook.cc.fastcall) void { lua_gc_shrink_memory(L); luaCallUserDataGC(L); + cycleFinish(g); return; } @@ -239,6 +345,8 @@ var collect_hook: hook.Detour(CollectFn) = .{}; pub fn install() u32 { var installed: u32 = 0; if (collect_hook.attach(0x6F7340, &collectGarbageDetour) == .ok) installed += 1; + // Generational write barrier + if (table_set_hook.attach(0x6FA840, &tableSetBarrier) == .ok) installed += 1; return installed; } @@ -266,7 +374,10 @@ pub fn remove() void { setBirthMark(false); } collect_hook.detach(); + table_set_hook.detach(); sweeping = false; swept_head = 0; swept_tail = 0; + touched_count = 0; + touched_overflow = false; }