luagc: generational machinery (write barrier + age bitmap + minor/major cycles)

Adds the infrastructure for generational GC on top of the incremental sweep:

- luaalloc: per-page age bitmap (1 bit per slot) lazily allocated with each
  slab page. Public isOld/setOld/setYoung/clearAllAges API. Segment-indexed
  for O(1) lookup in the write barrier.

- luagc: write barrier via Detour on lua_table_set_value. Records touched
  old tables in a 4096-entry set. Pure bitmap read on every table write.

- luagc: minor/major cycle distinction. First collection, touched overflow,
  and every 32nd minor are forced to major. Major clears the age bitmap and
  touched set; minor preserves both. cycleFinish repopulates the bitmap by
  walking rootgc after a full sweep completes.

Both cycles still run lua_gc_full_collection for now -- the custom minor
mark that actually skips untouched old objects is the next layer.

Previous crash investigations proved Detours on lua_table_set_value are
safe; earlier failures were from simultaneous marked-byte modifications,
not the Detour itself.

src/weirdperformance/luaalloc.zig

@@ -98,6 +98,62 @@ inline fn classFromPtr(ptr: u32) u8 {
 
 var free_lists: [NUM_CLASSES]u32 = .{0} ** NUM_CLASSES;
 
+// ============================================================================
+// Age bitmaps for generational GC
+//
+// One bit per slot: 0 = young, 1 = old.
+// Indexed by segment number (ptr >> 16). Each slab page gets a 512-byte
+// bitmap (enough for 4096 slots at the smallest 16-byte class).
+// Bitmaps are allocated alongside pages via VirtualAlloc.
+// ============================================================================
+
+const BITMAP_SIZE = 512; // 4096 bits = covers max slots per page
+var age_bitmaps: [65536]?[*]u8 = .{null} ** 65536;
+
+fn slotIndex(ptr: u32, class_idx: usize) u32 {
+    const page_base = ptr & ~@as(u32, PAGE_SIZE - 1);
+    return (ptr - page_base) / class_sizes[class_idx];
+}
+
+/// Check if an object is old. Returns false for non-slab pointers.
+pub fn isOld(ptr: u32) bool {
+    const seg = ptr >> SEGMENT_SHIFT;
+    const seg_val = segment_table[seg];
+    if (seg_val == 0 or seg_val == LARGE_CLASS) return false;
+    const bitmap = age_bitmaps[seg] orelse return false;
+    const idx = slotIndex(ptr, seg_val - 1);
+    return (bitmap[idx >> 3] & (@as(u8, 1) << @intCast(idx & 7))) != 0;
+}
+
+/// Mark an object as old.
+pub fn setOld(ptr: u32) void {
+    const seg = ptr >> SEGMENT_SHIFT;
+    const seg_val = segment_table[seg];
+    if (seg_val == 0 or seg_val == LARGE_CLASS) return;
+    const bitmap = age_bitmaps[seg] orelse return;
+    const idx = slotIndex(ptr, seg_val - 1);
+    bitmap[idx >> 3] |= @as(u8, 1) << @intCast(idx & 7);
+}
+
+/// Mark an object as young (clear old bit).
+pub fn setYoung(ptr: u32) void {
+    const seg = ptr >> SEGMENT_SHIFT;
+    const seg_val = segment_table[seg];
+    if (seg_val == 0 or seg_val == LARGE_CLASS) return;
+    const bitmap = age_bitmaps[seg] orelse return;
+    const idx = slotIndex(ptr, seg_val - 1);
+    bitmap[idx >> 3] &= ~(@as(u8, 1) << @intCast(idx & 7));
+}
+
+/// Clear all age bits for all pages (used before major collection).
+pub fn clearAllAges() void {
+    for (age_bitmaps) |bm_opt| {
+        if (bm_opt) |bm| {
+            @memset(bm[0..BITMAP_SIZE], 0);
+        }
+    }
+}
+
 /// Allocate a new 64KB page for the given class, register in segment table,
 /// and link all slots into the free list.
 fn refillClass(class_idx: usize) bool {
@@ -108,7 +164,15 @@ fn refillClass(class_idx: usize) bool {
 
     // Register this page in the segment table (class indices are 0-based,
     // store as idx+1 so 0 remains "unowned")
-    segment_table[base >> SEGMENT_SHIFT] = @intCast(class_idx + 1);
+    const seg = base >> SEGMENT_SHIFT;
+    segment_table[seg] = @intCast(class_idx + 1);
+
+    // Allocate age bitmap for this page (all zeros = all young)
+    if (age_bitmaps[seg] == null) {
+        if (VirtualAlloc(null, BITMAP_SIZE, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE)) |bm| {
+            age_bitmaps[seg] = bm;
+        }
+    }
 
     // Carve page into slots, chain from last to first
     const slots_per_page = PAGE_SIZE / slot_size;

src/weirdperformance/luagc.zig

@@ -1,19 +1,30 @@
-//! Incremental GC for Lua 5.0.
+//! Generational incremental GC for Lua 5.0.
 //!
 //! The original stop-the-world GC freezes the game for up to 5 seconds.
-//! This module splits the rootgc sweep across multiple GC triggers.
+//! This module layers two optimizations on top:
 //!
-//! Strategy:
-//!   1. Mark + udata sweep + string sweep run atomically (mark is fast)
-//!   2. Rootgc is split into chunks. Each chunk is swept by the original
-//!      lua_gc_remove_objects. After sweeping, surviving objects are moved
-//!      to a separate "swept" list so they can't be re-swept.
-//!   3. After all chunks are processed, the swept list is reconnected.
+//!   1. Incremental rootgc sweep: the rootgc list is swept in chunks across
+//!      multiple GC triggers. The 5s pause becomes ~9ms chunks.
+//!
+//!   2. Generational tracking: surviving rootgc objects are marked "old" in
+//!      an external age bitmap (managed by luaalloc). Writes to old tables
+//!      are captured by a write barrier (tableSetBarrier) and added to a
+//!      touched set, which will be re-traversed by future minor mark
+//!      implementations.
+//!
+//! Current minor/major distinction is only in the machinery: both cycles
+//! still run the full lua_gc_full_collection (full mark). The minor path
+//! preserves old objects in the age bitmap; the major path clears the
+//! bitmap and touched set so the full cycle starts from scratch.
+//!
+//! A custom minor mark phase that avoids traversing untouched old objects
+//! is the next optimization layer on top of this machinery.
 //!
 //! Birth-mark: binary patch in luaC_link makes new objects born marked=1
 //! during sweep so they survive until the next GC cycle.
 
 const hook = @import("zhook");
+const luaalloc = @import("luaalloc.zig");
 
 const GS_ROOTGC = 0x10;
 const GS_ROOTUDATA = 0x14;
@@ -33,6 +44,86 @@ const lua_gc_remove_objects: *const fn (u32, u32, u32) callconv(hook.cc.fastcall
 var sweeping: bool = false;
 var in_gc: bool = false;
 
+// ============================================================================
+// Generational cycle tracking
+// ============================================================================
+//
+// is_major: true if the current collection is a major (full) cycle. Set at
+//   the start of a collection based on cycle_count / touched overflow.
+// cycle_count: number of minor cycles since the last major. Reset to 0 at
+//   the end of a major.
+// MINORS_PER_MAJOR: force a major every N minor cycles to bound the amount
+//   of dead-old garbage that accumulates.
+// first_collection: flag to force the very first collection to be a major
+//   so we can populate the age bitmap from a clean slate.
+//
+var is_major: bool = false;
+var cycle_count: u32 = 0;
+const MINORS_PER_MAJOR: u32 = 32;
+var first_collection: bool = true;
+
+/// Decide minor vs major for this cycle. Called once at the start.
+fn selectCycleMode() void {
+    if (first_collection or touched_overflow or cycle_count >= MINORS_PER_MAJOR) {
+        is_major = true;
+    } else {
+        is_major = false;
+    }
+}
+
+/// Called at the start of a collection: clear the touched set (writes from
+/// the previous cycle). For a major cycle, also clear the entire age bitmap
+/// so survivors can be re-aged from zero.
+fn cycleStart() void {
+    if (is_major) {
+        luaalloc.clearAllAges();
+    }
+    touched_count = 0;
+    touched_overflow = false;
+}
+
+/// Called at the end of a completed collection. Repopulates the age bitmap
+/// (all survivors are now old) and advances cycle_count / first_collection.
+fn cycleFinish(g: u32) void {
+    markAllOld(g);
+    if (is_major) {
+        cycle_count = 0;
+    } else {
+        cycle_count += 1;
+    }
+    first_collection = false;
+}
+
+// ============================================================================
+// Write barrier for generational GC
+// Hooks lua_table_set_value to detect writes to old tables.
+// __fastcall(ECX=L, EDX=table, stack=key_TValue_ptr) -> u32 (value slot ptr), RET 0x4
+// ============================================================================
+
+const MAX_TOUCHED = 4096;
+var touched_set: [MAX_TOUCHED]u32 = .{0} ** MAX_TOUCHED;
+var touched_count: u32 = 0;
+var touched_overflow: bool = false;
+
+fn addTouched(table: u32) void {
+    if (touched_count >= MAX_TOUCHED) {
+        touched_overflow = true;
+        return;
+    }
+    touched_set[touched_count] = table;
+    touched_count += 1;
+}
+
+const TableSetFn = fn (u32, u32, u32) callconv(hook.cc.fastcall) u32;
+var table_set_hook: hook.Detour(TableSetFn) = .{};
+
+fn tableSetBarrier(L: u32, table: u32, key: u32) callconv(hook.cc.fastcall) u32 {
+    if (luaalloc.isOld(table)) {
+        addTouched(table);
+    }
+    return table_set_hook.callOriginal(.{ L, table, key });
+}
+
 // The "already swept" list: objects that survived sweep, detached from rootgc.
 // swept_head -> first swept survivor, swept_tail -> last (for O(1) append).
 var swept_head: u32 = 0;
@@ -85,6 +176,15 @@ fn getGlobalState(L: u32) u32 {
     return @as(*const u32, @ptrFromInt(L + 0x10)).*;
 }
 
+/// Walk rootgc and mark all objects as old in the external age bitmap.
+fn markAllOld(g: u32) void {
+    var obj = readU32(g + GS_ROOTGC);
+    while (obj != 0) {
+        luaalloc.setOld(obj);
+        obj = readU32(obj + OBJ_NEXT);
+    }
+}
+
 /// Walk the list from a head pointer, find the Nth object.
 /// Returns (obj_addr, count_walked). obj_addr=0 if list shorter than N.
 fn findNth(head: u32, n: u32) struct { obj: u32, count: u32 } {
@@ -137,6 +237,10 @@ fn collectGarbageDetour(L: u32) callconv(hook.cc.fastcall) void {
     const g = getGlobalState(L);
 
     if (!sweeping) {
+        // === Start of a new cycle: decide minor vs major ===
+        selectCycleMode();
+        cycleStart();
+
         // === Atomic: mark + udata sweep + string sweep ===
         const t0 = rdtsc();
         lua_gc_full_collection(L);
@@ -159,6 +263,7 @@ fn collectGarbageDetour(L: u32) callconv(hook.cc.fastcall) void {
             _ = lua_gc_remove_objects(L, g + GS_ROOTGC, 0);
             lua_gc_shrink_memory(L);
             luaCallUserDataGC(L);
+            cycleFinish(g);
             return;
         }
 
@@ -211,6 +316,7 @@ fn collectGarbageDetour(L: u32) callconv(hook.cc.fastcall) void {
 
         lua_gc_shrink_memory(L);
         luaCallUserDataGC(L);
+        cycleFinish(g);
         return;
     }
 
@@ -239,6 +345,8 @@ var collect_hook: hook.Detour(CollectFn) = .{};
 pub fn install() u32 {
     var installed: u32 = 0;
     if (collect_hook.attach(0x6F7340, &collectGarbageDetour) == .ok) installed += 1;
+    // Generational write barrier
+    if (table_set_hook.attach(0x6FA840, &tableSetBarrier) == .ok) installed += 1;
     return installed;
 }
 
@@ -266,7 +374,10 @@ pub fn remove() void {
         setBirthMark(false);
     }
     collect_hook.detach();
+    table_set_hook.detach();
     sweeping = false;
     swept_head = 0;
     swept_tail = 0;
+    touched_count = 0;
+    touched_overflow = false;
 }