diff --git a/src/luagc/luagc.zig b/src/luagc/luagc.zig index d707f16..278abba 100644 --- a/src/luagc/luagc.zig +++ b/src/luagc/luagc.zig @@ -103,8 +103,20 @@ var phase: Phase = .idle; var saved_L: u32 = 0; var saved_g: u32 = 0; -/// Objects to process per step. ~5000 = ~2ms target per step. -const CHUNK_SIZE: u32 = 5000; +/// Step size in bytes -- matches 5.1 GCSTEPSIZE (default 1024). +/// Controls how often GC triggers between steps: threshold = totalbytes + stepsize. +var gcstepsize: u32 = 1024; + +/// Step multiplier -- matches 5.1 gcstepmul (default 200). +/// Budget per trigger = (stepsize / 100) * stepmul bytes of work. +var gcstepmul: u32 = 200; + +/// Pause factor -- matches 5.1 gcpause (default 200 = 2x). +/// Between cycles: threshold = (totalbytes / 100) * gcpause. +var gcpause: u32 = 200; + +/// Max objects to sweep per singlestep (5.1 GCSWEEPMAX = 40). +const GCSWEEPMAX: u32 = 40; // ============================================================================= // Gray Stack @@ -361,6 +373,15 @@ inline fn makewhite(obj: u32) void { fn markObject(obj: u32) bool { if (!isWhite(obj)) return false; const tt = objType(obj); + if (tt > 10) { + // Invalid type tag -- this is garbage, not a GC object. + // Log and skip to prevent crash from following garbage pointers. + var buf: [160]u8 = undefined; + log.print(fmt(&buf, "BUG markObject(0x{x}) tt={d} marked=0x{x} caller=0x{x}\n", .{ + obj, tt, readU8(obj + offsets.OBJ_marked), @returnAddress(), + })); + return false; + } switch (tt) { offsets.LUA_TSTRING => { // Strings: just clear white bits, don't go to gray (no children) @@ -822,26 +843,115 @@ fn markroot(g: u32) void { } // ============================================================================= -// Incremental Mark Step +// Single Step (matches 5.1 singlestep lgc.c:556-607) // ============================================================================= +// +// Process one unit of GC work. Returns estimated memory cost. +// Called in a budget loop by collectGarbageDetour. -/// Process up to CHUNK_SIZE objects from the gray stack. -/// Returns true when gray stack is drained (mark complete). -fn markStep() bool { - const t0 = rdtsc(); - cur_mark_steps += 1; - var processed: u32 = 0; +const GCSWEEPCOST: u32 = 10; +const GCFINALIZECOST: u32 = 100; - while (gray_count > 0 and processed < CHUNK_SIZE) { - const obj = grayStackPop(); - if (obj == 0) break; - propagatemark(obj); - processed += 1; +fn singlestep(L: u32, g: u32) u32 { + switch (phase) { + .idle => { + // Start new cycle + cur_mark_steps = 0; + cur_sweep_steps = 0; + cur_dead_freed = 0; + cur_strings_freed = 0; + cur_max_step_cycles = 0; + cur_atomic_cycles = 0; + dbg_table_count = 0; + dbg_closure_count = 0; + dbg_thread_count = 0; + dbg_proto_count = 0; + dbg_udata_count = 0; + dbg_other_count = 0; + + markroot(g); + phase = .marking; + return 0; + }, + .marking => { + if (gray_count > 0) { + cur_mark_steps += 1; + const obj = grayStackPop(); + if (obj != 0) propagatemark(obj); + // Return estimated cost (rough, like 5.1 propagatemark) + return 100; + } else { + // Gray stack drained -- run atomic phase + phase = .atomic; + runAtomicAndStartSweep(L, g); + return 0; + } + }, + .atomic => { + // Should not be called (atomic runs to completion in runAtomicAndStartSweep) + runAtomicAndStartSweep(L, g); + return 0; + }, + .sweep_strings => { + cur_sweep_steps += 1; + const hash_array = readU32(g + offsets.GS_strt_hash); + const bucket_count = readU32(g + offsets.GS_strt_size); + if (hash_array == 0 or bucket_count == 0 or sweep_string_bucket >= bucket_count) { + // Strings done, start rootgc sweep + sweep_prev_next = g + offsets.GS_rootgc; + phase = .sweeping; + return GCSWEEPCOST; + } + // Sweep one bucket + var prev_next: u32 = hash_array + sweep_string_bucket * 4; + while (true) { + const obj = readU32(prev_next); + if (obj == 0) break; + if (!isDead(obj)) { + makewhite(obj); + prev_next = obj + offsets.OBJ_next; + } else { + const next = readU32(obj + offsets.OBJ_next); + writeU32(prev_next, next); + const nuse = readU32(g + offsets.GS_strt_nuse); + if (nuse > 0) writeU32(g + offsets.GS_strt_nuse, nuse - 1); + native_free_object(L, obj); + cur_strings_freed += 1; + } + } + sweep_string_bucket += 1; + return GCSWEEPCOST; + }, + .sweeping => { + // Sweep up to GCSWEEPMAX objects from rootgc + cur_sweep_steps += 1; + var count: u32 = 0; + while (count < GCSWEEPMAX) { + const obj = readU32(sweep_prev_next); + if (obj == 0) { + // rootgc sweep done + phase = .finalize; + finalizeCycle(L, g); + return 0; + } + if (!isDead(obj)) { + makewhite(obj); + sweep_prev_next = obj + offsets.OBJ_next; + } else { + const next = readU32(obj + offsets.OBJ_next); + writeU32(sweep_prev_next, next); + native_free_object(L, obj); + cur_dead_freed += 1; + } + count += 1; + } + return GCSWEEPMAX * GCSWEEPCOST; + }, + .finalize => { + finalizeCycle(L, g); + return GCFINALIZECOST; + }, } - - const elapsed = rdtsc() - t0; - if (elapsed > cur_max_step_cycles) cur_max_step_cycles = elapsed; - return gray_count == 0; } /// Drain the entire gray stack (used in atomic phase). @@ -1018,12 +1128,13 @@ fn isCleared(tv_tt: u8, tv_gc: u32) bool { // No list splitting. sweep_prev_next points to the address of the "next" // field that leads to the current object (initially &g->rootgc). +/// Legacy chunked sweep for DIAG_MODE only. fn sweepRootgcStep(L: u32) bool { const t0 = rdtsc(); cur_sweep_steps += 1; var processed: u32 = 0; - while (processed < CHUNK_SIZE) { + while (processed < 1000000) { const obj = readU32(sweep_prev_next); if (obj == 0) { const elapsed = rdtsc() - t0; @@ -1048,7 +1159,7 @@ fn sweepRootgcStep(L: u32) bool { const elapsed = rdtsc() - t0; if (elapsed > cur_max_step_cycles) cur_max_step_cycles = elapsed; - bumpThresholdHeadroom(saved_g); + // threshold managed by budget loop return false; } @@ -1082,9 +1193,11 @@ fn sweepRootudata(L: u32, g: u32) void { // Decrement strt.nuse for each freed string. // Chunks by processing N buckets per call. +/// Legacy chunked string sweep for DIAG_MODE only. fn sweepStringsStep(L: u32, g: u32) bool { const t0 = rdtsc(); cur_sweep_steps += 1; + const CHUNK_SIZE = 1000000; const hash_array = readU32(g + offsets.GS_strt_hash); const bucket_count = readU32(g + offsets.GS_strt_size); @@ -1117,7 +1230,7 @@ fn sweepStringsStep(L: u32, g: u32) bool { if (sweep_string_bucket >= bucket_count) return true; - bumpThresholdHeadroom(g); + // threshold managed by budget loop return false; } @@ -1125,19 +1238,15 @@ fn sweepStringsStep(L: u32, g: u32) bool { // Threshold Management // ============================================================================= -/// After sweep: threshold = 2 * totalbytes (matches Lua 5.0 checkSizes). -/// The native formula is `2 * nblocks - deadmem` but deadmem (size of -/// finalized udata) is typically negligible, so 2x is close enough. +/// After sweep: threshold = (totalbytes / 100) * gcpause. +/// Matches 5.1 setthreshold. Default gcpause=200 means 2x. fn setFinalThreshold(g: u32) void { - const totalbytes = readU32(g + offsets.GS_totalbytes); - writeU32(g + offsets.GS_gcthreshold, totalbytes *| 2); + const totalbytes: u64 = readU32(g + offsets.GS_totalbytes); + const threshold = (totalbytes / 100) * gcpause; + writeU32(g + offsets.GS_gcthreshold, @intCast(@min(threshold, 0xFFFFFFFF))); } -/// Between incremental steps: bump threshold just enough to not retrigger. -fn bumpThresholdHeadroom(g: u32) void { - const totalbytes = readU32(g + offsets.GS_totalbytes); - writeU32(g + offsets.GS_gcthreshold, totalbytes + offsets.BATCH_HEADROOM); -} +// bumpThresholdHeadroom removed -- threshold managed by 5.1-style budget loop. /// Shrink string table if load factor is low (nuse < size/4). fn maybeShrinkStringTable(L: u32, g: u32) void { @@ -1252,87 +1361,25 @@ fn collectGarbageDetour(L: u32) callconv(hook.cc.fastcall) void { return; } - // Log entry state for first 20 cycles - if (stats.cycles_total < 20 and phase == .idle) { - const tb = readU32(g + offsets.GS_totalbytes); - const thr = readU32(g + offsets.GS_gcthreshold); - var buf: [128]u8 = undefined; - log.print(fmt(&buf, " trigger: totalbytes={d} threshold={d}\n", .{ tb, thr })); + // 5.1-style budget loop (luaC_step, lgc.c:610-632). + // Each trigger processes (stepsize/100)*stepmul bytes of work. + const t0 = rdtsc(); + var lim: i32 = @intCast((gcstepsize / 100) * gcstepmul); + if (lim == 0) lim = @as(i32, @intCast((@as(u32, 0x7FFFFFFF)))); // no limit + + while (lim > 0) { + const cost = singlestep(L, g); + lim -= @as(i32, @intCast(cost)); + if (phase == .idle) break; } - switch (phase) { - .idle => { - // Start a new GC cycle - gray_count = 0; - weak_table_count = 0; - grayagain_count = 0; - cur_mark_steps = 0; - cur_sweep_steps = 0; - cur_dead_freed = 0; - cur_strings_freed = 0; - cur_max_step_cycles = 0; - cur_atomic_cycles = 0; - dbg_table_count = 0; - dbg_closure_count = 0; - dbg_thread_count = 0; - dbg_proto_count = 0; - dbg_udata_count = 0; - dbg_other_count = 0; + const elapsed = rdtsc() - t0; + if (elapsed > cur_max_step_cycles) cur_max_step_cycles = elapsed; - // Push roots and start marking. - // Birth mark stays OFF during mark and rootgc sweep. New rootgc - // objects prepend to the head (behind our forward cursor) and are - // safe. Birth mark is only needed during string sweep where new - // strings can land in unswept hash buckets ahead of the cursor. - markroot(g); - phase = .marking; - - // Do one mark step immediately - if (markStep()) { - phase = .atomic; - runAtomicAndStartSweep(L, g); - } else { - bumpThresholdHeadroom(g); - } - }, - - .marking => { - if (markStep()) { - phase = .atomic; - runAtomicAndStartSweep(L, g); - } else { - bumpThresholdHeadroom(g); - } - }, - - .atomic => { - // Should not be called in this phase (atomic runs to completion) - // but handle gracefully - runAtomicAndStartSweep(L, g); - }, - - .sweep_strings => { - if (sweepStringsStep(L, g)) { - sweep_prev_next = g + offsets.GS_rootgc; - phase = .sweeping; - if (sweepRootgcStep(L)) { - phase = .finalize; - finalizeCycle(L, g); - } - } - }, - - .sweeping => { - if (sweepRootgcStep(L)) { - phase = .finalize; - finalizeCycle(L, g); - } - }, - - .finalize => { - // Should not happen (finalize runs to completion), handle gracefully - finalizeCycle(L, g); - }, + if (phase != .idle) { + // Mid-cycle: set threshold for next trigger + const totalbytes = readU32(g + offsets.GS_totalbytes); + writeU32(g + offsets.GS_gcthreshold, totalbytes + gcstepsize); } } @@ -1350,18 +1397,10 @@ fn runAtomicAndStartSweep(L: u32, g: u32) void { // Sweep rootudata atomically (small list, fast) sweepRootudata(L, g); - // Start chunked string sweep + // Initialize sweep cursors, transition to string sweep. + // Actual sweep work is done by singlestep in the budget loop. sweep_string_bucket = 0; phase = .sweep_strings; - - if (sweepStringsStep(L, g)) { - sweep_prev_next = g + offsets.GS_rootgc; - phase = .sweeping; - if (sweepRootgcStep(L)) { - phase = .finalize; - finalizeCycle(L, g); - } - } } /// Final cleanup: shrink string table, set threshold, run finalizers. @@ -1545,6 +1584,17 @@ fn luaCLinkDetour(L: u32, obj: u32, tt: u32) callconv(hook.cc.fastcall) void { writeU8(obj + offsets.OBJ_tt, @intCast(tt)); } +// ============================================================================= +// Pre-allocation GC check (5.1 luaC_checkGC pattern) +// ============================================================================= +// +// 5.1 calls luaC_checkGC at the START of API functions that allocate objects, +// BEFORE the allocation. This ensures pending GC work completes before any +// half-initialized object is linked to rootgc. WoW's 5.0 triggers checkGC +// DURING allocation (inside luaM_realloc), after luaC_link but before the +// caller finishes initialization. We hook the API functions to do the +// pre-check, matching 5.1's order. + // ============================================================================= // Forward barriers (luaC_barrierf equivalent) // ============================================================================= @@ -1620,16 +1670,100 @@ fn setupvalDetour(L: u32, funcindex: u32, n: u32) callconv(hook.cc.fastcall) u32 return result; } -// --- lua_pushcclosure (0x6F3920) --- -// __fastcall(ECX=L, EDX=fn_ptr, stack=n_upvals). Creates C closure with upvalues. -const PushCClosureFn = fn (u32, u32, u32) callconv(hook.cc.fastcall) void; -var pushcclosure_hook: hook.Detour(PushCClosureFn) = .{}; -fn pushcclosureDetour(L: u32, fn_ptr: u32, n: u32) callconv(hook.cc.fastcall) void { - pushcclosure_hook.callOriginal(.{ L, fn_ptr, n }); - // The new closure is at L->top - 1. It was just created (WHITE). - // Its upvalues were set from the stack. The closure is WHITE, so - // no BLACK→WHITE edge (barrier only matters if container is BLACK). - // New closure is currentwhite → no barrier needed. +// ============================================================================= +// Pre-allocation GC hooks (5.1 luaC_checkGC at API entry) +// ============================================================================= +// +// 5.1 calls luaC_checkGC at the start of every API function that allocates, +// BEFORE any allocation. This ensures pending GC work completes before +// half-initialized objects exist in rootgc. WoW's 5.0 lacks these checks. +// We hook all 8 allocating API functions. + +/// Run pending GC work if threshold is exceeded. Called at API entry. +fn preAllocCheck(L: u32) void { + if (phase == .idle) return; + if (in_gc) return; + const g = getGlobalState(L); + const tb = readU32(g + offsets.GS_totalbytes); + const thr = readU32(g + offsets.GS_gcthreshold); + if (tb >= thr) { + collectGarbageDetour(L); + } +} + +// Generic detour types for different API signatures +const ApiFn1 = fn (u32, u32, u32) callconv(hook.cc.fastcall) void; // pushcclosure, pushlstring +const ApiFn0 = fn (u32) callconv(hook.cc.fastcall) void; // concat-like (1 arg) +const ApiFn0r = fn (u32) callconv(hook.cc.fastcall) u32; // newuserdata-like (returns ptr) +const ApiFn2 = fn (u32, u32) callconv(hook.cc.fastcall) void; // newtable-like +const ApiFn2r = fn (u32, u32) callconv(hook.cc.fastcall) u32; // pushstring-like (returns ptr) + +var prealloc_pushcclosure: hook.Detour(ApiFn1) = .{}; +var prealloc_pushlstring: hook.Detour(ApiFn1) = .{}; +var prealloc_createtable: hook.Detour(ApiFn1) = .{}; +var prealloc_pushvfstring: hook.Detour(ApiFn1) = .{}; +// pushfstring is cdecl (variadic), not fastcall -- cannot use standard Detour. +// var prealloc_pushfstring: hook.Detour(ApiFn1) = .{}; +var prealloc_pushstring: hook.Detour(ApiFn2) = .{}; +var prealloc_concat: hook.Detour(ApiFn2) = .{}; +var prealloc_newuserdata: hook.Detour(ApiFn2r) = .{}; +var prealloc_newthread: hook.Detour(ApiFn0r) = .{}; + +fn prealloc_pushcclosure_fn(L: u32, a: u32, b: u32) callconv(hook.cc.fastcall) void { + preAllocCheck(L); + prealloc_pushcclosure.callOriginal(.{ L, a, b }); +} +fn prealloc_pushlstring_fn(L: u32, a: u32, b: u32) callconv(hook.cc.fastcall) void { + preAllocCheck(L); + prealloc_pushlstring.callOriginal(.{ L, a, b }); +} +fn prealloc_createtable_fn(L: u32, a: u32, b: u32) callconv(hook.cc.fastcall) void { + preAllocCheck(L); + prealloc_createtable.callOriginal(.{ L, a, b }); +} +fn prealloc_pushvfstring_fn(L: u32, a: u32, b: u32) callconv(hook.cc.fastcall) void { + preAllocCheck(L); + prealloc_pushvfstring.callOriginal(.{ L, a, b }); +} +// pushfstring detour removed -- cdecl variadic, not fastcall +fn prealloc_pushstring_fn(L: u32, a: u32) callconv(hook.cc.fastcall) void { + preAllocCheck(L); + prealloc_pushstring.callOriginal(.{ L, a }); +} +fn prealloc_concat_fn(L: u32, a: u32) callconv(hook.cc.fastcall) void { + preAllocCheck(L); + prealloc_concat.callOriginal(.{ L, a }); +} +fn prealloc_newuserdata_fn(L: u32, a: u32) callconv(hook.cc.fastcall) u32 { + preAllocCheck(L); + return prealloc_newuserdata.callOriginal(.{ L, a }); +} +fn prealloc_newthread_fn(L: u32) callconv(hook.cc.fastcall) u32 { + preAllocCheck(L); + return prealloc_newthread.callOriginal(.{L}); +} + +fn installPreAllocHooks() void { + // Disabled for diagnosis -- re-enable one at a time + _ = prealloc_pushcclosure.attach(0x6F3920, &prealloc_pushcclosure_fn); + //_ = prealloc_pushlstring.attach(0x6F3840, &prealloc_pushlstring_fn); + //_ = prealloc_createtable.attach(0x6F3C90, &prealloc_createtable_fn); + //_ = prealloc_pushvfstring.attach(0x6F38C0, &prealloc_pushvfstring_fn); + //_ = prealloc_concat.attach(0x6F44E0, &prealloc_concat_fn); + //_ = prealloc_newuserdata.attach(0x6F4560, &prealloc_newuserdata_fn); + //_ = prealloc_newthread.attach(0x6F6B10, &prealloc_newthread_fn); +} + +fn removePreAllocHooks() void { + prealloc_pushcclosure.detach(); + prealloc_pushlstring.detach(); + prealloc_createtable.detach(); + prealloc_pushvfstring.detach(); + // prealloc_pushfstring not installed + prealloc_pushstring.detach(); + prealloc_concat.detach(); + prealloc_newuserdata.detach(); + prealloc_newthread.detach(); } // --- lua_setfenv (0x6F40D0) --- @@ -1913,6 +2047,35 @@ pub fn luaZGCStats(L: lua.State) callconv(hook.cc.fastcall) i32 { return 9; } +/// ZGCTune(stepsize, stepmul, pause) -- set GC tuning parameters. +/// stepsize: bytes between incremental steps (default 1024, 5.1 GCSTEPSIZE) +/// stepmul: work multiplier per step (default 200, 5.1 gcstepmul) +/// pause: cycle threshold factor (default 200 = 2x, 5.1 gcpause) +/// Returns previous values. +pub fn luaZGCTune(L: lua.State) callconv(hook.cc.fastcall) i32 { + // Return old values + lua.pushnumber(L, @floatFromInt(gcstepsize)); + lua.pushnumber(L, @floatFromInt(gcstepmul)); + lua.pushnumber(L, @floatFromInt(gcpause)); + + // Set new values if provided + const top = lua.gettop(L); + if (top >= 1) { + const v = lua.tonumber(L, 1); + if (v > 0) gcstepsize = @intFromFloat(v); + } + if (top >= 2) { + const v = lua.tonumber(L, 2); + if (v > 0) gcstepmul = @intFromFloat(v); + } + if (top >= 3) { + const v = lua.tonumber(L, 3); + if (v > 0) gcpause = @intFromFloat(v); + } + + return 3; +} + // ============================================================================= // Installation // ============================================================================= @@ -1984,6 +2147,10 @@ pub fn installHooks() void { installed += 1; } // Note: lua_pushcclosure doesn't need a barrier (new closures are WHITE) + // Pre-allocation GC check (5.1 pattern: luaC_checkGC before allocation). + // Ensures pending GC work completes before half-initialized objects exist. + installPreAllocHooks(); + // Compiler barriers: covered by proto grayagain in propagatemark. // Direct hooks removed -- proto is re-traversed in atomic phase instead. installSetupvalPatch(); @@ -2012,6 +2179,7 @@ pub fn removeHooks() void { lua_close_hook.detach(); luac_link_hook.detach(); lua_replace_hook.detach(); + removePreAllocHooks(); string_create_hook.detach(); upval_create_hook.detach(); setmeta_hook.detach(); diff --git a/src/main.zig b/src/main.zig index 642c3ec..07dec6a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -24,7 +24,7 @@ const build_opts = struct { const silicon = @import("build_options").enable_silicon; const weirdperformance = @import("build_options").enable_weirdperformance; const superweirdo = @import("build_options").enable_superweirdo; - const luavm = @import("build_options").enable_luavm; + const luagc = @import("build_options").enable_luagc; }; // Conditional module imports @@ -47,7 +47,7 @@ const ssemaths = if (build_opts.ssemaths) @import("ssemaths/ssemaths.zig") else const silicon = if (build_opts.silicon) @import("silicon/silicon.zig") else struct {}; const weirdperformance = if (build_opts.weirdperformance) @import("weirdperformance/weirdperformance.zig") else struct {}; const superweirdo = if (build_opts.superweirdo) @import("superweirdo/superweirdo.zig") else struct {}; -const luavm = if (build_opts.luavm) @import("luavm/luavm.zig") else struct {}; +const luagc_mod = if (build_opts.luagc) @import("luagc/luagc.zig") else struct {}; const module_active = @import("module_active.zig"); @@ -127,6 +127,10 @@ fn registerLuaFunctions() void { registerFunction("ResetAddOnCPUUsage", @intFromPtr(&addonperf.luaResetAddOnCPUUsage)); registerFunction("GetScriptCPUUsage", @intFromPtr(&addonperf.luaGetScriptCPUUsage)); } + if (build_opts.luagc and luagc_mod.isActive()) { + registerFunction("ZGCStats", @intFromPtr(&luagc_mod.luaZGCStats)); + registerFunction("ZGCTune", @intFromPtr(&luagc_mod.luaZGCTune)); + } if (build_opts.worldmarkers and markers.isActive()) { // User-facing functions stay global registerFunction("WorldMarker", @intFromPtr(&markers.luaWorldMarker)); @@ -788,7 +792,7 @@ const modules = [_]ModuleHooks{ if (build_opts.silicon) .{ .name = silicon.module_name, .install = silicon.installHooks, .remove = silicon.removeHooks, .is_active = silicon.isActive } else .{}, if (build_opts.weirdperformance) .{ .name = weirdperformance.module_name, .install = weirdperformance.installHooks, .remove = weirdperformance.removeHooks, .is_active = weirdperformance.isActive, .remove_on_shutdown = true } else .{}, if (build_opts.superweirdo) .{ .name = superweirdo.module_name, .install = superweirdo.installHooks, .remove = superweirdo.removeHooks, .is_active = superweirdo.isActive } else .{}, - if (build_opts.luavm) .{ .name = luavm.module_name, .install = luavm.installHooks, .remove = luavm.removeHooks, .is_active = luavm.isActive } else .{}, + if (build_opts.luagc) .{ .name = luagc_mod.module_name, .install = luagc_mod.installHooks, .remove = luagc_mod.removeHooks, .is_active = luagc_mod.isActive } else .{}, }; fn shutdownDetour() callconv(hook.cc.stdcall) void { diff --git a/src/weirdperformance/AllocBench/AllocBench.lua b/src/weirdperformance/AllocBench/AllocBench.lua index 395686a..e61abf0 100644 --- a/src/weirdperformance/AllocBench/AllocBench.lua +++ b/src/weirdperformance/AllocBench/AllocBench.lua @@ -185,7 +185,382 @@ SlashCmdList["GCCOMPARE"] = function(args) end -- ========================================================================= --- zluagen stats frame — visible by default, center screen, movable +-- /gclife [duration]: mixed-age residency stress test +-- Simulates realistic addon memory patterns: +-- - Long-lived config tables (persist entire test) +-- - Medium-lived frame data (replaced every few seconds) +-- - Short-lived closures and strings (created/discarded each frame) +-- - Weak table caches (entries die and get cleared) +-- - Nested tables with upvalue captures +-- Runs per-frame for [duration] seconds (default 20), reports timing. +-- ========================================================================= + +local gclife_frame = CreateFrame("Frame") +local gclife_running = false +local gclife_end_time = 0 +local gclife_frame_times = {} +local gclife_frame_count = 0 +local gclife_last_time = 0 + +-- Long-lived: addon config tables (never replaced during test) +local gclife_config = {} +-- Medium-lived: frame data (replaced every ~2 seconds) +local gclife_frames = {} +local gclife_frames_age = 0 +-- Weak cache: entries die when not referenced elsewhere +local gclife_weak_cache = setmetatable({}, { __mode = "v" }) +-- Closure registry: closures with upvalue captures +local gclife_closures = {} + +local function gclife_build_config() + -- 2000 addon configs with 20 callbacks each = 40k closures + 2k tables + for i = 1, 2000 do + gclife_config["addon_" .. i] = { + enabled = true, + settings = { scale = 1.0, alpha = 0.8, x = i, y = i * 2 }, + history = {}, + callbacks = {}, + data = {}, + } + local cfg = gclife_config["addon_" .. i] + -- Deep nested data (3 levels) + for j = 1, 10 do + cfg.data[j] = { + entries = {}, + meta = { created = i * 1000 + j, tag = "d" .. j }, + } + for k = 1, 5 do + cfg.data[j].entries[k] = { id = k, val = "item_" .. i .. "_" .. j .. "_" .. k } + end + end + -- 20 callback closures capturing different upvalues + for j = 1, 20 do + local slot = j + local data = cfg.data[math.mod(j, 10) + 1] + cfg.callbacks[j] = function() return cfg.settings.scale * slot + data.meta.created end + end + end +end + +local function gclife_rebuild_frames() + -- 800 frames with 20 children each = 16k tables + 1600 closures + gclife_frames = {} + for i = 1, 800 do + gclife_frames[i] = { + name = "Frame" .. i, + children = {}, + scripts = {}, + visible = math.mod(i, 3) ~= 0, + textures = {}, + } + for j = 1, 20 do + gclife_frames[i].children[j] = { + parent = gclife_frames[i], + id = i * 100 + j, + text = string.format("child_%d_%d", i, j), + tooltip = "Tooltip for " .. i .. ":" .. j, + } + end + -- Texture references (string heavy) + for j = 1, 5 do + gclife_frames[i].textures[j] = "Interface\\Icons\\Icon_" .. (i * 5 + j) + end + local f = gclife_frames[i] + f.scripts.OnUpdate = function() return f.name end + f.scripts.OnClick = function() f.visible = not f.visible end + end + gclife_frames_age = 0 +end + +local function gclife_per_frame() + -- Heavy short-lived: 500 event tables + strings each frame + local temps = {} + for i = 1, 500 do + local msg = "event_" .. math.random(10000) .. "_" .. GetTime() + temps[i] = { + type = "CHAT", + msg = msg, + time = GetTime(), + source = "Player" .. math.random(40), + args = { math.random(100), "spell_" .. math.random(500), math.random() > 0.5 }, + } + end + + -- 200 short-lived closures (simulates addon callbacks, iterators) + for i = 1, 200 do + local val = math.random(1000) + local name = "cb_" .. i + local fn = function() return val * 2 + string.len(name) end + fn() + end + + -- Heavy weak cache churn: 100 entries, some pinned + for i = 1, 100 do + local key = "cache_" .. math.random(2000) + local entry = { + data = math.random(), + ts = GetTime(), + payload = { math.random(), "str_" .. math.random(1000) }, + } + gclife_weak_cache[key] = entry + if math.mod(i, 10) == 0 then + local cfg_key = "addon_" .. math.random(2000) + if gclife_config[cfg_key] then + table.insert(gclife_config[cfg_key].history, entry) + if table.getn(gclife_config[cfg_key].history) > 20 then + table.remove(gclife_config[cfg_key].history, 1) + end + end + end + end + + -- 50 upvalue mutations on existing closures + for i = 1, 50 do + local idx = math.random(2000) + local cfg = gclife_config["addon_" .. idx] + if cfg then + cfg.settings.scale = math.random() * 2 + cfg.settings.alpha = math.random() + end + end + + -- String interning pressure: lookup existing + create new + for i = 1, 200 do + local _ = "addon_" .. math.random(2000) -- hits intern table + local _ = string.format("fmt_%d_%d_%s", i, math.random(100), GetTime()) + end + + -- Medium-lived replacement every ~2 seconds + gclife_frames_age = gclife_frames_age + 1 + if gclife_frames_age > 120 then + gclife_rebuild_frames() + end + + temps = nil +end + +local function gclife_start(args) + local duration = tonumber(args) or 20 + + DEFAULT_CHAT_FRAME:AddMessage(string.format( + "|cff00ff00GC Life|r: building mixed-age heap, running %d seconds...", duration)) + + -- Build long-lived structures + gclife_config = {} + gclife_build_config() + gclife_rebuild_frames() + gclife_weak_cache = setmetatable({}, { __mode = "v" }) + + -- Reset counters + gclife_frame_times = {} + gclife_frame_count = 0 + gclife_last_time = debugprofilestop and 0 or GetTime() + gclife_end_time = GetTime() + duration + gclife_running = true + + debugprofilestart() + gclife_frame:SetScript("OnUpdate", function() + if not gclife_running then return end + if GetTime() >= gclife_end_time then + gclife_running = false + gclife_frame:SetScript("OnUpdate", nil) + + -- Report results + local n = gclife_frame_count + if n == 0 then return end + table.sort(gclife_frame_times) + local p50 = gclife_frame_times[math.floor(n * 0.5)] or 0 + local p95 = gclife_frame_times[math.floor(n * 0.95)] or 0 + local p99 = gclife_frame_times[math.floor(n * 0.99)] or 0 + local max_t = gclife_frame_times[n] or 0 + + DEFAULT_CHAT_FRAME:AddMessage(string.format( + "|cff00ff00GC Life|r: %d frames over %ds", n, duration)) + DEFAULT_CHAT_FRAME:AddMessage(string.format( + " p50=|cffffd700%.1f ms|r p95=|cffffd700%.1f ms|r p99=|cffffd700%.1f ms|r max=|cffff0000%.1f ms|r", + p50, p95, p99, max_t)) + + if type(ZGCStats) == "function" then + local total, mark, sweep, gray, freed, fstr, ph, stepus, atomicus = ZGCStats() + DEFAULT_CHAT_FRAME:AddMessage(string.format( + " GC: step_max=%dus atomic=%dus last_mark=%d last_sweep=%d", + stepus, atomicus, mark, sweep)) + end + + -- Cleanup + gclife_config = {} + gclife_frames = {} + gclife_closures = {} + gclife_weak_cache = nil + collectgarbage() + return + end + + debugprofilestart() + gclife_per_frame() + local elapsed = debugprofilestop() + gclife_frame_count = gclife_frame_count + 1 + table.insert(gclife_frame_times, elapsed) + end) +end + +SLASH_GCLIFE1 = "/gclife" +SlashCmdList["GCLIFE"] = gclife_start + +-- ========================================================================= +-- /gcsweep [seconds_per_combo]: auto-tune GC parameters +-- Runs /gclife workload with many (stepsize, stepmul, pause) combinations, +-- reports p99 frame time for each. Finds the empirical best defaults. +-- ========================================================================= + +local gcsweep_combos = { + -- { stepsize, stepmul, pause, label } + -- Vary stepsize + { 256, 200, 200, "step=256" }, + { 512, 200, 200, "step=512" }, + { 1024, 200, 200, "step=1024 (5.1 default)" }, + { 2048, 200, 200, "step=2048" }, + { 4096, 200, 200, "step=4096" }, + { 8192, 200, 200, "step=8192" }, + -- Vary stepmul + { 1024, 100, 200, "mul=100" }, + { 1024, 200, 200, "mul=200 (5.1 default)" }, + { 1024, 400, 200, "mul=400" }, + { 1024, 800, 200, "mul=800" }, + -- Vary pause + { 1024, 200, 150, "pause=150 (1.5x)" }, + { 1024, 200, 200, "pause=200 (2x, default)" }, + { 1024, 200, 300, "pause=300 (3x)" }, + { 1024, 200, 400, "pause=400 (4x)" }, + -- Promising combos + { 2048, 400, 200, "step=2k mul=400" }, + { 4096, 400, 200, "step=4k mul=400" }, + { 2048, 200, 150, "step=2k pause=1.5x" }, + { 512, 100, 150, "small+aggressive" }, + { 4096, 800, 300, "big+lazy" }, +} + +local gcsweep_running = false +local gcsweep_combo_idx = 0 +local gcsweep_seconds = 0 +local gcsweep_results = {} +local gcsweep_frame = CreateFrame("Frame") + +-- Reuse gclife's per-frame workload +local gcsweep_end_time = 0 +local gcsweep_frame_times = {} +local gcsweep_frame_count = 0 + +local function gcsweep_run_next() + gcsweep_combo_idx = gcsweep_combo_idx + 1 + if gcsweep_combo_idx > table.getn(gcsweep_combos) then + -- All done, report results + gcsweep_running = false + gcsweep_frame:SetScript("OnUpdate", nil) + + -- Restore defaults + ZGCTune(1024, 200, 200) + + DEFAULT_CHAT_FRAME:AddMessage("|cff00ff00GC Sweep|r: results (sorted by p99):") + table.sort(gcsweep_results, function(a, b) return a.p99 < b.p99 end) + for _, r in ipairs(gcsweep_results) do + local color = "|cff00ff00" + if r.p99 > 2.0 then color = "|cffff0000" + elseif r.p99 > 1.0 then color = "|cffffff00" + end + DEFAULT_CHAT_FRAME:AddMessage(string.format( + " %sp50=%.1f p95=%.1f p99=%.1f max=%.1f|r %s", + color, r.p50, r.p95, r.p99, r.max, r.label)) + end + local best = gcsweep_results[1] + if best then + DEFAULT_CHAT_FRAME:AddMessage(string.format( + "|cff00ff00BEST|r: %s (p99=%.1f ms)", best.label, best.p99)) + end + + -- Cleanup + gclife_config = {} + gclife_frames = {} + gclife_closures = {} + gclife_weak_cache = nil + collectgarbage() + return + end + + local combo = gcsweep_combos[gcsweep_combo_idx] + ZGCTune(combo[1], combo[2], combo[3]) + + DEFAULT_CHAT_FRAME:AddMessage(string.format( + "|cff00ff00GC Sweep|r [%d/%d]: %s ...", + gcsweep_combo_idx, table.getn(gcsweep_combos), combo[4])) + + -- Rebuild heap fresh for each combo + gclife_config = {} + gclife_build_config() + gclife_rebuild_frames() + gclife_weak_cache = setmetatable({}, { __mode = "v" }) + collectgarbage() + + gcsweep_frame_times = {} + gcsweep_frame_count = 0 + gcsweep_end_time = GetTime() + gcsweep_seconds + + gcsweep_frame:SetScript("OnUpdate", function() + if not gcsweep_running then return end + + if GetTime() >= gcsweep_end_time then + -- Collect results for this combo + local n = gcsweep_frame_count + if n > 0 then + table.sort(gcsweep_frame_times) + local combo = gcsweep_combos[gcsweep_combo_idx] + table.insert(gcsweep_results, { + label = combo[4], + p50 = gcsweep_frame_times[math.floor(n * 0.5)] or 0, + p95 = gcsweep_frame_times[math.floor(n * 0.95)] or 0, + p99 = gcsweep_frame_times[math.floor(n * 0.99)] or 0, + max = gcsweep_frame_times[n] or 0, + }) + end + -- Next combo + gcsweep_run_next() + return + end + + debugprofilestart() + gclife_per_frame() + local elapsed = debugprofilestop() + gcsweep_frame_count = gcsweep_frame_count + 1 + table.insert(gcsweep_frame_times, elapsed) + end) +end + +SLASH_GCSWEEP1 = "/gcsweep" +SlashCmdList["GCSWEEP"] = function(args) + if gcsweep_running then + DEFAULT_CHAT_FRAME:AddMessage("|cffff0000GC Sweep already running|r") + return + end + if type(ZGCTune) ~= "function" then + DEFAULT_CHAT_FRAME:AddMessage("|cffff0000ZGCTune not available|r") + return + end + + gcsweep_seconds = tonumber(args) or 5 + gcsweep_combo_idx = 0 + gcsweep_results = {} + gcsweep_running = true + + DEFAULT_CHAT_FRAME:AddMessage(string.format( + "|cff00ff00GC Sweep|r: testing %d combos, %ds each (%ds total)...", + table.getn(gcsweep_combos), gcsweep_seconds, + table.getn(gcsweep_combos) * gcsweep_seconds)) + + gcsweep_run_next() +end + +-- ========================================================================= +-- zluagen stats frame -- visible by default, center screen, movable -- ZGCStats() is registered by zluagen on its first GC tick. -- Returns: cycles_total, cycles_major, cycles_minor, -- mark_steps_last, sweep_steps_last, diff --git a/src/weirdperformance/GC_WRITE_BARRIER.md b/src/weirdperformance/GC_WRITE_BARRIER.md new file mode 100644 index 0000000..8e51a14 --- /dev/null +++ b/src/weirdperformance/GC_WRITE_BARRIER.md @@ -0,0 +1,67 @@ +# Lua 5.0 GC Write Barrier in lua_vm_execute + +## Location + +`lua_vm_execute` (0x6F8720), the main VM interpreter loop. +The barrier appears after every TValue copy (~20 sites in the function). + +## Globals + +- `0xCEEAC0` -- GC barrier value (current white marker / gc object pointer) +- `0xCEEAC4` -- GC barrier flag (non-zero = barrier is active) + +## Pattern (from disassembly) + +After copying a 16-byte TValue (type_tag, gc_ptr, value_lo, value_hi): + +```asm +mov eax, [dst + 4] ; eax = copied gc_ptr field +test eax, eax +jz skip ; NULL gc_ptr -> no barrier needed +cmp dword ptr [0xCEEAC4], 0 +jz skip ; barrier disabled -> skip +mov [0xCEEAC0], eax ; mark: write gc_ptr into barrier global +skip: +``` + +## What it does + +The gc_ptr field at TValue+0x04 holds a pointer to a GC-managed object +(string, table, closure, userdata) or NULL for non-collectable types +(number, boolean, nil, lightuserdata). + +When a TValue is copied (MOVE, GETGLOBAL, GETTABLE, LOADK, etc.), the +barrier checks: +1. Is the copied value a GC object? (gc_ptr != NULL) +2. Is the write barrier active? (flag at 0xCEEAC4 != 0) +3. If both true, write the gc_ptr to the barrier global at 0xCEEAC0 + +This is Lua 5.0's incremental GC write barrier. It tracks which GC objects +have been moved/copied so the collector knows which objects are reachable +from newly-written locations. The barrier global accumulates the "last written" +gc object -- the actual GC uses this to avoid rescanning the full root set. + +## Opcodes that trigger the barrier + +Every opcode that writes a TValue to a register or table slot: +- MOVE (op 0) +- LOADK (op 1) -- loads constant, has gc_ptr for string constants +- GETUPVAL (op 4) +- GETGLOBAL (op 5) +- GETTABLE (op 6) +- SETGLOBAL (op 7) -- barrier on the table side +- SETTABLE (op 9) +- NEWTABLE (op 10) +- SELF (op 11) +- CONCAT (op 21) +- CLOSURE (op 30) +- FORLOOP (op 23) -- number only, but barrier still present + +## Relevance to luagc module + +Our luagc module (in weirdperformance) hooks `lua_gc_step` (0x6FAE00). +Understanding the barrier globals is useful for: +- Knowing when/how often the barrier fires during heavy addon activity +- Potentially batching barrier writes if we ever replace the GC step +- The flag at 0xCEEAC4 could be used to temporarily disable the barrier + during bulk operations (dangerous -- must re-enable before GC runs) diff --git a/src/weirdperformance/luastr.zig b/src/weirdperformance/luastr.zig new file mode 100644 index 0000000..9480e2b --- /dev/null +++ b/src/weirdperformance/luastr.zig @@ -0,0 +1,393 @@ +//! luastr -- Lua string pattern optimizations. +//! +//! Two production wins, both measured in real gameplay (see git history for +//! before/after numbers): +//! +//! 1. pattern_match_class (0x7FC930): replace Wine CRT's locale dispatch +//! (~300+ cyc/call) with a direct 256-entry table lookup (~30 cyc/call). +//! The table is built once at install time by calling the original +//! matchclass for every (byte, class_letter) pair, capturing the exact +//! Wine/locale classification. Zero correctness risk. +//! +//! 2. string.find / string.gfind-iterator / string.gsub literal pre-filter: +//! extract the longest mandatory literal run from each pattern and short- +//! circuit with memmem if absent from the subject. Kills the O(n^2) +//! backtracking that addon combat log parsers (MSBT, WIM, BigWigs, etc.) +//! inflict on every chat message. Measured impact: +//! - eliminates ~22-40% of real pattern-engine cycles per frame window +//! - ~5-17% overhead of its own (inside the real matcher's budget) +//! - net_if_live consistently positive across all measurement windows +//! +//! Correctness: the prefilter is correct by construction -- we only short- +//! circuit when a REQUIRED literal substring is PROVABLY absent from the +//! subject. Any parse ambiguity, complex construct (%b/%f), or run too short +//! to beat MIN_LITERAL_LEN falls through to the real matcher. + +const std = @import("std"); +const hook = @import("zhook"); +const lua = @import("../lua.zig"); +const logging = @import("../logging.zig"); + +var log: logging.Logger = .{}; + +// ============================================================================= +// pattern_match_class (0x7FC930) -- 256-entry table lookup +// __fastcall(ECX=character, EDX=class_letter) -> u32 (1=match, 0=no match) +// +// Wine sets DAT_00831710 >= 2, forcing every %d/%a/%s/etc. check through +// CheckCharacterProperties() instead of a direct table lookup. A single +// %d+ match against a 100-char string calls this 100+ times per frame. +// ============================================================================= + +// Runtime 256-entry classification table. Each entry is a u16 bitfield: +// bit 0 ( 1): %a -- alpha +// bit 1 ( 2): %c -- control +// bit 2 ( 4): %d -- digit +// bit 3 ( 8): %l -- lowercase +// bit 4 ( 16): %p -- punct +// bit 5 ( 32): %s -- space +// bit 6 ( 64): %u -- uppercase +// bit 7 (128): %w -- alnum +// bit 8 (256): %x -- xdigit +// +// Populated at installHooks time by calling the original pattern_match_class +// for every (c=0..255, class_letter) pair. Captures the exact Wine/locale +// classification -- correct for ASCII, UTF-8 multibyte bytes, Windows codepage, +// and any locale. 256 * 9 = 2304 calls at load, zero overhead thereafter. +var runtime_table: [256]u16 = [_]u16{0} ** 256; + +fn buildRuntimeTable() void { + const OrigFn = fn (u32, u32) callconv(hook.cc.fastcall) u32; + const orig: *const OrigFn = @ptrFromInt(0x7FC930); + const Entry = struct { cl: u32, bit: u16 }; + const entries = [_]Entry{ + .{ .cl = 'a', .bit = 1 }, + .{ .cl = 'c', .bit = 2 }, + .{ .cl = 'd', .bit = 4 }, + .{ .cl = 'l', .bit = 8 }, + .{ .cl = 'p', .bit = 16 }, + .{ .cl = 's', .bit = 32 }, + .{ .cl = 'u', .bit = 64 }, + .{ .cl = 'w', .bit = 128 }, + .{ .cl = 'x', .bit = 256 }, + }; + for (entries) |e| { + for (0..256) |c| { + if (orig(@intCast(c), e.cl) != 0) { + runtime_table[c] |= e.bit; + } + } + } +} + +// Reimplementation of matchclass() using the runtime-populated table. +// %z/%Z are special-cased (zero-byte semantics, not locale-dependent). +// Unknown class letters fall through to literal match, same as the original. +fn matchClassImpl(c: u32, cl: u32) u32 { + const lc: u32 = cl | 0x20; // ASCII tolower (safe for A-Z only) + const is_upper_cl: bool = cl >= 'A' and cl <= 'Z'; + + // %z / %Z: zero-byte check -- not a locale concept + if (lc == 'z') { + const m = c == 0; + return if (if (is_upper_cl) !m else m) 1 else 0; + } + + const bit: u16 = switch (lc) { + 'a' => 1, + 'c' => 2, + 'd' => 4, + 'l' => 8, + 'p' => 16, + 's' => 32, + 'u' => 64, + 'w' => 128, + 'x' => 256, + else => return if (cl == c) 1 else 0, + }; + + const entry: u16 = if (c < 256) runtime_table[c] else 0; + const m = (entry & bit) != 0; + return if (if (is_upper_cl) !m else m) 1 else 0; +} + +const MatchClassFn = fn (u32, u32) callconv(hook.cc.fastcall) u32; +var matchclass_hook: hook.Detour(MatchClassFn) = .{}; + +fn matchclassDetour(c: u32, cl: u32) callconv(hook.cc.fastcall) u32 { + // Preserve ESI/EDI/EBX across our pure-Zig body -- Wine's pattern matcher + // assumes these are callee-saved across the call. + asm volatile ("" ::: .{ .esi = true, .edi = true, .ebx = true }); + return matchClassImpl(c, cl); +} + +// ============================================================================= +// String function literal pre-filter +// ============================================================================= + +const LUA_TSTRING: i32 = 4; + +// Minimum literal run length before we bother running memmem. Shorter runs +// produce too many false-positive passes on typical chat/tooltip subjects to +// be worth the scan cost. 4 is empirically a good tradeoff. +const MIN_LITERAL_LEN: usize = 4; + +const FnTag = enum { find, match, gsub }; + +// Pseudo-index constants for the gfind iterator closure body (0x7FCFF0). +// Subject/pattern/position live in upvalues, not arg slots. +// LUA_GLOBALSINDEX = -10001, upvalue(i) = LUA_GLOBALSINDEX - i +const GFIND_SUBJ_IDX: i32 = -10002; // upvalue 1 +const GFIND_PAT_IDX: i32 = -10003; // upvalue 2 + +/// lua_strlen(L, idx) -> size_t. Address 0x6F36E0. Ghidra and earlier project +/// notes mislabeled this as `lua_tolstring`; disassembly shows it returns +/// `*(TString + 0xC)` which is the `len` field of Lua 5.0's TString header, +/// NOT a data pointer. 2-arg fastcall, ECX=L EDX=idx, `RET` (no stack cleanup). +fn luaStrLen(L: lua.State, idx: i32) usize { + const f: *const fn (lua.State, i32) callconv(hook.cc.fastcall) usize = @ptrFromInt(0x6F36E0); + return f(L, idx); +} + +/// Return the longest run of mandatory literal bytes in a Lua 5.0 pattern, +/// as a slice into `pat` itself. Zero-copy: no scratch buffer, no escape +/// translation, no out-parameter. +/// +/// Tradeoff: runs break at any `%X` escape item (even escaped punctuation +/// like `%.`). The literal `foo%.bar` yields best run "foo" or "bar" rather +/// than "foo.bar". For prefilter correctness this is fine -- any mandatory +/// literal substring being provably absent from the subject disproves the +/// pattern. A shorter run means slightly weaker filtering, never a wrong +/// answer. +/// +/// Parser notes: +/// - `+`, `-`, `*`, `?` at a fresh item-parse position are plain literal +/// bytes, matching Lua 5.0's own matcher. Example: `%d+-%d+` parses as +/// `%d+` class+quantifier, then `-` as literal dash, then `%d+`. +/// - `(` and `)` are zero-width capture delimiters; they break runs without +/// consuming a literal byte. +/// - `[set]` scans past the matching `]` as a non-literal item. +/// - `%b...` and `%f[...]` are complex; we bail. +fn longestLiteral(pat: []const u8) []const u8 { + var best_start: usize = 0; + var best_len: usize = 0; + var cur_start: usize = 0; + var cur_len: usize = 0; + + var i: usize = 0; + // Leading `^` is an anchor, skip it -- the rest still defines the literal. + if (pat.len > 0 and pat[0] == '^') i = 1; + + while (i < pat.len) { + const c = pat[i]; + var item_bytes: usize = 1; + var is_literal: bool = true; + + switch (c) { + '(', ')' => { + // Zero-width capture delim: break run, advance 1, no quantifier. + if (cur_len > best_len) { + best_start = cur_start; + best_len = cur_len; + } + cur_len = 0; + i += 1; + continue; + }, + '.', '$' => { + is_literal = false; + }, + '[' => { + // Scan past matching ]. + var j = i + 1; + if (j < pat.len and pat[j] == '^') j += 1; + if (j < pat.len and pat[j] == ']') j += 1; // first ] inside set is literal + while (j < pat.len and pat[j] != ']') : (j += 1) { + if (pat[j] == '%' and j + 1 < pat.len) j += 1; + } + if (j >= pat.len) break; // unbalanced -- bail + is_literal = false; + item_bytes = j + 1 - i; + }, + '%' => { + if (i + 1 >= pat.len) break; // trailing % -- bail + const n = pat[i + 1]; + if (n == 'b' or n == 'B' or n == 'f' or n == 'F') break; // complex + is_literal = false; + item_bytes = 2; + }, + ']' => break, // unbalanced close-bracket -- bail + else => {}, // plain literal byte (including +/-/*/? at item position) + } + + // Peek for a quantifier modifying this item. + const next_i = i + item_bytes; + var quant: u8 = 0; + if (next_i < pat.len) { + const q = pat[next_i]; + if (q == '+' or q == '*' or q == '-' or q == '?') quant = q; + } + + if (is_literal) { + if (quant == '*' or quant == '-' or quant == '?') { + // Optional -- doesn't contribute, break run. + if (cur_len > best_len) { + best_start = cur_start; + best_len = cur_len; + } + cur_len = 0; + } else { + // No quant or `+`: byte required at least once. + if (cur_len == 0) cur_start = i; + cur_len += 1; + if (quant == '+') { + // Required once, but run breaks: subject may contain extra + // repeats of this byte, so the next pattern byte isn't + // guaranteed contiguous in the subject. + if (cur_len > best_len) { + best_start = cur_start; + best_len = cur_len; + } + cur_len = 0; + } + } + } else { + // Non-literal item: break run. + if (cur_len > best_len) { + best_start = cur_start; + best_len = cur_len; + } + cur_len = 0; + } + + i = next_i + @intFromBool(quant != 0); + } + + // Finalize trailing run. + if (cur_len > best_len) { + best_start = cur_start; + best_len = cur_len; + } + + return pat[best_start..][0..best_len]; +} + +/// Returns nresults if the call was short-circuited (pattern provably can't +/// match), or null if the caller should proceed to the real matcher. +/// +/// subj_idx/pat_idx choose where to read subject and pattern from. For +/// string.find/gsub these are arg slots 1 and 2. For the string.gfind +/// iterator closure (0x7FCFF0) they are upvalue pseudo-indices -10002/-10003. +fn tryPrefilter(L: lua.State, tag: FnTag, subj_idx: i32, pat_idx: i32) ?u32 { + // Require string type (not number) to avoid triggering number -> string + // coercion side effects inside a detour. typeOf returns LUA_TNONE on + // invalid indices, which cleanly falls through. + if (lua.typeOf(L, subj_idx) != LUA_TSTRING) return null; + if (lua.typeOf(L, pat_idx) != LUA_TSTRING) return null; + + const subj_ptr_sent = lua.tostring(L, subj_idx) orelse return null; + const pat_ptr_sent = lua.tostring(L, pat_idx) orelse return null; + const subj_len = luaStrLen(L, subj_idx); + const pat_len = luaStrLen(L, pat_idx); + if (pat_len == 0 or subj_len == 0) return null; + + const subj_ptr: [*]const u8 = @ptrCast(subj_ptr_sent); + const pat_ptr: [*]const u8 = @ptrCast(pat_ptr_sent); + const pat_slice = pat_ptr[0..pat_len]; + + const lit = longestLiteral(pat_slice); + if (lit.len < MIN_LITERAL_LEN) return null; + + const subj_slice = subj_ptr[0..subj_len]; + if (std.mem.indexOf(u8, subj_slice, lit) != null) return null; + + // Literal provably absent -- pattern cannot match. Short-circuit. + switch (tag) { + .find, .match => { + lua.pushnil(L); + return 1; + }, + .gsub => { + // string.gsub on no match returns (subject_unchanged, 0) + lua.pushvalue(L, 1); + lua.pushnumber(L, 0); + return 2; + }, + } +} + +// ============================================================================= +// Hooked functions +// +// 0x7FC3B0 lua_string_find -- reads subject/pattern from stack args 1, 2 +// 0x7FCFF0 lua_string_gfind_iter -- the gfind iterator closure body; reads +// subject/pattern/position from upvalues. Ghidra mislabels this as +// "lua_string_match"; Lua 5.0 does NOT have string.match (5.1+). +// 0x7FD0E0 lua_string_gsub -- reads subject/pattern from stack args 1, 2 +// ============================================================================= + +const StringFn = fn (u32, u32) callconv(hook.cc.fastcall) u32; +var strfind_hook: hook.Detour(StringFn) = .{}; +var strmatch_hook: hook.Detour(StringFn) = .{}; +var strgsub_hook: hook.Detour(StringFn) = .{}; + +fn strfindDetour(state: u32, edx: u32) callconv(hook.cc.fastcall) u32 { + asm volatile ("" ::: .{ .esi = true, .edi = true, .ebx = true }); + const L: lua.State = @ptrFromInt(state); + if (tryPrefilter(L, .find, 1, 2)) |nres| return nres; + return strfind_hook.callOriginal(.{ state, edx }); +} + +fn strmatchDetour(state: u32, edx: u32) callconv(hook.cc.fastcall) u32 { + asm volatile ("" ::: .{ .esi = true, .edi = true, .ebx = true }); + const L: lua.State = @ptrFromInt(state); + if (tryPrefilter(L, .match, GFIND_SUBJ_IDX, GFIND_PAT_IDX)) |nres| return nres; + return strmatch_hook.callOriginal(.{ state, edx }); +} + +fn strgsubDetour(state: u32, edx: u32) callconv(hook.cc.fastcall) u32 { + asm volatile ("" ::: .{ .esi = true, .edi = true, .ebx = true }); + const L: lua.State = @ptrFromInt(state); + if (tryPrefilter(L, .gsub, 1, 2)) |nres| return nres; + return strgsub_hook.callOriginal(.{ state, edx }); +} + +// ============================================================================= +// Install / Remove +// ============================================================================= + +pub fn install() u32 { + log = logging.Logger.open("luastr", .console); + + buildRuntimeTable(); + log.print(" matchclass: runtime table built\n"); + + var installed: u32 = 0; + if (matchclass_hook.attach(0x7FC930, &matchclassDetour) == .ok) { + log.print(" matchclass: table lookup active\n"); + installed += 1; + } + if (strfind_hook.attach(0x7FC3B0, &strfindDetour) == .ok) { + log.print(" string.find: literal prefilter active\n"); + installed += 1; + } + if (strmatch_hook.attach(0x7FCFF0, &strmatchDetour) == .ok) { + log.print(" string.gfind: literal prefilter active\n"); + installed += 1; + } + if (strgsub_hook.attach(0x7FD0E0, &strgsubDetour) == .ok) { + log.print(" string.gsub: literal prefilter active\n"); + installed += 1; + } + + log.print("luastr: active\n"); + return installed; +} + +pub fn remove() void { + strgsub_hook.detach(); + strmatch_hook.detach(); + strfind_hook.detach(); + matchclass_hook.detach(); + log.close(); +} diff --git a/src/weirdperformance/luavm.zig b/src/weirdperformance/luavm.zig new file mode 100644 index 0000000..0dc9ba4 --- /dev/null +++ b/src/weirdperformance/luavm.zig @@ -0,0 +1,103 @@ +//! luavm -- Lua VM hotspot optimizations. +//! +//! luaS_newlstr (0x6F9D00, 1.35% CPU): hash pre-check before memcmp. +//! ~40% win vs original (validated by prior A/B). Production-only, no instrumentation. + +const hook = @import("zhook"); +const logging = @import("../logging.zig"); + +var log: logging.Logger = .{}; + +// ============================================================================= +// luaS_newlstr (0x6F9D00) +// __fastcall(ECX=lua_State*, EDX=str_ptr, stack=len) -> TString* +// RET 0x4 +// ============================================================================= + +const NewLStrFn = fn (u32, u32, u32) callconv(hook.cc.fastcall) u32; +var newlstr_hook: hook.Detour(NewLStrFn) = .{}; + +fn luaCreateStringObject(state: u32, str_ptr: u32, len: u32, hash_val: u32) u32 { + return hook.call( + fn (u32, u32, u32, u32) callconv(hook.cc.fastcall) u32, + 0x6F9D90, + .{ state, str_ptr, len, hash_val }, + ); +} + +fn newlstrDetour(state: u32, str_ptr: u32, len: u32) callconv(hook.cc.fastcall) u32 { + asm volatile ("" ::: .{ .esi = true, .edi = true, .ebx = true }); + return newlstrImpl(state, str_ptr, len); +} + +fn newlstrImpl(state: u32, str_ptr: u32, len: u32) u32 { + const str: [*]const u8 = @ptrFromInt(str_ptr); + var h: u32 = len; + const step: u32 = (len >> 5) + 1; + var l1: u32 = len; + while (l1 >= step) { + const c: u32 = str[l1 - 1]; + h = h ^ (c +% (h << 5) +% (h >> 2)); + l1 -= step; + } + + const global_state: u32 = hook.readMem(u32, state + 0x10); + const strt_hash: u32 = hook.readMem(u32, global_state + 0x04); + const strt_size: u32 = hook.readMem(u32, global_state + 0x0C); + + const bucket: u32 = h & (strt_size - 1); + var ts: u32 = hook.readMem(u32, strt_hash + bucket * 4); + + while (ts != 0) { + const ts_len: u32 = hook.readMem(u32, ts + 0x0C); + if (ts_len == len) { + const ts_hash: u32 = hook.readMem(u32, ts + 0x08); + if (ts_hash == h) { + if (len == 0 or strEqual(str_ptr, ts + 0x10, len)) { + return ts; + } + } + } + ts = hook.readMem(u32, ts); + } + + return luaCreateStringObject(state, str_ptr, len, h); +} + +fn strEqual(a_ptr: u32, b_ptr: u32, len: u32) bool { + const a: [*]const u8 = @ptrFromInt(a_ptr); + const b: [*]const u8 = @ptrFromInt(b_ptr); + + var i: u32 = 0; + while (i + 4 <= len) : (i += 4) { + const va = @as(*align(1) const u32, @ptrCast(a + i)).*; + const vb = @as(*align(1) const u32, @ptrCast(b + i)).*; + if (va != vb) return false; + } + while (i < len) : (i += 1) { + if (a[i] != b[i]) return false; + } + return true; +} + +// ============================================================================= +// Install / Remove +// ============================================================================= + +pub fn install() u32 { + log = logging.Logger.open("luavm", .console); + + var installed: u32 = 0; + if (newlstr_hook.attach(0x6F9D00, &newlstrDetour) == .ok) { + log.print(" newlstr: hash pre-check active\n"); + installed += 1; + } + + log.print("luavm: active\n"); + return installed; +} + +pub fn remove() void { + newlstr_hook.detach(); + log.close(); +} diff --git a/src/weirdperformance/weirdperformance.zig b/src/weirdperformance/weirdperformance.zig index cf9a595..a70cd3b 100644 --- a/src/weirdperformance/weirdperformance.zig +++ b/src/weirdperformance/weirdperformance.zig @@ -44,6 +44,8 @@ const cull_sse = @import("cull_sse.zig"); const silicon_sse = @import("silicon_sse.zig"); const luaalloc = @import("luaalloc.zig"); const luagc = @import("luagc.zig"); +const luastr = @import("luastr.zig"); +const luavm = @import("luavm.zig"); const renderParticleSprites_SSE = particle_sse.renderParticleSprites_SSE; const resetParticleCache = particle_sse.resetParticleCache; @@ -326,9 +328,14 @@ pub fn installHooks() void { // libdeflate inflate replacement if (inflate_hook.install()) installed += 1; - // Lua slab allocator + GC: owned by luagc module. - // installed += luaalloc.install(); - // installed += luagc.install(); + // Lua slab allocator replacement + installed += luaalloc.install(); + + installed += luagc.install(); + + installed += luastr.install(); + + installed += luavm.install(); } pub fn lateInit() void { @@ -338,6 +345,8 @@ pub fn lateInit() void { pub fn removeHooks() void { if (g_is_hook_owner) { + luavm.remove(); + luastr.remove(); luaalloc.dumpStats(); luagc.dumpStats(); filecache.remove();