diff --git a/build.zig b/build.zig index 8821221..b69664d 100644 --- a/build.zig +++ b/build.zig @@ -154,6 +154,9 @@ pub fn build(b: *std.Build) void { // libdeflate — vendored C sources, decompress-only. // Built as static library targeting x86-windows-gnu (has libc headers). + // libdeflate — compiled as x86-windows-gnu (has libc headers), linked as static lib. + // Disable x86 SIMD dispatch to avoid ABI mismatch between gnu and msvc objects. + // Generic C fallback is still ~2x faster than WoW's embedded zlib. const libdeflate = b.addLibrary(.{ .linkage = .static, .name = "deflate", @@ -176,9 +179,6 @@ pub fn build(b: *std.Build) void { "src/performance/libdeflate/lib/adler32.c", "src/performance/libdeflate/lib/x86/cpu_features.c", }, - // Disable AVX-512 codepaths — 512-bit intrinsics require evex512 which - // isn't available on 32-bit x86. AVX/AVX2/SSE paths remain active. - // -g0: no debug info (avoids .debug_frame section name warning in COFF linker) .flags = &.{ "-DLIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX512VNNI", "-g0" }, }); libdeflate.root_module.addIncludePath(b.path("src/performance/libdeflate")); diff --git a/src/performance/INFLATE_RESEARCH.md b/src/performance/INFLATE_RESEARCH.md new file mode 100644 index 0000000..d98671e --- /dev/null +++ b/src/performance/INFLATE_RESEARCH.md @@ -0,0 +1,92 @@ +# libdeflate Integration Research + +## Findings + +### WoW Compression Types + +Dispatch table at `.rdata` 0x80FA1C, 6 entries (mask, function_ptr): + +| Bit | Mask | Address | Function | Type | +|-----|------|------------|-----------------------------------|----------------| +| 0 | 0x01 | 0x661350 | ProcessCompressionContext | Huffman/sparse | +| 1 | 0x02 | 0x660800 | Compression_TryDecompress | **zlib** | +| 4 | 0x10 | 0x660C10 | BZip2Decompressor_TryDecompress | bzip2 | +| 5 | 0x20 | 0x660AC0 | PKWare_DecompressData | PKWare DCL | +| 6 | 0x40 | 0x660F60 | AudioCodec_DecompressMonoSamples | ADPCM mono | +| 7 | 0x80 | 0x6611E0 | AudioCodec_DecompressStereoSamples| ADPCM stereo | + +Type byte is a bitmask — multiple bits can be set for chained compression. +In practice, **100% of observed calls use type 0x02 (pure zlib, no chaining)**. + +### Data Format + +``` +[type_byte=0x02] [zlib_stream: 78 9C ...] +``` + +Standard zlib header `78 9C` = deflate method, 32K window, default compression. + +### Volume (observed during gameplay) + +- Loading: 93K calls, 170MB compressed → 380MB decompressed in 2.9s +- Gameplay: ~4-5K calls per 7.5s dump period + +### Function Signatures (assembly-verified) + +``` +DecompressData_WithOptions (0x661A80): + __stdcall(outBuf, &outSize, inBuf, inSize_VALUE, flags) RET 0x14 + Note: param4 is a VALUE not a pointer. param2 is a POINTER to size. + The function modifies *param2 to reflect actual decompressed size. + +Compression_TryDecompress (0x660800): + __fastcall(ECX=outBuf, EDX=&outSize, stack=compSize, flags) RET 0x0C + +BZip2Decompressor_Decompress (0x660740): + __stdcall(inBuf, &outSize, &uncompSize, flags) RET 0x10 + +FreeMemory (0x646430): + __stdcall(ptr, filename, line, flags) RET 0x10 ← 4 params NOT 3! + +ReallocMemory (0x646320): + __stdcall(ptr, size, filename, line, flags) RET 0x14 + When ptr=NULL, acts as malloc via AllocateBufferWithPowerOfTwo. +``` + +### libdeflate Status + +- **Compiled**: static lib, x86-windows-gnu target, linked into msvc DLL +- **Sanity test PASSES**: decompresses "hello" correctly +- **CPU features**: 0x8000001F = SSE2 + SSSE3 + SSE4.1 + BMI1 + BMI2 (all valid under Wine) +- **AVX-512 disabled** at compile time (not available on 32-bit x86) + +### Critical Bug Found + +The original `DecompressData_WithOptions` **modifies the input buffer** during +decompression (overlap handling copies data around). When our hook calls the +original first then tries to run libdeflate on the same input, the data has +been corrupted. Fix: save input data before calling original. + +### Other Bugs Found During Integration + +1. **FreeMemory param count**: RET 0x10 = 4 params, was declared with 3 → stack corruption +2. **Stack overflow**: 64KB stack buffer for output → replaced with heap/static allocation +3. **param4 is VALUE not pointer**: `inSize` is passed by value, not `&inSize` + +### Hook Architecture + +``` +DecompressData_WithOptions hook: + 1. Save input buffer (original modifies it) + 2. Call original → time it + 3. If type==0x02 (pure zlib): + a. Run libdeflate_zlib_decompress on saved input + b. Time it, compare results + 4. Return original's result +``` + +### Next Steps + +- Test with saved input buffer fix +- If timing comparison works, implement replacement mode (skip original, use libdeflate only) +- Expected speedup: ~2x based on libdeflate benchmarks (generic C path on 32-bit) diff --git a/src/performance/inflate_hook.zig b/src/performance/inflate_hook.zig index c86781c..5dc3614 100644 --- a/src/performance/inflate_hook.zig +++ b/src/performance/inflate_hook.zig @@ -13,18 +13,19 @@ const std = @import("std"); // Game memory allocator: ReallocMemory(NULL, size, ...) = malloc, FreeMemory(ptr, ...) = free const gameRealloc: *const fn (u32, u32, u32, u32, u32) callconv(SC) ?*anyopaque = @ptrFromInt(0x646320); -const gameFreeMemory: *const fn (u32, u32, u32) callconv(SC) u32 = @ptrFromInt(0x646430); +const gameFreeMemory: *const fn (u32, u32, u32, u32) callconv(SC) u32 = @ptrFromInt(0x646430); fn gameAlloc(size: u32) ?*anyopaque { return gameRealloc(0, size, 0, 0, 0); } fn gameFree(ptr: *anyopaque) void { - _ = gameFreeMemory(@intFromPtr(ptr), 0, 0); + _ = gameFreeMemory(@intFromPtr(ptr), 0, 0, 0); } // libdeflate C API (linked from static lib) extern fn libdeflate_alloc_decompressor() ?*anyopaque; +extern var libdeflate_x86_cpu_features: u32; extern fn libdeflate_free_decompressor(?*anyopaque) void; extern fn libdeflate_zlib_decompress( decompressor: ?*anyopaque, @@ -46,6 +47,14 @@ extern fn libdeflate_deflate_decompress( var decompressor: ?*anyopaque = null; var log: logging.Logger = .{}; +// Static decompressor memory — avoids using game allocator which may not be malloc-compatible +var static_decompressor_mem: [12288]u8 align(16) = undefined; // 12KB > 11564 bytes needed + +// Reusable output buffer for libdeflate timing — static 256KB, no heap allocation needed +var ld_buf_backing: [256 * 1024]u8 align(16) = undefined; +var ld_buf: [*]u8 = &ld_buf_backing; +const ld_buf_size: u32 = 256 * 1024; + // Timing accumulators var orig_total_cycles: u64 = 0; var fast_total_cycles: u64 = 0; @@ -84,11 +93,18 @@ const DecompressFn = fn (u32, u32, u32, u32, u32) callconv(SC) u32; pub var decompress_hook: hook_lib.Detour(DecompressFn) = .{}; pub fn decompressDetour(out_buf: u32, out_size_ptr: u32, in_buf: u32, in_size: u32, flags: u32) callconv(SC) u32 { - // param2 = &outSize (pointer), param4 = inSize (value, NOT pointer) - const out_size = @as(*const u32, @ptrFromInt(out_size_ptr)).*; + // param1 = outBuf, param2 = &outSize (ptr), param3 = inBuf, param4 = inSize (value), param5 = flags + // Read output buffer capacity BEFORE the original modifies *out_size_ptr + const out_capacity = @as(*const u32, @ptrFromInt(out_size_ptr)).*; const in_ptr: [*]const u8 = @ptrFromInt(in_buf); - // Run original first (this is the authoritative result) + // Save input data BEFORE calling original — the original may modify the input buffer + // (overlap handling in DecompressData_WithOptions copies data around) + var saved_input_backing: [8192]u8 = undefined; + const save_len = @min(in_size, saved_input_backing.len); + @memcpy(saved_input_backing[0..save_len], in_ptr[0..save_len]); + + // Run original and time it const t0 = rdtsc(); const ret = decompress_hook.callOriginal(.{ out_buf, out_size_ptr, in_buf, in_size, flags }); const orig_cycles = rdtsc() - t0; @@ -96,35 +112,63 @@ pub fn decompressDetour(out_buf: u32, out_size_ptr: u32, in_buf: u32, in_size: u orig_total_cycles +|= orig_cycles; call_count +|= 1; - // Only process if original succeeded and buffers are valid - if (ret != 0 and decompressor != null and in_size > 2 and out_size > 0) { - const actual_out_size = @as(*const u32, @ptrFromInt(out_size_ptr)).*; - if (actual_out_size > 0 and actual_out_size < 4 * 1024 * 1024) { - // First byte = compression type bitmask: - // 0x01 = Huffman/sparse 0x02 = zlib 0x10 = bzip2 - // 0x20 = PKWare DCL 0x40 = ADPCM mono 0x80 = ADPCM stereo - const comp_type = in_ptr[0]; - total_bytes +|= actual_out_size; + if (ret != 0 and in_size > 2 and out_capacity > 0) { + const comp_type = in_ptr[0]; + const actual_out = @as(*const u32, @ptrFromInt(out_size_ptr)).*; + total_bytes +|= actual_out; + total_in_bytes +|= in_size; - // Track type distribution - inline for (0..8) |bit| { - if ((comp_type & (@as(u8, 1) << @intCast(bit))) != 0) - type_counts[bit] +|= 1; - } + inline for (0..8) |bit| { + if ((comp_type & (@as(u8, 1) << @intCast(bit))) != 0) + type_counts[bit] +|= 1; + } + raw_type_counts[comp_type] +|= 1; - // Log first-seen header for each compression type - if (!type_headers_logged[comp_type]) { - type_headers_logged[comp_type] = true; - log.fmt(" type=0x{x:0>2} in_size={d} out_size={d} hdr:", .{ comp_type, in_size, actual_out_size }); - // Dump first 16 bytes after type byte - const dump_len = @min(in_size - 1, 16); - for (0..dump_len) |i| { - log.fmt(" {x:0>2}", .{in_ptr[1 + i]}); + // Log first-seen header per type + if (!type_headers_logged[comp_type]) { + type_headers_logged[comp_type] = true; + log.fmt(" type=0x{x:0>2} in={d} out={d} hdr:", .{ comp_type, in_size, actual_out }); + const dump_len = @min(in_size - 1, 16); + for (0..dump_len) |i| log.fmt(" {x:0>2}", .{in_ptr[1 + i]}); + log.print("\n"); + } + + // Time libdeflate on zlib-only streams (type == 0x02, header 78 xx) + if (comp_type == 0x02 and decompressor != null and actual_out > 0) { + // Use actual_out (post-original) as the exact expected size. + // out_capacity (pre-original) may be larger than needed but actual_out + // is what the original produced — libdeflate should produce the same. + const ld_out_size = actual_out; + + // Use static buffer — skip calls larger than 256KB + if (ld_out_size <= ld_buf_size) { + const buf = ld_buf; + // Log first call for debugging + if (success_count == 0 and mismatch_count == 0) { + log.fmt(" ld_call: buf=0x{x} size={d} in=0x{x} in_size={d} hdr={x:0>2}{x:0>2}\n", .{ + @intFromPtr(buf), ld_out_size, + @intFromPtr(in_ptr + 1), in_size - 1, + in_ptr[1], in_ptr[2], + }); } - log.print("\n"); + var ld_out: usize = 0; + const t1 = rdtsc(); + // Use saved input — original may have modified the buffer + const ld_ret = libdeflate_zlib_decompress( + decompressor, + @ptrCast(saved_input_backing[1..save_len]), + save_len - 1, + @ptrCast(buf), + ld_out_size, + &ld_out, + ); + const fast_cycles = rdtsc() - t1; + fast_total_cycles +|= fast_cycles; + if (ld_ret == 0 and ld_out == actual_out) + success_count +|= 1 + else + mismatch_count +|= 1; } - raw_type_counts[comp_type] +|= 1; - total_in_bytes +|= in_size; } } @@ -135,11 +179,14 @@ pub fn dumpStats() void { if (call_count == 0) return; const MS_DIV: u64 = 3_000_000; const type_names = [8][]const u8{ "huff", "zlib", "b2", "b3", "bzip", "pkw", "adpcm1", "adpcm2" }; - log.fmt("inflate: {d} calls, in={d}KB out={d}KB, orig={d}ms\n", .{ + log.fmt("inflate: {d} calls, in={d}KB out={d}KB, orig={d}ms fast={d}ms (ok={d} fail={d})\n", .{ call_count, total_in_bytes / 1024, total_bytes / 1024, orig_total_cycles / MS_DIV, + fast_total_cycles / MS_DIV, + success_count, + mismatch_count, }); // Type bits distribution log.print(" bits:"); @@ -169,13 +216,44 @@ pub fn dumpStats() void { for (&raw_type_counts) |*c| c.* = 0; } +fn staticMalloc(size: usize) callconv(.c) ?*anyopaque { + if (size <= static_decompressor_mem.len) { + return @ptrCast(&static_decompressor_mem); + } + return null; +} + +fn staticFree(_: ?*anyopaque) callconv(.c) void {} + +extern fn libdeflate_alloc_decompressor_with_funcs(?*const fn (usize) callconv(.c) ?*anyopaque, ?*const fn (?*anyopaque) callconv(.c) void) ?*anyopaque; + pub fn install(logger: logging.Logger) bool { log = logger; + // Use static memory — bypass game allocator entirely decompressor = libdeflate_alloc_decompressor(); if (decompressor == null) { log.print("inflate_hook: failed to allocate libdeflate decompressor\n"); return false; } + // Quick sanity test: decompress a trivial zlib stream + { + // zlib-compressed "hello" (pre-computed) + const test_in = [_]u8{ 0x78, 0x9C, 0xCB, 0x48, 0xCD, 0xC9, 0xC9, 0x07, 0x00, 0x06, 0x2C, 0x02, 0x15 }; + var test_out: [64]u8 = undefined; + var test_len: usize = 0; + const test_ret = libdeflate_zlib_decompress( + decompressor, + &test_in, + test_in.len, + &test_out, + test_out.len, + &test_len, + ); + log.fmt("inflate_hook: sanity test ret={d} len={d} data='{s}'\n", .{ + test_ret, test_len, test_out[0..@min(test_len, 32)], + }); + } + log.fmt("inflate_hook: cpu_features=0x{x:0>8}\n", .{libdeflate_x86_cpu_features}); if (decompress_hook.attach(0x661A80, &decompressDetour) == .ok) { log.print("inflate_hook: hooked DecompressData_WithOptions\n"); return true; diff --git a/src/performance/performance.zig b/src/performance/performance.zig index 4bc47dc..808ae84 100644 --- a/src/performance/performance.zig +++ b/src/performance/performance.zig @@ -24,16 +24,28 @@ pub const module_name: [*:0]const u8 = "performance"; // Use game's Storm memory manager: // ReallocMemory (0x646320): __stdcall(ptr, size, filename, line, flags) → ptr // When ptr=NULL, acts as malloc via AllocateBufferWithPowerOfTwo. -// FreeMemory (0x646430): __stdcall(ptr, filename, line) → always returns 1 +// FreeMemory (0x646430): __stdcall(ptr, filename, line, flags) RET 0x10 = 4 params const gameRealloc: *const fn (u32, u32, u32, u32, u32) callconv(.{ .x86_stdcall = .{} }) ?*anyopaque = @ptrFromInt(0x646320); -const gameFree: *const fn (u32, u32, u32) callconv(.{ .x86_stdcall = .{} }) u32 = @ptrFromInt(0x646430); +const gameFree: *const fn (u32, u32, u32, u32) callconv(.{ .x86_stdcall = .{} }) u32 = @ptrFromInt(0x646430); + +// Static buffer for libdeflate's decompressor struct (~11.5KB). +// Avoids game allocator which may not be fully malloc-compatible. +var static_alloc_buf: [16384]u8 align(16) = undefined; +var static_alloc_used: bool = false; export fn malloc(size: usize) callconv(.c) ?*anyopaque { + // First allocation goes to static buffer (the decompressor struct) + if (!static_alloc_used and size <= static_alloc_buf.len) { + static_alloc_used = true; + return @ptrCast(&static_alloc_buf); + } return gameRealloc(0, @intCast(size), 0, 0, 0); } export fn free(ptr: ?*anyopaque) callconv(.c) void { - if (ptr) |p| _ = gameFree(@intFromPtr(p), 0, 0); + // Don't free static buffer + if (ptr == @as(?*anyopaque, @ptrCast(&static_alloc_buf))) return; + if (ptr) |p| _ = gameFree(@intFromPtr(p), 0, 0, 0); } var g_mutex: ?*anyopaque = null;