Dusk/WeirdUtils
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

inflate: logging reveals 100% zlib (type 0x02), standard 78 9C header

Browse Source 
All 93K decompression calls during loading are pure zlib (type 0x02).
Header bytes 78 9C confirm standard zlib-wrapped deflate. No PKWare,
bzip2, or ADPCM observed.

Volume: 170MB compressed → 380MB decompressed in 2.9s during loading,
~5K calls/period during gameplay.

Format: [0x02] [zlib_stream...] — straightforward for libdeflate.
Previous crashes were from reading out_size AFTER original modified it
and from writing to out_buf unsafely.
This commit is contained in:
MarcelineVQ
2026-03-24 03:49:44 -07:00
parent 59ef452f22
commit 2b11ae5cf1
2 changed files with 81 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/performance/inflate_hook.zig
+73 -52
View File
@@ -11,6 +11,18 @@ const FC: std.builtin.CallingConvention = .{ .x86_fastcall = .{} };
const SC: std.builtin.CallingConvention = .{ .x86_stdcall = .{} };
const std = @import("std");
// Game memory allocator: ReallocMemory(NULL, size, ...) = malloc, FreeMemory(ptr, ...) = free
const gameRealloc: *const fn (u32, u32, u32, u32, u32) callconv(SC) ?*anyopaque = @ptrFromInt(0x646320);
const gameFreeMemory: *const fn (u32, u32, u32) callconv(SC) u32 = @ptrFromInt(0x646430);
fn gameAlloc(size: u32) ?*anyopaque {
return gameRealloc(0, size, 0, 0, 0);
}
fn gameFree(ptr: *anyopaque) void {
_ = gameFreeMemory(@intFromPtr(ptr), 0, 0);
}
// libdeflate C API (linked from static lib)
extern fn libdeflate_alloc_decompressor() ?*anyopaque;
extern fn libdeflate_free_decompressor(?*anyopaque) void;
@@ -39,7 +51,15 @@ var orig_total_cycles: u64 = 0;
var fast_total_cycles: u64 = 0;
var call_count: u64 = 0;
var mismatch_count: u64 = 0;
var success_count: u64 = 0;
var total_bytes: u64 = 0;
var total_in_bytes: u64 = 0;
// Per-type counters: index by compression mask bit
var type_counts: [8]u64 = .{0} ** 8; // bits 0-7
// Per raw type byte counter (full byte value)
var raw_type_counts: [256]u32 = .{0} ** 256;
// Track first-seen header bytes per type for format identification
var type_headers_logged: [256]bool = .{false} ** 256;
inline fn rdtsc() u64 {
var lo: u32 = undefined;
@@ -63,68 +83,49 @@ inline fn rdtsc() u64 {
const DecompressFn = fn (u32, u32, u32, u32, u32) callconv(SC) u32;
pub var decompress_hook: hook_lib.Detour(DecompressFn) = .{};
pub fn decompressDetour(out_buf: u32, out_size_ptr: u32, in_buf: u32, in_size_ptr: u32, flags: u32) callconv(SC) u32 {
// Read input/output sizes
const in_size = @as(*const u32, @ptrFromInt(in_size_ptr)).*;
pub fn decompressDetour(out_buf: u32, out_size_ptr: u32, in_buf: u32, in_size: u32, flags: u32) callconv(SC) u32 {
// param2 = &outSize (pointer), param4 = inSize (value, NOT pointer)
const out_size = @as(*const u32, @ptrFromInt(out_size_ptr)).*;
const in_ptr: [*]const u8 = @ptrFromInt(in_buf);
// Run original first (this is the authoritative result)
const t0 = rdtsc();
const ret = decompress_hook.callOriginal(.{ out_buf, out_size_ptr, in_buf, in_size_ptr, flags });
const ret = decompress_hook.callOriginal(.{ out_buf, out_size_ptr, in_buf, in_size, flags });
const orig_cycles = rdtsc() - t0;
// Only compare if original succeeded, we have a decompressor, and buffer is non-trivial
if (ret != 0 and decompressor != null and in_size > 16 and out_size > 0) {
// Read the compression type byte — first byte of compressed data
const comp_type = in_ptr[0];
orig_total_cycles +|= orig_cycles;
call_count +|= 1;
// Only test zlib/deflate streams (type byte has bit patterns for different compressors)
// The actual data starts at byte 1 (after the type byte)
// For now, test on all calls regardless of type — libdeflate handles raw deflate
_ = comp_type;
// Allocate temp buffer for libdeflate output
// Only process if original succeeded and buffers are valid
if (ret != 0 and decompressor != null and in_size > 2 and out_size > 0) {
const actual_out_size = @as(*const u32, @ptrFromInt(out_size_ptr)).*;
if (actual_out_size > 0 and actual_out_size < 4 * 1024 * 1024) {
// Use VirtualAlloc or stack for small buffers
var fast_buf: [65536]u8 = undefined;
const use_buf: [*]u8 = if (actual_out_size <= 65536) &fast_buf else return ret;
var actual_out: usize = 0;
const t1 = rdtsc();
const ld_ret = libdeflate_deflate_decompress(
decompressor,
in_ptr + 1, // skip type byte
in_size - 1,
use_buf,
actual_out_size,
&actual_out,
);
const fast_cycles = rdtsc() - t1;
orig_total_cycles +|= orig_cycles;
fast_total_cycles +|= fast_cycles;
call_count +|= 1;
// First byte = compression type bitmask:
// 0x01 = Huffman/sparse 0x02 = zlib 0x10 = bzip2
// 0x20 = PKWare DCL 0x40 = ADPCM mono 0x80 = ADPCM stereo
const comp_type = in_ptr[0];
total_bytes +|= actual_out_size;
// Check if libdeflate succeeded and produced same output
if (ld_ret == 0 and actual_out == actual_out_size) {
// Compare outputs
const orig_out: [*]const u8 = @ptrFromInt(out_buf);
var match = true;
for (0..actual_out_size) |i| {
if (orig_out[i] != use_buf[i]) {
match = false;
break;
}
}
if (!match) mismatch_count +|= 1;
// Track type distribution
inline for (0..8) |bit| {
if ((comp_type & (@as(u8, 1) << @intCast(bit))) != 0)
type_counts[bit] +|= 1;
}
// Log first-seen header for each compression type
if (!type_headers_logged[comp_type]) {
type_headers_logged[comp_type] = true;
log.fmt(" type=0x{x:0>2} in_size={d} out_size={d} hdr:", .{ comp_type, in_size, actual_out_size });
// Dump first 16 bytes after type byte
const dump_len = @min(in_size - 1, 16);
for (0..dump_len) |i| {
log.fmt(" {x:0>2}", .{in_ptr[1 + i]});
}
log.print("\n");
}
raw_type_counts[comp_type] +|= 1;
total_in_bytes +|= in_size;
}
} else {
orig_total_cycles +|= orig_cycles;
call_count +|= 1;
}
return ret;
@@ -133,19 +134,39 @@ pub fn decompressDetour(out_buf: u32, out_size_ptr: u32, in_buf: u32, in_size_pt
pub fn dumpStats() void {
if (call_count == 0) return;
const MS_DIV: u64 = 3_000_000;
log.fmt("inflate: {d} calls, {d}MB, orig={d}ms fast={d}ms ({d} mismatches)\n", .{
const type_names = [8][]const u8{ "huff", "zlib", "b2", "b3", "bzip", "pkw", "adpcm1", "adpcm2" };
log.fmt("inflate: {d} calls, in={d}KB out={d}KB, orig={d}ms\n", .{
call_count,
total_bytes / (1024 * 1024),
total_in_bytes / 1024,
total_bytes / 1024,
orig_total_cycles / MS_DIV,
fast_total_cycles / MS_DIV,
mismatch_count,
});
// Type bits distribution
log.print(" bits:");
for (type_names, 0..) |name, i| {
if (type_counts[i] > 0) {
log.fmt(" {s}={d}", .{ name, type_counts[i] });
}
}
log.print("\n");
// Raw type byte distribution (shows exact combos used)
log.print(" raw:");
for (raw_type_counts, 0..) |count, i| {
if (count > 0) {
log.fmt(" 0x{x:0>2}={d}", .{ i, count });
}
}
log.print("\n");
// Reset
orig_total_cycles = 0;
fast_total_cycles = 0;
call_count = 0;
success_count = 0;
mismatch_count = 0;
total_bytes = 0;
total_in_bytes = 0;
for (&type_counts) |*c| c.* = 0;
for (&raw_type_counts) |*c| c.* = 0;
}
pub fn install(logger: logging.Logger) bool {
src/performance/performance.zig
+8 -7
View File
@@ -21,18 +21,19 @@ const inflate_hook = @import("inflate_hook.zig");
pub const module_name: [*:0]const u8 = "performance";
// Provide malloc/free for libdeflate's default allocator (linked without libc).
// Use game's own SMemAlloc/SMemFree (Storm memory manager) which are always available.
// SMemAlloc at 0x6464B0: __stdcall(size, filename_str, line, flags) → ptr
// SMemFree at 0x646430: __stdcall(ptr, filename_str, flags) → void
const gameMalloc: *const fn (u32, u32, u32, u32) callconv(.{ .x86_stdcall = .{} }) ?*anyopaque = @ptrFromInt(0x6464B0);
const gameFree: *const fn (u32, u32, u32) callconv(.{ .x86_stdcall = .{} }) void = @ptrFromInt(0x646430);
// Use game's Storm memory manager:
// ReallocMemory (0x646320): __stdcall(ptr, size, filename, line, flags) → ptr
// When ptr=NULL, acts as malloc via AllocateBufferWithPowerOfTwo.
// FreeMemory (0x646430): __stdcall(ptr, filename, line) → always returns 1
const gameRealloc: *const fn (u32, u32, u32, u32, u32) callconv(.{ .x86_stdcall = .{} }) ?*anyopaque = @ptrFromInt(0x646320);
const gameFree: *const fn (u32, u32, u32) callconv(.{ .x86_stdcall = .{} }) u32 = @ptrFromInt(0x646430);
export fn malloc(size: usize) callconv(.c) ?*anyopaque {
return gameMalloc(@intCast(size), 0, 0, 0);
return gameRealloc(0, @intCast(size), 0, 0, 0);
}
export fn free(ptr: ?*anyopaque) callconv(.c) void {
if (ptr) |p| gameFree(@intFromPtr(p), 0, 0);
if (ptr) |p| _ = gameFree(@intFromPtr(p), 0, 0);
}
var g_mutex: ?*anyopaque = null;