From 3ec7796cc10c849a686dcf6db957c8ca9b8dff85 Mon Sep 17 00:00:00 2001 From: MarcelineVQ Date: Tue, 24 Feb 2026 02:28:13 -0800 Subject: [PATCH] Fix model rendering stutter: eliminate dummy D3D9 device, defer vtable hooks The dummy device technique (Direct3DCreate9 + CreateDevice) corrupted the d3d9 proxy's internal state when run on the main rendering thread, causing model rendering to update at ~10fps while camera remained smooth. Replace dummy device with direct vtable read from the game's existing device via GxDevice global (0xC0ED38 + 0x38A8), sourced from UnitXP_SP3. Also defer D3D9 hook installation until first rendered frame (triggered from renderDrawDetour) to guarantee the device exists, and add forceD24S8IfNeeded() in EndScene to force a Reset with D24S8 stencil format since the deferred install misses the initial device Reset. Other changes carried from prior session: - model_hook.zig: native thiscall/naked detours (thunks eliminated) - tracker.zig: tracked_obj_count made pub --- src/outline/api.zig | 17 +++-- src/outline/d3d9_hook.zig | 152 +++++++++++++++++++++---------------- src/outline/model_hook.zig | 104 ++++++++++++++++++------- src/outline/tracker.zig | 2 +- 4 files changed, 173 insertions(+), 102 deletions(-) diff --git a/src/outline/api.zig b/src/outline/api.zig index 36809d2..7bb9e2d 100644 --- a/src/outline/api.zig +++ b/src/outline/api.zig @@ -8,17 +8,22 @@ const tracker = @import("tracker.zig"); const model_hook = @import("model_hook.zig"); const d3d9_hook = @import("d3d9_hook.zig"); -/// Install all outline hooks (model pipeline + D3D9 vtable). -/// Called from main.zig during DLL_PROCESS_ATTACH or engine init. +/// Install model hooks immediately. D3D9 hooks are deferred until the first +/// model hook fires (i.e. the game is actively rendering), because creating a +/// dummy D3D9 device during engine init corrupts the d3d9 proxy's state and +/// causes model rendering to stutter at ~10fps. pub fn init() bool { if (!model_hook.installHooks()) return false; - if (!d3d9_hook.installHooks()) { - model_hook.removeHooks(); - return false; - } return true; } +/// Called from the first model hook callback, once rendering is active. +/// Safe to create the dummy D3D9 device now — the game's real device is +/// fully initialised and the proxy's state is stable. +pub fn initD3D9Deferred() void { + _ = d3d9_hook.installHooks(); +} + /// Remove all outline hooks. Called during DLL_PROCESS_DETACH. pub fn cleanup() void { d3d9_hook.removeHooks(); diff --git a/src/outline/d3d9_hook.zig b/src/outline/d3d9_hook.zig index 6060801..3ed2c83 100644 --- a/src/outline/d3d9_hook.zig +++ b/src/outline/d3d9_hook.zig @@ -22,25 +22,6 @@ const sc: std.builtin.CallingConvention = .{ .x86_stdcall = .{} }; extern "kernel32" fn LoadLibraryA(name: [*:0]const u8) callconv(WINAPI) ?*anyopaque; extern "kernel32" fn GetProcAddress(module: *anyopaque, name: [*:0]const u8) callconv(WINAPI) ?*anyopaque; -extern "kernel32" fn GetModuleHandleA(name: ?[*:0]const u8) callconv(WINAPI) ?*anyopaque; -extern "user32" fn RegisterClassExA(wc: *const types.WNDCLASSEXA) callconv(WINAPI) u16; -extern "user32" fn CreateWindowExA( - exStyle: u32, - cls: [*:0]const u8, - title: [*:0]const u8, - style: u32, - x: i32, - y: i32, - w: i32, - h: i32, - parent: ?*anyopaque, - menu: ?*anyopaque, - inst: ?*anyopaque, - param: ?*anyopaque, -) callconv(WINAPI) ?*anyopaque; -extern "user32" fn DestroyWindow(hwnd: *anyopaque) callconv(WINAPI) i32; -extern "user32" fn UnregisterClassA(name: [*:0]const u8, inst: ?*anyopaque) callconv(WINAPI) i32; -extern "user32" fn DefWindowProcA(hwnd: ?*anyopaque, msg: u32, wp: usize, lp: usize) callconv(WINAPI) usize; // ============================================================================= // COM helper: read vtable pointer, call method by index @@ -71,6 +52,9 @@ var orig_reset: usize = 0; var d3d9_vtable: ?[*]usize = null; var hooks_installed: bool = false; +/// True until the first EndScene verifies (and if needed, forces) D24S8 format. +var need_force_reset: bool = true; + // ============================================================================= // Outline shader resources (created on first use) // ============================================================================= @@ -96,6 +80,14 @@ const D3DXAssembleShaderFn = *const fn ( // ============================================================================= fn hkEndScene(device: *anyopaque) callconv(sc) i32 { + // One-time: check if depth/stencil surface has stencil bits. + // Because D3D9 hooks are installed after engine init (deferred), we miss + // the initial Reset. If the surface lacks stencil, force a Reset now. + if (need_force_reset) { + need_force_reset = false; + forceD24S8IfNeeded(device); + } + // Per-frame: scan objects for outline tracking tracker.scanObjects(); @@ -403,6 +395,61 @@ fn deviceClear(dev: *anyopaque, flags: u32) void { _ = f(dev, 0, null, flags, 0, 1.0, 0); } +// ============================================================================= +// Force D24S8 depth/stencil on first EndScene (deferred hooks miss initial Reset) +// ============================================================================= + +fn hasStencilBits(fmt: u32) bool { + return fmt == types.D3DFMT_D24S8 or fmt == types.D3DFMT_D24FS8 or + fmt == types.D3DFMT_D24X4S4 or fmt == types.D3DFMT_D15S1; +} + +fn forceD24S8IfNeeded(device: *anyopaque) void { + // GetDepthStencilSurface(ppSurface) + var pDS: ?*anyopaque = null; + const getDS: *const fn (*anyopaque, *?*anyopaque) callconv(sc) i32 = + @ptrFromInt(vt(device)[types.VT.GetDepthStencilSurface]); + if (getDS(device, &pDS) < 0) return; + const ds = pDS orelse return; + defer comRelease(ds); + + // IDirect3DSurface9::GetDesc — vtable index 12 + // (IUnknown 0-2, IDirect3DResource9 3-10, GetContainer 11, GetDesc 12) + var desc: types.D3DSURFACE_DESC = .{}; + const getDesc: *const fn (*anyopaque, *types.D3DSURFACE_DESC) callconv(sc) i32 = + @ptrFromInt(vt(ds)[12]); + if (getDesc(ds, &desc) < 0) return; + + if (hasStencilBits(desc.Format)) return; // already good + + // Get current present parameters from swap chain 0 + // GetSwapChain(0, ppSwapChain) + var pSwap: ?*anyopaque = null; + const getSC: *const fn (*anyopaque, u32, *?*anyopaque) callconv(sc) i32 = + @ptrFromInt(vt(device)[types.VT.GetSwapChain]); + if (getSC(device, 0, &pSwap) < 0) return; + const swap = pSwap orelse return; + defer comRelease(swap); + + // IDirect3DSwapChain9::GetPresentParameters — vtable index 9 + var pp: types.D3DPRESENT_PARAMETERS = .{}; + const getPP: *const fn (*anyopaque, *types.D3DPRESENT_PARAMETERS) callconv(sc) i32 = + @ptrFromInt(vt(swap)[9]); + if (getPP(swap, &pp) < 0) return; + + // Force D24S8 and reset + pp.AutoDepthStencilFormat = types.D3DFMT_D24S8; + pp.EnableAutoDepthStencil = 1; + + // Release shaders before reset (device state lost) + releaseShaders(); + + // Call Reset through our hook (which also enforces D24S8) + const resetFn: *const fn (*anyopaque, *types.D3DPRESENT_PARAMETERS) callconv(sc) i32 = + @ptrFromInt(vt(device)[types.VT.Reset]); + _ = resetFn(device, &pp); +} + // ============================================================================= // Shader creation (loaded dynamically from d3dx9_43.dll) // ============================================================================= @@ -574,59 +621,32 @@ fn restoreVtableEntry(vtable_ptr: [*]usize, idx: usize, old_fn: usize) void { } // ============================================================================= -// D3D9 vtable discovery via dummy device +// D3D9 vtable discovery from game's existing device // ============================================================================= +// Reads the device pointer from WoW's GxDevice global instead of creating a +// dummy device. Creating a dummy device through the d3d9 proxy on the main +// thread corrupts the proxy's internal state and causes model rendering to +// stutter at ~10fps. Reading the existing device avoids this entirely. +// +// Source: UnitXP_SP3 — vanilla1121_gxDevice() / vanilla1121_d3dDevice() +// gxDevice = *(uint32_t*)0xC0ED38 +// d3dDevice = *(void**)(gxDevice + 0x38A8) + +const GX_DEVICE_PTR: usize = 0xC0ED38; +const GX_DEVICE_D3D_OFFSET: usize = 0x38A8; fn getD3D9VTable() ?[*]usize { - const d3d9_mod = LoadLibraryA("d3d9.dll") orelse return null; - const create9_raw = GetProcAddress(d3d9_mod, "Direct3DCreate9") orelse return null; + const gx_device = hook.readMem(u32, GX_DEVICE_PTR); + if (gx_device == 0) return null; - // Direct3DCreate9(D3D_SDK_VERSION=32) → IDirect3D9* - const Direct3DCreate9: *const fn (u32) callconv(sc) ?*anyopaque = @ptrCast(create9_raw); - const pD3D = Direct3DCreate9(32) orelse return null; + const d3d_device = hook.readMem(u32, gx_device + GX_DEVICE_D3D_OFFSET); + if (d3d_device == 0) return null; - // Register dummy window class - const inst = GetModuleHandleA(null); - var wc = types.WNDCLASSEXA{}; - wc.lpfnWndProc = &DefWindowProcA; - wc.hInstance = inst; - wc.lpszClassName = "WU_DummyD3D9"; - _ = RegisterClassExA(&wc); + // First dword of the COM object is the vtable pointer + const vtable_addr = hook.readMem(u32, d3d_device); + if (vtable_addr == 0) return null; - const hwnd = CreateWindowExA(0, "WU_DummyD3D9", "D", 0, 0, 0, 100, 100, null, null, inst, null) orelse { - comRelease(pD3D); - return null; - }; - - // IDirect3D9::CreateDevice — vtable[16] - // (self, Adapter, DeviceType, hFocusWindow, BehaviorFlags, pPP, ppDevice) - var pp = types.D3DPRESENT_PARAMETERS{}; - pp.Windowed = 1; - pp.SwapEffect = 1; // D3DSWAPEFFECT_DISCARD - pp.hDeviceWindow = @intFromPtr(hwnd); - - var pDevice: ?*anyopaque = null; - const createDev: *const fn (*anyopaque, u32, u32, *anyopaque, u32, *types.D3DPRESENT_PARAMETERS, *?*anyopaque) callconv(sc) i32 = - @ptrFromInt(vt(pD3D)[16]); - const hr = createDev(pD3D, 0, 1, hwnd, 0x20, &pp, &pDevice); // HAL, SOFTWARE_VERTEXPROCESSING - - if (hr < 0 or pDevice == null) { - comRelease(pD3D); - _ = DestroyWindow(hwnd); - _ = UnregisterClassA("WU_DummyD3D9", inst); - return null; - } - - // Read vtable — shared across all IDirect3DDevice9 instances - const vtable_ptr: [*]usize = @ptrFromInt(hook.readMem(u32, @intFromPtr(pDevice.?))); - - // Cleanup dummy objects - comRelease(pDevice.?); - comRelease(pD3D); - _ = DestroyWindow(hwnd); - _ = UnregisterClassA("WU_DummyD3D9", inst); - - return vtable_ptr; + return @ptrFromInt(vtable_addr); } // ============================================================================= diff --git a/src/outline/model_hook.zig b/src/outline/model_hook.zig index e810432..372c1a1 100644 --- a/src/outline/model_hook.zig +++ b/src/outline/model_hook.zig @@ -4,13 +4,28 @@ //! - CM2SceneRenderDraw — reorders batches so outline targets render first. //! - CM2Model_ManageRenderListNode — classifies models on render-list add. //! - CM2Scene_DrawBatchProjected — flags the DIP hook for outline rendering. +//! +//! Calling conventions: +//! - RenderDraw & ManageRender use callconv(.x86_thiscall) — direct native detours. +//! - DrawBatchProj uses a callconv(.naked) entry point because Zig 0.15 has a +//! codegen bug with callconv(.x86_fastcall) that generates wrong ret instructions +//! for functions with ≤2 register params. The naked wrapper bridges to a cdecl +//! implementation function. +const std = @import("std"); const hook = @import("hook"); +const api = @import("api.zig"); const o = @import("offsets.zig"); const types = @import("types.zig"); const tracker = @import("tracker.zig"); const wow = @import("wow.zig"); +// ============================================================================= +// Calling convention constants +// ============================================================================= + +const THISCALL = std.builtin.CallingConvention{ .x86_thiscall = .{} }; + // ============================================================================= // Hook state // ============================================================================= @@ -19,6 +34,10 @@ var render_draw_hook: hook.Hook = .{}; var manage_render_hook: hook.Hook = .{}; var draw_batch_hook: hook.Hook = .{}; +/// D3D9 hooks are deferred until the first model hook fires, because creating +/// a dummy D3D9 device during engine init corrupts the proxy's state. +var d3d9_deferred_pending: bool = true; + // ============================================================================= // Volatile flags shared with d3d9_hook (read by DIP hook) // ============================================================================= @@ -47,10 +66,14 @@ var reordered_indices: [MAX_REORDER]i32 = undefined; // CM2SceneRenderDraw hook // ============================================================================= // __thiscall(this, viewMatrix, batchData, batchIndices, batchCount) -// Thunked to __cdecl(ecx_this, edx_unused, viewMatrix, batchData, batchIndices, batchCount) +// Native thiscall detour — no thunk needed. -fn renderDrawDetour(this: u32, _edx: u32, view_matrix: u32, batch_data: u32, batch_indices: u32, batch_count: u32) callconv(.c) void { - _ = _edx; +fn renderDrawDetour(this: u32, view_matrix: u32, batch_data: u32, batch_indices: u32, batch_count: u32) callconv(THISCALL) void { + // One-time: install D3D9 hooks now that the game is actively rendering. + if (d3d9_deferred_pending) { + d3d9_deferred_pending = false; + api.initD3D9Deferred(); + } // If outlines disabled or nothing tracked, fast-path to original if (!tracker.enabled or !tracker.hasTargets() or batch_count == 0 or @@ -125,11 +148,9 @@ fn callOrigRenderDraw(this: u32, view_matrix: u32, batch_data: u32, batch_indice // CM2Model_ManageRenderListNode hook // ============================================================================= // __thiscall(model_ECX, addToList_stack) -// Thunked to __cdecl(ecx_model, edx_unused, addToList) - -fn manageRenderDetour(model: u32, _edx: u32, add_to_list: u32) callconv(.c) void { - _ = _edx; +// Native thiscall detour — no thunk needed. +fn manageRenderDetour(model: u32, add_to_list: u32) callconv(THISCALL) void { // Classify the model when it's being ADDED to the render list if (add_to_list == 1 and model != 0 and tracker.enabled and tracker.hasTargets()) { tracker.classifyModel(model); @@ -151,11 +172,34 @@ fn manageRenderDetour(model: u32, _edx: u32, add_to_list: u32) callconv(.c) void // CM2Scene_DrawBatchProjected hook // ============================================================================= // __fastcall(renderContext_ECX) -// Thunked to __cdecl(ecx_ctx, edx_unused) +// +// Uses a naked entry point because Zig 0.15's x86_fastcall codegen generates +// wrong ret instructions for functions with ≤2 register params. The naked +// function bridges fastcall → cdecl and calls the implementation function. -fn drawBatchProjDetour(ctx: u32, _edx: u32) callconv(.c) void { +fn drawBatchProjEntry() callconv(.naked) void { + // __fastcall(ECX): ECX = render context, 0 stack args. + // Bridge to cdecl: push edx + ecx as args, call impl, cleanup, ret. + asm volatile ( + \\push %%edx + \\push %%ecx + \\call *%%eax + \\add $8, %%esp + \\ret + : + : [_] "{eax}" (@intFromPtr(&drawBatchProjImpl)) + ); +} + +fn drawBatchProjImpl(ctx: u32, _edx: u32) callconv(.c) void { _ = _edx; + // Fast path: no tracking enabled or nothing tracked → just call original + if (!tracker.enabled or !tracker.hasTargets()) { + callOrigDrawBatch(ctx); + return; + } + const model_ptr = if (wow.isValidPtr(ctx +% @as(u32, @intCast(o.RENDER_CONTEXT_MODEL_OFFSET)))) hook.readMem(u32, ctx + o.RENDER_CONTEXT_MODEL_OFFSET) else @@ -181,7 +225,7 @@ fn drawBatchProjDetour(ctx: u32, _edx: u32) callconv(.c) void { current_model = 0; rendering_unit = false; - if (tracker.hasTargets() and model_ptr != 0) { + if (model_ptr != 0) { rendering_unit = tracker.isUnitModel(model_ptr); } @@ -205,28 +249,30 @@ fn callOrigDrawBatch(ctx: u32) void { // ============================================================================= pub fn installHooks() bool { - // CM2SceneRenderDraw — __thiscall, 4 stack args → cdecl thunk + // CM2SceneRenderDraw — native thiscall detour, no thunk needed. // Prologue is 9 bytes: PUSH EBP (1) + MOV EBP,ESP (2) + SUB ESP,0x80 (6). - // 6 bytes would cut SUB ESP,0x80 mid-instruction. - if (render_draw_hook.prepare(o.FN_CM2SCENE_RENDER_DRAW, 9, &.{})) { - const thunk = render_draw_hook.mem.? + 32; - _ = hook.buildFastcallToCdeclThunk(thunk, @intFromPtr(&renderDrawDetour), 4); - render_draw_hook.activate(@intFromPtr(thunk)); - } else return false; + if (!render_draw_hook.install( + o.FN_CM2SCENE_RENDER_DRAW, + 9, + @intFromPtr(&renderDrawDetour), + &.{}, + )) return false; - // ManageRenderListNode — __thiscall, 1 stack arg → cdecl thunk - if (manage_render_hook.prepare(o.FN_CM2MODEL_MANAGE_RENDER_LIST, 6, &.{})) { - const thunk = manage_render_hook.mem.? + 32; - _ = hook.buildFastcallToCdeclThunk(thunk, @intFromPtr(&manageRenderDetour), 1); - manage_render_hook.activate(@intFromPtr(thunk)); - } else return false; + // ManageRenderListNode — native thiscall detour, no thunk needed. + if (!manage_render_hook.install( + o.FN_CM2MODEL_MANAGE_RENDER_LIST, + 6, + @intFromPtr(&manageRenderDetour), + &.{}, + )) return false; - // DrawBatchProj — __fastcall, 0 stack args → cdecl thunk - if (draw_batch_hook.prepare(o.FN_DRAW_BATCH_PROJ, 6, &.{})) { - const thunk = draw_batch_hook.mem.? + 32; - _ = hook.buildFastcallToCdeclThunk(thunk, @intFromPtr(&drawBatchProjDetour), 0); - draw_batch_hook.activate(@intFromPtr(thunk)); - } else return false; + // DrawBatchProj — naked entry bridges fastcall → cdecl, no thunk needed. + if (!draw_batch_hook.install( + o.FN_DRAW_BATCH_PROJ, + 6, + @intFromPtr(&drawBatchProjEntry), + &.{}, + )) return false; return true; } diff --git a/src/outline/tracker.zig b/src/outline/tracker.zig index 4ebf2ab..0c4c7a0 100644 --- a/src/outline/tracker.zig +++ b/src/outline/tracker.zig @@ -36,7 +36,7 @@ const TrackedObj = struct { }; var tracked_objs: [MAX_TRACKED_OBJS]TrackedObj = undefined; -var tracked_obj_count: usize = 0; +pub var tracked_obj_count: usize = 0; // All unit/player object pointers for stencil occlusion detection. var unit_obj_ptrs: [MAX_UNIT_OBJS]u32 = .{0} ** MAX_UNIT_OBJS;