From fbfe1cb545c397854479d893a6bcd712793178f3 Mon Sep 17 00:00:00 2001 From: MarcelineVQ Date: Wed, 25 Feb 2026 18:32:49 -0800 Subject: [PATCH] Outline render order, stencil protection, JFA sentinel fix, diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Render order: 3-way M2 batch partition — game objects + local player render first (write depth), then outline targets (stencil marks), then other players/gear/NPCs. Outlines show through other players but are occluded by world/WMO/game objects/local player. Stencil protection: set STENCILWRITEMASK=0 after outline target DIPs to prevent subsequent renders from overwriting stencil marks. JFA sentinel: changed from (1,1) to (-1,-1) to move it outside UV space. Did not fix banding but is correct regardless. Debug: added DEBUG_SHOW_SILHOUETTE comptime flag to bypass JFA and composite raw silhouette RT. Confirmed silhouette is clean — banding is in the JFA pipeline, not stale vertex buffers. Game object + local player tracking in tracker.zig for batch ordering. Added project README and outline subsystem README documenting render pipeline, architecture, known issues, and planned features. --- README.md | 171 +++++++++++++++++++++ docs/jfa-banding-investigation.md | 64 ++++++++ src/outline/README.md | 239 ++++++++++++++++++++++++++++++ src/outline/d3d9_hook.zig | 114 ++++++++++++-- src/outline/model_hook.zig | 45 +++--- src/outline/tracker.zig | 56 +++++++ 6 files changed, 656 insertions(+), 33 deletions(-) create mode 100644 README.md create mode 100644 docs/jfa-banding-investigation.md create mode 100644 src/outline/README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..7389551 --- /dev/null +++ b/README.md @@ -0,0 +1,171 @@ +# WeirdUtils + +All-in-one WoW 1.12.1 (build 5875) utility DLL. Injected as a 32-bit DLL into the +game process via Wine/DXVK on Linux. Provides screen-space outlines, screenshots, +interaction helpers, and an embedded addon with Lua API + keybindings. + +## Current Features + +| Module | Description | +|---|---| +| **Outline** | JFA-based screen-space outlines for targets, raid marks, dead players. See [src/outline/README.md](src/outline/README.md). | +| **Screenshot** | Hooks CTgaFile::Write for screenshot capture. | +| **Interact** | Nearest NPC/object interaction, bulk looting with queue processing. | +| **Embedded Addon** | Virtual addon loaded from DLL memory — .toc, .lua, .xml served via file I/O hook. Registers Lua commands and keybindings without any on-disk addon folder. | +| **Lua Protection Bypass** | Stubs the Lua callback address validator to allow C function registration. | + +## Consolidation Plan + +WeirdUtils replaces the standalone utility DLLs in the parent directory. All +development happens here — shared code, shared hooking infrastructure, one build +system. The standalone DLLs are being retired. + +| Standalone DLL | Purpose | Integration Status | +|---|---|---| +| `../assetfix` | Loose file loading, permissive MPQ glob patterns, pre-indexed file hash set | Not started | +| `../transmogfix` | Death frame drop fix — coalesces transmog durability update packets | Not started | +| `../interact` | Nearest interact + bulk loot | Partially integrated | + +### Compile-Time Feature Gating + +Each module is gated behind a build flag. The same codebase produces both the +all-in-one DLL and individual feature DLLs — just different compile flags. +Users can pick the full package or grab only the features they want. + +```zig +// build.zig options (planned) +const enable_assetfix = b.option(bool, "assetfix", "Enable asset/MPQ fixes") orelse true; +const enable_transmogfix = b.option(bool, "transmogfix", "Enable transmog coalesce fix") orelse true; +const enable_interact = b.option(bool, "interact", "Enable interact helpers") orelse true; +const enable_outline = b.option(bool, "outline", "Enable outline rendering") orelse true; +``` + +```sh +# Full build — all features in one DLL +zig build + +# Single-feature builds — one DLL per feature for individual distribution +zig build -Dassetfix=true -Dtransmogfix=false -Dinteract=false -Doutline=false +zig build -Dassetfix=false -Dtransmogfix=true -Dinteract=false -Doutline=false +# etc. +``` + +Release artifacts: +- `weirdutils.dll` — everything +- `assetfix.dll` — just asset/MPQ fixes +- `transmogfix.dll` — just transmog coalesce +- `interact.dll` — just interact/loot helpers +- `outline.dll` — just outline rendering + +All built from this repo, all sharing the same hook library and codebase. + +### Per-Feature Named Mutex + +A user might load the full DLL alongside one of the smaller single-feature DLLs +(e.g. they use `weirdutils.dll` for everything but also have `assetfix.dll` from +before they switched). Each feature module claims a **named mutex** on load — if +it's already held, that module skips hook installation. This way any combination +of DLLs coexists safely with no duplicate hooks. + +```zig +// Each module creates a process-specific named mutex on init +const mutex = CreateMutexA(null, 1, "Local\\WeirdUtils_AssetFix_{pid}"); +if (GetLastError() == ERROR_ALREADY_EXISTS) { + // Another DLL already owns this feature's hooks — skip + CloseHandle(mutex); + return; +} +// First to load wins — install hooks +``` + +This is per-feature, not per-DLL. The full DLL claims one mutex per enabled +feature. A single-feature DLL claims one mutex. Whichever loads first owns the +hooks; the duplicate gracefully becomes a no-op. + +## Planned: Ground-Projected Markers + +World-space markers projected onto terrain, similar to raid markers but driven +programmatically. Use cases: + +- Visual range indicators (spell range circles, aggro radius) +- Waypoint markers for navigation +- Area-of-effect visualization +- Custom raid positioning markers + +Implementation will require: +- Projecting screen-space or world-space coordinates onto the terrain mesh +- Rendering textured quads or circles that conform to terrain height +- Integration with the D3D9 hook pipeline (rendered during EndScene or as + additional geometry injected into the scene) + +## Distribution + +This repo is private (source not published to avoid empowering bad actors). +Distribution uses a separate **public release repo** that contains only a +user-facing README and binary releases — no source code. + +- **This repo** (private): all source, development, docs +- **Public repo** (e.g. `WeirdUtils`): README with feature descriptions + + GitHub Releases with DLL downloads + +Release workflow: +```sh +# Build all variants from this repo +zig build # weirdutils.dll (full) +zig build -Doutline=true -Deverything-else=false # outline.dll +# ... etc for each single-feature build + +# Publish to the public repo +gh release create v1.0 --repo YourName/WeirdUtils \ + --title "v1.0" --notes "Release notes" \ + ./zig-out/lib/weirdutils.dll \ + ./builds/outline.dll \ + ./builds/assetfix.dll +``` + +## Project Structure + +``` +weirdutils/ + build.zig Build configuration + src/ + main.zig DLL entry, Lua API, file I/O hook, embedded addon + screenshot.zig Screenshot capture hook + interact.zig Interact + loot helpers + png.zig PNG encoding for screenshots + outline/ Outline subsystem (see src/outline/README.md) + api.zig Public API, Lua command handler + d3d9_hook.zig D3D9 vtable hooks, JFA pipeline, shaders + model_hook.zig M2 batch reordering, rendering_outline flag + tracker.zig Per-frame object/model tracking + types.zig D3D9 constants, outline colors, categories + offsets.zig WoW memory addresses and struct offsets + wow.zig Game memory access wrappers + addon/ Embedded addon files (.toc, .lua, .xml) + libs/ + hook/ Shared x86 inline hooking library (trampoline, fastcall thunks) + docs/ Design docs, research notes, shader analysis + reference/ C reference implementations +``` + +## Build + +```sh +cd /media/storage/projects/zig/weirdutils +zig build +``` + +Target: x86-windows-msvc (32-bit DLL), Zig 0.15. +Host: Linux (Arch), game runs via Wine/DXVK. + +## Hook Installation Order + +Hooks are installed in a specific sequence to handle dependencies: + +1. **DLL_PROCESS_ATTACH** — Lua protection bypass, file I/O hook, LoadScriptFunctions, + LoadAddonsRecursively, interact hooks, GameEngine_MainInitialize, CGGameUI_Shutdown +2. **GameEngine_MainInitialize** (one-shot) — screenshot hook, outline model hooks +3. **First model hook callback** (deferred) — D3D9 vtable hooks (EndScene, DIP, Reset) + +D3D9 hooks are deferred because creating a dummy device during engine init +corrupts the d3d9 proxy's state. diff --git a/docs/jfa-banding-investigation.md b/docs/jfa-banding-investigation.md new file mode 100644 index 0000000..8b00e54 --- /dev/null +++ b/docs/jfa-banding-investigation.md @@ -0,0 +1,64 @@ +# JFA Outline Banding Investigation + +## Date: 2026-02-25 + +## Problem +Outlines rendered via the JFA pipeline have a "marching ants" pattern of missing outline pixels. The artifact depends on **which side of the camera** the target is on (screen-space position), not the left/right side of the model itself. Camera-angle dependent. + +## What Was Ruled Out + +### 1. Silhouette RT is clean (CONFIRMED) +Added `DEBUG_SHOW_SILHOUETTE` comptime flag in `d3d9_hook.zig` that skips JFA and composites the raw silhouette RT directly to the backbuffer. The silhouette was solid with no banding — the cached draw replay produces correct geometry. **Stale VB hypothesis is NOT the cause.** + +### 2. JFA sentinel value (1.0, 1.0) → (-1.0, -1.0) (NO IMPROVEMENT) +Changed the JFA init shader sentinel from `(1.0, 1.0)` to `(-1.0, -1.0)` so unflooded pixels can't act as false seeds near the right screen edge. This did NOT fix the marching ants pattern. The sentinel is currently set to `(-1.0, -1.0)` in the code (uncommitted). + +### 3. JFA pass count changes pattern but doesn't fix it +- `[2, 1]` → vertical bands of missing outline +- `[8, 4, 2, 1]` → marching ants alternating pattern (current uncommitted state) + +### 4. Mid-DIP silhouette drawing corrupts render state (ABANDONED) +Attempted drawing silhouettes directly in DIP hook. Corrupts WoW's GxDevice state. Reverted. + +### 5. Game object occlusion (FIXED, committed as 661e156) +Batch reorder puts outline targets last so depth buffer has full scene geometry when stencil marks are written. + +## Current Uncommitted Changes in d3d9_hook.zig +- JFA steps changed from `[2, 1]` to `[8, 4, 2, 1]` +- JFA sentinel changed from `(1.0, 1.0)` to `(-1.0, -1.0)` +- `DEBUG_SHOW_SILHOUETTE` comptime flag added (currently `false`), with `debug_sil_ps` shader +- Debug shader had a cmp operand inversion bug that was fixed (`c0.w, c0.x` not `c0.x, c0.w`) + +## Remaining Investigation — JFA Pipeline Bug + +Since the silhouette is clean, the bug is in the JFA shaders (Phase 2). Possible causes: + +### A. JFA propagation shader distance comparison logic +The propagation shader uses `cmp` (ps_3_0) which tests `>= 0` vs `< 0`. When `new_dist² == best_dist²` (exactly equal), `cmp` picks the OLD seed (`>= 0` branch). This tie-breaking might cause systematic bias where seeds from certain directions are always preferred, creating directional artifacts. Worth testing: swap `cmp` operands to prefer new seed on ties, or add a small epsilon. + +### B. dp2add precision on DXVK +`dp2add r2.z, r2, r2, c1.x` computes `r2.x*r2.x + r2.y*r2.y + 0.0`. DXVK translates this to Vulkan — there may be precision differences vs native D3D9 that affect distance comparisons, especially for pixels equidistant from multiple seeds. + +### C. Neighbor sampling at texture edges +When `mad r4.xy, offset, step_uv, v0.xy` goes outside [0,1], CLAMP addressing returns the edge texel. This could feed stale/wrong seed UVs into the comparison. Clamping the sample coordinate to valid range before comparison could help. + +### D. The JFA algorithm itself may not suit this use case +Consider alternative approaches: +- **Screen-space dilation** (iterative morphological expand of silhouette) — simpler, no distance field needed +- **Gaussian blur difference** — blur silhouette, subtract original, threshold +- **Sobel/edge detection** on the silhouette RT +- Docs in `/media/storage/projects/zig/weirdutils/docs/` describe these alternatives +- Reference articles: ameye.dev "5 ways to draw an outline", Ben Golus "Quest for Very Wide Outlines" + +## Key Files +- `src/outline/d3d9_hook.zig` — D3D9 hooks, JFA pipeline, all shaders +- `src/outline/model_hook.zig` — batch reordering, rendering_outline flag +- `src/outline/tracker.zig` — per-frame model tracking +- `src/outline/types.zig` — D3D9 constants +- `reference/c_overlay/d3d9_hook.cpp` — C reference (uses 3-pass shell extrusion, not JFA) + +## Build / Environment +- `zig build` from `/media/storage/projects/zig/weirdutils/` +- Zig 0.15, target x86-windows-msvc (32-bit DLL injected into WoW 3.3.5) +- Linux host (Arch, kernel 6.16.1), game via Wine/DXVK +- Git last commit: `661e156` (game object occlusion fix) diff --git a/src/outline/README.md b/src/outline/README.md new file mode 100644 index 0000000..9b11cc6 --- /dev/null +++ b/src/outline/README.md @@ -0,0 +1,239 @@ +# Outline Subsystem + +Screen-space outline rendering for WoW 1.12.1 (build 5875), injected as a 32-bit DLL +via Wine/DXVK on Linux. Outlines are rendered using a Jump Flood Algorithm (JFA) pipeline +that runs entirely in EndScene, composited on top of the final backbuffer. + +## Architecture Overview + +The outline system has three main phases per frame: + +1. **Object scan** (EndScene start) — identify which game objects should be outlined +2. **DIP hook** (during game rendering) — cache draw calls and write stencil marks +3. **JFA pipeline** (EndScene, after game rendering) — produce and composite outlines + +## Files + +| File | Purpose | +|---|---| +| `api.zig` | Public API: `init()`, `cleanup()`, `setEnabled()`, Lua command handler | +| `d3d9_hook.zig` | D3D9 vtable hooks (EndScene, DIP, Reset), JFA pipeline, all shaders | +| `model_hook.zig` | WoW model render hooks, batch reordering, `rendering_outline` flag | +| `tracker.zig` | Per-frame object/model tracking, classification, color/width queries | +| `types.zig` | D3D9 constants, vtable indices, outline colors, model categories | +| `offsets.zig` | WoW memory addresses and struct offsets (object manager, functions) | +| `wow.zig` | Game memory access wrappers (object traversal, GUID resolution, unit helpers) | + +## Render Order + +The game's rendering pipeline processes geometry in this order. The outline system +controls M2 batch ordering via `model_hook.zig` and uses the resulting depth/stencil +state to determine outline visibility. + +``` +1. World geometry + WMOs (game engine, writes depth) +2. Game object M2s (doodads) (batch group 1, writes depth) +3. Local player M2s (batch group 1, writes depth) +4. Outline target M2s (batch group 2, DIP hook writes stencil) +5. Other players + gear + NPCs (batch group 3, renders normally) +6. EndScene (JFA pipeline composites outlines) +``` + +### Batch reordering (model_hook.zig) + +`CM2SceneRenderDraw` receives a flat array of M2 batch indices. The hook partitions +them into 3 groups before calling the original function: + +- **Group 1 — depth-priority models**: game object M2s and the local player's M2s. + These render first so their depth is in the buffer when stencil marks are written. + The local player occludes outlines (they're the camera reference point). + If the local player IS an outline target, their models go in group 2 instead + (the outline check takes priority in the partition logic). + +- **Group 2 — outline targets**: models belonging to tracked entities (current target, + raid-marked units, dead friendly players). The DIP hook intercepts these draws to + cache parameters and write stencil=1 where they pass the depth test. + +- **Group 3 — everything else**: other players, their gear, NPCs, creatures. + These render last. Their depth is NOT in the buffer when stencil marks are written, + so outlines show through them. The outline composites on top in EndScene regardless. + +### What occludes outlines + +| Geometry | Occludes outlines? | Why | +|---|---|---| +| World terrain, WMOs | Yes | Rendered before any M2s, depth already in buffer | +| Game objects (M2 doodads) | Yes | Batch group 1, depth written before stencil | +| Local player | Yes | Batch group 1, unless self-outlined | +| Other players + gear | No | Batch group 3, render after stencil is written | +| NPCs / creatures | No | Batch group 3 | + +## Stencil System + +The DIP hook writes stencil marks during outline target rendering (group 2): + +- `STENCILFUNC = ALWAYS`, `STENCILPASS = REPLACE`, `STENCILREF = 1` +- `STENCILZFAIL = KEEP` — pixels behind depth-tested geometry keep stencil=0 +- After each outline DIP, `STENCILWRITEMASK` is set to 0 to protect marks from + subsequent draws (group 3 models could otherwise overwrite them) +- Exception: dead players skip stencil entirely (`STENCILENABLE = 0`) so their + outlines are visible through walls for corpse finding + +EndScene Phase 1 uses `STENCILFUNC = EQUAL`, `STENCILREF = 1` to gate the +silhouette replay — only pixels marked as visible get silhouette color. + +Stencil is cleared to 0 after Phase 1 to avoid affecting the next frame. + +## JFA Pipeline (EndScene Phase 2) + +After Phase 1 produces the silhouette RT (A8R8G8B8), the JFA pipeline generates +outlines via distance field: + +1. **JFA Init** — seed the distance field from the silhouette. Pixels with + silhouette content (alpha >= 0.002) output their own UV as a seed. + Empty pixels output sentinel (-1, -1) which is outside UV space [0,1] + so it never wins distance comparisons. + +2. **JFA Propagation** — 4 passes at step sizes [8, 4, 2, 1], ping-ponging + between two G16R16F render targets. Each pass does a 9-tap sample + (self + 8 neighbors at step distance) and keeps the nearest seed UV. + +3. **JFA Decode + Composite** — compute pixel-space distance from each pixel + to its nearest seed. If distance < outline width AND the pixel is outside + the silhouette interior, output the outline color with alpha blending. + +### Outline widths + +| Category | Pixels | Encoded alpha | +|---|---|---| +| Target | 2.25 | 0.5625 | +| Raid mark | 1.5 | 0.375 | +| Dead player | 2.5 | 0.625 | + +Width is encoded as `alpha = pixels / 4.0` in the silhouette, decoded as +`width = alpha * 4.0` in the decode shader. + +### Render targets + +| RT | Format | Purpose | +|---|---|---| +| `rt_silhouette_tex` | A8R8G8B8 | Flat-color silhouettes with width-encoded alpha | +| `rt_jfa_a_tex` | G16R16F | JFA ping buffer (seed UV coordinates) | +| `rt_jfa_b_tex` | G16R16F | JFA pong buffer | + +## DIP Hook — Draw Caching + +The DIP hook does NOT draw silhouettes inline (that corrupts WoW's GxDevice +internal render state). Instead it: + +1. Caches draw parameters (VB, IB, vertex decl, VS, VS constants, prim params) +2. AddRef's COM objects to keep them alive until EndScene +3. Writes stencil marks using the game's own depth buffer +4. Calls the original DIP exactly once (normal game rendering) + +EndScene Phase 1 replays cached draws to the silhouette RT with: +- The flat-color pixel shader (outputs PS constant c0) +- The game's original vertex shader + VS constants (bone matrices, transforms) +- Stencil gating (stencil=1 required, except dead players) +- Depth testing disabled, depth writes disabled + +## Object Tracking (tracker.zig) + +Each frame, `scanObjects()` iterates the WoW object manager and collects: + +- **Outline targets**: current target, raid-marked units/players, dead friendly players +- **Depth-priority objects**: game objects (type 5) and the local player + +When `CM2Model_ManageRenderListNode` fires for each model being added to the +render list, `classifyModel()` reads the model's owner back-pointers +(`model+0x28` direct, `model+0x3C0` callback) and matches them against the +collected object pointers. No pointer dereferencing of unknown memory — just +value comparison against the validated set from the object manager. + +## Outline Categories + +| Category | Color | Trigger | +|---|---|---| +| `target` | Golden amber (#FFC800) | Current target | +| `raid_marked` | Per-mark color (8 colors) | Unit has raid mark 1-8 | +| `dead_player` | Cyan (#00FFFF) | Dead friendly player or non-skeleton corpse | + +## Hook Installation + +1. `api.init()` installs model hooks immediately (ManageRenderListNode, + DrawBatchProjected, CM2SceneRenderDraw) +2. D3D9 hooks are **deferred** until the first model hook fires — creating a + dummy D3D9 device during engine init corrupts the proxy's state +3. `api.initD3D9Deferred()` patches the D3D9 vtable (EndScene, DIP, Reset) +4. Reset hook forces D24S8 depth/stencil format (8 stencil bits required) + +## Lua API + +```lua +OutlineCommand() -- returns current enabled state (bool) +OutlineCommand("on") -- enable outlines +OutlineCommand("off") -- disable outlines +``` + +## Known Issues + +- **JFA banding artifacts**: "marching ants" pattern in the JFA output, dependent + on screen-space position of the target. Confirmed the silhouette RT is clean + (no stale VB issue). Root cause is in the JFA propagation/decode shaders. + See `docs/jfa-banding-investigation.md` for full analysis. + +- **Outline target gear not included**: equipment M2s (shoulders, weapons, helms) + belonging to the outline target are not currently part of the silhouette. + The outline follows the body mesh contour only. Tracking equipment models + requires identifying them via owner back-pointers (planned). + +- **Local player outline (planned)**: a Gaussian blur outline mode for the local + player to improve visibility in combat when surrounded by mobs. Separate from + the JFA pipeline — will use blur difference (blur silhouette, subtract original, + threshold) for a softer glow effect and only apply to other players. + +- **Death tracking needs improvement**: currently uses `UNIT_FLAG_DEAD` which is + too simple. Needed changes: + - Dead players should be outlined (works now) + - Released bodies (corpse objects) should also be outlined (partially works via `.corpse` + type, but needs verification that released-but-not-skeleton corpses are caught) + - Feign Death must NOT trigger the dead outline — feign death sets the dead flag + but the player is alive. Need to check for the feign death aura/buff or use a + more specific death condition + - The local player should never get a death outline on themselves — the purpose + of the death outline is to help the player find and resurrect others, not to + highlight their own corpse + - Marker outlines should not persist on units after they die + - Targeting a dead body should use a distinct outline color/style from the normal + target outline, a player body should have the corpse outline. Only group/raid member + bodies/corpses should outline! + - Skeleton corpses should not outline. + +- **Shapeshift form breaks local player occlusion**: changing form (e.g. ghost wolf, + druid forms) causes the local player's model pointer to change. The new model + isn't matched against the local player's object pointer, so it falls into batch + group 3 instead of group 1 and no longer occludes outlines. + +- **Mount + rider outline (planned)**: mounted players should outline the full + mount+rider+gear as one unit. Requires understanding the M2 attachment hierarchy + (mount is parent, rider is attached, gear is attached to rider?). All child models + (rider body, gear) should be included in the silhouette. + +## Future Architecture Notes + +- **Hook "should render" function for model tracking**: the perfboost system uses a + "should render" callback that receives a direct live object reference for every + model considered for rendering each frame. Hooking this instead of (or alongside) + `ManageRenderListNode` would give us a more reliable way to stay current on what + objects and models are in the scene, with a direct object reference we can inspect + for owner, form, mount status, etc. This could solve the shapeshift/mount model + tracking issues since we'd always have the live object to check against. + +## Build + +```sh +cd /media/storage/projects/zig/weirdutils +zig build +``` + +Target: x86-windows-msvc (32-bit DLL), Zig 0.15. diff --git a/src/outline/d3d9_hook.zig b/src/outline/d3d9_hook.zig index a3798ad..2aa3c38 100644 --- a/src/outline/d3d9_hook.zig +++ b/src/outline/d3d9_hook.zig @@ -8,11 +8,10 @@ //! marks where models pass the terrain depth test (stencil=1 = visible). //! - **Reset**: forces D24S8 depth/stencil format, releases resources. //! -//! Batch reordering in model_hook.zig ensures outline targets render LAST in -//! CM2SceneRenderDraw, after all other M2 models (game objects, characters, -//! NPCs) have filled the depth buffer. The DIP hook writes stencil marks -//! using the game's own depth buffer; EndScene replay uses these marks to -//! gate silhouette drawing (full scene occlusion including game objects). +//! Batch reordering in model_hook.zig partitions M2 batches into 3 groups: +//! game objects first (write depth), then outline targets (DIP hook writes +//! stencil against that depth), then other players/gear/NPCs. Outlines are +//! occluded by world/WMO/game objects but show through other players. const std = @import("std"); const hook = @import("hook"); @@ -70,8 +69,14 @@ var outline_ps: ?*anyopaque = null; // flat-color PS (silhouettes) var jfa_init_ps: ?*anyopaque = null; // JFA seed init PS var jfa_prop_ps: ?*anyopaque = null; // JFA propagation PS var jfa_decode_ps: ?*anyopaque = null; // JFA decode + composite PS +var debug_sil_ps: ?*anyopaque = null; // debug: composite silhouette directly var shaders_attempted: bool = false; +// Debug: set to true to skip JFA and composite raw silhouette RT to backbuffer. +// Used to diagnose whether banding artifacts originate in the silhouette (Phase 1 +// replay / stale VB) or in the JFA pipeline (Phase 2 shader bug). +const DEBUG_SHOW_SILHOUETTE = false; + // D3DXAssembleShader function pointer (loaded dynamically) const D3DXAssembleShaderFn = *const fn ( [*]const u8, // pSrcData @@ -459,10 +464,11 @@ fn releaseResources() void { /// Flat colour pixel shader — outputs PS constant c0. const ps_flat_src = "ps_3_0\nmov oC0, c0\n"; -/// JFA init: sample silhouette, output own UV as seed or sentinel (1,1). +/// JFA init: sample silhouette, output own UV as seed or sentinel (-1,-1). +/// Sentinel must be outside [0,1] UV space so it never wins distance comparisons. const jfa_init_src = "ps_3_0\n" ++ - "def c0, 1.0, 1.0, -0.002, 0.0\n" ++ + "def c0, -1.0, -1.0, -0.002, 0.0\n" ++ "dcl_2d s0\n" ++ "dcl_texcoord0 v0\n" ++ "texld r0, v0, s0\n" ++ @@ -587,6 +593,18 @@ const jfa_decode_src = "mov r4.xyz, r2.xyz\n" ++ // outline colour from seed "mov oC0, r4\n"; +/// Debug: composite silhouette RT directly. Forces alpha to 1.0 where silhouette +/// has any content (alpha >= 0.002), 0.0 elsewhere. Bypasses JFA entirely. +const debug_sil_src = + "ps_3_0\n" ++ + "def c0, 0.0, 0.0, -0.002, 1.0\n" ++ + "dcl_2d s0\n" ++ + "dcl_texcoord0 v0\n" ++ + "texld r0, v0, s0\n" ++ + "add r1.x, r0.a, c0.z\n" ++ // alpha - 0.002 + "cmp r0.w, r1.x, c0.w, c0.x\n" ++ // >= 0 → 1.0 (opaque), < 0 → 0.0 (transparent) + "mov oC0, r0\n"; + // ============================================================================= // Shader creation // ============================================================================= @@ -620,6 +638,14 @@ fn ensureShaders(device: *anyopaque) void { releaseShaders(); return; }; + + // --- Debug silhouette composite PS (only when diagnostic enabled) --- + if (DEBUG_SHOW_SILHOUETTE) { + debug_sil_ps = assemblePS(device, assemble, debug_sil_src, debug_sil_src.len) orelse { + releaseShaders(); + return; + }; + } } /// Assemble a pixel shader from source text, create device PS object. @@ -645,7 +671,7 @@ fn assemblePS(device: *anyopaque, assemble: D3DXAssembleShaderFn, src: [*]const } fn releaseShaders() void { - inline for (.{ &outline_ps, &jfa_init_ps, &jfa_prop_ps, &jfa_decode_ps }) |ps| { + inline for (.{ &outline_ps, &jfa_init_ps, &jfa_prop_ps, &jfa_decode_ps, &debug_sil_ps }) |ps| { if (ps.*) |p| { comRelease(p); ps.* = null; } } shaders_attempted = false; @@ -806,7 +832,7 @@ fn hkDIP( const s_enable = deviceGetRS(device, types.D3DRS.STENCILENABLE); const s_func = deviceGetRS(device, types.D3DRS.STENCILFUNC); const s_ref = deviceGetRS(device, types.D3DRS.STENCILREF); - const s_wmask = deviceGetRS(device, types.D3DRS.STENCILWRITEMASK); + // (STENCILWRITEMASK not saved — intentionally set to 0 on restore) const s_pass = deviceGetRS(device, types.D3DRS.STENCILPASS); const s_fail = deviceGetRS(device, types.D3DRS.STENCILFAIL); const s_zfail = deviceGetRS(device, types.D3DRS.STENCILZFAIL); @@ -821,14 +847,19 @@ fn hkDIP( const result = origFn(device, prim_type, base_vtx, min_vtx, num_verts, start_idx, prim_count); - // Restore stencil state to match WoW's GxDevice cache + // Restore stencil state to match WoW's GxDevice cache, but lock + // stencil writes to protect our marks from subsequent draws (other + // players' gear, NPCs, etc. that render after outline targets). deviceSetRS(device, types.D3DRS.STENCILENABLE, s_enable); deviceSetRS(device, types.D3DRS.STENCILFUNC, s_func); deviceSetRS(device, types.D3DRS.STENCILREF, s_ref); - deviceSetRS(device, types.D3DRS.STENCILWRITEMASK, s_wmask); deviceSetRS(device, types.D3DRS.STENCILPASS, s_pass); deviceSetRS(device, types.D3DRS.STENCILFAIL, s_fail); deviceSetRS(device, types.D3DRS.STENCILZFAIL, s_zfail); + // Write mask 0 instead of restoring original — prevents any + // subsequent DIP from overwriting our stencil=1 marks. + // Restored properly in EndScene before the JFA pipeline. + deviceSetRS(device, types.D3DRS.STENCILWRITEMASK, 0); return result; } @@ -988,6 +1019,38 @@ fn runJfaPipeline(device: *anyopaque) void { _ = clearFn(device, 0, null, types.D3DCLEAR_STENCIL, 0, 1.0, 0); } + // ===================================================================== + // Debug: skip JFA, composite raw silhouette RT to see if banding is + // in the silhouette (stale VB / replay issue) or the JFA pipeline. + // ===================================================================== + if (DEBUG_SHOW_SILHOUETTE) { + if (debug_sil_ps) |dps| { + if (saved_rt0) |rt| deviceSetRenderTarget(device, 0, rt); + deviceSetPtrOrNull(device, types.VT.SetDepthStencilSurface, null); + deviceSetPtrOrNull(device, types.VT.SetVertexShader, null); + deviceSetFVF(device, types.D3DFVF_XYZRHW | types.D3DFVF_TEX1); + deviceSetRS(device, types.D3DRS.ZENABLE, types.D3DZB_FALSE); + deviceSetRS(device, types.D3DRS.ZWRITEENABLE, 0); + deviceSetRS(device, types.D3DRS.CULLMODE, types.D3DCULL_NONE); + deviceSetRS(device, types.D3DRS.ALPHATESTENABLE, 0); + deviceSetRS(device, types.D3DRS.COLORWRITEENABLE, 0x0F); + deviceSetRS(device, types.D3DRS.ALPHABLENDENABLE, 1); + deviceSetRS(device, types.D3DRS.SRCBLEND, types.D3DBLEND_SRCALPHA); + deviceSetRS(device, types.D3DRS.DESTBLEND, types.D3DBLEND_INVSRCALPHA); + deviceSetSamplerState(device, 0, types.D3DSAMP.ADDRESSU, types.D3DTADDRESS_CLAMP); + deviceSetSamplerState(device, 0, types.D3DSAMP.ADDRESSV, types.D3DTADDRESS_CLAMP); + deviceSetSamplerState(device, 0, types.D3DSAMP.MAGFILTER, types.D3DTEXF_POINT); + deviceSetSamplerState(device, 0, types.D3DSAMP.MINFILTER, types.D3DTEXF_POINT); + deviceSetSamplerState(device, 0, types.D3DSAMP.MIPFILTER, types.D3DTEXF_NONE); + deviceSetTexture(device, 0, rt_silhouette_tex); + deviceSetPtr(device, types.VT.SetPixelShader, dps); + + const quad = buildFullscreenQuad(vp.Width, vp.Height); + deviceDrawPrimitiveUP(device, types.D3DPT_TRIANGLESTRIP, 2, @ptrCast(&quad), @sizeOf(QuadVertex)); + } + // Skip JFA — jump straight to state restore + } else { + // ===================================================================== // Phase 2: JFA pipeline (silhouette → outline composite) // ===================================================================== @@ -1024,18 +1087,35 @@ fn runJfaPipeline(device: *anyopaque) void { deviceSetPtr(device, types.VT.SetPixelShader, jfa_init_ps.?); deviceDrawPrimitiveUP(device, types.D3DPT_TRIANGLESTRIP, 2, @ptrCast(&quad), qstride); - // Pass 2: JFA Propagation step=2 (JFA_A → JFA_B) + // JFA Propagation: steps [8, 4, 2, 1] ping-ponging between A and B. + deviceSetPtr(device, types.VT.SetPixelShader, jfa_prop_ps.?); + var c0: [4]f32 = undefined; + + // step=8 (JFA_A → JFA_B) deviceSetRenderTarget(device, 0, rt_jfa_b_surf.?); deviceSetTexture(device, 0, rt_jfa_a_tex); - var c0 = [4]f32{ 2.0 / fw, 2.0 / fh, 0.0, 0.0 }; + c0 = .{ 8.0 / fw, 8.0 / fh, 0.0, 0.0 }; deviceSetPSConstF(device, 0, &c0); - deviceSetPtr(device, types.VT.SetPixelShader, jfa_prop_ps.?); deviceDrawPrimitiveUP(device, types.D3DPT_TRIANGLESTRIP, 2, @ptrCast(&quad), qstride); - // Pass 3: JFA Propagation step=1 (JFA_B → JFA_A) + // step=4 (JFA_B → JFA_A) deviceSetRenderTarget(device, 0, rt_jfa_a_surf.?); deviceSetTexture(device, 0, rt_jfa_b_tex); - c0 = [4]f32{ 1.0 / fw, 1.0 / fh, 0.0, 0.0 }; + c0 = .{ 4.0 / fw, 4.0 / fh, 0.0, 0.0 }; + deviceSetPSConstF(device, 0, &c0); + deviceDrawPrimitiveUP(device, types.D3DPT_TRIANGLESTRIP, 2, @ptrCast(&quad), qstride); + + // step=2 (JFA_A → JFA_B) + deviceSetRenderTarget(device, 0, rt_jfa_b_surf.?); + deviceSetTexture(device, 0, rt_jfa_a_tex); + c0 = .{ 2.0 / fw, 2.0 / fh, 0.0, 0.0 }; + deviceSetPSConstF(device, 0, &c0); + deviceDrawPrimitiveUP(device, types.D3DPT_TRIANGLESTRIP, 2, @ptrCast(&quad), qstride); + + // step=1 (JFA_B → JFA_A) + deviceSetRenderTarget(device, 0, rt_jfa_a_surf.?); + deviceSetTexture(device, 0, rt_jfa_b_tex); + c0 = .{ 1.0 / fw, 1.0 / fh, 0.0, 0.0 }; deviceSetPSConstF(device, 0, &c0); deviceDrawPrimitiveUP(device, types.D3DPT_TRIANGLESTRIP, 2, @ptrCast(&quad), qstride); @@ -1051,6 +1131,8 @@ fn runJfaPipeline(device: *anyopaque) void { deviceSetRS(device, types.D3DRS.DESTBLEND, types.D3DBLEND_INVSRCALPHA); deviceDrawPrimitiveUP(device, types.D3DPT_TRIANGLESTRIP, 2, @ptrCast(&quad), qstride); + } // end else (normal JFA path) + // ===================================================================== // Restore ALL state // ===================================================================== diff --git a/src/outline/model_hook.zig b/src/outline/model_hook.zig index 5aa607a..be1b209 100644 --- a/src/outline/model_hook.zig +++ b/src/outline/model_hook.zig @@ -62,11 +62,14 @@ var reordered_indices: [MAX_REORDER]i32 = undefined; // __thiscall(this, viewMatrix, batchData, batchIndices, batchCount) // Native thiscall detour — no thunk needed. // -// Reorders batch indices so outline targets draw LAST. Non-outline models -// (game objects, other characters, NPCs) render first, filling the depth -// buffer with full scene geometry. When outline targets then render, the -// DIP hook writes stencil marks against this complete depth buffer, so -// outlines are properly occluded by all scene objects (not just terrain/WMOs). +// Reorders batch indices into 3 groups: +// 1. Game objects/doodads — render first, write depth so outlines respect them +// 2. Outline targets — render second, DIP hook writes stencil against depth +// 3. Other players, gear, NPCs — render last, draw over targets normally +// +// This gives outlines that are occluded by world/WMO/game objects but show +// through other players and gear (since those aren't in depth when stencil +// is written). The outline composites on top of everything in EndScene. fn renderDrawDetour(this: u32, view_matrix: u32, batch_data: u32, batch_indices: u32, batch_count: u32) callconv(THISCALL) void { // One-time: install D3D9 hooks now that the game is actively rendering. @@ -83,13 +86,18 @@ fn renderDrawDetour(this: u32, view_matrix: u32, batch_data: u32, batch_indices: const indices: [*]i32 = @ptrFromInt(batch_indices); - // Pass 1: count outline targets + // Pass 1: count outline targets and game objects for partitioning var outline_count: u32 = 0; + var game_obj_count: u32 = 0; for (0..batch_count) |i| { const idx_u: u32 = @bitCast(indices[i]); const model_ptr = hook.readMem(u32, batch_data +% idx_u *% 0x40 +% 4); - if (model_ptr != 0 and tracker.findOutlineEntry(model_ptr) != null) { - outline_count += 1; + if (model_ptr != 0) { + if (tracker.findOutlineEntry(model_ptr) != null) { + outline_count += 1; + } else if (tracker.isGameObjectModel(model_ptr)) { + game_obj_count += 1; + } } } @@ -98,13 +106,13 @@ fn renderDrawDetour(this: u32, view_matrix: u32, batch_data: u32, batch_indices: return; } - // Pass 2: partition — non-outline models first, outline targets last. - // Rendering non-outline models first fills the depth buffer with game - // objects, other characters, etc., so outline target stencil marks - // respect full scene occlusion (not just terrain+WMO). - const normal_count = batch_count - outline_count; - var normal_pos: u32 = 0; - var outline_pos: u32 = normal_count; + // Pass 2: 3-way partition: + // [0 .. game_obj_count) → game objects (write depth first) + // [game_obj_count .. +outline) → outline targets (stencil against depth) + // [remainder ..] → other players, gear, NPCs + var go_pos: u32 = 0; + var outline_pos: u32 = game_obj_count; + var rest_pos: u32 = game_obj_count + outline_count; for (0..batch_count) |i| { const batch_idx = indices[i]; const idx_u: u32 = @bitCast(batch_idx); @@ -112,9 +120,12 @@ fn renderDrawDetour(this: u32, view_matrix: u32, batch_data: u32, batch_indices: if (model_ptr != 0 and tracker.findOutlineEntry(model_ptr) != null) { reordered_indices[outline_pos] = batch_idx; outline_pos += 1; + } else if (model_ptr != 0 and tracker.isGameObjectModel(model_ptr)) { + reordered_indices[go_pos] = batch_idx; + go_pos += 1; } else { - reordered_indices[normal_pos] = batch_idx; - normal_pos += 1; + reordered_indices[rest_pos] = batch_idx; + rest_pos += 1; } } diff --git a/src/outline/tracker.zig b/src/outline/tracker.zig index 363c87b..efa4171 100644 --- a/src/outline/tracker.zig +++ b/src/outline/tracker.zig @@ -45,6 +45,21 @@ pub var tracked_obj_count: usize = 0; var frame_outlines: [MAX_OUTLINE_MODELS]types.OutlineEntry = undefined; var frame_outline_count: usize = 0; +// ============================================================================= +// Per-frame game object tracking (for render ordering — game objects first) +// ============================================================================= +// Game object M2 models need to render before outline targets so their depth +// is in the buffer when stencil marks are written. Tracked separately from +// outline entries since game objects don't get outlines. + +const MAX_GAME_OBJ_MODELS = 256; + +var game_obj_ptrs: [MAX_TRACKED_OBJS]u32 = undefined; +var game_obj_ptr_count: usize = 0; + +var game_obj_models: [MAX_GAME_OBJ_MODELS]u32 = undefined; +var game_obj_model_count: usize = 0; + // ============================================================================= // Global enable flag // ============================================================================= @@ -88,6 +103,14 @@ pub fn getModelCategory(model_ptr: u32) types.ModelCategory { return entry.category; } +/// Check if a model belongs to a game object (for render ordering). +pub fn isGameObjectModel(model_ptr: u32) bool { + for (game_obj_models[0..game_obj_model_count]) |m| { + if (m == model_ptr) return true; + } + return false; +} + /// Get screen-space outline thickness in pixels for a category. pub fn getOutlinePixels(cat: types.ModelCategory) f32 { return switch (cat) { @@ -124,6 +147,24 @@ pub fn classifyModel(model_ptr: u32) void { } } } + + // Match against game object pointers (for render ordering). + if (game_obj_ptr_count > 0 and game_obj_model_count < MAX_GAME_OBJ_MODELS) { + for (game_obj_ptrs[0..game_obj_ptr_count]) |go_ptr| { + if (owner_callback == go_ptr or owner_direct == go_ptr) { + // Deduplicate + var found = false; + for (game_obj_models[0..game_obj_model_count]) |m| { + if (m == model_ptr) { found = true; break; } + } + if (!found) { + game_obj_models[game_obj_model_count] = model_ptr; + game_obj_model_count += 1; + } + break; + } + } + } } fn addOutlineEntry(model_ptr: u32, cat: types.ModelCategory, mark: u8) void { @@ -155,12 +196,21 @@ pub fn scanObjects() void { // Clear per-frame sets frame_outline_count = 0; tracked_obj_count = 0; + game_obj_ptr_count = 0; + game_obj_model_count = 0; resetDiag(); if (!wow.isInGame()) return; const local_player = wow.getLocalPlayer(); if (local_player == 0) return; + // Local player renders before outline targets (occludes outlines) unless + // the local player IS an outline target (partition logic checks outline first). + if (game_obj_ptr_count < MAX_TRACKED_OBJS) { + game_obj_ptrs[game_obj_ptr_count] = local_player; + game_obj_ptr_count += 1; + } + // Cache raid target GUIDs wow.cacheRaidTargets(); @@ -197,6 +247,12 @@ pub fn scanObjects() void { addTrackedObj(obj, .dead_player, 0); } }, + .game_object => { + if (game_obj_ptr_count < MAX_TRACKED_OBJS) { + game_obj_ptrs[game_obj_ptr_count] = obj; + game_obj_ptr_count += 1; + } + }, else => {}, } }