benilla-octo/scripts/dll-surface.py

#!/usr/bin/env python3
"""What Lua surface the injected DLLs beside the client publish, and how much of it benilla has.

    scripts/dll-surface.py                 # the report
    scripts/dll-surface.py --names         # + every name per DLL
    scripts/dll-surface.py --wanted        # + only the names grom's addon corpus calls
    scripts/dll-surface.py --unpack DLL DIR  # write out an addon the DLL carries in .rsrc

benilla loads no DLLs: it reimplements what they add (`docs/ADDON_COMPAT.md`). The problem that
creates is knowing *what* they add, and the answer is not their export tables — every one of these
exports a single `DllMain`/`Load` and registers with the client at runtime.

**They do point at every name they register, though.** A DLL that publishes a Lua function hands
the registrar a `const char*`, so the name sits in its data as a NUL-delimited string and the
image holds a little-endian pointer to it, in the registration table or at the push. So: collect
identifier-shaped strings that are NUL-delimited on both sides, convert each to its virtual
address through the section headers, and keep the ones the file contains a pointer to. The pointer
test is what makes this usable — on `ClassicAPI.dll` it cuts 2872 CamelCase candidates to 706.

What it is NOT: proof of a Lua API. Any pointed-at string survives, so a statically linked library
contributes its own (freetype's `BlueValues`, the CRT's `FlsAlloc`, month names). The filters below
remove the classes we can name, and the 1.12 census removes the client's own; what is left is a
*candidate* list to read, not a specification. Treat a name as real when something else agrees —
the addon corpus calling it, or the DLL's own docs.

Reading the DLL's code for one name is `benilla-octo-exe-patches-are-readable`'s recipe, which is
how the `.octoql` and `.tdata` patches in `WoW.exe` were read.

**`--unpack` is the better question for ClassicAPI**, and the answer the registration scan cannot
give: that DLL carries its whole `!!!ClassicAPI` addon — 173 Lua/XML files — gzipped in `.rsrc`,
and extracts it into `Interface/AddOns` at load. The decompressed blob is a plain container (decimal
byte lengths, one per line, a blank line, then the files concatenated) and the paths are the
`!!!ClassicAPI\…` strings in the pool, in the same order. So the authoritative spec for the build
sitting beside the client is on disk already; see `docs/ADDON_COMPAT.md`. Nothing else in this
bundle ships resources this way, so `--unpack` on another DLL simply finds none.
"""

import argparse
import re
import struct
import subprocess
import sys
from pathlib import Path

REPO = Path(__file__).resolve().parent.parent
# The client's folder: the DLLs sit beside `WoW.exe`, one level above this checkout.
CLIENT = REPO.parent

# Lua 5.0's own keywords and standard library, which a DLL links rather than publishes.
LUA_OWN = {
    "and", "break", "do", "else", "elseif", "end", "false", "for", "function", "if", "in",
    "local", "nil", "not", "or", "repeat", "return", "then", "true", "until", "while",
    "assert", "collectgarbage", "dofile", "error", "gcinfo", "getfenv", "getmetatable",
    "ipairs", "loadfile", "loadstring", "newproxy", "next", "pairs", "pcall", "print",
    "rawequal", "rawget", "rawset", "require", "setfenv", "setmetatable", "tonumber",
    "tostring", "type", "unpack", "xpcall", "abs", "acos", "asin", "atan", "atan2", "ceil",
    "cos", "deg", "exp", "floor", "fmod", "frexp", "ldexp", "log", "log10", "max", "min",
    "mod", "modf", "pow", "rad", "random", "randomseed", "sin", "sqrt", "tan", "byte", "char",
    "find", "format", "gfind", "gmatch", "gsub", "len", "lower", "rep", "reverse", "sub",
    "upper", "concat", "foreach", "foreachi", "getn", "insert", "remove", "setn", "sort",
    "clock", "date", "difftime", "time", "close", "flush", "lines", "open", "read", "seek",
    "write", "operator",
}


def sections(image: bytes):
    """`(image_base, [(virtual_address, virtual_size, raw_offset, raw_size)])` from the PE headers."""
    pe = struct.unpack_from("<I", image, 0x3C)[0]
    if image[pe : pe + 4] != b"PE\0\0":
        raise ValueError("not a PE image")
    count = struct.unpack_from("<H", image, pe + 6)[0]
    optional_size = struct.unpack_from("<H", image, pe + 20)[0]
    base = struct.unpack_from("<I", image, pe + 24 + 28)[0]
    table = pe + 24 + optional_size
    out = []
    for i in range(count):
        _, vsize, vaddr, rsize, raddr = struct.unpack_from("<8sIIII", image, table + i * 40)
        out.append((vaddr, vsize, raddr, rsize))
    return base, out


def pointed_at(path: Path) -> set[str]:
    """Identifier strings the image holds a pointer to; see the module docstring."""
    image = path.read_bytes()
    base, secs = sections(image)

    def virtual(offset: int):
        for vaddr, _vsize, raddr, rsize in secs:
            if rsize and raddr <= offset < raddr + rsize:
                return base + vaddr + (offset - raddr)
        return None

    by_address: dict[int, str] = {}
    for match in re.finditer(rb"\x00([A-Za-z_][A-Za-z0-9_]{3,48})\x00", image):
        address = virtual(match.start() + 1)
        if address is not None:
            by_address.setdefault(address, match.group(1).decode("latin1"))
    pointers = {
        struct.unpack_from("<I", image, i)[0] for i in range(0, max(len(image) - 4, 0))
    }
    return {name for address, name in by_address.items() if address in pointers}


def census(tsv: Path) -> set[str]:
    """The 1.12 client's own globals, which a DLL points at without adding."""
    names = set()
    for line in tsv.read_text(errors="replace").splitlines():
        if line.startswith("#") or not line.strip():
            continue
        names.add(line.split("\t")[0])
    return names


def ours() -> set[str]:
    """benilla's real global surface, the same way `api-coverage.sh` asks for it."""
    run = subprocess.run(
        ["cargo", "run", "-q", "-p", "benilla-ui", "--example", "dump_globals"],
        cwd=REPO,
        capture_output=True,
        text=True,
    )
    if run.returncode != 0:
        sys.exit("dump_globals failed — build the workspace first")
    return {line.split("\t")[0] for line in run.stdout.splitlines() if line.strip()}


def addon_calls(root: Path) -> set[str]:
    """Every identifier the addon corpus calls, as a crude `name(` scan — enough to rank."""
    if not root.is_dir():
        return set()
    called = set()
    pattern = re.compile(rb"\b([A-Za-z_][A-Za-z0-9_]{2,48})\s*\(")
    for lua in root.rglob("*.lua"):
        try:
            body = lua.read_bytes()
        except OSError:
            continue
        for match in pattern.finditer(body):
            called.add(match.group(1).decode("latin1"))
    return called


def unpack(dll: Path, out: Path) -> None:
    """Write out the addon a DLL carries in `.rsrc`, one file per path in its string pool."""
    import gzip
    import os

    image = dll.read_bytes()
    pe = struct.unpack_from("<I", image, 0x3C)[0]
    nsec = struct.unpack_from("<H", image, pe + 6)[0]
    opt = struct.unpack_from("<H", image, pe + 20)[0]
    base = pe + 24 + opt
    rsrc = None
    for i in range(nsec):
        off = base + 40 * i
        name = image[off : off + 8].rstrip(b"\0").decode("latin1")
        vsize, va, rsize, raw = struct.unpack_from("<IIII", image, off + 8)
        if name == ".rsrc":
            rsrc = (va, raw, rsize)
    if rsrc is None:
        sys.exit(f"{dll.name}: no .rsrc section")
    rva, raw, rsize = rsrc
    blob = image[raw : raw + rsize]

    entries: list[bytes] = []

    def walk(off: int) -> None:
        nnamed, nid = struct.unpack_from("<HH", blob, off + 12)
        for i in range(nnamed + nid):
            _, dataoff = struct.unpack_from("<II", blob, off + 16 + 8 * i)
            if dataoff & 0x80000000:
                walk(dataoff & 0x7FFFFFFF)
            else:
                drva, dsize = struct.unpack_from("<II", blob, dataoff)
                start = raw + (drva - rva)
                entries.append(image[start : start + dsize])

    walk(0)
    # The paths, in the order the pool holds them — which is the order the files inside one blob
    # are concatenated. Grouped by top-level folder, because each blob is one addon and the pool
    # also holds paths that belong to none of them (`Interface\\FrameXML\\Bindings.xml`, the
    # SavedVariables templates). A blob is matched to the group whose file count it equals, which
    # is unambiguous here and refuses rather than guesses when it is not.
    groups: dict[str, list[str]] = {}
    for m in re.finditer(rb"[A-Za-z!][\w!.-]*\\[^\x00\n]*?\.(?:lua|xml|toc|tga|blp)", image):
        path = m.group(0).decode("latin1").replace("\\", "/")
        if "%" in path or path.startswith(("Interface/", "WTF/")):
            continue
        groups.setdefault(path.split("/", 1)[0], []).append(path)

    written = 0
    for data in entries:
        if data[:2] != b"\x1f\x8b":
            continue
        body = gzip.decompress(data)
        nl = body.find(b"\n\n")
        if nl < 0:
            continue
        try:
            sizes = [int(x) for x in body[:nl].split(b"\n")]
        except ValueError:
            continue
        files = body[nl + 2 :]
        if sum(sizes) != len(files):
            continue
        hits = [k for k, v in groups.items() if len(v) == len(sizes)]
        if len(hits) != 1:
            print(f"  skipped a {len(sizes)}-file blob: {len(hits)} folders match that count")
            continue
        paths, pos = groups.pop(hits[0]), 0
        for path, size in zip(paths, sizes):
            dst = out / path
            dst.parent.mkdir(parents=True, exist_ok=True)
            dst.write_bytes(files[pos : pos + size])
            pos += size
            written += 1
    if written == 0:
        sys.exit(f"{dll.name}: no packed addon found in .rsrc")
    print(f"{dll.name}: wrote {written} files to {out}")


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--names", action="store_true", help="list every candidate per DLL")
    ap.add_argument("--wanted", action="store_true", help="list only what the addon corpus calls")
    ap.add_argument(
        "--unpack",
        nargs=2,
        metavar=("DLL", "DIR"),
        help="write out the addon a DLL carries in .rsrc (ClassicAPI.dll) and exit",
    )
    args = ap.parse_args()

    if args.unpack:
        dll = Path(args.unpack[0])
        unpack(dll if dll.exists() else CLIENT / args.unpack[0], Path(args.unpack[1]))
        return

    vanilla = census(REPO / "reference" / "1.12-globals.tsv")
    have = ours()
    called = addon_calls(REPO / "benilla-config" / "AddOns")
    print(f"1.12 census {len(vanilla)} · benilla exposes {len(have)} · addon corpus calls {len(called)}\n")

    dlls = sorted(p for p in CLIENT.glob("*.dll"))
    if not dlls:
        sys.exit(f"no DLLs beside the client at {CLIENT}")
    for path in dlls:
        try:
            names = pointed_at(path)
        except Exception as err:  # a packed or malformed image: say so, do not guess
            print(f"{path.name}: unreadable ({err})")
            continue
        # A Lua global these DLLs add is CapitalCase or SCREAMING_CASE by convention; the
        # lower-case tail is overwhelmingly struct fields and C symbols.
        candidates = {
            n
            for n in names
            if n not in vanilla and n not in LUA_OWN and n[:1].isupper()
        }
        missing = sorted(n for n in candidates if n not in have)
        wanted = sorted(n for n in missing if n in called)
        print(
            f"{path.name}: {len(names)} pointed-at · {len(candidates)} beyond 1.12 · "
            f"{len(missing)} not in benilla · {len(wanted)} of those the addons call"
        )
        for name in wanted if args.wanted else (missing if args.names else []):
            print(f"    {name}{'' if name in called else '   (no addon calls it)'}")


if __name__ == "__main__":
    main()