Initial: MPQ patch packager (Gitea Action, StormLib-based)

A BigWigs-style CI packager for WoW MPQ patch archives. Reads an mpq.yaml
manifest, stages files, builds a vanilla-1.12-compatible patch-X.mpq via
StormLib, and (in the example workflow) attaches it to a Gitea release on tag.
This commit is contained in:
2026-06-25 21:34:12 -05:00
parent df6372ac94
commit 4293e90a97
12 changed files with 628 additions and 0 deletions
+35
View File
@@ -0,0 +1,35 @@
# CI for the packager itself: build the Docker image and smoke-test that it
# produces a non-empty patch-Z.mpq from the test fixture.
name: CI
on:
push:
branches: [main]
pull_request:
jobs:
build-and-smoke:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build image
run: docker build -t mpq-packager:ci .
- name: Package the fixture
run: |
set -euo pipefail
docker run --rm \
-e INPUT_MANIFEST=mpq.yaml \
-e INPUT_VERSION=v0.0.0-ci \
-e INPUT_OUT_DIR=/work/dist \
-e GITHUB_WORKSPACE=/work \
-v "$PWD/test:/work" \
mpq-packager:ci
- name: Assert artifact exists
run: |
set -euo pipefail
test -s test/dist/patch-Z.mpq
echo "OK: $(stat -c%s test/dist/patch-Z.mpq) bytes"
+4
View File
@@ -0,0 +1,4 @@
dist/
test/dist/
*.mpq
*.o
+42
View File
@@ -0,0 +1,42 @@
# mpq-packager — build image for the Gitea Action.
#
# Stage 1 builds StormLib (the MoPaQ library MPQEditor itself is built on) and
# our small `mpqpack` binary against it. Stage 2 is a slim runtime carrying the
# binary plus Python for the orchestration layer.
FROM debian:bookworm-slim AS build
RUN apt-get update && apt-get install -y --no-install-recommends \
git cmake g++ make zlib1g-dev libbz2-dev ca-certificates \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /build
# Pin StormLib to a known-good tag for reproducible builds.
ARG STORMLIB_REF=v9.25
RUN git clone --depth 1 --branch "${STORMLIB_REF}" \
https://github.com/ladislav-zezula/StormLib.git
RUN cmake -S StormLib -B StormLib/build \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_SHARED_LIBS=OFF \
-DWITH_LIBTOMCRYPT=OFF \
&& cmake --build StormLib/build --target storm -j "$(nproc)" \
&& cmake --install StormLib/build
COPY src/mpqpack.c .
# StormLib installs headers to /usr/local/include and libstorm.a to
# /usr/local/lib. Link zlib/bz2 for sector (de)compression.
RUN g++ -O2 -std=c++17 -o /usr/local/bin/mpqpack mpqpack.c \
-I/usr/local/include -lstorm -lz -lbz2 \
&& /usr/local/bin/mpqpack 2>/dev/null; \
test -x /usr/local/bin/mpqpack
FROM debian:bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends \
python3 python3-yaml zlib1g libbz2-1.0 ca-certificates \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /usr/local/bin/mpqpack /usr/local/bin/mpqpack
COPY package.py /opt/mpq-packager/package.py
COPY entrypoint.sh /opt/mpq-packager/entrypoint.sh
RUN chmod +x /opt/mpq-packager/entrypoint.sh
ENTRYPOINT ["/opt/mpq-packager/entrypoint.sh"]
+118
View File
@@ -1,2 +1,120 @@
# mpq-packager
A CI packager for World of Warcraft **MPQ patch archives** — the MPQ equivalent
of [BigWigsMods/packager](https://github.com/BigWigsMods/packager). Point it at a
WoW interface repo and it produces a vanilla-1.12-compatible `patch-X.mpq`,
ready to attach to a Gitea release.
It runs as a **Gitea Action** on a Linux runner. MPQs are built with
[StormLib](https://github.com/ladislav-zezula/StormLib) (the MoPaQ library
Ladik's MPQ Editor is itself built on) so no Windows host or Wine is needed.
## What it does
1. Reads a manifest (`mpq.yaml`) describing what to pack and where it lands
inside the archive.
2. Stages the selected files, honouring ignore globs.
3. Optionally substitutes `@project-version@` / `@project-revision@` /
`@project-date-iso@` tokens in text files.
4. Names the output `patch-<letter>.mpq` — vanilla 1.12's loader only accepts a
**single-character** suffix (`patch-Z.mpq` ✅, `patch-myproject.mpq` ❌).
5. Builds the archive as **MPQ format v1** with zlib-compressed sectors and a
`(listfile)`.
## Usage in a consuming repo
Add a manifest at the repo root (`mpq.yaml`):
```yaml
name: GlueXML
patch-letter: Z # high letters sort last → win on conflict
substitute:
enabled: true
extensions: [".lua", ".xml", ".toc"]
contents:
- src: . # repo root
dest: Interface/GlueXML # in-MPQ path (slashes become backslashes)
ignore:
- ".git/**"
- ".gitea/**"
- "mpq.yaml"
- "*.md"
```
Add a release workflow at `.gitea/workflows/release.yml` (see
[`examples/release.yml`](examples/release.yml)). Pushing a tag like `v1.0.0`
builds the MPQ and attaches it to a matching Gitea release:
```yaml
- uses: paste/mpq-packager@v1
id: pack
with:
manifest: mpq.yaml
version: ${{ github.ref_name }}
# steps.pack.outputs.mpq-path / mpq-name point at the built archive
```
Working examples for the GlueXML repo live in [`examples/`](examples/).
## Manifest reference
| Key | Default | Meaning |
|-----------------|--------------------|----------------------------------------------------------------|
| `name` | — | Cosmetic, used in logs. |
| `patch-letter` | `Z` | Single char; becomes `patch-<letter>.mpq`. |
| `max-files` | file count + 16 | Hash-table sizing hint (StormLib rounds up to a power of two). |
| `substitute` | disabled | `enabled` + `extensions` list for `@token@` replacement. |
| `contents[].src` | `.` | Repo-relative file or directory to pack. |
| `contents[].dest` | — | In-MPQ destination path (forward slashes; converted to `\`). |
| `contents[].ignore` | `[]` | Glob patterns matched against repo-relative paths. |
## Action inputs / outputs
| Input | Default | Description |
|------------|------------|----------------------------------------------------------|
| `manifest` | `mpq.yaml` | Manifest path, relative to repo root. |
| `version` | ref name | Stamped into `@project-version@`. |
| `out-dir` | `dist` | Where the `.mpq` is written. |
| Output | Description |
|------------|--------------------------------------|
| `mpq-path` | Filesystem path of the built `.mpq`. |
| `mpq-name` | Filename, e.g. `patch-Z.mpq`. |
## Prerequisites
- **Actions enabled** on the consuming repo (repo Settings → Actions).
- An **`act_runner`** registered to your Gitea instance advertising a label your
workflow's `runs-on` uses (e.g. `ubuntu-latest`), with Docker available.
## Local build & test
```bash
docker build -t mpq-packager .
docker run --rm \
-e INPUT_MANIFEST=mpq.yaml -e INPUT_VERSION=v0.0.0 \
-e INPUT_OUT_DIR=/work/dist -e GITHUB_WORKSPACE=/work \
-v "$PWD/test:/work" mpq-packager
# -> test/dist/patch-Z.mpq
```
## Layout
| Path | Role |
|---------------------|-----------------------------------------------------------|
| `src/mpqpack.c` | StormLib-based packer: staging dir → MPQ v1. |
| `package.py` | Orchestration: manifest, staging, ignores, substitution. |
| `entrypoint.sh` | Action entrypoint wiring `INPUT_*` env to `package.py`. |
| `Dockerfile` | Builds StormLib + `mpqpack`; ships the runtime image. |
| `action.yml` | Gitea/GitHub Docker action definition. |
| `examples/` | Manifest + release workflow for a consuming repo. |
| `test/` | Fixture + manifest exercised by CI. |
## Notes & caveats
- **`Blizzard_*` addons** still go through the engine's `.toc.sig` check
regardless of MPQ residency — packaging them here does not bypass it.
- Compression is zlib, which vanilla 1.12.1 decompresses. The output is MPQ v1
specifically; v2+ headers are not read by the 1.12 loader.
- See ClassicAPI `docs/MPQBuilding.md` for the empirical background on the
naming constraint and load-order behaviour.
+35
View File
@@ -0,0 +1,35 @@
name: "MPQ Packager"
description: "Pack a WoW interface repo into a vanilla-compatible patch-X.mpq using StormLib."
author: "paste"
inputs:
manifest:
description: "Path to the MPQ manifest, relative to the repo root."
required: false
default: "mpq.yaml"
version:
description: "Release version stamped into @project-version@ (defaults to the tag/ref name)."
required: false
default: ""
out-dir:
description: "Directory the built .mpq is written to."
required: false
default: "dist"
outputs:
mpq-path:
description: "Filesystem path of the built .mpq."
mpq-name:
description: "Filename of the built .mpq (e.g. patch-Z.mpq)."
runs:
using: "docker"
image: "Dockerfile"
env:
INPUT_MANIFEST: ${{ inputs.manifest }}
INPUT_VERSION: ${{ inputs.version }}
INPUT_OUT_DIR: ${{ inputs.out-dir }}
branding:
icon: "package"
color: "purple"
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
# Gitea Action entrypoint. Reads action inputs from the INPUT_* environment
# variables Gitea/GitHub Actions set, then runs the packager.
set -euo pipefail
manifest="${INPUT_MANIFEST:-mpq.yaml}"
repo="${INPUT_REPO:-${GITHUB_WORKSPACE:-.}}"
out_dir="${INPUT_OUT_DIR:-dist}"
version="${INPUT_VERSION:-${GITHUB_REF_NAME:-}}"
# Derive revision/date from git when available; harmless if absent.
revision=""
date_iso=""
if command -v git >/dev/null 2>&1 && git -C "$repo" rev-parse --short HEAD >/dev/null 2>&1; then
revision="$(git -C "$repo" rev-parse --short HEAD)"
date_iso="$(git -C "$repo" show -s --format=%cI HEAD)"
fi
echo "mpq-packager: manifest=$manifest repo=$repo version=${version:-<none>}"
exec python3 /opt/mpq-packager/package.py \
--manifest "$manifest" \
--repo "$repo" \
--out-dir "$out_dir" \
--version "$version" \
--revision "$revision" \
--date "$date_iso"
+20
View File
@@ -0,0 +1,20 @@
# Example manifest for the GlueXML repo. Copy this to the repo root as
# `mpq.yaml`. It packs the whole repo into Interface\GlueXML\ inside a
# patch-Z.mpq (Z sorts after Blizzard's stock MPQs, so its files win on conflict).
name: GlueXML
patch-letter: Z
substitute:
enabled: true
extensions: [".lua", ".xml", ".toc"]
contents:
- src: .
dest: Interface/GlueXML
ignore:
- ".git/**"
- ".gitea/**"
- "mpq.yaml"
- "*.md"
- "dist/**"
+50
View File
@@ -0,0 +1,50 @@
# Example consuming workflow. Drop this in a WoW interface repo (e.g. GlueXML)
# at .gitea/workflows/release.yml. Pushing a tag like `v1.0.0` builds the MPQ
# and attaches it to a matching Gitea release.
#
# Requires: Actions enabled on the repo, and a runner advertising `ubuntu-latest`
# (or change runs-on to a label your act_runner registers).
name: Release MPQ
on:
push:
tags:
- "v*"
jobs:
package:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build patch MPQ
id: pack
uses: paste/mpq-packager@v1
with:
manifest: mpq.yaml
version: ${{ github.ref_name }}
- name: Create release and upload asset
env:
TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
API="${GITHUB_SERVER_URL}/api/v1/repos/${GITHUB_REPOSITORY}"
TAG="${GITHUB_REF_NAME}"
MPQ="${{ steps.pack.outputs.mpq-path }}"
NAME="${{ steps.pack.outputs.mpq-name }}"
echo "Creating release $TAG"
RID=$(curl -fsSL -X POST "$API/releases" \
-H "Authorization: token $TOKEN" \
-H "Content-Type: application/json" \
-d "{\"tag_name\":\"$TAG\",\"name\":\"$TAG\",\"draft\":false,\"prerelease\":false}" \
| python3 -c 'import sys,json; print(json.load(sys.stdin)["id"])')
echo "Uploading $NAME to release $RID"
curl -fsSL -X POST "$API/releases/$RID/assets?name=$NAME" \
-H "Authorization: token $TOKEN" \
-H "Content-Type: application/octet-stream" \
--data-binary "@$MPQ"
echo "Done."
+172
View File
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
"""Stage a repo into an MPQ patch according to a manifest, then invoke mpqpack.
This is the orchestration layer (the "BigWigs release.sh" equivalent). It:
1. reads a manifest (mpq.yaml) describing what to pack and where it lands
inside the archive,
2. copies the selected files into a temp staging tree, honouring ignore
globs,
3. optionally substitutes @keyword@ tokens (version/revision/date) in text
files,
4. names the output `patch-<letter>.mpq` per the vanilla 1.12 loader
constraint (single-character suffix only), and
5. calls the `mpqpack` binary to build the archive with StormLib.
Manifest schema (all keys optional except `contents`):
name: GlueXML # cosmetic, used in logs / release naming
patch-letter: Z # single char A-Z; high letters sort last & win
max-files: 4096 # hash-table sizing hint
substitute:
enabled: true
extensions: [".lua", ".xml", ".toc"]
contents:
- src: . # path in the repo (file or dir)
dest: Interface/GlueXML # in-MPQ path (forward slashes; converted to \)
ignore: # glob patterns, matched against repo-relative paths
- ".git/**"
- ".gitea/**"
- "mpq.yaml"
- "*.md"
"""
from __future__ import annotations
import argparse
import fnmatch
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
try:
import yaml
except ImportError:
sys.exit("package.py: PyYAML is required (pip install pyyaml)")
DEFAULT_IGNORES = [".git", ".git/**", ".gitea", ".gitea/**", "mpq.yaml"]
def log(msg: str) -> None:
print(f"package.py: {msg}", flush=True)
def matches_any(rel_posix: str, patterns: list[str]) -> bool:
"""True if rel_posix matches a glob, or sits under an ignored directory."""
first = rel_posix.split("/", 1)[0]
for pat in patterns:
if fnmatch.fnmatch(rel_posix, pat):
return True
# bare directory name (e.g. ".git") ignores the whole subtree
if "/" not in pat and "*" not in pat and first == pat:
return True
return False
def stage_entry(repo: Path, stage: Path, entry: dict) -> int:
src = (repo / entry.get("src", ".")).resolve()
dest_rel = entry["dest"].replace("\\", "/").strip("/")
ignore = DEFAULT_IGNORES + list(entry.get("ignore", []))
if not src.exists():
sys.exit(f"package.py: source not found: {src}")
copied = 0
files = [src] if src.is_file() else sorted(p for p in src.rglob("*") if p.is_file())
for f in files:
rel = f.name if src.is_file() else f.relative_to(src).as_posix()
if src.is_dir() and matches_any(rel, ignore):
continue
target = stage / dest_rel / rel
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(f, target)
copied += 1
log(f"staged {copied} files: {entry.get('src', '.')} -> {dest_rel.replace('/', chr(92))}")
return copied
def substitute_tokens(stage: Path, exts: list[str], tokens: dict[str, str]) -> None:
pattern = re.compile("|".join(re.escape(k) for k in tokens))
exts = {e.lower() for e in exts}
n = 0
for f in stage.rglob("*"):
if not f.is_file() or f.suffix.lower() not in exts:
continue
try:
text = f.read_text(encoding="utf-8")
except UnicodeDecodeError:
continue
new = pattern.sub(lambda m: tokens[m.group(0)], text)
if new != text:
f.write_text(new, encoding="utf-8")
n += 1
if n:
log(f"substituted version tokens in {n} files")
def main() -> int:
ap = argparse.ArgumentParser(description="Build an MPQ patch from a manifest.")
ap.add_argument("--manifest", default="mpq.yaml")
ap.add_argument("--repo", default=".", help="repo root to package")
ap.add_argument("--out-dir", default="dist")
ap.add_argument("--version", default="", help="release version, e.g. v1.0.0")
ap.add_argument("--revision", default="", help="git short hash")
ap.add_argument("--date", default="", help="ISO build date")
ap.add_argument("--mpqpack", default="mpqpack", help="path to mpqpack binary")
args = ap.parse_args()
repo = Path(args.repo).resolve()
manifest_path = repo / args.manifest
if not manifest_path.exists():
sys.exit(f"package.py: manifest not found: {manifest_path}")
manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
contents = manifest.get("contents")
if not contents:
sys.exit("package.py: manifest has no 'contents'")
letter = str(manifest.get("patch-letter", "Z")).strip()
if len(letter) != 1 or not letter.isalnum():
sys.exit(f"package.py: patch-letter must be a single char, got {letter!r}")
out_dir = Path(args.out_dir).resolve()
out_dir.mkdir(parents=True, exist_ok=True)
out_mpq = out_dir / f"patch-{letter}.mpq"
with tempfile.TemporaryDirectory(prefix="mpqstage-") as tmp:
stage = Path(tmp)
total = sum(stage_entry(repo, stage, e) for e in contents)
if total == 0:
sys.exit("package.py: nothing staged — check your manifest paths/ignores")
sub = manifest.get("substitute", {})
if sub.get("enabled"):
tokens = {
"@project-version@": args.version,
"@project-revision@": args.revision,
"@project-date-iso@": args.date,
}
substitute_tokens(stage, sub.get("extensions", [".lua", ".xml", ".toc"]), tokens)
max_files = str(manifest.get("max-files", total + 16))
log(f"building {out_mpq.name} from {total} files")
rc = subprocess.run(
[args.mpqpack, str(out_mpq), str(stage), max_files]
).returncode
if rc != 0:
sys.exit(f"package.py: mpqpack failed (exit {rc})")
log(f"done -> {out_mpq}")
# Expose the artifact path to a Gitea/GitHub Actions step.
if gh_out := os.environ.get("GITHUB_OUTPUT"):
with open(gh_out, "a", encoding="utf-8") as fh:
fh.write(f"mpq-path={out_mpq}\n")
fh.write(f"mpq-name={out_mpq.name}\n")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+112
View File
@@ -0,0 +1,112 @@
/*
* mpqpack — pack a staging directory tree into a vanilla-compatible MPQ patch.
*
* mpqpack <output.mpq> <staging-dir> [maxFiles]
*
* Every regular file under <staging-dir> is added to the archive. Its internal
* (in-MPQ) name is the path relative to <staging-dir> with '/' rewritten to the
* '\' separator WoW expects. So a staging layout of
*
* stage/Interface/GlueXML/CharacterSelect.lua
*
* is stored in the MPQ as Interface\GlueXML\CharacterSelect.lua.
*
* The archive is created as MPQ format v1 (32-bit) with zlib-compressed sectors
* and a (listfile) — the combination vanilla 1.12.1 reads. See docs/MPQBuilding.md
* in ClassicAPI for the empirical notes behind these choices.
*/
#include <StormLib.h>
#include <dirent.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
static HANDLE g_mpq;
static char g_root[PATH_MAX];
static size_t g_rootlen;
static unsigned g_added;
static void die(const char *what, const char *detail) {
fprintf(stderr, "mpqpack: %s: %s (err %u)\n", what, detail, GetLastError());
exit(1);
}
/* Build the in-MPQ name for a staged file and add it. */
static void add_file(const char *fullpath) {
const char *rel = fullpath + g_rootlen;
while (*rel == '/') rel++; /* trim leading separator */
char internal[PATH_MAX];
snprintf(internal, sizeof internal, "%s", rel);
for (char *p = internal; *p; ++p)
if (*p == '/') *p = '\\'; /* WoW wants backslashes */
DWORD flags = MPQ_FILE_COMPRESS | MPQ_FILE_REPLACEEXISTING;
if (!SFileAddFileEx(g_mpq, fullpath, internal, flags,
MPQ_COMPRESSION_ZLIB, MPQ_COMPRESSION_ZLIB))
die("add", internal);
printf(" + %s\n", internal);
g_added++;
}
/* Recursively walk a directory. mode 0 = count only, mode 1 = add. */
static unsigned walk(const char *dir, int add) {
DIR *d = opendir(dir);
if (!d) die("opendir", dir);
unsigned count = 0;
struct dirent *e;
while ((e = readdir(d))) {
if (!strcmp(e->d_name, ".") || !strcmp(e->d_name, "..")) continue;
char path[PATH_MAX];
snprintf(path, sizeof path, "%s/%s", dir, e->d_name);
struct stat st;
if (stat(path, &st)) die("stat", path);
if (S_ISDIR(st.st_mode)) {
count += walk(path, add);
} else if (S_ISREG(st.st_mode)) {
if (add) add_file(path);
count++;
}
}
closedir(d);
return count;
}
int main(int argc, char **argv) {
if (argc < 3) {
fprintf(stderr, "usage: %s <output.mpq> <staging-dir> [maxFiles]\n", argv[0]);
return 2;
}
const char *out = argv[1];
const char *stage = argv[2];
if (!realpath(stage, g_root)) die("resolve", stage);
g_rootlen = strlen(g_root);
/* Count first so the hash table is sized to fit (v1 tables are fixed). */
unsigned files = walk(g_root, 0);
DWORD maxFiles = argc > 3 ? (DWORD)strtoul(argv[3], NULL, 10) : 0;
if (maxFiles < files) maxFiles = files;
if (maxFiles < 16) maxFiles = 16; /* StormLib rounds up to pow2 */
remove(out);
DWORD createFlags = MPQ_CREATE_LISTFILE | MPQ_CREATE_ARCHIVE_V1;
if (!SFileCreateArchive(out, createFlags, maxFiles, &g_mpq))
die("create", out);
walk(g_root, 1);
if (!SFileFlushArchive(g_mpq)) die("flush", out);
if (!SFileCloseArchive(g_mpq)) die("close", out);
printf("mpqpack: wrote %s (%u files)\n", out, g_added);
return 0;
}
+4
View File
@@ -0,0 +1,4 @@
-- Fixture file packed by the CI smoke test. Version token below is substituted
-- by the packager when substitute.enabled is true.
local VERSION = "@project-version@"
print("mpq-packager fixture, version " .. VERSION)
+9
View File
@@ -0,0 +1,9 @@
# Manifest used by the CI smoke test.
name: fixture
patch-letter: Z
substitute:
enabled: true
extensions: [".lua"]
contents:
- src: fixture
dest: Interface/AddOns/Fixture