build: pack each embedded addon into one gzip stream

Each embedded file was its own gzipped RCDATA resource, 175 in all.
Every stream started with an empty deflate window, so no file could
match text it shares with its neighbours, and 97 of the files are under
4 KB, where that costs the most. Each resource also cost about 50 bytes
of resource directory and padding.

cmake/pack_files.cmake, which replaces gzip_file.cmake, now joins each
addon's files at build time into one gzip stream. It holds an index,
each file's size as a decimal line and then an empty line, followed by
the files' bytes in path order. `cmake -E cat` joins them byte for
byte, in batches of 100 so the Windows command-line limit cannot fail
the build. The index goes through file(CONFIGURE NEWLINE_STYLE UNIX)
because file(WRITE) writes \r\n on Windows. The resources are now three:
Bindings.xml as a pack of one, !!!ClassicAPI, and ClassicAPI_DebugTools.

The manifest entries become {path, resourceId, index}, and
Embed::Resource(id) becomes Embed::File(id, index). On first use it
inflates the whole pack with the engine's zlib, splits it into views,
and keeps it for the life of the DLL, as before. A pack whose sizes do
not cover its bytes exactly is rejected. The load-on-demand DebugTools
pack is still inflated only when it loads.

Embedded data 323,947 -> 272,825 bytes; .rsrc 334,336 -> 274,432;
DLL 1,632,768 -> 1,573,888 bytes.

cmake/embed_resources.cmake

@@ -3,8 +3,8 @@
 #
 # Runs at configure time. It writes two files into OUT_DIR:
 #
-#   embedded.rc           one `<id> RCDATA "<path>"` line per file; rc.exe
-#                         reads each file's gzipped copy from disk.
+#   embedded.rc           one `<id> RCDATA "<path>"` line per pack; rc.exe
+#                         reads each gzipped pack from disk.
 #   embedded_resources.h  the IDs the DLL looks up at runtime through
 #                         src/embed/Resource.h:
 #
@@ -12,14 +12,24 @@
 #     namespace Addons::Embedded::Manifest {
 #         constexpr const char *kAddons[] = { ... };  // in ADDONS order
 #         constexpr size_t kAddonCount = ...;
-#         struct File { const char *path; int resourceId; };
+#         struct File { const char *path; int resourceId; int index; };
 #         constexpr File kFiles[] = { ... };
 #         constexpr size_t kFileCount = ...;
 #     }
 #
+# Each resource is a pack: one gzip stream of several files, which
+# pack_files.cmake joins at build time into `OUT_DIR/embed/<name>.pack.gz`.
+# Resource 1 is the bindings XML, a pack of one; then each addon is one
+# pack, in ADDONS order. A pack compresses each file against the files
+# before it, so the addon text packs to about 22% of its size, against 27%
+# for one gzip stream per file. The DLL inflates a whole pack with the
+# engine's zlib on first use (src/embed/Resource.cpp), so the
+# load-on-demand addon costs nothing until it loads.
+#
 # File paths are relative to ADDONS_ROOT, in backslash form
 # (`!!!ClassicAPI\Util\Print.lua`) — the suffix the engine builds after
-# `Interface\AddOns\`. The DLL compares them ignoring case.
+# `Interface\AddOns\`. The DLL compares them ignoring case. A file's index
+# is its place in its addon's pack, which is in path order.
 #
 # Each addon's `.toc` is the one file that is not embedded verbatim: its
 # `## Version:` line is stamped with ADDON_VERSION into a copy under
@@ -28,30 +38,37 @@
 # never embedded: it must reflect on-disk presence only (see
 # DiskHasDevMarker in src/addons/Embedded.cpp).
 #
-# Every resource is gzipped at build time (gzip_file.cmake) into
-# `OUT_DIR/embed/<path>.gz`, and the DLL inflates it with the engine's zlib on
-# first use (src/embed/Resource.cpp). The addon text compresses to about a
-# quarter of its size.
-#
 # Rebuild triggers: adding or removing a file reconfigures through the
 # CONFIGURE_DEPENDS glob; editing a toc reconfigures through
-# CMAKE_CONFIGURE_DEPENDS; editing any other file re-gzips it through its
-# custom command, which recompiles the .rc through its OBJECT_DEPENDS.
+# CMAKE_CONFIGURE_DEPENDS; editing any other file re-packs its addon through
+# the pack's custom command, which recompiles the .rc through its
+# OBJECT_DEPENDS.
 #
 # Usage:
 #   embed_resources(TARGET <t> ADDONS_ROOT <dir> ADDONS <folder names...>
 #                   ADDON_VERSION <v> BINDINGS_XML <file> OUT_DIR <dir>)
 
-set(_EMBED_GZIP_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/gzip_file.cmake")
+set(_EMBED_PACK_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/pack_files.cmake")
 
-# Add the build-time command that gzips `src` into `gz`.
-function(_embed_gzip src gz)
+# Add the build-time command that packs `ARGN` (absolute paths, in pack
+# order) into `<out_dir>/embed/<name>.pack.gz`, and set `gz` in the caller
+# to that path. The paths reach the script through a generated list file:
+# an addon's paths are too long for one command line.
+function(_embed_pack out_dir name)
+    set(pack "${out_dir}/embed/${name}.pack")
+    set(list "set(FILES\n")
+    foreach(path ${ARGN})
+        string(APPEND list "    [==[${path}]==]\n")
+    endforeach()
+    string(APPEND list ")\n")
+    _embed_write_if_changed("${pack}.cmake" "${list}")
     add_custom_command(
-        OUTPUT "${gz}"
-        COMMAND "${CMAKE_COMMAND}" "-DIN=${src}" "-DOUT=${gz}"
-                -P "${_EMBED_GZIP_SCRIPT}"
-        DEPENDS "${src}" "${_EMBED_GZIP_SCRIPT}"
+        OUTPUT "${pack}.gz"
+        COMMAND "${CMAKE_COMMAND}" "-DLIST=${pack}.cmake" "-DOUT=${pack}.gz"
+                -P "${_EMBED_PACK_SCRIPT}"
+        DEPENDS ${ARGN} "${pack}.cmake" "${_EMBED_PACK_SCRIPT}"
         VERBATIM)
+    set(gz "${pack}.gz" PARENT_SCOPE)
 endfunction()
 
 # Write `content` to `path` only when it differs, so an unchanged
@@ -71,56 +88,55 @@ function(embed_resources)
     set(addon_names "")
     set(manifest "")
     set(inputs "")
-    set(id 1)
 
-    set(gz "${E_OUT_DIR}/embed/Bindings.xml.gz")
-    _embed_gzip("${E_BINDINGS_XML}" "${gz}")
-    string(APPEND rc "${id} RCDATA \"${gz}\"\n")
+    set(bindings_id 1)
+    _embed_pack("${E_OUT_DIR}" "Bindings.xml" "${E_BINDINGS_XML}")
+    string(APPEND rc "${bindings_id} RCDATA \"${gz}\"\n")
     list(APPEND inputs "${gz}")
-    set(bindings_id ${id})
-    math(EXPR id "${id} + 1")
 
-    set(files "")
+    set(id 2)
     foreach(addon ${E_ADDONS})
         string(APPEND addon_names "    \"${addon}\",\n")
-        file(GLOB_RECURSE addon_files CONFIGURE_DEPENDS RELATIVE
+        file(GLOB_RECURSE files CONFIGURE_DEPENDS RELATIVE
              "${E_ADDONS_ROOT}" "${E_ADDONS_ROOT}/${addon}/*")
-        list(FILTER addon_files EXCLUDE REGEX "(^|/)\\.classicapi-dev$")
-        list(APPEND files ${addon_files})
-    endforeach()
-    list(SORT files) # deterministic IDs
-    foreach(rel ${files})
-        set(abs "${E_ADDONS_ROOT}/${rel}")
-        if(rel MATCHES "\\.toc$")
-            file(READ "${abs}" toc)
-            string(REGEX REPLACE "## Version:[^\n\r]*"
-                   "## Version: ${E_ADDON_VERSION}" toc "${toc}")
-            set(stamped "${E_OUT_DIR}/embed/${rel}")
-            _embed_write_if_changed("${stamped}" "${toc}")
-            set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${abs}")
-            set(abs "${stamped}")
-        endif()
-        set(gz "${E_OUT_DIR}/embed/${rel}.gz")
-        _embed_gzip("${abs}" "${gz}")
+        list(FILTER files EXCLUDE REGEX "(^|/)\\.classicapi-dev$")
+        list(SORT files) # deterministic indexes
+        set(paths "")
+        set(index 0)
+        foreach(rel ${files})
+            set(abs "${E_ADDONS_ROOT}/${rel}")
+            if(rel MATCHES "\\.toc$")
+                file(READ "${abs}" toc)
+                string(REGEX REPLACE "## Version:[^\n\r]*"
+                       "## Version: ${E_ADDON_VERSION}" toc "${toc}")
+                set(stamped "${E_OUT_DIR}/embed/${rel}")
+                _embed_write_if_changed("${stamped}" "${toc}")
+                set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${abs}")
+                set(abs "${stamped}")
+            endif()
+            list(APPEND paths "${abs}")
+            string(REPLACE "/" "\\\\" rel_bs "${rel}")
+            string(APPEND manifest "    { \"${rel_bs}\", ${id}, ${index} },\n")
+            math(EXPR index "${index} + 1")
+        endforeach()
+        _embed_pack("${E_OUT_DIR}" "${addon}" ${paths})
         string(APPEND rc "${id} RCDATA \"${gz}\"\n")
         list(APPEND inputs "${gz}")
-        string(REPLACE "/" "\\\\" rel_bs "${rel}")
-        string(APPEND manifest "    { \"${rel_bs}\", ${id} },\n")
         math(EXPR id "${id} + 1")
     endforeach()
 
     _embed_write_if_changed("${E_OUT_DIR}/embedded.rc" "${rc}")
     _embed_write_if_changed("${E_OUT_DIR}/embedded_resources.h"
 "// Generated by cmake/embed_resources.cmake — do not edit.
-// Resource IDs for the RCDATA entries in embedded.rc. Load them through
-// src/embed/Resource.h.
+// Resource IDs for the RCDATA packs in embedded.rc, and each embedded file's
+// place in them. Load them through src/embed/Resource.h.
 
 #pragma once
 
 #include <cstddef>
 
 namespace Embed::Ids {
-constexpr int kBindingsXml = ${bindings_id};
+constexpr int kBindingsXml = ${bindings_id}; // a pack of the one file
 } // namespace Embed::Ids
 
 namespace Addons::Embedded::Manifest {
@@ -133,7 +149,8 @@ constexpr size_t kAddonCount = sizeof(kAddons) / sizeof(kAddons[0]);
 
 struct File {
     const char *path;
-    int resourceId;
+    int resourceId; // its addon's pack
+    int index;      // its place in the pack
 };
 
 constexpr File kFiles[] = {

cmake/gzip_file.cmake

@@ -1,8 +0,0 @@
-# gzip_file.cmake — gzip one file at build time, for embed_resources.cmake.
-#
-#   cmake -DIN=<file> -DOUT=<file.gz> -P gzip_file.cmake
-#
-# The raw format holds the one file's bytes with no archive entry around
-# them, so OUT is a plain gzip stream of IN.
-file(ARCHIVE_CREATE OUTPUT "${OUT}" PATHS "${IN}" FORMAT raw
-     COMPRESSION GZip COMPRESSION_LEVEL 9)

cmake/pack_files.cmake

@@ -0,0 +1,50 @@
+# pack_files.cmake — join files into one gzipped pack at build time, for
+# embed_resources.cmake.
+#
+#   cmake -DLIST=<list.cmake> -DOUT=<pack.gz> -P pack_files.cmake
+#
+# LIST sets FILES to the paths to join, in order. The pack is one gzip
+# stream, so each file compresses against the files before it: the addon
+# text packs about 16% smaller than when each file is gzipped alone. Inside
+# the stream is an index, each file's size as a decimal line and then an
+# empty line, followed by the files' bytes in FILES order. The DLL splits
+# it in src/embed/Resource.cpp.
+
+include("${LIST}")
+
+set(index "")
+foreach(file IN LISTS FILES)
+    file(SIZE "${file}" size)
+    string(APPEND index "${size}\n")
+endforeach()
+# Not file(WRITE): on Windows it writes each \n as \r\n.
+file(CONFIGURE OUTPUT "${OUT}.index" CONTENT "${index}\n" @ONLY NEWLINE_STYLE UNIX)
+
+# Join `ARGN` byte for byte into `out`.
+function(cat out)
+    execute_process(COMMAND "${CMAKE_COMMAND}" -E cat ${ARGN}
+                    OUTPUT_FILE "${out}" RESULT_VARIABLE rc)
+    if(NOT rc EQUAL 0)
+        message(FATAL_ERROR "pack_files.cmake: cannot join into ${out}: ${rc}")
+    endif()
+endfunction()
+
+# `cmake -E cat` takes its files on the command line, which Windows limits
+# to 32767 characters: join in batches of 100, then join the batches.
+set(parts "${OUT}.index")
+list(LENGTH FILES count)
+set(first 0)
+while(first LESS count)
+    list(SUBLIST FILES ${first} 100 batch)
+    set(part "${OUT}.part${first}")
+    cat("${part}" ${batch})
+    list(APPEND parts "${part}")
+    math(EXPR first "${first} + 100")
+endwhile()
+cat("${OUT}.raw" ${parts})
+
+# The raw format holds the one file's bytes with no archive entry around
+# them, so OUT is a plain gzip stream.
+file(ARCHIVE_CREATE OUTPUT "${OUT}" PATHS "${OUT}.raw" FORMAT raw
+     COMPRESSION GZip COMPRESSION_LEVEL 9)
+file(REMOVE ${parts} "${OUT}.raw")

src/addons/Embedded.cpp

@@ -24,9 +24,9 @@
 //
 // How:
 //
-//   1. The CMake build embeds every file of every listed addon as an
-//      RCDATA resource, plus a generated `{path, resourceId}` manifest
-//      (`embedded_resources.h`).
+//   1. The CMake build embeds each listed addon as one RCDATA resource,
+//      a gzipped pack of its files, plus a generated
+//      `{path, resourceId, index}` manifest (`embedded_resources.h`).
 //
 //   2. We hook the engine's file-read function at `FUN_FILE_READ`.
 //      When the engine asks for a path under
@@ -153,7 +153,8 @@ int MatchAddonPath(const char *path, const char **relPath) {
 Embed::Bytes LookupEmbedded(const char *relPath) {
     for (size_t i = 0; i < Manifest::kFileCount; ++i) {
         if (PathEqualsCI(relPath, Manifest::kFiles[i].path))
-            return Embed::Resource(Manifest::kFiles[i].resourceId);
+            return Embed::File(Manifest::kFiles[i].resourceId,
+                               Manifest::kFiles[i].index);
     }
     return {nullptr, 0};
 }

src/bindings/Inject.cpp

@@ -133,7 +133,7 @@ bool ExtractHeaderAttr(const char *tagStart, const char *tagEnd,
 
 EmbeddedConfig ResolveEmbeddedConfig() {
     EmbeddedConfig out = {};
-    const Embed::Bytes xml = Embed::Resource(Embed::Ids::kBindingsXml);
+    const Embed::Bytes xml = Embed::File(Embed::Ids::kBindingsXml, 0);
     if (xml.data == nullptr) return out;
     const char *content = reinterpret_cast<const char *>(xml.data);
     const char *end = content + xml.size;

src/embed/Resource.cpp

@@ -26,10 +26,46 @@ extern "C" IMAGE_DOS_HEADER __ImageBase;
 
 namespace Embed {
 
-Bytes Resource(int id) {
-    static std::unordered_map<int, std::vector<uint8_t>> inflated;
-    auto it = inflated.find(id);
-    if (it == inflated.end()) {
+namespace {
+
+struct Pack {
+    std::vector<uint8_t> bytes;
+    std::vector<Bytes> files; // views into `bytes`
+};
+
+// Split the inflated `pack.bytes` into `pack.files`. The pack opens with an
+// index, each file's size as a decimal `\n`-ended line and then an empty
+// line, followed by the files' bytes in index order. False unless the sizes
+// cover the bytes exactly.
+bool Split(Pack &pack) {
+    const uint8_t *p = pack.bytes.data();
+    const uint8_t *end = p + pack.bytes.size();
+    while (p < end && *p != '\n') {
+        size_t size = 0;
+        while (p < end && *p >= '0' && *p <= '9')
+            size = size * 10 + (*p++ - '0');
+        if (p == end || *p++ != '\n')
+            return false;
+        pack.files.push_back({nullptr, size});
+    }
+    if (p == end)
+        return false;
+    ++p; // the empty line
+    for (Bytes &file : pack.files) {
+        if (file.size > static_cast<size_t>(end - p))
+            return false;
+        file.data = p;
+        p += file.size;
+    }
+    return p == end;
+}
+
+} // namespace
+
+Bytes File(int id, size_t index) {
+    static std::unordered_map<int, Pack> packs;
+    auto it = packs.find(id);
+    if (it == packs.end()) {
         const HMODULE self = reinterpret_cast<HMODULE>(&__ImageBase);
         HRSRC info = FindResourceW(self, MAKEINTRESOURCEW(id), MAKEINTRESOURCEW(10)); // RT_RCDATA
         if (info == nullptr)
@@ -37,16 +73,19 @@ Bytes Resource(int id) {
         HGLOBAL handle = LoadResource(self, info);
         if (handle == nullptr)
             return {nullptr, 0};
-        std::vector<uint8_t> bytes;
+        Pack pack;
         if (!Encoding::Compress::Inflate(static_cast<const uint8_t *>(LockResource(handle)),
                                          SizeofResource(self, info),
-                                         Encoding::Compress::kGzipWindowBits, bytes) ||
-            bytes.empty())
+                                         Encoding::Compress::kGzipWindowBits, pack.bytes))
             return {nullptr, 0};
-        bytes.shrink_to_fit();
-        it = inflated.emplace(id, std::move(bytes)).first;
+        pack.bytes.shrink_to_fit();
+        if (!Split(pack))
+            return {nullptr, 0};
+        // Moving the vector keeps its buffer, so the views stay valid.
+        it = packs.emplace(id, std::move(pack)).first;
     }
-    return {it->second.data(), it->second.size()};
+    const std::vector<Bytes> &files = it->second.files;
+    return index < files.size() ? files[index] : Bytes{nullptr, 0};
 }
 
 } // namespace Embed

src/embed/Resource.h

@@ -22,11 +22,12 @@ struct Bytes {
     size_t size;
 };
 
-// The bytes of RCDATA resource `id` in this DLL (IDs come from the generated
-// `embedded_resources.h`). Each resource is stored gzipped
-// (cmake/embed_resources.cmake); the first call inflates it and keeps the
-// bytes, so they stay valid for the life of the DLL and must not be freed.
-// `{nullptr, 0}` when the resource is missing or does not inflate.
-Bytes Resource(int id);
+// The bytes of file `index` in the pack that is RCDATA resource `id` in this
+// DLL (both come from the generated `embedded_resources.h`). Each resource
+// is one gzipped pack of files (cmake/pack_files.cmake); the first call for
+// a resource inflates the whole pack and keeps it, so the bytes stay valid
+// for the life of the DLL and must not be freed. `{nullptr, 0}` when the
+// resource is missing, does not inflate or split, or has no file `index`.
+Bytes File(int id, size_t index);
 
 } // namespace Embed