diff --git a/cmake/embed_resources.cmake b/cmake/embed_resources.cmake index 06b4b443..8b40ef40 100644 --- a/cmake/embed_resources.cmake +++ b/cmake/embed_resources.cmake @@ -3,8 +3,8 @@ # # Runs at configure time. It writes two files into OUT_DIR: # -# embedded.rc one ` RCDATA ""` line per file; rc.exe -# reads each file's gzipped copy from disk. +# embedded.rc one ` RCDATA ""` line per pack; rc.exe +# reads each gzipped pack from disk. # embedded_resources.h the IDs the DLL looks up at runtime through # src/embed/Resource.h: # @@ -12,14 +12,24 @@ # namespace Addons::Embedded::Manifest { # constexpr const char *kAddons[] = { ... }; // in ADDONS order # constexpr size_t kAddonCount = ...; -# struct File { const char *path; int resourceId; }; +# struct File { const char *path; int resourceId; int index; }; # constexpr File kFiles[] = { ... }; # constexpr size_t kFileCount = ...; # } # +# Each resource is a pack: one gzip stream of several files, which +# pack_files.cmake joins at build time into `OUT_DIR/embed/.pack.gz`. +# Resource 1 is the bindings XML, a pack of one; then each addon is one +# pack, in ADDONS order. A pack compresses each file against the files +# before it, so the addon text packs to about 22% of its size, against 27% +# for one gzip stream per file. The DLL inflates a whole pack with the +# engine's zlib on first use (src/embed/Resource.cpp), so the +# load-on-demand addon costs nothing until it loads. +# # File paths are relative to ADDONS_ROOT, in backslash form # (`!!!ClassicAPI\Util\Print.lua`) — the suffix the engine builds after -# `Interface\AddOns\`. The DLL compares them ignoring case. +# `Interface\AddOns\`. The DLL compares them ignoring case. A file's index +# is its place in its addon's pack, which is in path order. # # Each addon's `.toc` is the one file that is not embedded verbatim: its # `## Version:` line is stamped with ADDON_VERSION into a copy under @@ -28,30 +38,37 @@ # never embedded: it must reflect on-disk presence only (see # DiskHasDevMarker in src/addons/Embedded.cpp). # -# Every resource is gzipped at build time (gzip_file.cmake) into -# `OUT_DIR/embed/.gz`, and the DLL inflates it with the engine's zlib on -# first use (src/embed/Resource.cpp). The addon text compresses to about a -# quarter of its size. -# # Rebuild triggers: adding or removing a file reconfigures through the # CONFIGURE_DEPENDS glob; editing a toc reconfigures through -# CMAKE_CONFIGURE_DEPENDS; editing any other file re-gzips it through its -# custom command, which recompiles the .rc through its OBJECT_DEPENDS. +# CMAKE_CONFIGURE_DEPENDS; editing any other file re-packs its addon through +# the pack's custom command, which recompiles the .rc through its +# OBJECT_DEPENDS. # # Usage: # embed_resources(TARGET ADDONS_ROOT ADDONS # ADDON_VERSION BINDINGS_XML OUT_DIR ) -set(_EMBED_GZIP_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/gzip_file.cmake") +set(_EMBED_PACK_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/pack_files.cmake") -# Add the build-time command that gzips `src` into `gz`. -function(_embed_gzip src gz) +# Add the build-time command that packs `ARGN` (absolute paths, in pack +# order) into `/embed/.pack.gz`, and set `gz` in the caller +# to that path. The paths reach the script through a generated list file: +# an addon's paths are too long for one command line. +function(_embed_pack out_dir name) + set(pack "${out_dir}/embed/${name}.pack") + set(list "set(FILES\n") + foreach(path ${ARGN}) + string(APPEND list " [==[${path}]==]\n") + endforeach() + string(APPEND list ")\n") + _embed_write_if_changed("${pack}.cmake" "${list}") add_custom_command( - OUTPUT "${gz}" - COMMAND "${CMAKE_COMMAND}" "-DIN=${src}" "-DOUT=${gz}" - -P "${_EMBED_GZIP_SCRIPT}" - DEPENDS "${src}" "${_EMBED_GZIP_SCRIPT}" + OUTPUT "${pack}.gz" + COMMAND "${CMAKE_COMMAND}" "-DLIST=${pack}.cmake" "-DOUT=${pack}.gz" + -P "${_EMBED_PACK_SCRIPT}" + DEPENDS ${ARGN} "${pack}.cmake" "${_EMBED_PACK_SCRIPT}" VERBATIM) + set(gz "${pack}.gz" PARENT_SCOPE) endfunction() # Write `content` to `path` only when it differs, so an unchanged @@ -71,56 +88,55 @@ function(embed_resources) set(addon_names "") set(manifest "") set(inputs "") - set(id 1) - set(gz "${E_OUT_DIR}/embed/Bindings.xml.gz") - _embed_gzip("${E_BINDINGS_XML}" "${gz}") - string(APPEND rc "${id} RCDATA \"${gz}\"\n") + set(bindings_id 1) + _embed_pack("${E_OUT_DIR}" "Bindings.xml" "${E_BINDINGS_XML}") + string(APPEND rc "${bindings_id} RCDATA \"${gz}\"\n") list(APPEND inputs "${gz}") - set(bindings_id ${id}) - math(EXPR id "${id} + 1") - set(files "") + set(id 2) foreach(addon ${E_ADDONS}) string(APPEND addon_names " \"${addon}\",\n") - file(GLOB_RECURSE addon_files CONFIGURE_DEPENDS RELATIVE + file(GLOB_RECURSE files CONFIGURE_DEPENDS RELATIVE "${E_ADDONS_ROOT}" "${E_ADDONS_ROOT}/${addon}/*") - list(FILTER addon_files EXCLUDE REGEX "(^|/)\\.classicapi-dev$") - list(APPEND files ${addon_files}) - endforeach() - list(SORT files) # deterministic IDs - foreach(rel ${files}) - set(abs "${E_ADDONS_ROOT}/${rel}") - if(rel MATCHES "\\.toc$") - file(READ "${abs}" toc) - string(REGEX REPLACE "## Version:[^\n\r]*" - "## Version: ${E_ADDON_VERSION}" toc "${toc}") - set(stamped "${E_OUT_DIR}/embed/${rel}") - _embed_write_if_changed("${stamped}" "${toc}") - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${abs}") - set(abs "${stamped}") - endif() - set(gz "${E_OUT_DIR}/embed/${rel}.gz") - _embed_gzip("${abs}" "${gz}") + list(FILTER files EXCLUDE REGEX "(^|/)\\.classicapi-dev$") + list(SORT files) # deterministic indexes + set(paths "") + set(index 0) + foreach(rel ${files}) + set(abs "${E_ADDONS_ROOT}/${rel}") + if(rel MATCHES "\\.toc$") + file(READ "${abs}" toc) + string(REGEX REPLACE "## Version:[^\n\r]*" + "## Version: ${E_ADDON_VERSION}" toc "${toc}") + set(stamped "${E_OUT_DIR}/embed/${rel}") + _embed_write_if_changed("${stamped}" "${toc}") + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${abs}") + set(abs "${stamped}") + endif() + list(APPEND paths "${abs}") + string(REPLACE "/" "\\\\" rel_bs "${rel}") + string(APPEND manifest " { \"${rel_bs}\", ${id}, ${index} },\n") + math(EXPR index "${index} + 1") + endforeach() + _embed_pack("${E_OUT_DIR}" "${addon}" ${paths}) string(APPEND rc "${id} RCDATA \"${gz}\"\n") list(APPEND inputs "${gz}") - string(REPLACE "/" "\\\\" rel_bs "${rel}") - string(APPEND manifest " { \"${rel_bs}\", ${id} },\n") math(EXPR id "${id} + 1") endforeach() _embed_write_if_changed("${E_OUT_DIR}/embedded.rc" "${rc}") _embed_write_if_changed("${E_OUT_DIR}/embedded_resources.h" "// Generated by cmake/embed_resources.cmake — do not edit. -// Resource IDs for the RCDATA entries in embedded.rc. Load them through -// src/embed/Resource.h. +// Resource IDs for the RCDATA packs in embedded.rc, and each embedded file's +// place in them. Load them through src/embed/Resource.h. #pragma once #include namespace Embed::Ids { -constexpr int kBindingsXml = ${bindings_id}; +constexpr int kBindingsXml = ${bindings_id}; // a pack of the one file } // namespace Embed::Ids namespace Addons::Embedded::Manifest { @@ -133,7 +149,8 @@ constexpr size_t kAddonCount = sizeof(kAddons) / sizeof(kAddons[0]); struct File { const char *path; - int resourceId; + int resourceId; // its addon's pack + int index; // its place in the pack }; constexpr File kFiles[] = { diff --git a/cmake/gzip_file.cmake b/cmake/gzip_file.cmake deleted file mode 100644 index e7ef2242..00000000 --- a/cmake/gzip_file.cmake +++ /dev/null @@ -1,8 +0,0 @@ -# gzip_file.cmake — gzip one file at build time, for embed_resources.cmake. -# -# cmake -DIN= -DOUT= -P gzip_file.cmake -# -# The raw format holds the one file's bytes with no archive entry around -# them, so OUT is a plain gzip stream of IN. -file(ARCHIVE_CREATE OUTPUT "${OUT}" PATHS "${IN}" FORMAT raw - COMPRESSION GZip COMPRESSION_LEVEL 9) diff --git a/cmake/pack_files.cmake b/cmake/pack_files.cmake new file mode 100644 index 00000000..58c44d4f --- /dev/null +++ b/cmake/pack_files.cmake @@ -0,0 +1,50 @@ +# pack_files.cmake — join files into one gzipped pack at build time, for +# embed_resources.cmake. +# +# cmake -DLIST= -DOUT= -P pack_files.cmake +# +# LIST sets FILES to the paths to join, in order. The pack is one gzip +# stream, so each file compresses against the files before it: the addon +# text packs about 16% smaller than when each file is gzipped alone. Inside +# the stream is an index, each file's size as a decimal line and then an +# empty line, followed by the files' bytes in FILES order. The DLL splits +# it in src/embed/Resource.cpp. + +include("${LIST}") + +set(index "") +foreach(file IN LISTS FILES) + file(SIZE "${file}" size) + string(APPEND index "${size}\n") +endforeach() +# Not file(WRITE): on Windows it writes each \n as \r\n. +file(CONFIGURE OUTPUT "${OUT}.index" CONTENT "${index}\n" @ONLY NEWLINE_STYLE UNIX) + +# Join `ARGN` byte for byte into `out`. +function(cat out) + execute_process(COMMAND "${CMAKE_COMMAND}" -E cat ${ARGN} + OUTPUT_FILE "${out}" RESULT_VARIABLE rc) + if(NOT rc EQUAL 0) + message(FATAL_ERROR "pack_files.cmake: cannot join into ${out}: ${rc}") + endif() +endfunction() + +# `cmake -E cat` takes its files on the command line, which Windows limits +# to 32767 characters: join in batches of 100, then join the batches. +set(parts "${OUT}.index") +list(LENGTH FILES count) +set(first 0) +while(first LESS count) + list(SUBLIST FILES ${first} 100 batch) + set(part "${OUT}.part${first}") + cat("${part}" ${batch}) + list(APPEND parts "${part}") + math(EXPR first "${first} + 100") +endwhile() +cat("${OUT}.raw" ${parts}) + +# The raw format holds the one file's bytes with no archive entry around +# them, so OUT is a plain gzip stream. +file(ARCHIVE_CREATE OUTPUT "${OUT}" PATHS "${OUT}.raw" FORMAT raw + COMPRESSION GZip COMPRESSION_LEVEL 9) +file(REMOVE ${parts} "${OUT}.raw") diff --git a/src/addons/Embedded.cpp b/src/addons/Embedded.cpp index 570455c9..5622d4f5 100644 --- a/src/addons/Embedded.cpp +++ b/src/addons/Embedded.cpp @@ -24,9 +24,9 @@ // // How: // -// 1. The CMake build embeds every file of every listed addon as an -// RCDATA resource, plus a generated `{path, resourceId}` manifest -// (`embedded_resources.h`). +// 1. The CMake build embeds each listed addon as one RCDATA resource, +// a gzipped pack of its files, plus a generated +// `{path, resourceId, index}` manifest (`embedded_resources.h`). // // 2. We hook the engine's file-read function at `FUN_FILE_READ`. // When the engine asks for a path under @@ -153,7 +153,8 @@ int MatchAddonPath(const char *path, const char **relPath) { Embed::Bytes LookupEmbedded(const char *relPath) { for (size_t i = 0; i < Manifest::kFileCount; ++i) { if (PathEqualsCI(relPath, Manifest::kFiles[i].path)) - return Embed::Resource(Manifest::kFiles[i].resourceId); + return Embed::File(Manifest::kFiles[i].resourceId, + Manifest::kFiles[i].index); } return {nullptr, 0}; } diff --git a/src/bindings/Inject.cpp b/src/bindings/Inject.cpp index 3fe096c0..d4c7d8a3 100644 --- a/src/bindings/Inject.cpp +++ b/src/bindings/Inject.cpp @@ -133,7 +133,7 @@ bool ExtractHeaderAttr(const char *tagStart, const char *tagEnd, EmbeddedConfig ResolveEmbeddedConfig() { EmbeddedConfig out = {}; - const Embed::Bytes xml = Embed::Resource(Embed::Ids::kBindingsXml); + const Embed::Bytes xml = Embed::File(Embed::Ids::kBindingsXml, 0); if (xml.data == nullptr) return out; const char *content = reinterpret_cast(xml.data); const char *end = content + xml.size; diff --git a/src/embed/Resource.cpp b/src/embed/Resource.cpp index e5430066..453eae2c 100644 --- a/src/embed/Resource.cpp +++ b/src/embed/Resource.cpp @@ -26,10 +26,46 @@ extern "C" IMAGE_DOS_HEADER __ImageBase; namespace Embed { -Bytes Resource(int id) { - static std::unordered_map> inflated; - auto it = inflated.find(id); - if (it == inflated.end()) { +namespace { + +struct Pack { + std::vector bytes; + std::vector files; // views into `bytes` +}; + +// Split the inflated `pack.bytes` into `pack.files`. The pack opens with an +// index, each file's size as a decimal `\n`-ended line and then an empty +// line, followed by the files' bytes in index order. False unless the sizes +// cover the bytes exactly. +bool Split(Pack &pack) { + const uint8_t *p = pack.bytes.data(); + const uint8_t *end = p + pack.bytes.size(); + while (p < end && *p != '\n') { + size_t size = 0; + while (p < end && *p >= '0' && *p <= '9') + size = size * 10 + (*p++ - '0'); + if (p == end || *p++ != '\n') + return false; + pack.files.push_back({nullptr, size}); + } + if (p == end) + return false; + ++p; // the empty line + for (Bytes &file : pack.files) { + if (file.size > static_cast(end - p)) + return false; + file.data = p; + p += file.size; + } + return p == end; +} + +} // namespace + +Bytes File(int id, size_t index) { + static std::unordered_map packs; + auto it = packs.find(id); + if (it == packs.end()) { const HMODULE self = reinterpret_cast(&__ImageBase); HRSRC info = FindResourceW(self, MAKEINTRESOURCEW(id), MAKEINTRESOURCEW(10)); // RT_RCDATA if (info == nullptr) @@ -37,16 +73,19 @@ Bytes Resource(int id) { HGLOBAL handle = LoadResource(self, info); if (handle == nullptr) return {nullptr, 0}; - std::vector bytes; + Pack pack; if (!Encoding::Compress::Inflate(static_cast(LockResource(handle)), SizeofResource(self, info), - Encoding::Compress::kGzipWindowBits, bytes) || - bytes.empty()) + Encoding::Compress::kGzipWindowBits, pack.bytes)) return {nullptr, 0}; - bytes.shrink_to_fit(); - it = inflated.emplace(id, std::move(bytes)).first; + pack.bytes.shrink_to_fit(); + if (!Split(pack)) + return {nullptr, 0}; + // Moving the vector keeps its buffer, so the views stay valid. + it = packs.emplace(id, std::move(pack)).first; } - return {it->second.data(), it->second.size()}; + const std::vector &files = it->second.files; + return index < files.size() ? files[index] : Bytes{nullptr, 0}; } } // namespace Embed diff --git a/src/embed/Resource.h b/src/embed/Resource.h index 8c06e4be..74e1df5d 100644 --- a/src/embed/Resource.h +++ b/src/embed/Resource.h @@ -22,11 +22,12 @@ struct Bytes { size_t size; }; -// The bytes of RCDATA resource `id` in this DLL (IDs come from the generated -// `embedded_resources.h`). Each resource is stored gzipped -// (cmake/embed_resources.cmake); the first call inflates it and keeps the -// bytes, so they stay valid for the life of the DLL and must not be freed. -// `{nullptr, 0}` when the resource is missing or does not inflate. -Bytes Resource(int id); +// The bytes of file `index` in the pack that is RCDATA resource `id` in this +// DLL (both come from the generated `embedded_resources.h`). Each resource +// is one gzipped pack of files (cmake/pack_files.cmake); the first call for +// a resource inflates the whole pack and keeps it, so the bytes stay valid +// for the life of the DLL and must not be freed. `{nullptr, 0}` when the +// resource is missing, does not inflate or split, or has no file `index`. +Bytes File(int id, size_t index); } // namespace Embed