diff --git a/README.md b/README.md index 5349902a..a16851f6 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ The flagship feature: run modern Lua 5.1 addon code on 1.12's Lua 5.0 VM. | Feature | Effect | |---------|--------| | [Syntax](docs/API.md#lua-51-syntax) | Compiles the Lua 5.1 length (`#`), modulo (`%`), `...`-expression, `0x` hex-literal, and leveled long bracket (`[=[ ]=]`) syntax that vanilla's Lua 5.0 rejects, by rewriting addon source before it compiles. Each addon file also receives its `(name, table)` through `...` (`local name, tbl = ...`). | +| [Upvalue limit](docs/API.md#upvalue-limit) | A function can use up to 255 upvalues, the outer locals it refers to. Lua 5.0 stops at 32 and rejects the whole file. Lua 5.1 allows 60, so a large handler that reads many file-level locals can be valid 5.1 and still fail to load here. Nothing changes for a function within 32. | | [String methods](docs/API.md#string-methods-supper-sformat) | Every string value accepts method calls — `("asd"):upper()`, `("%d gold"):format(n)`, `msg:match("^!(%w+)")` — resolving through the `string` table, the way Lua 5.1 works. Works on literals and variables, in-world and on the login screen, and inside coroutines. | | [Script-handler arguments](docs/API.md#setmodernscriptargsenable--getmodernscriptargs) | Frame-script handlers receive their values as positional arguments — `OnMouseWheel(self, delta)`, `OnClick(self, button)`, `OnEvent(self, event, ...)`, etc. — so modern addon ports work unmodified. The `this` / `arg1` globals stay set. A handler that declares no parameters is unaffected. A handler that declared a parameter and expected nil now receives its real value. On by default. `SetModernScriptArgs(false)` restores exact vanilla dispatch. | diff --git a/docs/API.md b/docs/API.md index 4932d82b..5053758b 100644 --- a/docs/API.md +++ b/docs/API.md @@ -390,6 +390,7 @@ build instructions. - [Lua](#lua) - [Lua 5.1 syntax](#lua-51-syntax) + - [Upvalue limit](#upvalue-limit) - [String methods (`s:upper()`, `s:format(...)`)](#string-methods-supper-sformat) - [`getfenv` / `setfenv` environment protection](#getfenv--setfenv-environment-protection) - [`select(index, ...)`](#selectindex-) @@ -9282,6 +9283,18 @@ needs that addon's opt-in. reverts affected chunks to the state that fails to compile, so use it only to answer "is the rewrite breaking this addon?". +### Upvalue limit + +A function can use up to 255 upvalues — the outer locals it refers to. + +Lua 5.0 stops at 32 and rejects the whole file with `too many upvalues +(limit=32)`. Lua 5.1 allows 60. A large handler that reads many file-level +locals can be valid Lua 5.1 and still fail to load here for that reason +alone. ClassicAPI raises the limit to 255, the most a closure can hold. + +Nothing changes for a function within the old limit. Past 255 the error is +the same message with the new number: `too many upvalues (limit=255)`. + ### String methods (`s:upper()`, `s:format(...)`) In Lua 5.1, every string value accepts method calls: `s:method(...)` diff --git a/src/Offsets.h b/src/Offsets.h index d8f3c7f0..c621a766 100644 --- a/src/Offsets.h +++ b/src/Offsets.h @@ -5155,6 +5155,76 @@ enum Offsets { // follows the usual Lua convention: `@name` is shown as a filename, `=name` // verbatim. This is what `RunScript` runs through. FUN_LUA_RUN_STRING = 0x00704AE0, + + // --- Lua 5.0 parser internals: the upvalue limit -------------------------- + // Stock lparser.c compiled as-is (the `lua\src\lparser.c` path string is + // still in .rdata). The 32-upvalue limit is NOT just the `PUSH 0x20` + // immediate in indexupvalue: it is the size of a fixed `expdesc + // upvalues[32]` array inside the parser's FuncState, and the array ends + // exactly where the next field begins — 0x38 + 32 × 0x14 = 0x2B8 = + // OFF_LUA_FUNCSTATE_ACTVAR, the declared-variable stack. Raising the + // immediate alone makes the 33rd upvalue overwrite actvar[0] and miscompile. + // In 5.0 the array has exactly two readers, both below; everything past the + // parser is dynamic (nups and LClosure.nupvalues are bytes, OP_GETUPVAL's B + // field is 9 bits, closures are sized per instance, the debug-name array + // grows with luaM_growaux). LuaSyntax::Upvalues co-hooks the two readers and + // spills entries 33+ into side storage, raising the ceiling to the byte. + // + // indexupvalue(fs, name, v) -> index. `__fastcall(ecx = FuncState*, edx = + // TString *name, [stack] expdesc *v)`, RET 4. Scans fs->upvalues for + // {k, info} equal to v's, else checklimit(nups+1, 32, "upvalues"), grows + // f->upvalues (names) via luaM_growaux(L, block, &f->sizeupvalues, 4, + // 0x7FFFFFFD, ""), stores the name and the expdesc, returns nups++. + FUN_LUA_PARSER_INDEXUPVALUE = 0x006FDF80, + // pushclosure(ls, func, v). `__fastcall(ecx = LexState*, edx = FuncState + // *func, [stack] expdesc *v)`, RET 4. Grows f->p via luaM_growaux(L, f->p, + // &f->sizep, 4, 0x3FFFF, "constant table overflow"), stores func->f at + // f->p[np++], emits OP_CLOSURE (0x22) with luaK_codeABx(fs, op, 0, np-1), + // init_exp(v, VRELOCABLE = 10, pc), then one luaK_codeABC(fs, k == VLOCAL + // (5) ? OP_MOVE (0) : OP_GETUPVAL (4), 0, info, 0) per func->upvalues[i]. + FUN_LUA_PARSER_PUSHCLOSURE = 0x006FD980, + // luaY_checklimit(ls, value, limit, what). `__fastcall(ecx = LexState*, + // edx = value, [stack] limit, what)`, RET 8. Raises "too many %s (limit=%d)" + // through luaO_pushfstring(ls->L, …) when limit < value; never returns then. + // NOTE: takes the LexState in this build, not the FuncState stock 5.0 passes. + FUN_LUA_PARSER_CHECKLIMIT = 0x006FF440, + // init_exp(v, k, info): `__fastcall(ecx = expdesc*, edx = k, [stack] info)`. + FUN_LUA_PARSER_INIT_EXP = 0x006FD3E0, + // luaM_growaux(L, block, &size, elemSize, limit, errMsg) -> new block. + // `__fastcall(ecx = L, edx = block, [stack] int *size, elemSize, limit, + // msg)`, RET 0x10. + FUN_LUA_M_GROWAUX = 0x006FC900, + // luaK_codeABx(fs, op, A, Bx) -> pc and luaK_codeABC(fs, op, A, B, C) -> pc. + // `__fastcall(ecx = fs, edx = op, [stack] …)`; RET 8 / RET 0xC. + FUN_LUA_K_CODEABX = 0x00701910, + FUN_LUA_K_CODEABC = 0x007018E0, + // FuncState (lparser.h), verified from indexupvalue / pushclosure / + // new_localvar: f@+0, ls@+0xC, L@+0x10, np@+0x2C, nactvar@+0x34, then the + // fixed expdesc upvalues[32]@+0x38 (stride 0x14) running into + // actvar[200]@+0x2B8. + OFF_LUA_FUNCSTATE_F = 0x00, + OFF_LUA_FUNCSTATE_LS = 0x0C, + OFF_LUA_FUNCSTATE_L = 0x10, + OFF_LUA_FUNCSTATE_NP = 0x2C, + OFF_LUA_FUNCSTATE_UPVALUES = 0x38, + OFF_LUA_FUNCSTATE_ACTVAR = 0x2B8, + LUA_PARSER_NATIVE_UPVALUE_SLOTS = 32, // sizeof(fs->upvalues) / sizeof(expdesc) + SIZEOF_LUA_EXPDESC = 0x14, // { k, info, aux, t, f } + // LexState: fs@+0x30, L@+0x34 (new_localvar reads both; checklimit reads L). + OFF_LUA_LEXSTATE_FS = 0x30, + OFF_LUA_LEXSTATE_L = 0x34, + // Proto (lobject.h). p/sizep and upvalues/sizeupvalues from pushclosure and + // indexupvalue; the four packed bytes at +0x44 are nups, numparams, + // is_vararg, maxstacksize — numparams/is_vararg were verified separately by + // Frame::ScriptArgs, which reads them to decide whether a handler declares + // parameters, and they pin nups to the byte before them. + OFF_LUA_PROTO_P = 0x10, + OFF_LUA_PROTO_UPVALUES = 0x1C, // TString **, the debug names + OFF_LUA_PROTO_SIZEUPVALUES = 0x24, + OFF_LUA_PROTO_SIZEP = 0x34, + OFF_LUA_PROTO_NUPS = 0x44, + OFF_LUA_PROTO_NUMPARAMS = 0x45, + OFF_LUA_PROTO_IS_VARARG = 0x46, // Shared environment-protection predicate for `getfenv`/`setfenv` // (`bool __fastcall(L /*ecx*/)`). Pushes the function-at-top's environment // via `lua_getfenv(L, -1)`, then pushes the protection marker diff --git a/src/frame/ScriptArgs.cpp b/src/frame/ScriptArgs.cpp index 3c968a27..7b4e42b2 100644 --- a/src/frame/ScriptArgs.cpp +++ b/src/frame/ScriptArgs.cpp @@ -287,14 +287,13 @@ bool RunModern(int handlerRef, void *frame, const char *fmt, const void *vaPtr) // lua_State: top @ +0x08 (TValue *, next free slot) // TValue: tag @ +0x00 (int); GC/closure pointer @ +0x08 (16-byte TValue) // Closure: isC byte @ +0x06; union member (C func / Proto) @ +0x0C -// Proto: numparams (u8) @ +0x45, is_vararg (u8) @ +0x46 // Single-use here, so kept local (like the lua_State field offsets elsewhere). +// The Proto fields this reads (numparams @ +0x45, is_vararg @ +0x46) are shared +// with LuaSyntax::Upvalues and live in Offsets.h as OFF_LUA_PROTO_*. constexpr uintptr_t OFF_LUASTATE_TOP = 0x08; constexpr uintptr_t OFF_TVALUE_GC = 0x08; constexpr uintptr_t OFF_CLOSURE_ISC = 0x06; constexpr uintptr_t OFF_LCLOSURE_PROTO = 0x0C; -constexpr uintptr_t OFF_PROTO_NUMPARAMS = 0x45; -constexpr uintptr_t OFF_PROTO_IS_VARARG = 0x46; constexpr int kLuaTFunction = 6; constexpr uintptr_t kTValueSize = 0x10; @@ -322,8 +321,8 @@ bool HandlerWantsArgs(void *L, int handlerRef) { const void *proto = reinterpret_cast( Game::Read(cl, OFF_LCLOSURE_PROTO)); if (proto != nullptr) - wants = Game::Read(proto, OFF_PROTO_NUMPARAMS) > 0 || - Game::Read(proto, OFF_PROTO_IS_VARARG) != 0; + wants = Game::Read(proto, Offsets::OFF_LUA_PROTO_NUMPARAMS) > 0 || + Game::Read(proto, Offsets::OFF_LUA_PROTO_IS_VARARG) != 0; } } Game::Lua::SetTop(L, -2); // pop the handler diff --git a/src/luasyntax/Upvalues.cpp b/src/luasyntax/Upvalues.cpp new file mode 100644 index 00000000..89bd8bdc --- /dev/null +++ b/src/luasyntax/Upvalues.cpp @@ -0,0 +1,222 @@ +// This file is part of ClassicAPI. +// +// ClassicAPI is free software: you can redistribute it and/or modify it under the terms +// of the GNU General Public License as published by the Free Software Foundation, either +// version 3 of the License, or (at your option) any later version. +// +// ClassicAPI is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +// PURPOSE. See the GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along with +// ClassicAPI. If not, see . + +// Lifts the parser's limit of 32 upvalues per function to 255. +// +// Lua 5.0 refuses the 33rd upvalue with "too many upvalues (limit=32)". Lua 5.1 +// allows 60, so a function written against a modern client can be perfectly +// valid there and unloadable here -- and it fails at compile time, as a syntax +// error on the whole file, which is the one class of 5.1 difference the source +// transpiler cannot paper over because the limit sits in the parser itself. +// +// WHY THE IMMEDIATE CANNOT SIMPLY BE RAISED. The limit is the size of a fixed +// array. The parser's FuncState carries `expdesc upvalues[32]` at +0x38, and the +// next field -- actvar[200], the stack of locals currently in scope -- begins at +// +0x2B8, which is exactly 0x38 + 32 * 0x14. Patch the `PUSH 0x20` and the 33rd +// upvalue is written over actvar[0]: the function then compiles with the wrong +// idea of which locals exist, and nothing raises. See Offsets.h at +// FUN_LUA_PARSER_INDEXUPVALUE for the derivation. +// +// WHAT IS ACTUALLY LOAD-BEARING. That array has exactly two readers in 5.0, both +// located in this binary: indexupvalue, which scans it for an existing entry and +// appends new ones, and pushclosure, which walks it once the function body is +// complete to emit one OP_MOVE / OP_GETUPVAL per entry after OP_CLOSURE. +// Everything downstream is sized dynamically -- f->nups and LClosure.nupvalues +// are bytes, OP_GETUPVAL's B field is 9 bits, closures are allocated per +// instance, the debug-name array grows on demand -- so the byte is the real +// ceiling and the array is the only obstacle. +// +// THE APPROACH. Co-hook both readers and keep entries 33 and up in side storage +// keyed by the FuncState's address. A function that stays within 32 upvalues +// never leaves engine code: both hooks call the original for it, so the +// ordinary case is the engine's own bytes. Only a function the engine would +// have rejected takes the reimplemented path, and that path calls the same +// engine helpers the original does -- luaY_checklimit with the byte's ceiling +// in place of 32, so an overflow still raises the engine's own message; +// luaM_growaux for the name array; luaK_codeABx / luaK_codeABC for emission. +// No bytecode format or VM behaviour changes: a closure with 40 upvalues is +// exactly the closure 5.0's own VM already knows how to build and run. +// +// STALENESS. A parse error longjmps out of the parser, so side storage keyed by +// a C-stack address can outlive its function, and the next chunk may put a new +// FuncState at the same address. Two facts make this self-correcting without a +// hook on open_func: side entries exist only while nups > 32, and indexupvalue +// is the only writer. So a FuncState arriving at indexupvalue with nups <= 32 +// cannot own any side entries, and whatever is filed under its address is a +// leftover -- cleared there, on the new function's first upvalue, before +// anything could read it. pushclosure clears too, since a completed FuncState +// is dead once it has emitted its closure. +// +// Compile-time only, so no per-frame cost, and nothing else hooks lparser +// internals -- LuaSyntax's transpiler hooks luaL_loadbuffer, a different +// function, and the other Octo DLLs do not touch the compiler. + +#include "Game.h" +#include "Offsets.h" + +#include +#include +#include + +namespace LuaSyntax::Upvalues { + +namespace { + +// lparser.h: { expkind k; int info; int aux; int t; int f; } +struct Expdesc { + int32_t k; + int32_t info; + int32_t aux; + int32_t t; + int32_t f; +}; +static_assert(sizeof(Expdesc) == Offsets::SIZEOF_LUA_EXPDESC, + "expdesc must match the parser's 20-byte stride"); + +constexpr int kNativeSlots = Offsets::LUA_PARSER_NATIVE_UPVALUE_SLOTS; +constexpr int kLimit = 255; // f->nups and LClosure.nupvalues are lu_byte + +constexpr int32_t VLOCAL = 5; +constexpr int32_t VRELOCABLE = 10; +constexpr int32_t OP_MOVE = 0; +constexpr int32_t OP_GETUPVAL = 4; +constexpr int32_t OP_CLOSURE = 0x22; + +// Entries 33+ of every FuncState currently being compiled, keyed by its address. +std::unordered_map> g_spill; + +using IndexUpvalue_t = int(__fastcall *)(void *fs, void *name, Expdesc *v); +using PushClosure_t = void(__fastcall *)(void *ls, void *func, Expdesc *v); +using CheckLimit_t = void(__fastcall *)(void *ls, int value, int limit, const char *what); +using GrowAux_t = void *(__fastcall *)(void *L, void *block, int *size, int elemSize, int limit, + const char *errMsg); +using CodeABx_t = int(__fastcall *)(void *fs, int op, int a, int bx); +using CodeABC_t = int(__fastcall *)(void *fs, int op, int a, int b, int c); +using InitExp_t = void(__fastcall *)(Expdesc *v, int k, int info); + +IndexUpvalue_t IndexUpvalue_o = nullptr; +PushClosure_t PushClosure_o = nullptr; + +template T *At(void *base, uintptr_t offset) { + return reinterpret_cast(static_cast(base) + offset); +} + +uint8_t &Nups(void *proto) { return *At(proto, Offsets::OFF_LUA_PROTO_NUPS); } + +Expdesc *NativeSlots(void *fs) { + return At(fs, Offsets::OFF_LUA_FUNCSTATE_UPVALUES); +} + +bool SameUpvalue(const Expdesc &a, const Expdesc &b) { return a.k == b.k && a.info == b.info; } + +int __fastcall IndexUpvalue_h(void *fs, void *name, Expdesc *v) { + void *proto = *At(fs, Offsets::OFF_LUA_FUNCSTATE_F); + const int nups = Nups(proto); + + // See STALENESS above: with nups <= 32 nothing filed under this address can + // be ours, so drop it before it can be read. + if (nups <= kNativeSlots) + g_spill.erase(fs); + // Within the engine's own array: the engine's own code, untouched. + if (nups < kNativeSlots) + return IndexUpvalue_o(fs, name, v); + + // From here the original would raise. Same scan it performs, over both + // halves of the array. + const Expdesc *native = NativeSlots(fs); + for (int i = 0; i < kNativeSlots; ++i) + if (SameUpvalue(native[i], *v)) + return i; + std::vector &spill = g_spill[fs]; + for (size_t i = 0; i < spill.size(); ++i) + if (SameUpvalue(spill[i], *v)) + return kNativeSlots + static_cast(i); + + // The engine's own limit check with the byte's ceiling: over it, this raises + // "too many upvalues (limit=255)" exactly as the original raises at 32, and + // does not return. + void *ls = *At(fs, Offsets::OFF_LUA_FUNCSTATE_LS); + reinterpret_cast(Offsets::FUN_LUA_PARSER_CHECKLIMIT)(ls, nups + 1, kLimit, + "upvalues"); + + // Grow the debug-name array the way the original does, then record the name. + int *sizeUpvalues = At(proto, Offsets::OFF_LUA_PROTO_SIZEUPVALUES); + void **&names = *At(proto, Offsets::OFF_LUA_PROTO_UPVALUES); + if (*sizeUpvalues < nups + 1) { + void *L = *At(fs, Offsets::OFF_LUA_FUNCSTATE_L); + names = static_cast(reinterpret_cast(Offsets::FUN_LUA_M_GROWAUX)( + L, names, sizeUpvalues, sizeof(void *), 0x7FFFFFFD, "")); + } + names[nups] = name; + + spill.push_back(*v); + Nups(proto) = static_cast(nups + 1); + return nups; +} + +void __fastcall PushClosure_h(void *ls, void *func, Expdesc *v) { + void *funcProto = *At(func, Offsets::OFF_LUA_FUNCSTATE_F); + const int nups = Nups(funcProto); + if (nups <= kNativeSlots) { + PushClosure_o(ls, func, v); + g_spill.erase(func); // finished with; cannot own entries anyway + return; + } + + // Every spilled entry must be present: indexupvalue is the only thing that + // grows nups past 32 and it always spills as it does so. If they are not, + // fall back to the engine's own refusal rather than read past the array. + auto it = g_spill.find(func); + if (it == g_spill.end() || it->second.size() < static_cast(nups - kNativeSlots)) { + reinterpret_cast(Offsets::FUN_LUA_PARSER_CHECKLIMIT)(ls, nups, kNativeSlots, + "upvalues"); + return; // unreachable: checklimit raised + } + + // The original, step for step, with the emission loop reading both halves. + void *fs = *At(ls, Offsets::OFF_LUA_LEXSTATE_FS); + void *proto = *At(fs, Offsets::OFF_LUA_FUNCSTATE_F); + int *np = At(fs, Offsets::OFF_LUA_FUNCSTATE_NP); + int *sizep = At(proto, Offsets::OFF_LUA_PROTO_SIZEP); + void **&protos = *At(proto, Offsets::OFF_LUA_PROTO_P); + if (*sizep < *np + 1) { + void *L = *At(ls, Offsets::OFF_LUA_LEXSTATE_L); + protos = static_cast(reinterpret_cast(Offsets::FUN_LUA_M_GROWAUX)( + L, protos, sizep, sizeof(void *), 0x3FFFF, "constant table overflow")); + } + protos[*np] = funcProto; + const int index = (*np)++; + const int pc = + reinterpret_cast(Offsets::FUN_LUA_K_CODEABX)(fs, OP_CLOSURE, 0, index); + reinterpret_cast(Offsets::FUN_LUA_PARSER_INIT_EXP)(v, VRELOCABLE, pc); + + const Expdesc *native = NativeSlots(func); + const std::vector &spill = it->second; + auto codeABC = reinterpret_cast(Offsets::FUN_LUA_K_CODEABC); + for (int i = 0; i < nups; ++i) { + const Expdesc &e = (i < kNativeSlots) ? native[i] : spill[i - kNativeSlots]; + codeABC(fs, e.k == VLOCAL ? OP_MOVE : OP_GETUPVAL, 0, e.info, 0); + } + g_spill.erase(it); +} + +const Game::HookAutoRegister _indexUpvalueHook{Offsets::FUN_LUA_PARSER_INDEXUPVALUE, + reinterpret_cast(&IndexUpvalue_h), + reinterpret_cast(&IndexUpvalue_o)}; +const Game::HookAutoRegister _pushClosureHook{Offsets::FUN_LUA_PARSER_PUSHCLOSURE, + reinterpret_cast(&PushClosure_h), + reinterpret_cast(&PushClosure_o)}; + +} // namespace + +} // namespace LuaSyntax::Upvalues