diff --git a/crates/benilla-mpq/src/crypto.rs b/crates/benilla-mpq/src/crypto.rs index 5210eb8c..75e5d1be 100644 --- a/crates/benilla-mpq/src/crypto.rs +++ b/crates/benilla-mpq/src/crypto.rs @@ -1,6 +1,6 @@ //! The Storm (MPQ) crypto, byte-exact with the client: the string hash and the block decryptor, -//! both over one 0x500-entry table. 1.12.1 archives carry no encrypted files, so only the hash and -//! block tables are ever decrypted. +//! both over one 0x500-entry table. 1.12.1 archives encrypt only their hash and block tables; this +//! realm's patch archives also encrypt a few files, their `(listfile)` among them. /// The 0x500-entry encryption table, built at compile time by the seed walk the client runs. const fn encryption_table() -> [u32; 0x500] { @@ -75,6 +75,28 @@ pub fn decrypt_block(data: &mut [u32], mut key: u32) { } } +/// [`decrypt_block`] over a byte buffer: every whole little-endian `u32` is decrypted, and a tail +/// shorter than four bytes is left as stored, as Storm's `DecryptMpqBlock` leaves it. +pub fn decrypt_bytes(data: &mut [u8], key: u32) { + let (words, _) = data.as_chunks_mut::<4>(); + let mut buf: Vec = words.iter().map(|c| u32::from_le_bytes(*c)).collect(); + decrypt_block(&mut buf, key); + for (dst, w) in words.iter_mut().zip(buf) { + *dst = w.to_le_bytes(); + } +} + +/// The inverse of [`decrypt_bytes`], for tests that build encrypted files. +#[cfg(test)] +pub(crate) fn encrypt_bytes(data: &mut [u8], key: u32) { + let (words, _) = data.as_chunks_mut::<4>(); + let mut buf: Vec = words.iter().map(|c| u32::from_le_bytes(*c)).collect(); + encrypt_block(&mut buf, key); + for (dst, w) in words.iter_mut().zip(buf) { + *dst = w.to_le_bytes(); + } +} + /// The inverse of [`decrypt_block`], for tests that build the encrypted tables an archive stores. #[cfg(test)] pub(crate) fn encrypt_block(data: &mut [u32], mut key: u32) { @@ -118,6 +140,18 @@ mod tests { assert_eq!(a, b); } + #[test] + fn decrypt_bytes_inverts_encrypt_and_keeps_the_short_tail() { + let key = hash_string("(listfile)", hash_type::FILE_KEY); + let original = *b"Interface\\a.txt"; + let mut buf = original; + encrypt_bytes(&mut buf, key); + assert_ne!(buf[..12], original[..12], "the whole words are encrypted"); + assert_eq!(buf[12..], original[12..], "the 3-byte tail is stored plain"); + decrypt_bytes(&mut buf, key); + assert_eq!(buf, original); + } + #[test] fn decrypt_inverts_encrypt() { let key = hash_string("(hash table)", hash_type::FILE_KEY); diff --git a/crates/benilla-mpq/src/lib.rs b/crates/benilla-mpq/src/lib.rs index 360ecbd5..05263073 100644 --- a/crates/benilla-mpq/src/lib.rs +++ b/crates/benilla-mpq/src/lib.rs @@ -1,5 +1,8 @@ //! A read-only MPQ reader for the 1.12.1 `Data/` chain: format V1/V2, every file `COMPRESS`-flagged -//! and sectored, no encrypted or single-unit files, no PTCH patches. Anything else is a hard error. +//! and sectored or stored as one unit, no PTCH patches. Anything else is a hard error. An encrypted +//! file reads by name, as its key derives from the name: 1.12.1 ships none, but this realm's patch +//! archives encrypt their `(listfile)` and a few content files, and `Patch-Q.mpq` stores every file +//! it carries as a single unit. //! //! The patch archives carry delete markers (flag `0x02000000`, size 0): the path is deleted from //! the composite chain. [`Archive::contains`] reports one present and [`Archive::read_file`] @@ -19,7 +22,7 @@ use benilla_bytes::{capped, ByteExt}; use flate2::{Decompress, FlushDecompress}; mod crypto; -use crypto::{decrypt_block, hash_string, hash_type}; +use crypto::{decrypt_block, decrypt_bytes, hash_string, hash_type}; /// `MPQ\x1A` as a little-endian `u32`: the archive header signature. const SIGNATURE: u32 = 0x1A51_504D; @@ -29,6 +32,8 @@ const USERDATA_SIGNATURE: u32 = 0x1B51_504D; const FLAG_IMPLODE: u32 = 0x0000_0100; const FLAG_COMPRESS: u32 = 0x0000_0200; const FLAG_ENCRYPTED: u32 = 0x0001_0000; +/// The file key is offset by the block position and XOR'd with the size (Storm's `MPQ_FILE_FIX_KEY`). +const FLAG_FIX_KEY: u32 = 0x0002_0000; const FLAG_SINGLE_UNIT: u32 = 0x0100_0000; /// A patch archive's delete marker (size 0): the path is deleted from the composite chain. const FLAG_DELETE_MARKER: u32 = 0x0200_0000; @@ -50,6 +55,9 @@ struct HashEntry { #[derive(Clone, Copy)] struct BlockEntry { file_pos: u32, + /// Stored size. The sector offset table sizes a sectored file's sectors, so this is read only + /// for a `SINGLE_UNIT` entry, which has no offset table and is exactly this many bytes. + compressed_size: u32, file_size: u32, flags: u32, } @@ -196,7 +204,8 @@ impl Archive { &self.index.path } - /// Read and decompress a file by its internal path (`/` or `\`, case-insensitive). + /// Read and decompress a file by its internal path (`/` or `\`, case-insensitive), decrypting + /// an encrypted one with the key its name derives. pub fn read_file(&self, name: &str) -> Result> { let idx = &self.index; let block = idx.find(name).ok_or_else(|| Error::NotFound(name.into()))?; @@ -208,14 +217,12 @@ impl Archive { if block.flags & FLAG_DELETE_MARKER != 0 { return Err(Error::NotFound(name.into())); } - // Outside the 1.12.1 envelope: refuse rather than guess. - if block.flags & FLAG_ENCRYPTED != 0 { - return Err(Error::Unsupported(format!("encrypted file {name}"))); - } - if block.flags & FLAG_SINGLE_UNIT != 0 { - return Err(Error::Unsupported(format!("single-unit file {name}"))); - } - + // An encrypted file's key derives from its name, which this read has. 1.12.1 ships none; + // this realm's patch archives encrypt their `(listfile)` (patch-1, -2, -4, -5 and -A) and + // two content files in patch-N, and the `Unsupported` this used to raise took the whole + // listfile union down with it — the macro-icon catalog and the archive addon pass both + // walk it (`benilla_formats::Chain::list_under`). + let key = (block.flags & FLAG_ENCRYPTED != 0).then(|| file_key(name, &block)); let mut file = File::open(&idx.path)?; let file_pos = idx.archive_offset + block.file_pos as u64; let file_size = block.file_size as usize; @@ -225,6 +232,60 @@ impl Archive { let file_len = file.metadata()?.len(); let avail = avail_from(file_len, file_pos); + // SINGLE_UNIT: the whole file is one unit at `file_pos` with no sector offset table — + // `compressed_size` bytes, a codec-mask byte first where it actually shrank. 1.12.1 ships + // none, so this used to be refused; this realm's `Patch-Q.mpq` stores all 93 of its files + // this way, 92 of them `ITEM\TextureComponents\` armour art, and it mounts above every + // other patch. `Chain::read` answers from the winning archive and does not fall through on + // an error, so the refusal failed the armour composite's read outright. + if block.flags & FLAG_SINGLE_UNIT != 0 { + let comp_len = block.compressed_size as usize; + if capped(comp_len, 1, avail) < comp_len { + return Err(Error::Corrupt(format!( + "{name}: single-unit size ({comp_len}) larger than the archive" + ))); + } + file.seek(SeekFrom::Start(file_pos))?; + let mut raw = vec![0u8; comp_len]; // proven <= avail above + file.read_exact(&mut raw)?; + // One unit is sector 0, so the key does not step. + if let Some(key) = key { + decrypt_bytes(&mut raw, key); + } + if !compressed || comp_len >= file_size { + // Stored, or a unit that would not shrink: verbatim, cut to the file's size. + raw.truncate(file_size); + return Ok(raw); + } + if raw.is_empty() { + return Err(Error::Corrupt(format!("{name}: single unit is empty"))); + } + if implode_only { + // IMPLODE units carry no method byte; 1.12.1 data has none. + return Err(Error::Unsupported(format!( + "{name}: single-unit codec 0x08 not yet implemented" + ))); + } + let mut out = Vec::with_capacity(capped(file_size, 1, avail)); + match raw[0] { + // non-empty: refused above + 0x02 => inflate_sector( + &mut Decompress::new(true), + &mut Vec::new(), + &raw[1..], + file_size, + name, + &mut out, + )?, + other => { + return Err(Error::Unsupported(format!( + "{name}: single-unit codec 0x{other:02X} not yet implemented" + ))) + } + } + return Ok(out); + } + if !compressed { // Stored: exactly `file_size` raw bytes, refused up front if the file cannot hold them. if capped(file_size, 1, avail) < file_size { @@ -235,6 +296,12 @@ impl Archive { file.seek(SeekFrom::Start(file_pos))?; let mut out = vec![0u8; file_size]; // proven <= avail above file.read_exact(&mut out)?; + // A stored file is still encrypted sector by sector, sector i with `key + i`. + if let Some(key) = key { + for (i, sector) in out.chunks_mut(idx.sector_size).enumerate() { + decrypt_bytes(sector, key.wrapping_add(i as u32)); + } + } return Ok(out); } @@ -253,6 +320,10 @@ impl Archive { file.seek(SeekFrom::Start(file_pos))?; let mut otab = vec![0u8; otab_cap * 4]; // == otab_len * 4, proven <= avail above file.read_exact(&mut otab)?; + // An encrypted file's offset table is encrypted with `key - 1`. + if let Some(key) = key { + decrypt_bytes(&mut otab, key.wrapping_sub(1)); + } let offsets: Vec = otab .as_chunks::<4>() .0 @@ -294,6 +365,9 @@ impl Archive { let mut out = Vec::with_capacity(capped(file_size, 1, avail)); let mut inflate = Decompress::new(true); let mut scratch = Vec::new(); + // Only an encrypted file needs a per-sector copy to decrypt into; the plain path keeps + // reading straight out of the one bulk buffer. + let mut plain: Vec = Vec::new(); for i in 0..sector_count { let start = offsets[i]; let end = offsets[i + 1]; @@ -308,7 +382,17 @@ impl Archive { return Err(Error::Corrupt(format!("{name}: sector {i} is empty"))); } // Within the span by the fold above, so this slice cannot panic. - let raw = §ors[(start - lo) as usize..(end - lo) as usize]; + let stored = §ors[(start - lo) as usize..(end - lo) as usize]; + // Sector i is encrypted with `key + i`, over its stored (compressed) bytes. + let raw: &[u8] = match key { + None => stored, + Some(key) => { + plain.clear(); + plain.extend_from_slice(stored); + decrypt_bytes(&mut plain, key.wrapping_add(i as u32)); + &plain + } + }; if comp_len >= want { // A sector that would not shrink is stored verbatim. @@ -364,6 +448,18 @@ impl Index { } } +/// The key Storm decrypts an encrypted file with: the name's last path component under the +/// `FILE_KEY` hash, offset by the block position and XOR'd with the size for a `FIX_KEY` entry. +fn file_key(name: &str, block: &BlockEntry) -> u32 { + let base = name.rsplit(['\\', '/']).next().unwrap_or(name); + let key = hash_string(base, hash_type::FILE_KEY); + if block.flags & FLAG_FIX_KEY != 0 { + key.wrapping_add(block.file_pos) ^ block.file_size + } else { + key + } +} + /// The `MPQ\x1A` header's offset: the first 512-aligned match, or where a `MPQ\x1B` user-data /// header points. fn find_header(file: &mut File, file_len: u64) -> Result { @@ -449,7 +545,7 @@ fn read_block_table( let o = i * 4; BlockEntry { file_pos: words[o], - // words[o + 1] is compressed_size, unread: the offset table sizes the sectors. + compressed_size: words[o + 1], file_size: words[o + 2], flags: words[o + 3], } @@ -608,6 +704,221 @@ mod tests { bytes } + /// A `SINGLE_UNIT` entry stored verbatim, the shape `Patch-Q.mpq` gives all 93 of its files: + /// one unit, `compressed_size == file_size`, no offset table and no codec byte. + #[test] + fn a_stored_single_unit_entry_reads_its_bytes() { + let name = "ITEM\\TextureComponents\\HandTexture\\tm_cloth_01_Glove_HA_U.blp"; + let body = b"BLP2\x01single unit, stored verbatim"; + let (arc, path) = open_temp_kept( + "single_unit_stored", + &archive_with_one_entry(name, FLAG_EXISTS | FLAG_SINGLE_UNIT, body), + ); + assert_eq!(arc.read_file(name).unwrap(), body); + let _ = std::fs::remove_file(&path); + } + + /// A compressed `SINGLE_UNIT` entry: one codec-mask byte then one zlib stream for the whole + /// file, inflated to `file_size` — not to a sector's share, because there are no sectors. + #[test] + fn a_compressed_single_unit_entry_inflates_whole() { + use flate2::{write::ZlibEncoder, Compression}; + use std::io::Write; + let name = "ITEM\\TextureComponents\\FootTexture\\tm_plate_01_Boot_FO_U.blp"; + // Longer than one 512-byte test sector, to pin that nothing sectors it. + let plain = b"single unit, deflated whole, over a sector's worth of it. ".repeat(20); + let mut unit = vec![0x02u8]; + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(&plain).unwrap(); + unit.extend_from_slice(&e.finish().unwrap()); + assert!(unit.len() < plain.len(), "the unit must actually shrink"); + + let (arc, path) = open_temp_kept( + "single_unit_zlib", + &archive_with_one_block( + name, + FLAG_EXISTS | FLAG_COMPRESS | FLAG_SINGLE_UNIT, + &unit, + plain.len() as u32, + ), + ); + assert_eq!(arc.read_file(name).unwrap(), plain); + let _ = std::fs::remove_file(&path); + } + + /// Over the install's own archives: every file a `SINGLE_UNIT` archive names in its own + /// `(listfile)` reads, and reads to the size its block entry promises. `Patch-Q.mpq` mounts + /// above every other patch here, so each of these 92 armour textures is the copy the composite + /// would get. Skipped without an install (`$WOW_DATA`). + #[test] + fn single_unit_files_in_the_install_read_to_their_promised_size() { + let Some(data) = std::env::var_os("WOW_DATA").filter(|v| !v.is_empty()) else { + eprintln!("skipping: $WOW_DATA not set"); + return; + }; + let dir = PathBuf::from(data); + let Ok(entries) = std::fs::read_dir(&dir) else { + eprintln!("skipping: {} is not a directory", dir.display()); + return; + }; + let mut archives: Vec = entries + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|e| e.eq_ignore_ascii_case("mpq"))) + .collect(); + archives.sort(); + + let mut checked = 0usize; + for path in &archives { + let Ok(arc) = Archive::open(path) else { + continue; + }; + let Ok(listfile) = arc.read_file("(listfile)") else { + continue; + }; + for raw in String::from_utf8_lossy(&listfile).split([';', '\r', '\n']) { + let name = raw.trim(); + if name.is_empty() { + continue; + } + let Some(block) = arc.index.find(name) else { + continue; + }; + if block.flags & FLAG_SINGLE_UNIT == 0 || block.flags & FLAG_DELETE_MARKER != 0 { + continue; + } + let bytes = arc.read_file(name).unwrap_or_else(|e| { + panic!("{}: {name} did not read: {e}", path.display()); + }); + assert_eq!( + bytes.len(), + block.file_size as usize, + "{}: {name} read {} bytes, block entry says {}", + path.display(), + bytes.len(), + block.file_size + ); + checked += 1; + } + } + // `Patch-Q.mpq` alone names 92. A near-empty run would mean the loop silently did nothing. + assert!(checked > 50, "only checked {checked} single-unit files"); + } + + /// An encrypted, stored entry reads back by name: the key derives from the name's last + /// component (`hello.txt`), and a stored file is encrypted sector by sector all the same. + #[test] + fn an_encrypted_stored_entry_reads_by_name() { + use crypto::encrypt_bytes; + let name = "Interface\\hello.txt"; + let plain = b"hello, encrypted world"; + let mut data = plain.to_vec(); + encrypt_bytes(&mut data, hash_string("hello.txt", hash_type::FILE_KEY)); + let (arc, path) = open_temp_kept( + "encrypted_stored", + &archive_with_one_entry(name, FLAG_EXISTS | FLAG_ENCRYPTED, &data), + ); + assert_eq!(arc.read_file(name).unwrap(), plain); + let _ = std::fs::remove_file(&path); + } + + /// The shape this realm's patch archives give their `(listfile)`: `COMPRESS | ENCRYPTED | + /// FIX_KEY`, the offset table under `key - 1` and sector 0 under `key + 0`, with the key offset + /// by the block position and XOR'd with the size. + #[test] + fn an_encrypted_fix_key_compressed_entry_reads_by_name() { + use crypto::encrypt_bytes; + use flate2::{write::ZlibEncoder, Compression}; + use std::io::Write; + let name = "(listfile)"; + let plain = + b"Interface\\FrameXML\\LFT\\LFT.lua\r\nDBFilesClient\\ChrRaces.dbc\r\n".repeat(4); + let mut sector = vec![0x02u8]; + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(&plain).unwrap(); + sector.extend_from_slice(&e.finish().unwrap()); + + // Laid out as `archive_with_one_block` lays it: 4 hash slots and one block entry after the + // 32-byte header, so the data starts at 32 + 64 + 16. + let data_pos = 32u32 + 4 * 16 + 16; + let key = + hash_string(name, hash_type::FILE_KEY).wrapping_add(data_pos) ^ plain.len() as u32; + let mut otab = Vec::new(); + otab.extend_from_slice(&8u32.to_le_bytes()); + otab.extend_from_slice(&(8 + sector.len() as u32).to_le_bytes()); + encrypt_bytes(&mut otab, key.wrapping_sub(1)); + encrypt_bytes(&mut sector, key); + let mut data = otab; + data.extend_from_slice(§or); + + let (arc, path) = open_temp_kept( + "encrypted_fix_key", + &archive_with_one_block( + name, + FLAG_EXISTS | FLAG_COMPRESS | FLAG_ENCRYPTED | FLAG_FIX_KEY, + &data, + plain.len() as u32, + ), + ); + assert_eq!(arc.read_file(name).unwrap(), plain); + let _ = std::fs::remove_file(&path); + } + + /// Over the install's own archives: every `(listfile)` the chain carries reads, encrypted or + /// not. This is the regression the two synthetic tests above stand in for — before the key was + /// derived, an encrypted `(listfile)` raised `Unsupported` and took the whole listfile union + /// down with it (the macro-icon catalog, the archive addon pass). The count of encrypted ones + /// is asserted non-zero so an install with none cannot let this pass while proving nothing. + /// Skipped without an install (`$WOW_DATA`), like every other corpus-backed test here. + #[test] + fn every_listfile_in_the_install_reads_including_the_encrypted_ones() { + let Some(data) = std::env::var_os("WOW_DATA").filter(|v| !v.is_empty()) else { + eprintln!("skipping: $WOW_DATA not set"); + return; + }; + let dir = PathBuf::from(data); + let Ok(entries) = std::fs::read_dir(&dir) else { + eprintln!("skipping: {} is not a directory", dir.display()); + return; + }; + let mut archives: Vec = entries + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|e| e.eq_ignore_ascii_case("mpq"))) + .collect(); + archives.sort(); + if archives.is_empty() { + eprintln!("skipping: no archives in {}", dir.display()); + return; + } + + let (mut with, mut encrypted) = (0usize, 0usize); + for path in &archives { + let Ok(arc) = Archive::open(path) else { + continue; + }; + let Some(block) = arc.index.find("(listfile)") else { + continue; + }; + with += 1; + if block.flags & FLAG_ENCRYPTED != 0 { + encrypted += 1; + } + let bytes = arc.read_file("(listfile)").unwrap_or_else(|e| { + panic!("{}: (listfile) did not read: {e}", path.display()); + }); + let text = String::from_utf8_lossy(&bytes); + let names = text + .split([';', '\r', '\n']) + .filter(|n| !n.trim().is_empty()) + .count(); + assert!(names > 0, "{}: (listfile) named nothing", path.display()); + } + assert!(with > 0, "no archive in {} has a listfile", dir.display()); + assert!( + encrypted > 0, + "{with} listfiles read but none was encrypted — this install cannot prove the fix" + ); + } + /// The control for the delete-marker test: the same builder's plain entry reads back. #[test] fn stored_entry_reads_its_bytes() {