benilla-octo/crates/benilla-app/src/item_persist.rs

//! Item templates kept across sessions: benilla's `WDB/itemcache.wdb`.
//!
//! The reference caches every `SMSG_ITEM_QUERY_SINGLE_RESPONSE` to disk and reads it back at
//! login, so `GetItemInfo` answers a known item **immediately**. benilla asked the server afresh
//! every session, and that one gap is visible in three different places at once:
//!
//! - OctoWoW's shop draws an item's icon from `GetItemInfo(entry)`; a cold cache draws the frame
//!   with no icon.
//! - Its model cards branch on that same call (`elseif loc and loc ~= ""`), so for an uncached
//!   item **neither** branch runs and the pane keeps the previous category's model — which is why
//!   the Fashion tab showed companions.
//! - Tmog queues the page's item ids and spins in an `OnUpdate` until every one answers, calling
//!   `EnableMouse(false)` on every preview button and re-showing its "Loading" overlay *each
//!   frame* while it waits (`Tmog.lua:1446-1481`). On a cold cache that wait is long enough to
//!   read as the window flashing and going unresponsive, which is exactly what it was.
//!
//! None of those is a rendering fault, and none is fixable addon-side: they are all one missing
//! file.
//!
//! The shape is [`crate::name_persist`]'s, which does this for creature names — realm-scoped
//! under `benilla-config/cache/` rather than `WDB/` in the read-only install, written on a
//! debounce and again at exit. The record is the parsed [`ItemInfo`] as JSON rather than the
//! reference's raw response block: the wire reader is the only parser, so re-encoding by hand
//! would be a second copy of 52 fields free to drift from it.
//!
//! **The file is append-only**, one `[id, {…}]` record per line, and the append runs on the IO
//! pool. The first build of this wrote the whole map every save, which is the kind of thing that
//! looks free until the cache fills: at the 10 415 templates this realm answers, re-encoding it
//! measured **51 ms of JSON on the main thread**, plus an 11.6 MB atomic rewrite, every ten
//! seconds for as long as answers kept landing. That is a frame of ~60 ms, which is both a
//! visible hitch and long enough to swallow a keystroke — and it fired in precisely the windows
//! this module exists to help, because those are the ones that ask about items in bulk. Writing
//! only what is new makes a save proportional to the burst instead of to the cache, and putting
//! it on the IO pool keeps even a large burst out of the frame.
//!
//! Loading reads the append file *and* the legacy whole-map `.json` if one is there, so an
//! upgrade keeps what it already paid for; the legacy file is never written again. Every id
//! already on disk is remembered ([`ItemCacheFile::written`]), so nothing is appended twice.

use std::collections::{HashMap, HashSet};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};

use benilla_protocol::ItemInfo;
use bevy::prelude::*;
use bevy::tasks::IoTaskPool;

use crate::char_select::Roster;
use crate::items::Items;

/// Seconds between writes: answers arrive in bursts when a window opens, and a crash costs only
/// re-asking.
const SAVE_DEBOUNCE: f32 = 10.0;

pub(crate) struct ItemPersistPlugin;

impl Plugin for ItemPersistPlugin {
    fn build(&self, app: &mut App) {
        // `Last`, not `Update`: both systems touch `Items`, which a great many `Update` systems
        // read, and putting them there added 78 undeclared orders against systems they have no
        // real relationship with. Neither is frame-critical — the load runs once per realm and the
        // save is on a 10 s debounce — so a frame's delay at login costs nothing and the schedule
        // keeps its meaning.
        app.init_resource::<ItemCacheFile>()
            .add_systems(Last, (load_item_cache, save_item_cache).chain());
        // The exit write catches the last burst, which the debounce would not.
        crate::shutdown::on_app_exit(app, save_on_exit.into_configs());
    }
}

/// Where this realm's cache lives and what we last wrote there.
#[derive(Resource, Default)]
pub(crate) struct ItemCacheFile {
    /// The realm the loaded file belongs to; a change triggers a load.
    realm: Option<String>,
    path: Option<PathBuf>,
    /// [`Items::template_epoch`] at the last write: the dirty bit, which ticks on every answer.
    saved_epoch: u64,
    since_save: f32,
    /// Every entry id already on disk, so a save appends only what is new. Seeded by the load
    /// from both files, which is what makes the append-only file free of duplicates.
    written: HashSet<u32>,
    /// Held across the appends so two overlapping saves cannot interleave their lines. The
    /// appends are seconds apart and short, so this is a formality — but a torn line would cost
    /// the whole record, and the loader would be right to reject it.
    gate: Arc<Mutex<()>>,
}

/// Loads the realm's cache the first frame its identity is known, and on a realm change.
fn load_item_cache(roster: Res<Roster>, mut file: ResMut<ItemCacheFile>, mut items: ResMut<Items>) {
    let Some((realm, _)) = crate::ui_macro::identity(&roster) else {
        return;
    };
    if file.realm.as_deref() == Some(realm.as_str()) {
        return;
    }
    let path = crate::local_state::item_cache_path(&realm);
    let legacy = crate::local_state::legacy_item_cache_path(&realm);
    file.realm = Some(realm.clone());
    file.path = path.clone();
    file.written.clear();
    file.saved_epoch = items.template_epoch();
    let Some(path) = path else {
        return; // a hermetic capture, or no state folder: session-only
    };
    // The legacy whole-map file first, so an append-file record of the same id wins: it is the
    // later answer of the two.
    let mut loaded = 0usize;
    if let Some(legacy) = legacy.filter(|p| p.exists()) {
        loaded += read_legacy(&legacy, &mut file, &mut items);
    }
    loaded += read_appended(&path, &mut file, &mut items);
    file.saved_epoch = items.template_epoch();
    if loaded > 0 {
        info!(
            "items: loaded {loaded} cached template(s) for {realm} from {}",
            path.display()
        );
    }
}

/// Read the pre-append whole-map file. A file this build cannot read is discarded rather than
/// repaired: the cost of being wrong is one session of re-asking.
fn read_legacy(path: &Path, file: &mut ItemCacheFile, items: &mut Items) -> usize {
    let Ok(text) = std::fs::read_to_string(path) else {
        return 0;
    };
    match serde_json::from_str::<HashMap<u32, ItemInfo>>(&text) {
        Ok(map) => {
            let n = map.len();
            for (entry, info) in map {
                items.insert_template(entry, Some(info));
                file.written.insert(entry);
            }
            n
        }
        Err(e) => {
            warn!("items: ignoring {} — {e}", path.display());
            0
        }
    }
}

/// Read the append-only file, one record per line. A line that does not parse is skipped and
/// counted, not fatal: a torn tail from a kill costs that record and nothing else, which is the
/// reason this format was chosen over one the whole file has to be valid for.
fn read_appended(path: &Path, file: &mut ItemCacheFile, items: &mut Items) -> usize {
    let Ok(text) = std::fs::read_to_string(path) else {
        return 0; // no cache yet
    };
    let (mut ok, mut bad) = (0usize, 0usize);
    for line in text.lines().filter(|l| !l.trim().is_empty()) {
        match serde_json::from_str::<(u32, ItemInfo)>(line) {
            Ok((entry, info)) => {
                items.insert_template(entry, Some(info));
                file.written.insert(entry);
                ok += 1;
            }
            Err(_) => bad += 1,
        }
    }
    if bad > 0 {
        warn!(
            "items: skipped {bad} unreadable line(s) in {} — the rest loaded",
            path.display()
        );
    }
    ok
}

/// Appends the templates that have landed since the last write, once the debounce has elapsed.
fn save_item_cache(time: Res<Time>, mut file: ResMut<ItemCacheFile>, items: Res<Items>) {
    file.since_save += time.delta_secs();
    if file.since_save < SAVE_DEBOUNCE {
        return;
    }
    file.since_save = 0.0;
    let Some(batch) = take_unwritten(&mut file, &items) else {
        return;
    };
    // The encode and the write both go to the IO pool: a burst can be thousands of templates, and
    // none of this is worth a frame.
    let (path, gate) = (file.path.clone(), file.gate.clone());
    let Some(path) = path else {
        return;
    };
    IoTaskPool::get()
        .spawn(async move {
            if let Err(e) = append_records(&path, &batch, &gate) {
                warn!("items: cannot append to {}: {e}", path.display());
            }
        })
        .detach();
}

/// The exit write, regardless of the debounce, and on this thread: the IO pool does not outlive
/// the app, so a detached task here would be a cache that silently loses the last burst.
fn save_on_exit(mut file: ResMut<ItemCacheFile>, items: Res<Items>) {
    let Some(batch) = take_unwritten(&mut file, &items) else {
        return;
    };
    let (Some(path), gate) = (file.path.clone(), file.gate.clone()) else {
        return;
    };
    if let Err(e) = append_records(&path, &batch, &gate) {
        warn!("items: cannot append to {}: {e}", path.display());
    }
}

/// The answered templates not yet on disk, cloned out and marked written; `None` when there is
/// nothing to write. Marking here rather than after the append is deliberate: a failed write
/// should cost the records, not re-queue them behind a path that is not going to start working.
fn take_unwritten(file: &mut ItemCacheFile, items: &Items) -> Option<Vec<(u32, ItemInfo)>> {
    if items.template_epoch() == file.saved_epoch {
        return None; // nothing landed since the last write
    }
    file.saved_epoch = items.template_epoch();
    // Only the answered ones: an entry the server called unknown is not worth a record, and
    // re-asking it costs one packet.
    let batch: Vec<(u32, ItemInfo)> = items
        .cached_template_ids()
        .into_iter()
        .filter(|id| !file.written.contains(id))
        .filter_map(|id| items.template_cached(id).map(|t| (id, t.clone())))
        .collect();
    if batch.is_empty() {
        return None;
    }
    for (id, _) in &batch {
        file.written.insert(*id);
    }
    Some(batch)
}

/// Encode the batch and append it in one write, under the gate.
fn append_records(path: &Path, batch: &[(u32, ItemInfo)], gate: &Mutex<()>) -> std::io::Result<()> {
    let mut buf = String::new();
    for record in batch {
        match serde_json::to_string(record) {
            // One line per record, which is what lets the loader skip a torn one.
            Ok(line) => {
                buf.push_str(&line);
                buf.push('\n');
            }
            Err(e) => warn!("items: cannot encode template {}: {e}", record.0),
        }
    }
    if buf.is_empty() {
        return Ok(());
    }
    if let Some(dir) = path.parent() {
        std::fs::create_dir_all(dir)?;
    }
    let _held = gate.lock().unwrap_or_else(|e| e.into_inner());
    let mut handle = std::fs::OpenOptions::new()
        .create(true)
        .append(true)
        .open(path)?;
    handle.write_all(buf.as_bytes())?;
    debug!(
        "items: appended {} template(s) to {}",
        batch.len(),
        path.display()
    );
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The record round-trips through the on-disk form unchanged.
    ///
    /// The cache is only worth having if what comes back is what the wire said: an item whose
    /// template read differently after a restart would be worse than no cache at all, because
    /// `GetItemInfo` would then answer *confidently* and wrongly. `ItemInfo` derives its codec,
    /// so this pins the contract rather than a hand-written copy of 52 fields.
    #[test]
    fn a_template_survives_the_round_trip_unchanged() {
        let mut info = crate::items::test_template("Gnomish Army Knife");
        info.class = 15;
        info.display_info_id = 12345;
        info.required_level = 40;

        let map: HashMap<u32, ItemInfo> = [(7005, info.clone())].into_iter().collect();
        let text = serde_json::to_string(&map).expect("encodes");
        let back: HashMap<u32, ItemInfo> = serde_json::from_str(&text).expect("decodes");

        assert_eq!(back.get(&7005), Some(&info), "the template is unchanged");
    }

    /// A legacy whole-map file this build cannot read is refused whole rather than half-applied.
    #[test]
    fn a_corrupt_file_is_refused_and_an_empty_one_is_valid() {
        assert!(serde_json::from_str::<HashMap<u32, ItemInfo>>("{not json").is_err());
        assert!(
            serde_json::from_str::<HashMap<u32, ItemInfo>>("{}").is_ok_and(|m| m.is_empty()),
            "an empty cache is a valid one, not an error"
        );
    }

    /// The append format's own round trip: a record is one line, and the line restores it.
    #[test]
    fn a_record_survives_the_line_round_trip() {
        let info = crate::items::test_template("Gnomish Army Knife");
        let line = serde_json::to_string(&(7005u32, info.clone())).expect("encodes");
        assert!(
            !line.contains('\n'),
            "a record must occupy exactly one line"
        );
        let (entry, back): (u32, ItemInfo) = serde_json::from_str(&line).expect("decodes");
        assert_eq!((entry, &back), (7005, &info));
    }

    /// The reason for the format: a torn tail costs its own record and nothing else. The
    /// whole-map file this replaced had the opposite property — a kill mid-write cost the lot.
    #[test]
    fn a_torn_line_costs_only_its_own_record() {
        let dir = std::env::temp_dir().join(format!("benilla-items-{}", std::process::id()));
        std::fs::create_dir_all(&dir).expect("temp dir");
        let path = dir.join("torn.jsonl");
        let good = |id: u32, name: &str| {
            serde_json::to_string(&(id, crate::items::test_template(name))).expect("encodes")
        };
        let text = format!(
            "{}\n{}\n{}",
            good(1, "First"),
            // A line cut off mid-record, as a kill during the append would leave it.
            &good(2, "Torn")[..20],
            good(3, "Third"),
        );
        std::fs::write(&path, text).expect("write");

        let mut file = ItemCacheFile::default();
        let mut items = Items::default();
        let loaded = read_appended(&path, &mut file, &mut items);

        assert_eq!(loaded, 2, "the two whole records load");
        assert!(items.template_cached(1).is_some(), "before the torn line");
        assert!(items.template_cached(3).is_some(), "after the torn line");
        assert!(
            items.template_cached(2).is_none(),
            "the torn one is skipped"
        );
        std::fs::remove_dir_all(&dir).ok();
    }

    /// The two files together: the legacy map is the base and an appended record of the same id
    /// wins, because it is the later answer of the two. Also the upgrade path — the templates a
    /// previous build paid for are still there, and are not appended a second time.
    #[test]
    fn an_appended_record_wins_over_the_legacy_map() {
        let dir = std::env::temp_dir().join(format!("benilla-items-both-{}", std::process::id()));
        std::fs::create_dir_all(&dir).expect("temp dir");
        let (legacy_path, append_path) = (dir.join("old.json"), dir.join("new.jsonl"));

        let legacy: HashMap<u32, ItemInfo> = [
            (1, crate::items::test_template("Stale Name")),
            (2, crate::items::test_template("Only In Legacy")),
        ]
        .into_iter()
        .collect();
        std::fs::write(
            &legacy_path,
            serde_json::to_string(&legacy).expect("encodes"),
        )
        .expect("write legacy");
        let fresh = serde_json::to_string(&(1u32, crate::items::test_template("Fresh Name")))
            .expect("encodes");
        std::fs::write(&append_path, format!("{fresh}\n")).expect("write append");

        let mut file = ItemCacheFile::default();
        let mut items = Items::default();
        read_legacy(&legacy_path, &mut file, &mut items);
        read_appended(&append_path, &mut file, &mut items);

        assert_eq!(
            items.template_cached(1).map(|t| t.name.as_str()),
            Some("Fresh Name"),
            "the appended record is the later answer"
        );
        assert_eq!(
            items.template_cached(2).map(|t| t.name.as_str()),
            Some("Only In Legacy"),
            "and the legacy-only template survives the upgrade"
        );
        assert!(
            take_unwritten(&mut file, &items).is_none(),
            "everything loaded is already on disk, so a save appends nothing"
        );
        std::fs::remove_dir_all(&dir).ok();
    }

    /// A save appends only what landed since the last one, which is what keeps a save
    /// proportional to the burst instead of to the cache.
    #[test]
    fn only_new_templates_are_appended() {
        let mut file = ItemCacheFile::default();
        let mut items = Items::default();
        items.insert_template(1, Some(crate::items::test_template("First")));
        items.insert_template(2, Some(crate::items::test_template("Second")));

        let first = take_unwritten(&mut file, &items).expect("two to write");
        assert_eq!(first.len(), 2);

        // Nothing landed: the epoch has not moved, so there is nothing to do at all.
        assert!(
            take_unwritten(&mut file, &items).is_none(),
            "an unchanged store writes nothing"
        );

        // One more answer appends one record, not the whole cache again.
        items.insert_template(3, Some(crate::items::test_template("Third")));
        let second = take_unwritten(&mut file, &items).expect("one to write");
        assert_eq!(
            second.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
            vec![3],
            "only the new id"
        );
    }
}