benilla-octo/crates/benilla-app/src/gpu_errors.rs

//! Survive a wgpu validation error instead of dying of one.
//!
//! wgpu's default uncaptured-error handler **panics**, so a single bad pipeline takes the whole
//! process. That is the right default for a tool and the wrong one for a game client: a shader
//! interface this client got wrong is a missing shadow, not a reason to end a session — and on a
//! hardcore realm, dying of it mid-combat costs the character, not the frame.
//!
//! This replaces the handler with one that logs and lets the GPU resource fail. wgpu returns a
//! poisoned (error) resource rather than a valid one, so the draw that wanted it is skipped by
//! the backend; Bevy's `PipelineCache` keeps the pipeline in its error state and simply never
//! renders through it.
//!
//! **What this does not do** is hide the problem. Every error is logged at `ERROR` with its full
//! text, which is what wgpu would have panicked with, and each distinct one is counted. The
//! counter exists because the fatal handler had one property worth keeping: you noticed. A
//! validation error that repeats every frame would otherwise scroll past as noise, so the first
//! of each distinct message is loud and the rest are tallied.
//!
//! `WOW_WGPU_FATAL=1` restores the panic, for a session spent hunting one of these: the log line
//! names the error but not which pipeline specialization asked for it, and a panic brings the
//! backtrace and the crash report with it.

use std::collections::HashSet;
use std::sync::{Arc, Mutex};

use bevy::prelude::*;
use bevy::render::renderer::RenderDevice;
use bevy::render::{RenderApp, RenderStartup};

/// Install the handler on the render device once it exists.
pub(crate) struct GpuErrorPlugin;

impl Plugin for GpuErrorPlugin {
    fn build(&self, app: &mut App) {
        let Some(render_app) = app.get_sub_app_mut(RenderApp) else {
            return;
        };
        render_app.add_systems(RenderStartup, install_handler);
    }
}

/// The distinct errors already reported, so a per-frame repeat is counted rather than printed.
#[derive(Default)]
struct Seen {
    messages: HashSet<String>,
    repeats: u64,
}

fn install_handler(device: Res<RenderDevice>) {
    if std::env::var_os("WOW_WGPU_FATAL").is_some() {
        info!("gpu: WOW_WGPU_FATAL is set — leaving wgpu's own panicking handler in place");
        return;
    }
    let seen: Arc<Mutex<Seen>> = Arc::new(Mutex::new(Seen::default()));
    device
        .wgpu_device()
        .on_uncaptured_error(Arc::new(move |err: wgpu::Error| {
            // The whole chain, not the `Display` head: wgpu's `Display` prints only the
            // `description`, which for a validation error is often the bare words "Validation
            // Error". The pipeline label and the shader location that did not line up are in the
            // `source()` chain underneath it, and an error we cannot name is one we cannot fix.
            let text = error_chain(&err);
            let mut seen = seen.lock().unwrap_or_else(|e| e.into_inner());
            if seen.messages.insert(text.clone()) {
                error!(
                    "gpu: wgpu error (non-fatal, the resource that wanted it is skipped): {text}\n\
                 gpu: set WOW_WGPU_FATAL=1 to panic here instead and get a backtrace"
                );
            } else {
                seen.repeats += 1;
                // Loud once per power of ten, so a per-frame repeat neither scrolls the log nor
                // disappears from it.
                if seen.repeats.is_power_of_two() {
                    warn!(
                        "gpu: the same wgpu error has now repeated {} times",
                        seen.repeats
                    );
                }
            }
        }));
    info!("gpu: wgpu errors are logged, not fatal (WOW_WGPU_FATAL=1 to restore the panic)");
}

/// `err`, then each `source()` under it, one per line.
fn error_chain(err: &wgpu::Error) -> String {
    use std::fmt::Write;

    let mut text = format!("{err}");
    let mut source = std::error::Error::source(err);
    while let Some(cause) = source {
        let _ = write!(text, "\n  caused by: {cause}");
        source = cause.source();
    }
    text
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The repeat gate: the first of a distinct message is loud, and the repeats are reported on
    /// a widening scale rather than every frame or never.
    #[test]
    fn a_repeating_error_is_reported_on_a_widening_scale() {
        let mut seen = Seen::default();
        let mut loud = 0usize;
        for _ in 0..1000 {
            if seen.messages.insert("same error".to_string()) {
                loud += 1;
            } else {
                seen.repeats += 1;
                if seen.repeats.is_power_of_two() {
                    loud += 1;
                }
            }
        }
        // 1 first sighting + powers of two up to 512.
        assert_eq!(loud, 11, "once, then 1,2,4,…,512");
        assert_eq!(seen.repeats, 999);
    }

    /// A layered error for the chain test: `Display` gives one line, `source` the next.
    #[derive(Debug)]
    struct Layered(&'static str, Option<Box<Layered>>);

    impl std::fmt::Display for Layered {
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            f.write_str(self.0)
        }
    }

    impl std::error::Error for Layered {
        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
            self.1
                .as_deref()
                .map(|e| e as &(dyn std::error::Error + 'static))
        }
    }

    /// The whole chain is reported, not just wgpu's terse `description`. This is the difference
    /// between "Validation Error" and knowing which pipeline asked for what.
    #[test]
    fn the_report_carries_every_cause_under_the_description() {
        let err = wgpu::Error::Validation {
            source: Box::new(Layered(
                "in the shadow pass",
                Some(Box::new(Layered("location 3 is not written", None))),
            )),
            description: "Validation Error".to_string(),
        };

        let text = error_chain(&err);

        assert!(text.starts_with("Validation Error"), "the head is first");
        assert!(
            text.contains("caused by: in the shadow pass"),
            "the first cause survives: {text}"
        );
        assert!(
            text.contains("caused by: location 3 is not written"),
            "and so does the one under it: {text}"
        );
    }

    /// Distinct errors each get their own first sighting; one does not mask another.
    #[test]
    fn a_second_distinct_error_is_not_swallowed_by_the_first() {
        let mut seen = Seen::default();
        assert!(seen.messages.insert("pipeline A".to_string()));
        assert!(seen.messages.insert("pipeline B".to_string()));
        assert!(!seen.messages.insert("pipeline A".to_string()));
        assert_eq!(seen.messages.len(), 2);
    }
}