feat(bootstrap): implement launch crash watchdog and safe mode auto-recovery

This commit is contained in:
katR 2026-09-29 08:06:15 +05:30
parent b4be13d9f2
commit 5e45657cd9
3 changed files with 118 additions and 0 deletions

View file

@ -5,3 +5,4 @@ pub mod error;
pub mod fflags;
pub mod mods;
pub mod sandbox;
pub mod watchdog;

View file

@ -0,0 +1,74 @@
// SPDX-License-Identifier: GPL-3.0-or-later
use crate::error::MolotovError;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LaunchStatus {
Healthy,
RequiresSafeMode,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
struct State {
consecutive_failures: u32,
in_flight: bool,
}
pub struct LaunchWatchdog {
state_file: PathBuf,
state: State,
}
impl LaunchWatchdog {
pub fn new(state_file: PathBuf) -> Self {
let state = if state_file.exists() {
std::fs::read_to_string(&state_file)
.ok()
.and_then(|data| serde_json::from_str(&data).ok())
.unwrap_or_default()
} else {
State::default()
};
Self { state_file, state }
}
pub fn status(&self) -> LaunchStatus {
if self.state.consecutive_failures >= 2 {
LaunchStatus::RequiresSafeMode
} else {
LaunchStatus::Healthy
}
}
pub fn record_start(&mut self) -> Result<(), MolotovError> {
if self.state.in_flight {
self.state.consecutive_failures += 1;
}
self.state.in_flight = true;
self.persist()
}
pub fn record_crash(&mut self) -> Result<(), MolotovError> {
self.state.consecutive_failures += 1;
self.state.in_flight = false;
self.persist()
}
pub fn record_success(&mut self) -> Result<(), MolotovError> {
self.state.consecutive_failures = 0;
self.state.in_flight = false;
self.persist()
}
fn persist(&self) -> Result<(), MolotovError> {
if let Some(parent) = self.state_file.parent() {
std::fs::create_dir_all(parent)?;
}
let json = serde_json::to_string(&self.state)
.map_err(|e| MolotovError::FastFlag(e.to_string()))?;
std::fs::write(&self.state_file, json)?;
Ok(())
}
}

View file

@ -0,0 +1,43 @@
// SPDX-License-Identifier: GPL-3.0-or-later
use molotov_bootstrap::watchdog::{LaunchStatus, LaunchWatchdog};
use tempfile::tempdir;
#[test]
fn test_safe_mode_activates_after_two_crashes() {
let temp = tempdir().unwrap();
let state_file = temp.path().join("launch_state.json");
let mut watchdog = LaunchWatchdog::new(state_file.clone());
// Crash 1
watchdog.record_start().unwrap();
watchdog.record_crash().unwrap();
assert_eq!(watchdog.status(), LaunchStatus::Healthy);
// Crash 2
watchdog.record_start().unwrap();
watchdog.record_crash().unwrap();
assert_eq!(watchdog.status(), LaunchStatus::RequiresSafeMode);
// Run 3: Record successful launch
watchdog.record_success().unwrap();
assert_eq!(watchdog.status(), LaunchStatus::Healthy);
}
#[test]
fn test_safe_mode_activates_on_interrupted_launches() {
let temp = tempdir().unwrap();
let state_file = temp.path().join("launch_state_interrupted.json");
let mut watchdog = LaunchWatchdog::new(state_file);
// Launch 1 starts and dies (in_flight stays true)
watchdog.record_start().unwrap();
assert_eq!(watchdog.status(), LaunchStatus::Healthy);
// Launch 2 starts (detects launch 1 died in flight) and dies
watchdog.record_start().unwrap();
assert_eq!(watchdog.status(), LaunchStatus::Healthy);
// Launch 3 starts (detects launch 2 died in flight -> 2 failures reached)
watchdog.record_start().unwrap();
assert_eq!(watchdog.status(), LaunchStatus::RequiresSafeMode);
}