feat(bootstrap): implement launch crash watchdog and safe mode auto-recovery
This commit is contained in:
parent
b4be13d9f2
commit
5e45657cd9
3 changed files with 118 additions and 0 deletions
|
|
@ -5,3 +5,4 @@ pub mod error;
|
|||
pub mod fflags;
|
||||
pub mod mods;
|
||||
pub mod sandbox;
|
||||
pub mod watchdog;
|
||||
|
|
|
|||
74
crates/molotov-bootstrap/src/watchdog.rs
Normal file
74
crates/molotov-bootstrap/src/watchdog.rs
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
use crate::error::MolotovError;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LaunchStatus {
|
||||
Healthy,
|
||||
RequiresSafeMode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
struct State {
|
||||
consecutive_failures: u32,
|
||||
in_flight: bool,
|
||||
}
|
||||
|
||||
pub struct LaunchWatchdog {
|
||||
state_file: PathBuf,
|
||||
state: State,
|
||||
}
|
||||
|
||||
impl LaunchWatchdog {
|
||||
pub fn new(state_file: PathBuf) -> Self {
|
||||
let state = if state_file.exists() {
|
||||
std::fs::read_to_string(&state_file)
|
||||
.ok()
|
||||
.and_then(|data| serde_json::from_str(&data).ok())
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
State::default()
|
||||
};
|
||||
|
||||
Self { state_file, state }
|
||||
}
|
||||
|
||||
pub fn status(&self) -> LaunchStatus {
|
||||
if self.state.consecutive_failures >= 2 {
|
||||
LaunchStatus::RequiresSafeMode
|
||||
} else {
|
||||
LaunchStatus::Healthy
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_start(&mut self) -> Result<(), MolotovError> {
|
||||
if self.state.in_flight {
|
||||
self.state.consecutive_failures += 1;
|
||||
}
|
||||
self.state.in_flight = true;
|
||||
self.persist()
|
||||
}
|
||||
|
||||
pub fn record_crash(&mut self) -> Result<(), MolotovError> {
|
||||
self.state.consecutive_failures += 1;
|
||||
self.state.in_flight = false;
|
||||
self.persist()
|
||||
}
|
||||
|
||||
pub fn record_success(&mut self) -> Result<(), MolotovError> {
|
||||
self.state.consecutive_failures = 0;
|
||||
self.state.in_flight = false;
|
||||
self.persist()
|
||||
}
|
||||
|
||||
fn persist(&self) -> Result<(), MolotovError> {
|
||||
if let Some(parent) = self.state_file.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let json = serde_json::to_string(&self.state)
|
||||
.map_err(|e| MolotovError::FastFlag(e.to_string()))?;
|
||||
std::fs::write(&self.state_file, json)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
43
crates/molotov-bootstrap/tests/watchdog_test.rs
Normal file
43
crates/molotov-bootstrap/tests/watchdog_test.rs
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
use molotov_bootstrap::watchdog::{LaunchStatus, LaunchWatchdog};
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn test_safe_mode_activates_after_two_crashes() {
|
||||
let temp = tempdir().unwrap();
|
||||
let state_file = temp.path().join("launch_state.json");
|
||||
let mut watchdog = LaunchWatchdog::new(state_file.clone());
|
||||
|
||||
// Crash 1
|
||||
watchdog.record_start().unwrap();
|
||||
watchdog.record_crash().unwrap();
|
||||
assert_eq!(watchdog.status(), LaunchStatus::Healthy);
|
||||
|
||||
// Crash 2
|
||||
watchdog.record_start().unwrap();
|
||||
watchdog.record_crash().unwrap();
|
||||
assert_eq!(watchdog.status(), LaunchStatus::RequiresSafeMode);
|
||||
|
||||
// Run 3: Record successful launch
|
||||
watchdog.record_success().unwrap();
|
||||
assert_eq!(watchdog.status(), LaunchStatus::Healthy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_safe_mode_activates_on_interrupted_launches() {
|
||||
let temp = tempdir().unwrap();
|
||||
let state_file = temp.path().join("launch_state_interrupted.json");
|
||||
let mut watchdog = LaunchWatchdog::new(state_file);
|
||||
|
||||
// Launch 1 starts and dies (in_flight stays true)
|
||||
watchdog.record_start().unwrap();
|
||||
assert_eq!(watchdog.status(), LaunchStatus::Healthy);
|
||||
|
||||
// Launch 2 starts (detects launch 1 died in flight) and dies
|
||||
watchdog.record_start().unwrap();
|
||||
assert_eq!(watchdog.status(), LaunchStatus::Healthy);
|
||||
|
||||
// Launch 3 starts (detects launch 2 died in flight -> 2 failures reached)
|
||||
watchdog.record_start().unwrap();
|
||||
assert_eq!(watchdog.status(), LaunchStatus::RequiresSafeMode);
|
||||
}
|
||||
Reference in a new issue