fix: bound queued alerts, surface refresh errors, and shut down gracefully
- Cap each subscription's pending queue at MAX_PENDING (50) so a channel that can no longer receive messages cannot grow state.json without limit. prune_pending also runs after appending, so a single poll cannot push the queue past the limit. - Log the actual error from refresh() and deliver() in the poll loop and from the framework on_error handler; both discarded it, leaving a stalled refresh with no diagnosable cause. - Handle SIGTERM as well as Ctrl+C so container runtimes can stop the bot between commits instead of killing it mid-write. Adds regression tests for the queue bound in both prune_pending and updated().
This commit is contained in:
parent
6ffdb4ca4f
commit
f08ec5c02c
1 changed files with 107 additions and 5 deletions
112
src/main.rs
112
src/main.rs
|
|
@ -154,9 +154,14 @@ struct WatchPatch {
|
||||||
clear_role: bool,
|
clear_role: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Upper bound on queued alerts per subscription. A channel that can no longer receive
|
||||||
|
/// messages (deleted channel, revoked Send Messages permission) keeps every detection
|
||||||
|
/// pending so it can retry, which without a bound would grow `state.json` forever.
|
||||||
|
const MAX_PENDING: usize = 50;
|
||||||
|
|
||||||
fn prune_pending(sub: &mut Subscription) {
|
fn prune_pending(sub: &mut Subscription) {
|
||||||
let mut seen = BTreeSet::new();
|
let mut seen = BTreeSet::new();
|
||||||
sub.pending = sub
|
let mut kept: Vec<_> = sub
|
||||||
.pending
|
.pending
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|event| {
|
.filter(|event| {
|
||||||
|
|
@ -164,6 +169,17 @@ fn prune_pending(sub: &mut Subscription) {
|
||||||
})
|
})
|
||||||
.cloned()
|
.cloned()
|
||||||
.collect();
|
.collect();
|
||||||
|
if kept.len() > MAX_PENDING {
|
||||||
|
let dropped = kept.len() - MAX_PENDING;
|
||||||
|
// Keep the newest detections; the oldest queued alerts are the least useful.
|
||||||
|
kept.drain(..dropped);
|
||||||
|
tracing::warn!(
|
||||||
|
channel_id = sub.channel_id,
|
||||||
|
dropped,
|
||||||
|
"Queued alert limit reached; dropped oldest detections"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
sub.pending = kept;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn patch_subscription(
|
fn patch_subscription(
|
||||||
|
|
@ -549,6 +565,9 @@ fn updated(mut state: State, current: BTreeMap<String, Model>) -> State {
|
||||||
.cloned()
|
.cloned()
|
||||||
.collect();
|
.collect();
|
||||||
sub.pending.extend(additions);
|
sub.pending.extend(additions);
|
||||||
|
// Bound the queue again after appending so one poll cannot push it past the
|
||||||
|
// limit that prune_pending enforces.
|
||||||
|
prune_pending(sub);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
state.catalog = current;
|
state.catalog = current;
|
||||||
|
|
@ -645,6 +664,21 @@ fn support_url() -> Result<Option<String>, Error> {
|
||||||
Ok(Some(url.to_string()))
|
Ok(Some(url.to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
async fn shutdown_signal() -> std::io::Result<()> {
|
||||||
|
use tokio::signal::unix::{SignalKind, signal};
|
||||||
|
let mut terminate = signal(SignalKind::terminate())?;
|
||||||
|
tokio::select! {
|
||||||
|
result = tokio::signal::ctrl_c() => result,
|
||||||
|
_ = terminate.recv() => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
async fn shutdown_signal() -> std::io::Result<()> {
|
||||||
|
tokio::signal::ctrl_c().await
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<(), Error> {
|
async fn main() -> Result<(), Error> {
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
|
|
@ -731,7 +765,7 @@ async fn main() -> Result<(), Error> {
|
||||||
if let Some(ctx) = error.ctx() {
|
if let Some(ctx) = error.ctx() {
|
||||||
let _ = reply(ctx, embed("Request failed").description("Check your permissions and arguments, or retry shortly. The operator can check bot logs.")).await;
|
let _ = reply(ctx, embed("Request failed").description("Check your permissions and arguments, or retry shortly. The operator can check bot logs.")).await;
|
||||||
}
|
}
|
||||||
tracing::warn!("Discord command/framework error");
|
tracing::warn!(%error, "Discord command/framework error");
|
||||||
}),
|
}),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
})
|
})
|
||||||
|
|
@ -748,8 +782,12 @@ async fn main() -> Result<(), Error> {
|
||||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||||
loop {
|
loop {
|
||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
if refresh(&poll_store, &http_client).await.is_err() { tracing::warn!("Catalog refresh failed; preserving previous snapshot"); }
|
if let Err(error) = refresh(&poll_store, &http_client).await {
|
||||||
if deliver(&poll_store, &http).await.is_err() { tracing::warn!("Saving delivery progress failed; alerts may retry"); }
|
tracing::warn!(%error, "Catalog refresh failed; preserving previous snapshot");
|
||||||
|
}
|
||||||
|
if let Err(error) = deliver(&poll_store, &http).await {
|
||||||
|
tracing::warn!(%error, "Saving delivery progress failed; alerts may retry");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
tracing::info!(poll_seconds, "Model Radar connected; catalog polling configured");
|
tracing::info!(poll_seconds, "Model Radar connected; catalog polling configured");
|
||||||
|
|
@ -761,7 +799,11 @@ async fn main() -> Result<(), Error> {
|
||||||
.await?;
|
.await?;
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
result = client.start() => result?,
|
result = client.start() => result?,
|
||||||
result = tokio::signal::ctrl_c() => { result?; client.shard_manager.shutdown_all().await; }
|
result = shutdown_signal() => {
|
||||||
|
result?;
|
||||||
|
tracing::info!("Shutdown signal received; stopping shards");
|
||||||
|
client.shard_manager.shutdown_all().await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
@ -903,6 +945,66 @@ mod tests {
|
||||||
assert_eq!(reset.pending, sub.pending);
|
assert_eq!(reset.pending, sub.pending);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pending_queue_is_bounded_and_keeps_newest_detections() {
|
||||||
|
let mut sub = subscription();
|
||||||
|
assert!(!sub.free_only);
|
||||||
|
sub.pending = (0..MAX_PENDING + 10)
|
||||||
|
.map(|index| {
|
||||||
|
let mut event = event(false);
|
||||||
|
event.model.id = format!("model/{index}");
|
||||||
|
event.model.name = format!("Model {index}");
|
||||||
|
event
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
prune_pending(&mut sub);
|
||||||
|
assert_eq!(sub.pending.len(), MAX_PENDING);
|
||||||
|
assert_eq!(sub.pending[0].model.id, "model/10");
|
||||||
|
assert_eq!(
|
||||||
|
sub.pending[MAX_PENDING - 1].model.id,
|
||||||
|
format!("model/{}", MAX_PENDING + 9)
|
||||||
|
);
|
||||||
|
let mut small = subscription();
|
||||||
|
small.pending = vec![event(true), event(false)];
|
||||||
|
small.pending[0].kind = "paid_to_free".into();
|
||||||
|
let before = small.pending.clone();
|
||||||
|
prune_pending(&mut small);
|
||||||
|
assert_eq!(small.pending, before);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn updated_bounds_queued_alerts_for_unreachable_channels() {
|
||||||
|
let mut state = State {
|
||||||
|
initialized: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut sub = subscription();
|
||||||
|
sub.pending = (0..MAX_PENDING)
|
||||||
|
.map(|index| {
|
||||||
|
let mut event = event(false);
|
||||||
|
event.model.id = format!("model/{index}");
|
||||||
|
event
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
state.subscriptions.insert(1, sub);
|
||||||
|
let template = catalog(false).into_values().next().unwrap();
|
||||||
|
let current: BTreeMap<String, Model> = (0..10)
|
||||||
|
.map(|index| {
|
||||||
|
let mut model = template.clone();
|
||||||
|
model.id = format!("extra/{index}");
|
||||||
|
model.name = model.id.clone();
|
||||||
|
(model.id.clone(), model)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let state = updated(state, current);
|
||||||
|
assert_eq!(state.subscriptions[&1].pending.len(), MAX_PENDING);
|
||||||
|
assert_eq!(state.subscriptions[&1].pending[0].model.id, "model/10");
|
||||||
|
assert_eq!(
|
||||||
|
state.subscriptions[&1].pending[MAX_PENDING - 1].model.id,
|
||||||
|
"extra/9"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn role_patch_rules() {
|
fn role_patch_rules() {
|
||||||
assert!(
|
assert!(
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue