test: add comprehensive integration tests and WireMock test harness
This commit is contained in:
parent
fe3fc74e1c
commit
b7b88c6daa
14 changed files with 706 additions and 60 deletions
|
|
@ -163,8 +163,7 @@ impl BudgetTracker {
|
||||||
last_day_utc: self.last_day_utc.clone(),
|
last_day_utc: self.last_day_utc.clone(),
|
||||||
recent_calls: self.recent_calls.clone(),
|
recent_calls: self.recent_calls.clone(),
|
||||||
};
|
};
|
||||||
let serialized = serde_json::to_string_pretty(&state)
|
let serialized = serde_json::to_string_pretty(&state).map_err(std::io::Error::other)?;
|
||||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
|
|
||||||
|
|
||||||
// Atomic write via tempfile in the same parent directory
|
// Atomic write via tempfile in the same parent directory
|
||||||
let parent_dir = path.parent().unwrap_or_else(|| Path::new("."));
|
let parent_dir = path.parent().unwrap_or_else(|| Path::new("."));
|
||||||
|
|
|
||||||
13
src/cli.rs
13
src/cli.rs
|
|
@ -15,10 +15,19 @@ pub struct Cli {
|
||||||
#[arg(long, global = true, help = "Path to personas.toml file")]
|
#[arg(long, global = true, help = "Path to personas.toml file")]
|
||||||
pub personas_file: Option<PathBuf>,
|
pub personas_file: Option<PathBuf>,
|
||||||
|
|
||||||
#[arg(long, global = true, env = "OPENROUTER_API_KEY", help = "OpenRouter API Key (overrides keyring & env)")]
|
#[arg(
|
||||||
|
long,
|
||||||
|
global = true,
|
||||||
|
env = "OPENROUTER_API_KEY",
|
||||||
|
help = "OpenRouter API Key (overrides keyring & env)"
|
||||||
|
)]
|
||||||
pub api_key: Option<String>,
|
pub api_key: Option<String>,
|
||||||
|
|
||||||
#[arg(long, global = true, help = "Output structured logs in JSON format to stderr")]
|
#[arg(
|
||||||
|
long,
|
||||||
|
global = true,
|
||||||
|
help = "Output structured logs in JSON format to stderr"
|
||||||
|
)]
|
||||||
pub json_logs: bool,
|
pub json_logs: bool,
|
||||||
|
|
||||||
#[arg(long, short, global = true, help = "Enable verbose debug logs")]
|
#[arg(long, short, global = true, help = "Enable verbose debug logs")]
|
||||||
|
|
|
||||||
|
|
@ -136,7 +136,11 @@ impl OpenRouterClient {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_status(status: reqwest::StatusCode, body: &str, headers: &reqwest::header::HeaderMap) -> Result<(), ClientError> {
|
fn check_status(
|
||||||
|
status: reqwest::StatusCode,
|
||||||
|
body: &str,
|
||||||
|
headers: &reqwest::header::HeaderMap,
|
||||||
|
) -> Result<(), ClientError> {
|
||||||
match status.as_u16() {
|
match status.as_u16() {
|
||||||
200..=299 => Ok(()),
|
200..=299 => Ok(()),
|
||||||
401 => Err(ClientError::Unauthorized),
|
401 => Err(ClientError::Unauthorized),
|
||||||
|
|
@ -149,7 +153,9 @@ impl OpenRouterClient {
|
||||||
.unwrap_or(5);
|
.unwrap_or(5);
|
||||||
Err(ClientError::RateLimited(retry_after))
|
Err(ClientError::RateLimited(retry_after))
|
||||||
}
|
}
|
||||||
404 => Err(ClientError::ModelUnavailable(format!("Model not found (404): {body}"))),
|
404 => Err(ClientError::ModelUnavailable(format!(
|
||||||
|
"Model not found (404): {body}"
|
||||||
|
))),
|
||||||
s if s >= 500 => Err(ClientError::ServerError {
|
s if s >= 500 => Err(ClientError::ServerError {
|
||||||
status: s,
|
status: s,
|
||||||
message: body.to_string(),
|
message: body.to_string(),
|
||||||
|
|
@ -223,7 +229,8 @@ impl OpenRouterClient {
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl ModelClient for OpenRouterClient {
|
impl ModelClient for OpenRouterClient {
|
||||||
async fn complete(&self, model: &str, system: &str, user: &str) -> Result<String, ClientError> {
|
async fn complete(&self, model: &str, system: &str, user: &str) -> Result<String, ClientError> {
|
||||||
self.complete_with_params(model, system, user, None, None).await
|
self.complete_with_params(model, system, user, None, None)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn complete_stream(
|
async fn complete_stream(
|
||||||
|
|
@ -276,7 +283,7 @@ impl ModelClient for OpenRouterClient {
|
||||||
let mut buffer = String::new();
|
let mut buffer = String::new();
|
||||||
|
|
||||||
while let Some(item) = stream.next().await {
|
while let Some(item) = stream.next().await {
|
||||||
let bytes = item.map_err(|e| ClientError::Network(e))?;
|
let bytes = item.map_err(ClientError::Network)?;
|
||||||
let chunk_str = String::from_utf8_lossy(&bytes);
|
let chunk_str = String::from_utf8_lossy(&bytes);
|
||||||
buffer.push_str(&chunk_str);
|
buffer.push_str(&chunk_str);
|
||||||
|
|
||||||
|
|
@ -331,27 +338,47 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_check_status_mappings() {
|
fn test_check_status_mappings() {
|
||||||
let mut headers = reqwest::header::HeaderMap::new();
|
let mut headers = reqwest::header::HeaderMap::new();
|
||||||
headers.insert("retry-after", reqwest::header::HeaderValue::from_static("12"));
|
headers.insert(
|
||||||
|
"retry-after",
|
||||||
|
reqwest::header::HeaderValue::from_static("12"),
|
||||||
|
);
|
||||||
|
|
||||||
let err_401 = OpenRouterClient::check_status(reqwest::StatusCode::UNAUTHORIZED, "bad key", &headers).unwrap_err();
|
let err_401 =
|
||||||
|
OpenRouterClient::check_status(reqwest::StatusCode::UNAUTHORIZED, "bad key", &headers)
|
||||||
|
.unwrap_err();
|
||||||
match err_401 {
|
match err_401 {
|
||||||
ClientError::Unauthorized => (),
|
ClientError::Unauthorized => (),
|
||||||
_ => panic!("Expected Unauthorized, got {:?}", err_401),
|
_ => panic!("Expected Unauthorized, got {:?}", err_401),
|
||||||
}
|
}
|
||||||
|
|
||||||
let err_402 = OpenRouterClient::check_status(reqwest::StatusCode::PAYMENT_REQUIRED, "low balance", &headers).unwrap_err();
|
let err_402 = OpenRouterClient::check_status(
|
||||||
|
reqwest::StatusCode::PAYMENT_REQUIRED,
|
||||||
|
"low balance",
|
||||||
|
&headers,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
match err_402 {
|
match err_402 {
|
||||||
ClientError::PaymentRequired => (),
|
ClientError::PaymentRequired => (),
|
||||||
_ => panic!("Expected PaymentRequired, got {:?}", err_402),
|
_ => panic!("Expected PaymentRequired, got {:?}", err_402),
|
||||||
}
|
}
|
||||||
|
|
||||||
let err_429 = OpenRouterClient::check_status(reqwest::StatusCode::TOO_MANY_REQUESTS, "rate limited", &headers).unwrap_err();
|
let err_429 = OpenRouterClient::check_status(
|
||||||
|
reqwest::StatusCode::TOO_MANY_REQUESTS,
|
||||||
|
"rate limited",
|
||||||
|
&headers,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
match err_429 {
|
match err_429 {
|
||||||
ClientError::RateLimited(secs) => assert_eq!(secs, 12),
|
ClientError::RateLimited(secs) => assert_eq!(secs, 12),
|
||||||
_ => panic!("Expected RateLimited(12), got {:?}", err_429),
|
_ => panic!("Expected RateLimited(12), got {:?}", err_429),
|
||||||
}
|
}
|
||||||
|
|
||||||
let err_503 = OpenRouterClient::check_status(reqwest::StatusCode::SERVICE_UNAVAILABLE, "outage", &headers).unwrap_err();
|
let err_503 = OpenRouterClient::check_status(
|
||||||
|
reqwest::StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"outage",
|
||||||
|
&headers,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
match err_503 {
|
match err_503 {
|
||||||
ClientError::ServerError { status, .. } => assert_eq!(status, 503),
|
ClientError::ServerError { status, .. } => assert_eq!(status, 503),
|
||||||
_ => panic!("Expected ServerError 503, got {:?}", err_503),
|
_ => panic!("Expected ServerError 503, got {:?}", err_503),
|
||||||
|
|
|
||||||
|
|
@ -235,6 +235,6 @@ mod tests {
|
||||||
fn test_bundled_personas_valid() {
|
fn test_bundled_personas_valid() {
|
||||||
let parsed: PersonasConfigFile = toml::from_str(DEFAULT_PERSONAS_TOML).unwrap();
|
let parsed: PersonasConfigFile = toml::from_str(DEFAULT_PERSONAS_TOML).unwrap();
|
||||||
assert!(parsed.personas.len() >= 3);
|
assert!(parsed.personas.len() >= 3);
|
||||||
assert!(parsed.pipelines.len() >= 1);
|
assert!(!parsed.pipelines.is_empty());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
82
src/main.rs
82
src/main.rs
|
|
@ -103,7 +103,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
|
||||||
AppConfig::save_api_key_to_keyring(&api_key)
|
AppConfig::save_api_key_to_keyring(&api_key)
|
||||||
.map_err(|e| format!("Failed to save API key to OS credential store: {e}"))?;
|
.map_err(|e| format!("Failed to save API key to OS credential store: {e}"))?;
|
||||||
println!("{}", "✔ OpenRouter API key saved securely in OS credential store.".bright_green().bold());
|
println!(
|
||||||
|
"{}",
|
||||||
|
"✔ OpenRouter API key saved securely in OS credential store."
|
||||||
|
.bright_green()
|
||||||
|
.bold()
|
||||||
|
);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -115,7 +120,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let badge = get_persona_color_tag(&persona.name);
|
let badge = get_persona_color_tag(&persona.name);
|
||||||
println!("\n {} - {}", badge, persona.system_prompt.dimmed());
|
println!("\n {} - {}", badge, persona.system_prompt.dimmed());
|
||||||
if !persona.trigger_words.is_empty() {
|
if !persona.trigger_words.is_empty() {
|
||||||
println!(" Triggers: {}", persona.trigger_words.join(", ").italic());
|
println!(
|
||||||
|
" Triggers: {}",
|
||||||
|
persona.trigger_words.join(", ").italic()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
println!(" Candidates:");
|
println!(" Candidates:");
|
||||||
for (i, c) in persona.candidates.iter().enumerate() {
|
for (i, c) in persona.candidates.iter().enumerate() {
|
||||||
|
|
@ -128,7 +136,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
} else {
|
} else {
|
||||||
String::new()
|
String::new()
|
||||||
};
|
};
|
||||||
println!(" {}. {} ({}){}", i + 1, c.id.bold(), tier_tag, stealth_tag);
|
println!(
|
||||||
|
" {}. {} ({}){}",
|
||||||
|
i + 1,
|
||||||
|
c.id.bold(),
|
||||||
|
tier_tag,
|
||||||
|
stealth_tag
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -146,7 +160,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
println!(" • {} ➔ {}", pipe.name.bright_cyan().bold(), steps_str.join(" ➔ "));
|
println!(
|
||||||
|
" • {} ➔ {}",
|
||||||
|
pipe.name.bright_cyan().bold(),
|
||||||
|
steps_str.join(" ➔ ")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
println!();
|
println!();
|
||||||
|
|
@ -171,20 +189,39 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
};
|
};
|
||||||
|
|
||||||
if model_count > 0 {
|
if model_count > 0 {
|
||||||
println!(" Live catalog: {} available models cached", model_count.to_string().bright_green().bold());
|
println!(
|
||||||
|
" Live catalog: {} available models cached",
|
||||||
|
model_count.to_string().bright_green().bold()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
match budget_tracker.check_budget() {
|
match budget_tracker.check_budget() {
|
||||||
Ok(status) => {
|
Ok(status) => {
|
||||||
let pct = (status.daily_requests * 100) / status.daily_limit;
|
let pct = (status.daily_requests * 100) / status.daily_limit;
|
||||||
let daily_color = if pct >= 80 {
|
let daily_color = if pct >= 80 {
|
||||||
format!("{}/{} ({}%)", status.daily_requests, status.daily_limit, pct).yellow().bold().to_string()
|
format!(
|
||||||
|
"{}/{} ({}%)",
|
||||||
|
status.daily_requests, status.daily_limit, pct
|
||||||
|
)
|
||||||
|
.yellow()
|
||||||
|
.bold()
|
||||||
|
.to_string()
|
||||||
} else {
|
} else {
|
||||||
format!("{}/{} ({}%)", status.daily_requests, status.daily_limit, pct).green().bold().to_string()
|
format!(
|
||||||
|
"{}/{} ({}%)",
|
||||||
|
status.daily_requests, status.daily_limit, pct
|
||||||
|
)
|
||||||
|
.green()
|
||||||
|
.bold()
|
||||||
|
.to_string()
|
||||||
};
|
};
|
||||||
|
|
||||||
println!(" Account tier: {:?}", config.credit_tier);
|
println!(" Account tier: {:?}", config.credit_tier);
|
||||||
println!(" Rolling 1-min requests: {}/{}", status.minute_requests.to_string().bold(), status.minute_limit);
|
println!(
|
||||||
|
" Rolling 1-min requests: {}/{}",
|
||||||
|
status.minute_requests.to_string().bold(),
|
||||||
|
status.minute_limit
|
||||||
|
);
|
||||||
println!(" Daily requests today: {}", daily_color);
|
println!(" Daily requests today: {}", daily_color);
|
||||||
if status.approaching_daily_limit {
|
if status.approaching_daily_limit {
|
||||||
println!(" {}", "⚠ Notice: Approaching daily free-tier limit. Non-urgent calls may be deferred.".yellow().bold());
|
println!(" {}", "⚠ Notice: Approaching daily free-tier limit. Non-urgent calls may be deferred.".yellow().bold());
|
||||||
|
|
@ -204,7 +241,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
|
||||||
None => {
|
None => {
|
||||||
// Default to chat if no subcommand passed
|
// Default to chat if no subcommand passed
|
||||||
run_chat(ChatArgs::default(), &config, &mut budget_tracker, model_registry).await?;
|
run_chat(
|
||||||
|
ChatArgs::default(),
|
||||||
|
&config,
|
||||||
|
&mut budget_tracker,
|
||||||
|
model_registry,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -261,8 +304,14 @@ async fn run_chat(
|
||||||
}
|
}
|
||||||
|
|
||||||
// Otherwise, run interactive REPL loop
|
// Otherwise, run interactive REPL loop
|
||||||
println!("\n{}", "BrokeCrew Interactive Team Chat".bright_cyan().bold());
|
println!(
|
||||||
println!("{}", "Type your request and press Enter. Type 'exit' or Ctrl+C to quit.\n".dimmed());
|
"\n{}",
|
||||||
|
"BrokeCrew Interactive Team Chat".bright_cyan().bold()
|
||||||
|
);
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
"Type your request and press Enter. Type 'exit' or Ctrl+C to quit.\n".dimmed()
|
||||||
|
);
|
||||||
|
|
||||||
let stdin = io::stdin();
|
let stdin = io::stdin();
|
||||||
let mut reader = stdin.lock();
|
let mut reader = stdin.lock();
|
||||||
|
|
@ -375,14 +424,20 @@ async fn execute_prompt(
|
||||||
(dec.persona, Some(info))
|
(dec.persona, Some(info))
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("Router warning: {}, falling back to {}", e, config.default_persona);
|
eprintln!(
|
||||||
|
"Router warning: {}, falling back to {}",
|
||||||
|
e, config.default_persona
|
||||||
|
);
|
||||||
(config.default_persona.clone(), None)
|
(config.default_persona.clone(), None)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(info) = route_info {
|
if let Some(info) = route_info {
|
||||||
eprintln!("{}", format!("⚡ [BrokeCrew: {info} ➔ {}]", target_persona).dimmed());
|
eprintln!(
|
||||||
|
"{}",
|
||||||
|
format!("⚡ [BrokeCrew: {info} ➔ {}]", target_persona).dimmed()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let (chunk_tx, mut chunk_rx) = mpsc::channel::<String>(100);
|
let (chunk_tx, mut chunk_rx) = mpsc::channel::<String>(100);
|
||||||
|
|
@ -435,6 +490,7 @@ async fn execute_prompt(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
async fn run_pipeline(
|
async fn run_pipeline(
|
||||||
pipeline: &PipelineConfig,
|
pipeline: &PipelineConfig,
|
||||||
config: &AppConfig,
|
config: &AppConfig,
|
||||||
|
|
|
||||||
|
|
@ -49,7 +49,15 @@ impl PersonaRegistry {
|
||||||
pub fn new(personas: Vec<Persona>) -> Self {
|
pub fn new(personas: Vec<Persona>) -> Self {
|
||||||
Self { personas }
|
Self { personas }
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Vec<Persona>> for PersonaRegistry {
|
||||||
|
fn from(personas: Vec<Persona>) -> Self {
|
||||||
|
Self::new(personas)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PersonaRegistry {
|
||||||
pub fn get(&self, name: &str) -> Option<&Persona> {
|
pub fn get(&self, name: &str) -> Option<&Persona> {
|
||||||
self.personas.iter().find(|p| p.name == name)
|
self.personas.iter().find(|p| p.name == name)
|
||||||
}
|
}
|
||||||
|
|
@ -152,6 +160,8 @@ mod tests {
|
||||||
assert_eq!(paid_only[0].id, "gemini-paid");
|
assert_eq!(paid_only[0].id, "gemini-paid");
|
||||||
|
|
||||||
// Non-existent persona returns error
|
// Non-existent persona returns error
|
||||||
assert!(registry.get_eligible_candidates("unknown", false, false).is_err());
|
assert!(registry
|
||||||
|
.get_eligible_candidates("unknown", false, false)
|
||||||
|
.is_err());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,7 @@ pub struct PipelineExecutor;
|
||||||
|
|
||||||
impl PipelineExecutor {
|
impl PipelineExecutor {
|
||||||
/// Executes a single persona query with candidate fallback walk
|
/// Executes a single persona query with candidate fallback walk
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub async fn dispatch_persona(
|
pub async fn dispatch_persona(
|
||||||
persona_name: &str,
|
persona_name: &str,
|
||||||
persona_registry: &PersonaRegistry,
|
persona_registry: &PersonaRegistry,
|
||||||
|
|
@ -55,9 +56,9 @@ impl PipelineExecutor {
|
||||||
require_paid: bool,
|
require_paid: bool,
|
||||||
chunk_sender: Option<mpsc::Sender<String>>,
|
chunk_sender: Option<mpsc::Sender<String>>,
|
||||||
) -> Result<DispatchResult, ClientError> {
|
) -> Result<DispatchResult, ClientError> {
|
||||||
let persona = persona_registry
|
let persona = persona_registry.get(persona_name).ok_or_else(|| {
|
||||||
.get(persona_name)
|
ClientError::ModelUnavailable(format!("Persona '{persona_name}' not found"))
|
||||||
.ok_or_else(|| ClientError::ModelUnavailable(format!("Persona '{persona_name}' not found")))?;
|
})?;
|
||||||
|
|
||||||
// Filter candidates against live registry and options
|
// Filter candidates against live registry and options
|
||||||
let eligible = model_registry
|
let eligible = model_registry
|
||||||
|
|
@ -68,7 +69,11 @@ impl PipelineExecutor {
|
||||||
eligible
|
eligible
|
||||||
} else {
|
} else {
|
||||||
// If live registry filtering yielded nothing (e.g. offline/strict), fall back to raw persona candidates
|
// If live registry filtering yielded nothing (e.g. offline/strict), fall back to raw persona candidates
|
||||||
persona.candidates.iter().filter(|c| allow_stealth || !c.is_stealth).collect()
|
persona
|
||||||
|
.candidates
|
||||||
|
.iter()
|
||||||
|
.filter(|c| allow_stealth || !c.is_stealth)
|
||||||
|
.collect()
|
||||||
};
|
};
|
||||||
|
|
||||||
if candidates_to_try.is_empty() {
|
if candidates_to_try.is_empty() {
|
||||||
|
|
@ -166,6 +171,7 @@ impl PipelineExecutor {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Executes an entire pipeline sequentially, passing outputs between steps
|
/// Executes an entire pipeline sequentially, passing outputs between steps
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub async fn execute(
|
pub async fn execute(
|
||||||
pipeline: &PipelineConfig,
|
pipeline: &PipelineConfig,
|
||||||
persona_registry: &PersonaRegistry,
|
persona_registry: &PersonaRegistry,
|
||||||
|
|
@ -181,9 +187,9 @@ impl PipelineExecutor {
|
||||||
let total_steps = pipeline.steps.len();
|
let total_steps = pipeline.steps.len();
|
||||||
|
|
||||||
for (step_idx, step) in pipeline.steps.iter().enumerate() {
|
for (step_idx, step) in pipeline.steps.iter().enumerate() {
|
||||||
let persona = persona_registry
|
let persona = persona_registry.get(&step.persona).ok_or_else(|| {
|
||||||
.get(&step.persona)
|
ClientError::ModelUnavailable(format!("Persona '{}' not found", step.persona))
|
||||||
.ok_or_else(|| ClientError::ModelUnavailable(format!("Persona '{}' not found", step.persona)))?;
|
})?;
|
||||||
|
|
||||||
// Prepare prompt for this step
|
// Prepare prompt for this step
|
||||||
let step_prompt = if step.pass_previous_output && !previous_output.is_empty() {
|
let step_prompt = if step.pass_previous_output && !previous_output.is_empty() {
|
||||||
|
|
@ -286,7 +292,12 @@ mod tests {
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl ModelClient for MockPipelineClient {
|
impl ModelClient for MockPipelineClient {
|
||||||
async fn complete(&self, model: &str, _system: &str, user: &str) -> Result<String, ClientError> {
|
async fn complete(
|
||||||
|
&self,
|
||||||
|
model: &str,
|
||||||
|
_system: &str,
|
||||||
|
user: &str,
|
||||||
|
) -> Result<String, ClientError> {
|
||||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||||
Ok(format!("Output from {} for: {}", model, user))
|
Ok(format!("Output from {} for: {}", model, user))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -98,7 +98,10 @@ impl ModelRegistry {
|
||||||
.map_err(|e| format!("Network error fetching models catalog: {e}"))?;
|
.map_err(|e| format!("Network error fetching models catalog: {e}"))?;
|
||||||
|
|
||||||
if !res.status().is_success() {
|
if !res.status().is_success() {
|
||||||
return Err(format!("OpenRouter models endpoint returned HTTP {}", res.status()));
|
return Err(format!(
|
||||||
|
"OpenRouter models endpoint returned HTTP {}",
|
||||||
|
res.status()
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let body = res
|
let body = res
|
||||||
|
|
@ -184,10 +187,17 @@ mod tests {
|
||||||
]
|
]
|
||||||
}"#;
|
}"#;
|
||||||
|
|
||||||
let count = registry.update_from_json(json_data).await.expect("update failed");
|
let count = registry
|
||||||
|
.update_from_json(json_data)
|
||||||
|
.await
|
||||||
|
.expect("update failed");
|
||||||
assert_eq!(count, 2);
|
assert_eq!(count, 2);
|
||||||
|
|
||||||
assert!(registry.is_model_available("qwen/qwen-2.5-coder-32b-instruct:free").await);
|
assert!(
|
||||||
|
registry
|
||||||
|
.is_model_available("qwen/qwen-2.5-coder-32b-instruct:free")
|
||||||
|
.await
|
||||||
|
);
|
||||||
assert!(!registry.is_model_available("unlisted/model").await);
|
assert!(!registry.is_model_available("unlisted/model").await);
|
||||||
|
|
||||||
let candidates = vec![
|
let candidates = vec![
|
||||||
|
|
|
||||||
|
|
@ -70,13 +70,24 @@ mod tests {
|
||||||
Persona {
|
Persona {
|
||||||
name: "coder".into(),
|
name: "coder".into(),
|
||||||
system_prompt: "coder prompt".into(),
|
system_prompt: "coder prompt".into(),
|
||||||
trigger_words: vec!["build".into(), "code".into(), "script".into(), "website".into(), "api".into()],
|
trigger_words: vec![
|
||||||
|
"build".into(),
|
||||||
|
"code".into(),
|
||||||
|
"script".into(),
|
||||||
|
"website".into(),
|
||||||
|
"api".into(),
|
||||||
|
],
|
||||||
candidates: vec![],
|
candidates: vec![],
|
||||||
},
|
},
|
||||||
Persona {
|
Persona {
|
||||||
name: "reviewer".into(),
|
name: "reviewer".into(),
|
||||||
system_prompt: "reviewer prompt".into(),
|
system_prompt: "reviewer prompt".into(),
|
||||||
trigger_words: vec!["review".into(), "audit".into(), "critique".into(), "lint".into()],
|
trigger_words: vec![
|
||||||
|
"review".into(),
|
||||||
|
"audit".into(),
|
||||||
|
"critique".into(),
|
||||||
|
"lint".into(),
|
||||||
|
],
|
||||||
candidates: vec![],
|
candidates: vec![],
|
||||||
},
|
},
|
||||||
Persona {
|
Persona {
|
||||||
|
|
@ -97,7 +108,10 @@ mod tests {
|
||||||
("write a python script to parse logs", Some("coder")),
|
("write a python script to parse logs", Some("coder")),
|
||||||
("please review this pull request for bugs", Some("reviewer")),
|
("please review this pull request for bugs", Some("reviewer")),
|
||||||
("audit our authentication system", Some("reviewer")),
|
("audit our authentication system", Some("reviewer")),
|
||||||
("design a high scale distributed cache architecture", Some("architect")),
|
(
|
||||||
|
"design a high scale distributed cache architecture",
|
||||||
|
Some("architect"),
|
||||||
|
),
|
||||||
("hello, what is the weather today?", None),
|
("hello, what is the weather today?", None),
|
||||||
("how are you doing?", None),
|
("how are you doing?", None),
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -15,9 +15,15 @@ impl LlmRouter {
|
||||||
known_personas.join(", ")
|
known_personas.join(", ")
|
||||||
);
|
);
|
||||||
|
|
||||||
let user_prompt = format!("Classify this request to the most appropriate persona:\n\"{}\"", prompt);
|
let user_prompt = format!(
|
||||||
|
"Classify this request to the most appropriate persona:\n\"{}\"",
|
||||||
|
prompt
|
||||||
|
);
|
||||||
|
|
||||||
let raw_output = match client.complete(router_model, &system_prompt, &user_prompt).await {
|
let raw_output = match client
|
||||||
|
.complete(router_model, &system_prompt, &user_prompt)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(out) => out,
|
Ok(out) => out,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("Router LLM call failed, falling back to default persona: {e}");
|
tracing::warn!("Router LLM call failed, falling back to default persona: {e}");
|
||||||
|
|
@ -26,9 +32,15 @@ impl LlmRouter {
|
||||||
};
|
};
|
||||||
|
|
||||||
let cleaned = Self::sanitize_output(&raw_output);
|
let cleaned = Self::sanitize_output(&raw_output);
|
||||||
if known_personas.iter().any(|p| p.eq_ignore_ascii_case(&cleaned)) {
|
if known_personas
|
||||||
|
.iter()
|
||||||
|
.any(|p| p.eq_ignore_ascii_case(&cleaned))
|
||||||
|
{
|
||||||
// Find matched casing
|
// Find matched casing
|
||||||
if let Some(matched) = known_personas.iter().find(|p| p.eq_ignore_ascii_case(&cleaned)) {
|
if let Some(matched) = known_personas
|
||||||
|
.iter()
|
||||||
|
.find(|p| p.eq_ignore_ascii_case(&cleaned))
|
||||||
|
{
|
||||||
return Ok(matched.clone());
|
return Ok(matched.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -94,7 +106,12 @@ mod tests {
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl ModelClient for MockRouterClient {
|
impl ModelClient for MockRouterClient {
|
||||||
async fn complete(&self, _model: &str, _system: &str, _user: &str) -> Result<String, ClientError> {
|
async fn complete(
|
||||||
|
&self,
|
||||||
|
_model: &str,
|
||||||
|
_system: &str,
|
||||||
|
_user: &str,
|
||||||
|
) -> Result<String, ClientError> {
|
||||||
Ok(self.reply.clone())
|
Ok(self.reply.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -117,27 +134,45 @@ mod tests {
|
||||||
let client_valid = MockRouterClient {
|
let client_valid = MockRouterClient {
|
||||||
reply: "coder".into(),
|
reply: "coder".into(),
|
||||||
};
|
};
|
||||||
let res = LlmRouter::classify(&client_valid, "router-model", &known, "general", "build something")
|
let res = LlmRouter::classify(
|
||||||
.await
|
&client_valid,
|
||||||
.unwrap();
|
"router-model",
|
||||||
|
&known,
|
||||||
|
"general",
|
||||||
|
"build something",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
assert_eq!(res, "coder");
|
assert_eq!(res, "coder");
|
||||||
|
|
||||||
// 2. Extra whitespace & quotes
|
// 2. Extra whitespace & quotes
|
||||||
let client_messy = MockRouterClient {
|
let client_messy = MockRouterClient {
|
||||||
reply: " \"REVIEWER\" \n".into(),
|
reply: " \"REVIEWER\" \n".into(),
|
||||||
};
|
};
|
||||||
let res = LlmRouter::classify(&client_messy, "router-model", &known, "general", "audit this")
|
let res = LlmRouter::classify(
|
||||||
.await
|
&client_messy,
|
||||||
.unwrap();
|
"router-model",
|
||||||
|
&known,
|
||||||
|
"general",
|
||||||
|
"audit this",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
assert_eq!(res, "reviewer");
|
assert_eq!(res, "reviewer");
|
||||||
|
|
||||||
// 3. Hallucinated / unrecognized persona defaults to default_persona
|
// 3. Hallucinated / unrecognized persona defaults to default_persona
|
||||||
let client_hallucinated = MockRouterClient {
|
let client_hallucinated = MockRouterClient {
|
||||||
reply: "I believe a poet would be best for this query".into(),
|
reply: "I believe a poet would be best for this query".into(),
|
||||||
};
|
};
|
||||||
let res = LlmRouter::classify(&client_hallucinated, "router-model", &known, "general", "write poem")
|
let res = LlmRouter::classify(
|
||||||
.await
|
&client_hallucinated,
|
||||||
.unwrap();
|
"router-model",
|
||||||
|
&known,
|
||||||
|
"general",
|
||||||
|
"write poem",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
assert_eq!(res, "general");
|
assert_eq!(res, "general");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -87,7 +87,12 @@ mod tests {
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl ModelClient for DummyClient {
|
impl ModelClient for DummyClient {
|
||||||
async fn complete(&self, _model: &str, _system: &str, _user: &str) -> Result<String, ClientError> {
|
async fn complete(
|
||||||
|
&self,
|
||||||
|
_model: &str,
|
||||||
|
_system: &str,
|
||||||
|
_user: &str,
|
||||||
|
) -> Result<String, ClientError> {
|
||||||
Ok("general".into())
|
Ok("general".into())
|
||||||
}
|
}
|
||||||
async fn complete_stream(
|
async fn complete_stream(
|
||||||
|
|
@ -118,17 +123,27 @@ mod tests {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
let registry = PersonaRegistry::new(personas);
|
let registry = PersonaRegistry::new(personas);
|
||||||
let router = Router::new("general".into(), "meta-llama/llama-3.2-3b-instruct:free".into(), 7);
|
let router = Router::new(
|
||||||
|
"general".into(),
|
||||||
|
"meta-llama/llama-3.2-3b-instruct:free".into(),
|
||||||
|
7,
|
||||||
|
);
|
||||||
let client = DummyClient;
|
let client = DummyClient;
|
||||||
|
|
||||||
// Matches Stage A (build) without escalation
|
// Matches Stage A (build) without escalation
|
||||||
let dec1 = router.route(®istry, &client, "build a script", false).await.unwrap();
|
let dec1 = router
|
||||||
|
.route(®istry, &client, "build a script", false)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
assert_eq!(dec1.persona, "coder");
|
assert_eq!(dec1.persona, "coder");
|
||||||
assert_eq!(dec1.stage_used, RouteStage::StageAKeyword);
|
assert_eq!(dec1.stage_used, RouteStage::StageAKeyword);
|
||||||
assert!(!dec1.escalate_to_paid);
|
assert!(!dec1.escalate_to_paid);
|
||||||
|
|
||||||
// Matches Stage A with forced escalation
|
// Matches Stage A with forced escalation
|
||||||
let dec2 = router.route(®istry, &client, "build a script", true).await.unwrap();
|
let dec2 = router
|
||||||
|
.route(®istry, &client, "build a script", true)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
assert_eq!(dec2.persona, "coder");
|
assert_eq!(dec2.persona, "coder");
|
||||||
assert!(dec2.escalate_to_paid);
|
assert!(dec2.escalate_to_paid);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
174
tests/fallback_tests.rs
Normal file
174
tests/fallback_tests.rs
Normal file
|
|
@ -0,0 +1,174 @@
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use brokecrew::client::{ClientError, ModelClient};
|
||||||
|
use brokecrew::persona::{ModelCandidate, Persona, PersonaRegistry, Tier};
|
||||||
|
use brokecrew::pipeline::PipelineExecutor;
|
||||||
|
use brokecrew::registry::ModelRegistry;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
|
||||||
|
struct FaultInjectingClient {
|
||||||
|
candidate1_error: Option<ClientError>,
|
||||||
|
calls: Arc<AtomicUsize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl ModelClient for FaultInjectingClient {
|
||||||
|
async fn complete(&self, model: &str, system: &str, user: &str) -> Result<String, ClientError> {
|
||||||
|
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if model == "candidate-1" {
|
||||||
|
if let Some(ref err) = self.candidate1_error {
|
||||||
|
return match err {
|
||||||
|
ClientError::RateLimited(s) => Err(ClientError::RateLimited(*s)),
|
||||||
|
ClientError::ServerError { status, message } => Err(ClientError::ServerError {
|
||||||
|
status: *status,
|
||||||
|
message: message.clone(),
|
||||||
|
}),
|
||||||
|
ClientError::Unauthorized => Err(ClientError::Unauthorized),
|
||||||
|
ClientError::PaymentRequired => Err(ClientError::PaymentRequired),
|
||||||
|
_ => Err(ClientError::ModelUnavailable("simulated".into())),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(format!(
|
||||||
|
"Success from {model} for prompt '{user}' (system: {system})"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn complete_stream(
|
||||||
|
&self,
|
||||||
|
model: &str,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
chunk_sender: mpsc::Sender<String>,
|
||||||
|
) -> Result<String, ClientError> {
|
||||||
|
let reply = self.complete(model, system, user).await?;
|
||||||
|
let _ = chunk_sender.send(reply.clone()).await;
|
||||||
|
Ok(reply)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_resilience_persona() -> Persona {
|
||||||
|
Persona {
|
||||||
|
name: "resilient-coder".into(),
|
||||||
|
system_prompt: "You are coder".into(),
|
||||||
|
trigger_words: vec!["code".into()],
|
||||||
|
candidates: vec![
|
||||||
|
ModelCandidate {
|
||||||
|
id: "candidate-1".into(),
|
||||||
|
tier: Tier::Free,
|
||||||
|
is_stealth: false,
|
||||||
|
},
|
||||||
|
ModelCandidate {
|
||||||
|
id: "candidate-2".into(),
|
||||||
|
tier: Tier::Free,
|
||||||
|
is_stealth: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_candidate_cascade_on_503_service_unavailable() {
|
||||||
|
let persona = create_resilience_persona();
|
||||||
|
let persona_reg = PersonaRegistry::new(vec![persona]);
|
||||||
|
let model_reg = ModelRegistry::new();
|
||||||
|
|
||||||
|
let calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let client = FaultInjectingClient {
|
||||||
|
candidate1_error: Some(ClientError::ServerError {
|
||||||
|
status: 503,
|
||||||
|
message: "Overloaded".into(),
|
||||||
|
}),
|
||||||
|
calls: calls.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = PipelineExecutor::dispatch_persona(
|
||||||
|
"resilient-coder",
|
||||||
|
&persona_reg,
|
||||||
|
&model_reg,
|
||||||
|
&client,
|
||||||
|
"write rust code",
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("fallback to candidate-2 should succeed");
|
||||||
|
|
||||||
|
assert_eq!(result.persona, "resilient-coder");
|
||||||
|
assert_eq!(result.model_used, "candidate-2");
|
||||||
|
assert!(result.fell_back, "Result must indicate fallback occurred");
|
||||||
|
assert_eq!(result.attempts.len(), 1);
|
||||||
|
assert_eq!(result.attempts[0].model_id, "candidate-1");
|
||||||
|
assert_eq!(
|
||||||
|
calls.load(Ordering::SeqCst),
|
||||||
|
2,
|
||||||
|
"Both candidates must have been called"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_candidate_cascade_on_429_rate_limited() {
|
||||||
|
let persona = create_resilience_persona();
|
||||||
|
let persona_reg = PersonaRegistry::new(vec![persona]);
|
||||||
|
let model_reg = ModelRegistry::new();
|
||||||
|
|
||||||
|
let calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let client = FaultInjectingClient {
|
||||||
|
candidate1_error: Some(ClientError::RateLimited(15)),
|
||||||
|
calls: calls.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = PipelineExecutor::dispatch_persona(
|
||||||
|
"resilient-coder",
|
||||||
|
&persona_reg,
|
||||||
|
&model_reg,
|
||||||
|
&client,
|
||||||
|
"write python code",
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("fallback to candidate-2 should succeed");
|
||||||
|
|
||||||
|
assert_eq!(result.model_used, "candidate-2");
|
||||||
|
assert!(result.fell_back);
|
||||||
|
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_fail_fast_on_401_unauthorized() {
|
||||||
|
let persona = create_resilience_persona();
|
||||||
|
let persona_reg = PersonaRegistry::new(vec![persona]);
|
||||||
|
let model_reg = ModelRegistry::new();
|
||||||
|
|
||||||
|
let calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let client = FaultInjectingClient {
|
||||||
|
candidate1_error: Some(ClientError::Unauthorized),
|
||||||
|
calls: calls.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let err = PipelineExecutor::dispatch_persona(
|
||||||
|
"resilient-coder",
|
||||||
|
&persona_reg,
|
||||||
|
&model_reg,
|
||||||
|
&client,
|
||||||
|
"test query",
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
|
||||||
|
match err {
|
||||||
|
ClientError::Unauthorized => (),
|
||||||
|
_ => panic!("Expected Unauthorized fail-fast, got {:?}", err),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Must NOT call candidate-2 when 401 Unauthorized occurs!
|
||||||
|
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
148
tests/router_tests.rs
Normal file
148
tests/router_tests.rs
Normal file
|
|
@ -0,0 +1,148 @@
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use brokecrew::client::{ClientError, ModelClient};
|
||||||
|
use brokecrew::persona::{ModelCandidate, Persona, PersonaRegistry, Tier};
|
||||||
|
use brokecrew::router::{RouteDecision, RouteStage, Router};
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
|
||||||
|
struct TestRouterClient {
|
||||||
|
llm_choice: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl ModelClient for TestRouterClient {
|
||||||
|
async fn complete(
|
||||||
|
&self,
|
||||||
|
_model: &str,
|
||||||
|
_system: &str,
|
||||||
|
_user: &str,
|
||||||
|
) -> Result<String, ClientError> {
|
||||||
|
Ok(self.llm_choice.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn complete_stream(
|
||||||
|
&self,
|
||||||
|
model: &str,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
_chunk_sender: mpsc::Sender<String>,
|
||||||
|
) -> Result<String, ClientError> {
|
||||||
|
self.complete(model, system, user).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn setup_test_registry() -> PersonaRegistry {
|
||||||
|
vec![
|
||||||
|
Persona {
|
||||||
|
name: "coder".into(),
|
||||||
|
system_prompt: "You write code".into(),
|
||||||
|
trigger_words: vec![
|
||||||
|
"build".into(),
|
||||||
|
"code".into(),
|
||||||
|
"script".into(),
|
||||||
|
"website".into(),
|
||||||
|
"api".into(),
|
||||||
|
],
|
||||||
|
candidates: vec![ModelCandidate {
|
||||||
|
id: "qwen-free".into(),
|
||||||
|
tier: Tier::Free,
|
||||||
|
is_stealth: false,
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
Persona {
|
||||||
|
name: "reviewer".into(),
|
||||||
|
system_prompt: "You review code".into(),
|
||||||
|
trigger_words: vec!["review".into(), "audit".into(), "critique".into()],
|
||||||
|
candidates: vec![ModelCandidate {
|
||||||
|
id: "llama-free".into(),
|
||||||
|
tier: Tier::Free,
|
||||||
|
is_stealth: false,
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
Persona {
|
||||||
|
name: "general".into(),
|
||||||
|
system_prompt: "You are general assistant".into(),
|
||||||
|
trigger_words: vec!["help".into(), "hello".into()],
|
||||||
|
candidates: vec![ModelCandidate {
|
||||||
|
id: "general-free".into(),
|
||||||
|
tier: Tier::Free,
|
||||||
|
is_stealth: false,
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
.into()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_table_driven_classification() {
|
||||||
|
let registry = setup_test_registry();
|
||||||
|
let client = TestRouterClient {
|
||||||
|
llm_choice: "reviewer".into(),
|
||||||
|
};
|
||||||
|
let router = Router::new("general".into(), "router-model".into(), 7);
|
||||||
|
|
||||||
|
let cases = vec![
|
||||||
|
(
|
||||||
|
"build me a proxy checker site",
|
||||||
|
"coder",
|
||||||
|
RouteStage::StageAKeyword,
|
||||||
|
false,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"please audit this auth module",
|
||||||
|
"reviewer",
|
||||||
|
RouteStage::StageAKeyword,
|
||||||
|
false,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"how does photosynthesis work?",
|
||||||
|
"reviewer",
|
||||||
|
RouteStage::StageBLlm,
|
||||||
|
false,
|
||||||
|
), // LLM choice returned "reviewer"
|
||||||
|
];
|
||||||
|
|
||||||
|
for (prompt, expected_persona, expected_stage, expected_escalate) in cases {
|
||||||
|
let decision: RouteDecision = router
|
||||||
|
.route(®istry, &client, prompt, false)
|
||||||
|
.await
|
||||||
|
.expect("routing should succeed");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
decision.persona, expected_persona,
|
||||||
|
"Prompt failed: '{}'",
|
||||||
|
prompt
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decision.stage_used, expected_stage,
|
||||||
|
"Stage mismatch for: '{}'",
|
||||||
|
prompt
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decision.escalate_to_paid, expected_escalate,
|
||||||
|
"Escalate mismatch for: '{}'",
|
||||||
|
prompt
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_complexity_escalation_trigger() {
|
||||||
|
let registry = setup_test_registry();
|
||||||
|
let client = TestRouterClient {
|
||||||
|
llm_choice: "coder".into(),
|
||||||
|
};
|
||||||
|
let router = Router::new("general".into(), "router-model".into(), 7);
|
||||||
|
|
||||||
|
let heavy_prompt = "Refactor all microservices across the entire project with database migration and distributed architecture:\n```rust\nfn main() {}\n```";
|
||||||
|
let decision = router
|
||||||
|
.route(®istry, &client, heavy_prompt, false)
|
||||||
|
.await
|
||||||
|
.expect("routing should succeed");
|
||||||
|
|
||||||
|
assert_eq!(decision.persona, "coder");
|
||||||
|
assert!(
|
||||||
|
decision.escalate_to_paid,
|
||||||
|
"Heavy prompt must trigger escalation"
|
||||||
|
);
|
||||||
|
assert!(decision.complexity_score >= 7);
|
||||||
|
}
|
||||||
138
tests/wiremock_client.rs
Normal file
138
tests/wiremock_client.rs
Normal file
|
|
@ -0,0 +1,138 @@
|
||||||
|
use brokecrew::client::{ClientError, ModelClient, OpenRouterClient};
|
||||||
|
use brokecrew::registry::ModelRegistry;
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
use wiremock::matchers::{method, path};
|
||||||
|
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_wiremock_sse_streaming() {
|
||||||
|
let mock_server = MockServer::start().await;
|
||||||
|
|
||||||
|
let sse_body = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello \"}}]}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"from \"}}]}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"WireMock!\"}}]}\n\ndata: [DONE]\n\n";
|
||||||
|
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.and(path("/chat/completions"))
|
||||||
|
.respond_with(
|
||||||
|
ResponseTemplate::new(200)
|
||||||
|
.insert_header("content-type", "text/event-stream")
|
||||||
|
.set_body_string(sse_body),
|
||||||
|
)
|
||||||
|
.mount(&mock_server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let client = OpenRouterClient::with_base_url("test-key".into(), mock_server.uri());
|
||||||
|
|
||||||
|
let (tx, mut rx) = mpsc::channel::<String>(10);
|
||||||
|
let collect_task = tokio::spawn(async move {
|
||||||
|
let mut chunks = Vec::new();
|
||||||
|
while let Some(c) = rx.recv().await {
|
||||||
|
chunks.push(c);
|
||||||
|
}
|
||||||
|
chunks
|
||||||
|
});
|
||||||
|
|
||||||
|
let full_reply = client
|
||||||
|
.complete_stream("mock-model", "system", "user", tx)
|
||||||
|
.await
|
||||||
|
.expect("streaming should succeed");
|
||||||
|
|
||||||
|
let chunks = collect_task.await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(full_reply, "Hello from WireMock!");
|
||||||
|
assert_eq!(chunks, vec!["Hello ", "from ", "WireMock!"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_wiremock_401_unauthorized() {
|
||||||
|
let mock_server = MockServer::start().await;
|
||||||
|
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.and(path("/chat/completions"))
|
||||||
|
.respond_with(ResponseTemplate::new(401).set_body_string("Invalid API key"))
|
||||||
|
.mount(&mock_server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let client = OpenRouterClient::with_base_url("bad-key".into(), mock_server.uri());
|
||||||
|
let err = client
|
||||||
|
.complete("mock-model", "sys", "user")
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
|
||||||
|
match err {
|
||||||
|
ClientError::Unauthorized => (),
|
||||||
|
_ => panic!("Expected ClientError::Unauthorized, got {:?}", err),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_wiremock_402_payment_required() {
|
||||||
|
let mock_server = MockServer::start().await;
|
||||||
|
|
||||||
|
Mock::given(method("POST"))
|
||||||
|
.and(path("/chat/completions"))
|
||||||
|
.respond_with(ResponseTemplate::new(402).set_body_string("Account balance depleted"))
|
||||||
|
.mount(&mock_server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let client = OpenRouterClient::with_base_url("key".into(), mock_server.uri());
|
||||||
|
let err = client
|
||||||
|
.complete("mock-model", "sys", "user")
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
|
||||||
|
match err {
|
||||||
|
ClientError::PaymentRequired => (),
|
||||||
|
_ => panic!("Expected ClientError::PaymentRequired, got {:?}", err),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_wiremock_models_catalog_fetch() {
|
||||||
|
let mock_server = MockServer::start().await;
|
||||||
|
|
||||||
|
let catalog_json = r#"{
|
||||||
|
"data": [
|
||||||
|
{
|
||||||
|
"id": "qwen/qwen-2.5-coder-32b-instruct:free",
|
||||||
|
"name": "Qwen 2.5 Coder",
|
||||||
|
"context_length": 32768,
|
||||||
|
"pricing": { "prompt": "0", "completion": "0" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "meta-llama/llama-3.3-70b-instruct:free",
|
||||||
|
"name": "Llama 3.3 70B",
|
||||||
|
"context_length": 131072,
|
||||||
|
"pricing": { "prompt": "0", "completion": "0" }
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}"#;
|
||||||
|
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/models"))
|
||||||
|
.respond_with(
|
||||||
|
ResponseTemplate::new(200)
|
||||||
|
.insert_header("content-type", "application/json")
|
||||||
|
.set_body_string(catalog_json),
|
||||||
|
)
|
||||||
|
.mount(&mock_server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let registry = ModelRegistry::new();
|
||||||
|
let count = registry
|
||||||
|
.fetch_live(&mock_server.uri())
|
||||||
|
.await
|
||||||
|
.expect("catalog fetch failed");
|
||||||
|
|
||||||
|
assert_eq!(count, 2);
|
||||||
|
assert!(
|
||||||
|
registry
|
||||||
|
.is_model_available("qwen/qwen-2.5-coder-32b-instruct:free")
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
registry
|
||||||
|
.is_model_available("meta-llama/llama-3.3-70b-instruct:free")
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
assert!(!registry.is_model_available("non-existent-model").await);
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue