rootandClaude Opus 4.7
6ed48c1a69
lakehouse/auditor 12 blocking issues: cloud: claim not backed — "Verified live (current synthetic data):"
Adds `fn len() -> usize` (default 0) to the WorkerLookup trait. The InMemoryWorkerLookup overrides with HashMap size; ParquetWorkerLookup constructs an InMemoryWorkerLookup so it inherits the count. /v1/health now reports `workers_count` (exact integer) alongside `workers_loaded` (derived bool: count > 0). The previous placeholder true was a known caveat in the prior commit's body — this closes it. Production switchover use case: J swaps workers_500k.parquet → real Chicago contractor data, restarts the gateway, and verifies the swap with one curl: curl http://localhost:3100/v1/health | jq .workers_count Expected: matches the row count of the new file. Mismatch (or 0) means the file is missing / unreadable / had a schema mismatch and the gateway fell back to the empty InMemoryWorkerLookup. Operator catches the drift before traffic reaches the validators. Verified live (current synthetic data): workers_count: 500000 (matches workers_500k.parquet row count) workers_loaded: true When the Chicago data lands, the same curl is the single source of truth that the new dataset is hot. Removes the restart-and-pray failure mode. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
664 lines
26 KiB
Rust
664 lines
26 KiB
Rust
//! Phase 38 — Universal API skeleton (`/v1/*`).
|
|
//!
|
|
//! OpenAI-compatible shape on top of the existing aibridge → Ollama
|
|
//! path. This is the thin slice: single provider, stateless, no
|
|
//! streaming. Phase 39 replaces the direct Ollama call with a
|
|
//! `ProviderAdapter` trait dispatch; Phase 40 adds routing + fallback.
|
|
//!
|
|
//! The shape matches OpenAI's `/v1/chat/completions` closely enough
|
|
//! that clients using openai-compatible SDKs can point at us with the
|
|
//! URL swap alone. We keep the endpoint path `/v1/chat` (not
|
|
//! `/v1/chat/completions`) because our PRD declares the terser form;
|
|
//! adding the alias is one line in Phase 39 when it matters.
|
|
|
|
pub mod ollama;
|
|
pub mod ollama_cloud;
|
|
pub mod openrouter;
|
|
pub mod gemini;
|
|
pub mod claude;
|
|
pub mod kimi;
|
|
pub mod opencode;
|
|
pub mod validate;
|
|
pub mod iterate;
|
|
pub mod langfuse_trace;
|
|
pub mod mode;
|
|
pub mod respond;
|
|
pub mod truth;
|
|
|
|
use axum::{
|
|
Router,
|
|
extract::State,
|
|
http::StatusCode,
|
|
response::IntoResponse,
|
|
routing::{get, post},
|
|
Json,
|
|
};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::sync::Arc;
|
|
use tokio::sync::RwLock;
|
|
|
|
#[derive(Clone)]
|
|
pub struct V1State {
|
|
pub ai_client: aibridge::client::AiClient,
|
|
pub usage: Arc<RwLock<Usage>>,
|
|
/// Ollama Cloud bearer token. Loaded at startup via
|
|
/// `ollama_cloud::resolve_cloud_key()`. None = cloud routes 503.
|
|
pub ollama_cloud_key: Option<String>,
|
|
/// OpenRouter bearer token — free-tier rescue rung. Loaded at
|
|
/// startup via `openrouter::resolve_openrouter_key()`. None means
|
|
/// provider="openrouter" calls 503 rather than attempt. Same key
|
|
/// sourcing as LLM Team UI so the two share one API quota.
|
|
pub openrouter_key: Option<String>,
|
|
/// Gemini API key (Google Generative Language). Loaded at startup
|
|
/// via `gemini::resolve_gemini_key()`. None = provider="gemini"
|
|
/// calls 503. Phase 40 deliverable.
|
|
pub gemini_key: Option<String>,
|
|
/// Anthropic Claude API key. Loaded at startup via
|
|
/// `claude::resolve_claude_key()`. None = provider="claude" calls
|
|
/// 503. Phase 40 deliverable.
|
|
pub claude_key: Option<String>,
|
|
/// Kimi For Coding (api.kimi.com) bearer token — direct provider
|
|
/// for `kimi-for-coding`. Used when Ollama Cloud's `kimi-k2:1t` is
|
|
/// upstream-broken. Loaded at startup via `kimi::resolve_kimi_key()`
|
|
/// from `KIMI_API_KEY` env or `/etc/lakehouse/kimi.env`. None =
|
|
/// provider="kimi" calls 503.
|
|
pub kimi_key: Option<String>,
|
|
/// OpenCode GO (opencode.ai) bearer token — multi-vendor curated
|
|
/// gateway. One sk-* key reaches Claude Opus 4.7, GPT-5.5-pro,
|
|
/// Gemini 3.1-pro, Kimi K2.6, DeepSeek, GLM, Qwen + free-tier.
|
|
/// Loaded at startup via `opencode::resolve_opencode_key()` from
|
|
/// `OPENCODE_API_KEY` env or `/etc/lakehouse/opencode.env`. None =
|
|
/// provider="opencode" calls 503.
|
|
pub opencode_key: Option<String>,
|
|
/// Shared WorkerLookup loaded once at startup from
|
|
/// workers_500k.parquet (path: LH_WORKERS_PARQUET env, default
|
|
/// data/datasets/workers_500k.parquet). Used by /v1/validate to
|
|
/// run FillValidator/EmailValidator with worker-existence checks.
|
|
/// Falls back to an empty InMemoryWorkerLookup if the file is
|
|
/// missing — validators still run schema/PII checks but every
|
|
/// worker-existence check fails (Consistency error), which is
|
|
/// the correct behavior when the roster isn't configured.
|
|
pub validate_workers: std::sync::Arc<dyn validator::WorkerLookup>,
|
|
/// Phase 40 early deliverable — Langfuse client. None = tracing
|
|
/// disabled (keys missing or container unreachable). Traces are
|
|
/// fire-and-forget: never block the response path.
|
|
pub langfuse: Option<langfuse_trace::LangfuseClient>,
|
|
}
|
|
|
|
#[derive(Default, Clone, Serialize)]
|
|
pub struct Usage {
|
|
pub requests: u64,
|
|
pub prompt_tokens: u64,
|
|
pub completion_tokens: u64,
|
|
pub total_tokens: u64,
|
|
#[serde(default)]
|
|
pub by_provider: std::collections::HashMap<String, ProviderUsage>,
|
|
}
|
|
|
|
#[derive(Default, Clone, Serialize)]
|
|
pub struct ProviderUsage {
|
|
pub requests: u64,
|
|
pub prompt_tokens: u64,
|
|
pub completion_tokens: u64,
|
|
pub total_tokens: u64,
|
|
}
|
|
|
|
pub fn router(state: V1State) -> Router {
|
|
Router::new()
|
|
.route("/chat", post(chat))
|
|
// Canonical OpenAI path alias — lets any client built on the
|
|
// openai SDK (pi-ai, langchain-js, etc.) treat the gateway as
|
|
// a drop-in middleware via OPENAI_BASE_URL=http://gw/v1 alone.
|
|
// Same handler as /chat; same OpenAI-compatible request shape.
|
|
.route("/chat/completions", post(chat))
|
|
.route("/respond", post(respond::respond))
|
|
.route("/usage", get(usage))
|
|
.route("/sessions", get(sessions))
|
|
.route("/context", get(truth::context))
|
|
.route("/mode", post(mode::route))
|
|
.route("/mode/list", get(mode::list))
|
|
.route("/mode/execute", post(mode::execute))
|
|
.route("/validate", post(validate::validate))
|
|
.route("/iterate", post(iterate::iterate))
|
|
.route("/health", get(health))
|
|
.with_state(state)
|
|
}
|
|
|
|
// -- Shared types (OpenAI-compatible) --
|
|
|
|
/// OpenAI-compatible message. `content` accepts either a plain string or
|
|
/// an array of content parts (the modern multimodal shape:
|
|
/// `[{type:"text", text:"..."}, {type:"image_url", ...}]`). We store as
|
|
/// `serde_json::Value` to preserve client shape on forward; downstream
|
|
/// providers can take it verbatim. `Message::text()` flattens for
|
|
/// places that need a plain string (Ollama prompt assembly, char
|
|
/// counts, the assistant's own response synthesis).
|
|
#[derive(Serialize, Deserialize, Clone, Debug)]
|
|
pub struct Message {
|
|
pub role: String,
|
|
pub content: serde_json::Value,
|
|
}
|
|
|
|
impl Message {
|
|
/// Construct a plain text message — the common shape for callers
|
|
/// that don't need multimodal content. Wraps the body in
|
|
/// `serde_json::Value::String` so downstream serializers see the
|
|
/// canonical OpenAI shape.
|
|
pub fn new_text(role: impl Into<String>, body: impl Into<String>) -> Self {
|
|
Self {
|
|
role: role.into(),
|
|
content: serde_json::Value::String(body.into()),
|
|
}
|
|
}
|
|
/// Flatten content to a plain string. Strings pass through; content-
|
|
/// part arrays concatenate the `text` fields with newlines and skip
|
|
/// non-text parts (images etc.) — Phase 38/39 callers are text-only,
|
|
/// real multimodal forwarding is queued.
|
|
pub fn text(&self) -> String {
|
|
match &self.content {
|
|
serde_json::Value::String(s) => s.clone(),
|
|
serde_json::Value::Array(parts) => {
|
|
let mut out = String::new();
|
|
for p in parts {
|
|
if let Some(t) = p.get("text").and_then(|v| v.as_str()) {
|
|
if !out.is_empty() { out.push('\n'); }
|
|
out.push_str(t);
|
|
}
|
|
}
|
|
out
|
|
}
|
|
other => other.to_string(),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Deserialize, Debug, Clone)]
|
|
pub struct ChatRequest {
|
|
pub model: String,
|
|
pub messages: Vec<Message>,
|
|
#[serde(default)]
|
|
pub temperature: Option<f64>,
|
|
#[serde(default)]
|
|
pub max_tokens: Option<u32>,
|
|
/// Accepted for shape-compat but ignored in the thin slice —
|
|
/// Phase 38 returns non-streaming even when the client asked for it.
|
|
/// Phase 39+ wires real streaming.
|
|
#[serde(default)]
|
|
pub stream: Option<bool>,
|
|
/// Non-OpenAI extension. Passes through to the provider's thinking
|
|
/// toggle. Default: **false** — hot-path discipline for thinking
|
|
/// models (qwen3.5, qwen3, gpt-oss) that otherwise burn the token
|
|
/// budget on hidden reasoning before visible output starts,
|
|
/// producing empty responses. Set true explicitly when calling an
|
|
/// overseer / reasoning-heavy path.
|
|
#[serde(default)]
|
|
pub think: Option<bool>,
|
|
/// Non-OpenAI extension. Selects the provider adapter. Accepted:
|
|
/// - None / "ollama" / "local" → local Ollama via sidecar (default)
|
|
/// - "ollama_cloud" / "cloud" → Ollama Cloud direct HTTPS
|
|
/// Phase 40 adds a routing engine that picks this automatically
|
|
/// from the model name; Phase 38/39 requires explicit selection.
|
|
#[serde(default)]
|
|
pub provider: Option<String>,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
pub struct ChatResponse {
|
|
pub id: String,
|
|
pub object: &'static str,
|
|
pub created: i64,
|
|
pub model: String,
|
|
pub choices: Vec<Choice>,
|
|
pub usage: UsageBlock,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
pub struct Choice {
|
|
pub index: u32,
|
|
pub message: Message,
|
|
pub finish_reason: String,
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize, Clone)]
|
|
pub struct UsageBlock {
|
|
pub prompt_tokens: u32,
|
|
pub completion_tokens: u32,
|
|
pub total_tokens: u32,
|
|
}
|
|
|
|
// -- Handlers --
|
|
|
|
/// Phase 39: resolve (provider, effective_model) from a ChatRequest.
|
|
///
|
|
/// Explicit `req.provider` wins. If absent, infer from a model-name
|
|
/// prefix: "openrouter/..." → openrouter (strip prefix), "cloud/..." →
|
|
/// ollama_cloud (strip prefix). Bare names default to "ollama".
|
|
///
|
|
/// The stripped model is what the upstream adapter expects:
|
|
/// OpenRouter's API wants "openai/gpt-4o-mini", not
|
|
/// "openrouter/openai/gpt-4o-mini".
|
|
fn resolve_provider(req: &ChatRequest) -> (String, String) {
|
|
if let Some(p) = req.provider.as_deref() {
|
|
return (p.to_ascii_lowercase(), req.model.clone());
|
|
}
|
|
if let Some(rest) = req.model.strip_prefix("openrouter/") {
|
|
return ("openrouter".to_string(), rest.to_string());
|
|
}
|
|
if let Some(rest) = req.model.strip_prefix("cloud/") {
|
|
return ("ollama_cloud".to_string(), rest.to_string());
|
|
}
|
|
if let Some(rest) = req.model.strip_prefix("gemini/") {
|
|
return ("gemini".to_string(), rest.to_string());
|
|
}
|
|
if let Some(rest) = req.model.strip_prefix("claude/") {
|
|
return ("claude".to_string(), rest.to_string());
|
|
}
|
|
if let Some(rest) = req.model.strip_prefix("kimi/") {
|
|
return ("kimi".to_string(), rest.to_string());
|
|
}
|
|
if let Some(rest) = req.model.strip_prefix("opencode/") {
|
|
return ("opencode".to_string(), rest.to_string());
|
|
}
|
|
// Bare `vendor/model` shape (e.g. `x-ai/grok-4.1-fast`,
|
|
// `moonshotai/kimi-k2`, `openai/gpt-oss-120b:free`) → OpenRouter.
|
|
// This makes the gateway a drop-in OpenAI-compatible middleware:
|
|
// clients using the official `openai` SDK only set OPENAI_BASE_URL
|
|
// + a model name and get correct upstream routing without needing
|
|
// our custom `provider` field. Ollama models in J's stack use
|
|
// `model:tag` form with NO slash (`qwen3.5:latest`, `kimi-k2:1t`),
|
|
// so a slash here unambiguously means "namespaced provider/model".
|
|
if req.model.contains('/') {
|
|
return ("openrouter".to_string(), req.model.clone());
|
|
}
|
|
// Vendor-bare model names (no slash, no colon) — `gpt-4o-mini`,
|
|
// `claude-3-5-sonnet-20241022`, etc. Tools like pi-ai validate
|
|
// models against an OpenAI-style catalog (no namespace prefix),
|
|
// so they send the bare name. Map to OpenRouter's namespaced form
|
|
// by inferring the vendor from the leading token. Falls through to
|
|
// ollama if no pattern matches — preserves existing behavior.
|
|
if !req.model.contains(':') && !req.model.contains('/') {
|
|
let m = req.model.as_str();
|
|
if m.starts_with("gpt-") || m.starts_with("o1-") || m.starts_with("o3-") || m.starts_with("o4-") || m == "o1" || m == "o3" || m == "o4-mini" {
|
|
return ("openrouter".to_string(), format!("openai/{}", m));
|
|
}
|
|
if m.starts_with("claude-") {
|
|
return ("openrouter".to_string(), format!("anthropic/{}", m));
|
|
}
|
|
if m.starts_with("grok-") {
|
|
return ("openrouter".to_string(), format!("x-ai/{}", m));
|
|
}
|
|
}
|
|
("ollama".to_string(), req.model.clone())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod resolve_provider_tests {
|
|
use super::*;
|
|
|
|
fn mk_req(provider: Option<&str>, model: &str) -> ChatRequest {
|
|
ChatRequest {
|
|
model: model.to_string(),
|
|
messages: vec![],
|
|
temperature: None,
|
|
max_tokens: None,
|
|
stream: None,
|
|
think: None,
|
|
provider: provider.map(|s| s.to_string()),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn explicit_provider_wins() {
|
|
let r = mk_req(Some("openrouter"), "qwen3.5:latest");
|
|
assert_eq!(resolve_provider(&r), ("openrouter".into(), "qwen3.5:latest".into()));
|
|
}
|
|
|
|
#[test]
|
|
fn bare_model_defaults_to_ollama() {
|
|
let r = mk_req(None, "qwen3.5:latest");
|
|
assert_eq!(resolve_provider(&r), ("ollama".into(), "qwen3.5:latest".into()));
|
|
}
|
|
|
|
#[test]
|
|
fn openrouter_prefix_infers_and_strips() {
|
|
let r = mk_req(None, "openrouter/openai/gpt-4o-mini");
|
|
assert_eq!(resolve_provider(&r), ("openrouter".into(), "openai/gpt-4o-mini".into()));
|
|
}
|
|
|
|
#[test]
|
|
fn cloud_prefix_infers_and_strips() {
|
|
let r = mk_req(None, "cloud/kimi-k2:1t");
|
|
assert_eq!(resolve_provider(&r), ("ollama_cloud".into(), "kimi-k2:1t".into()));
|
|
}
|
|
|
|
#[test]
|
|
fn explicit_provider_preserves_full_model_even_with_prefix() {
|
|
// If caller provides both provider and a model with a prefix,
|
|
// trust them — don't strip. The adapter will get the full model
|
|
// string as-is.
|
|
let r = mk_req(Some("openrouter"), "openrouter/openai/gpt-4o-mini");
|
|
assert_eq!(resolve_provider(&r), ("openrouter".into(), "openrouter/openai/gpt-4o-mini".into()));
|
|
}
|
|
|
|
#[test]
|
|
fn gemini_prefix_infers_and_strips() {
|
|
let r = mk_req(None, "gemini/gemini-2.0-flash");
|
|
assert_eq!(resolve_provider(&r), ("gemini".into(), "gemini-2.0-flash".into()));
|
|
}
|
|
|
|
#[test]
|
|
fn claude_prefix_infers_and_strips() {
|
|
let r = mk_req(None, "claude/claude-3-5-sonnet-latest");
|
|
assert_eq!(resolve_provider(&r), ("claude".into(), "claude-3-5-sonnet-latest".into()));
|
|
}
|
|
|
|
#[test]
|
|
fn kimi_prefix_infers_and_strips() {
|
|
let r = mk_req(None, "kimi/kimi-for-coding");
|
|
assert_eq!(resolve_provider(&r), ("kimi".into(), "kimi-for-coding".into()));
|
|
}
|
|
}
|
|
|
|
async fn chat(
|
|
State(state): State<V1State>,
|
|
Json(req): Json<ChatRequest>,
|
|
) -> Result<Json<ChatResponse>, (StatusCode, String)> {
|
|
if req.messages.is_empty() {
|
|
return Err((StatusCode::BAD_REQUEST, "messages must be non-empty".into()));
|
|
}
|
|
if req.stream.unwrap_or(false) {
|
|
tracing::warn!("/v1/chat: stream=true requested but Phase 38 returns non-streaming");
|
|
}
|
|
|
|
// Provider resolution: explicit `req.provider` wins; otherwise
|
|
// infer from a model-name prefix. Phase 39 PRD gate example:
|
|
// `model: "openrouter/openai/gpt-4o-mini"` → provider "openrouter",
|
|
// adapter gets the stripped "openai/gpt-4o-mini".
|
|
let (provider, effective_model) = resolve_provider(&req);
|
|
let start_time = chrono::Utc::now();
|
|
let start_instant = std::time::Instant::now();
|
|
|
|
// If we stripped a prefix, clone req with the effective model so
|
|
// the adapter sees what the upstream provider expects (OpenRouter
|
|
// wants "openai/gpt-4o-mini", not "openrouter/openai/gpt-4o-mini").
|
|
let req_for_adapter: std::borrow::Cow<'_, ChatRequest> =
|
|
if effective_model == req.model {
|
|
std::borrow::Cow::Borrowed(&req)
|
|
} else {
|
|
let mut cloned = req.clone();
|
|
cloned.model = effective_model.clone();
|
|
std::borrow::Cow::Owned(cloned)
|
|
};
|
|
|
|
let (resp, used_provider) = match provider.as_str() {
|
|
"ollama" | "local" | "" => {
|
|
let r = ollama::chat(&state.ai_client, &*req_for_adapter)
|
|
.await
|
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("ollama local: {e}")))?;
|
|
(r, "ollama".to_string())
|
|
}
|
|
"ollama_cloud" | "cloud" => {
|
|
let key = state.ollama_cloud_key.as_deref().ok_or((
|
|
StatusCode::SERVICE_UNAVAILABLE,
|
|
"OLLAMA_CLOUD_KEY not configured".to_string(),
|
|
))?;
|
|
let r = ollama_cloud::chat(key, &*req_for_adapter)
|
|
.await
|
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("ollama cloud: {e}")))?;
|
|
(r, "ollama_cloud".to_string())
|
|
}
|
|
"openrouter" | "openrouter_free" => {
|
|
// Free-tier rescue rung. Added 2026-04-24 after iter 5
|
|
// repeated Ollama Cloud 502s on kimi-k2:1t — OpenRouter
|
|
// gives a different provider backbone as fallback.
|
|
let key = state.openrouter_key.as_deref().ok_or((
|
|
StatusCode::SERVICE_UNAVAILABLE,
|
|
"OPENROUTER_API_KEY not configured".to_string(),
|
|
))?;
|
|
let r = openrouter::chat(key, &*req_for_adapter)
|
|
.await
|
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("openrouter: {e}")))?;
|
|
(r, "openrouter".to_string())
|
|
}
|
|
"gemini" => {
|
|
// Phase 40 provider adapter. Google Generative Language
|
|
// API via query-string key auth (not bearer).
|
|
let key = state.gemini_key.as_deref().ok_or((
|
|
StatusCode::SERVICE_UNAVAILABLE,
|
|
"GEMINI_API_KEY not configured".to_string(),
|
|
))?;
|
|
let r = gemini::chat(key, &*req_for_adapter)
|
|
.await
|
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("gemini: {e}")))?;
|
|
(r, "gemini".to_string())
|
|
}
|
|
"claude" | "anthropic" => {
|
|
// Phase 40 provider adapter. Anthropic Messages API via
|
|
// x-api-key header + anthropic-version:2023-06-01.
|
|
let key = state.claude_key.as_deref().ok_or((
|
|
StatusCode::SERVICE_UNAVAILABLE,
|
|
"ANTHROPIC_API_KEY not configured".to_string(),
|
|
))?;
|
|
let r = claude::chat(key, &*req_for_adapter)
|
|
.await
|
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("claude: {e}")))?;
|
|
(r, "claude".to_string())
|
|
}
|
|
"kimi" => {
|
|
// Direct Kimi For Coding provider — bypasses Ollama Cloud's
|
|
// upstream-broken kimi-k2:1t and OpenRouter's rate-limited
|
|
// moonshotai/kimi-k2.6. Uses sk-kimi-* keys from the Kimi
|
|
// membership console.
|
|
let key = state.kimi_key.as_deref().ok_or((
|
|
StatusCode::SERVICE_UNAVAILABLE,
|
|
"KIMI_API_KEY not configured".to_string(),
|
|
))?;
|
|
let r = kimi::chat(key, &*req_for_adapter)
|
|
.await
|
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("kimi: {e}")))?;
|
|
(r, "kimi".to_string())
|
|
}
|
|
"opencode" => {
|
|
// OpenCode GO multi-vendor gateway — Claude Opus 4.7,
|
|
// GPT-5.5-pro, Gemini 3.1-pro, Kimi K2.6, DeepSeek, GLM,
|
|
// Qwen, free-tier. OpenAI-compat at opencode.ai/zen/go/v1.
|
|
let key = state.opencode_key.as_deref().ok_or((
|
|
StatusCode::SERVICE_UNAVAILABLE,
|
|
"OPENCODE_API_KEY not configured".to_string(),
|
|
))?;
|
|
let r = opencode::chat(key, &*req_for_adapter)
|
|
.await
|
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("opencode: {e}")))?;
|
|
(r, "opencode".to_string())
|
|
}
|
|
other => {
|
|
return Err((
|
|
StatusCode::BAD_REQUEST,
|
|
format!("unknown provider '{other}' — supported: ollama, ollama_cloud, openrouter, gemini, claude, kimi, opencode"),
|
|
));
|
|
}
|
|
};
|
|
|
|
let end_time = chrono::Utc::now();
|
|
let latency_ms = start_instant.elapsed().as_millis() as u64;
|
|
|
|
// Phase 40 — emit Langfuse trace. Fire-and-forget: the clone is
|
|
// cheap (Arc inside), the tokio::spawn never blocks us, a dead
|
|
// Langfuse just logs a warn. Client-visible response latency is
|
|
// untouched.
|
|
if let Some(lf) = &state.langfuse {
|
|
let output = resp.choices.first()
|
|
.map(|c| c.message.text())
|
|
.unwrap_or_default();
|
|
lf.emit_chat(langfuse_trace::ChatTrace {
|
|
provider: used_provider.clone(),
|
|
model: resp.model.clone(),
|
|
input: req.messages.clone(),
|
|
output,
|
|
prompt_tokens: resp.usage.prompt_tokens,
|
|
completion_tokens: resp.usage.completion_tokens,
|
|
temperature: req.temperature,
|
|
max_tokens: req.max_tokens,
|
|
think: req.think,
|
|
start_time: start_time.to_rfc3339(),
|
|
end_time: end_time.to_rfc3339(),
|
|
latency_ms,
|
|
});
|
|
}
|
|
|
|
// Phase 40 part 2 — fire-and-forget /event to observer at :3800.
|
|
// Same ring-buffer that scrum + scenario events land in, so any
|
|
// tool-routed-through-our-gateway (Pi, Archon, openai SDK clients)
|
|
// shows up alongside scrum_master events for KB consolidation +
|
|
// pathway-memory + bug-fingerprint compounding. Best-effort:
|
|
// observer being down doesn't block the chat response.
|
|
{
|
|
let provider = used_provider.clone();
|
|
let model = resp.model.clone();
|
|
let prompt_tokens = resp.usage.prompt_tokens;
|
|
let completion_tokens = resp.usage.completion_tokens;
|
|
let success = true;
|
|
tokio::spawn(async move {
|
|
let body = serde_json::json!({
|
|
"endpoint": "/v1/chat",
|
|
"source": "v1.chat",
|
|
"event_kind": "chat_completion",
|
|
"input_summary": format!(
|
|
"{} {} prompt={}t",
|
|
provider, model, prompt_tokens
|
|
),
|
|
"output_summary": format!(
|
|
"completion={}t {}ms",
|
|
completion_tokens, latency_ms
|
|
),
|
|
"success": success,
|
|
"duration_ms": latency_ms,
|
|
});
|
|
let client = reqwest::Client::builder()
|
|
.timeout(std::time::Duration::from_secs(2))
|
|
.build()
|
|
.unwrap_or_else(|_| reqwest::Client::new());
|
|
let _ = client
|
|
.post("http://localhost:3800/event")
|
|
.json(&body)
|
|
.send()
|
|
.await;
|
|
});
|
|
}
|
|
|
|
// Phase 40: per-provider usage tracking
|
|
{
|
|
let mut u = state.usage.write().await;
|
|
u.requests += 1;
|
|
u.prompt_tokens += resp.usage.prompt_tokens as u64;
|
|
u.completion_tokens += resp.usage.completion_tokens as u64;
|
|
u.total_tokens += resp.usage.total_tokens as u64;
|
|
|
|
let provider_usage = u.by_provider.entry(used_provider).or_default();
|
|
provider_usage.requests += 1;
|
|
provider_usage.prompt_tokens += resp.usage.prompt_tokens as u64;
|
|
provider_usage.completion_tokens += resp.usage.completion_tokens as u64;
|
|
provider_usage.total_tokens += resp.usage.total_tokens as u64;
|
|
}
|
|
|
|
Ok(Json(resp))
|
|
}
|
|
|
|
async fn usage(State(state): State<V1State>) -> impl IntoResponse {
|
|
let snapshot = state.usage.read().await.clone();
|
|
Json(snapshot)
|
|
}
|
|
|
|
/// Production operational health endpoint.
|
|
///
|
|
/// `/v1/health` reports per-subsystem status as a JSON object so an
|
|
/// operator (or the lakehouse-auditor service, or a load balancer)
|
|
/// can verify the gateway is fully booted, has its provider keys
|
|
/// loaded, the worker roster is hot, and Langfuse is reachable.
|
|
/// Returns 200 always — fields are observed-state, not pass/fail
|
|
/// gates. A monitoring tool should evaluate the booleans + counts
|
|
/// against its own thresholds.
|
|
async fn health(State(state): State<V1State>) -> impl IntoResponse {
|
|
// Honest worker count via WorkerLookup::len. Production switchover
|
|
// verification: after swapping workers_500k.parquet → real Chicago
|
|
// data and restarting, this number should match the row count of
|
|
// the new file. 0 means the file was missing / unreadable / had a
|
|
// schema mismatch and the gateway booted with the empty fallback.
|
|
let workers_count = state.validate_workers.len();
|
|
let providers_configured = serde_json::json!({
|
|
"ollama_cloud": state.ollama_cloud_key.is_some(),
|
|
"openrouter": state.openrouter_key.is_some(),
|
|
"kimi": state.kimi_key.is_some(),
|
|
"opencode": state.opencode_key.is_some(),
|
|
"gemini": state.gemini_key.is_some(),
|
|
"claude": state.claude_key.is_some(),
|
|
});
|
|
let langfuse_configured = state.langfuse.is_some();
|
|
let usage_snapshot = state.usage.read().await.clone();
|
|
Json(serde_json::json!({
|
|
"status": "ok",
|
|
"workers_count": workers_count,
|
|
"workers_loaded": workers_count > 0,
|
|
"providers_configured": providers_configured,
|
|
"langfuse_configured": langfuse_configured,
|
|
"usage_total_requests": usage_snapshot.requests,
|
|
"usage_by_provider": usage_snapshot.by_provider.keys().collect::<Vec<_>>(),
|
|
}))
|
|
}
|
|
|
|
// Phase 38 is stateless — no session persistence yet. Return an empty
|
|
// list in OpenAI-ish shape so clients that probe this endpoint don't
|
|
// 404. Real session state lands in Phase 41 with the profile-system
|
|
// expansion.
|
|
async fn sessions() -> impl IntoResponse {
|
|
Json(serde_json::json!({
|
|
"data": [],
|
|
"object": "list",
|
|
"note": "Phase 38: stateless. Session state lands in Phase 41.",
|
|
}))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn chat_request_parses_openai_shape() {
|
|
let raw = r#"{
|
|
"model": "qwen3.5:latest",
|
|
"messages": [
|
|
{"role": "system", "content": "You are helpful."},
|
|
{"role": "user", "content": "Hi"}
|
|
],
|
|
"temperature": 0.2,
|
|
"max_tokens": 100
|
|
}"#;
|
|
let r: ChatRequest = serde_json::from_str(raw).unwrap();
|
|
assert_eq!(r.model, "qwen3.5:latest");
|
|
assert_eq!(r.messages.len(), 2);
|
|
assert_eq!(r.messages[0].role, "system");
|
|
assert_eq!(r.messages[1].text(), "Hi");
|
|
assert_eq!(r.temperature, Some(0.2));
|
|
assert_eq!(r.max_tokens, Some(100));
|
|
}
|
|
|
|
#[test]
|
|
fn chat_request_accepts_minimal() {
|
|
let raw = r#"{
|
|
"model": "any",
|
|
"messages": [{"role": "user", "content": "hi"}]
|
|
}"#;
|
|
let r: ChatRequest = serde_json::from_str(raw).unwrap();
|
|
assert_eq!(r.temperature, None);
|
|
assert_eq!(r.max_tokens, None);
|
|
assert_eq!(r.stream, None);
|
|
}
|
|
|
|
#[test]
|
|
fn usage_counter_default_is_zero() {
|
|
let u = Usage::default();
|
|
assert_eq!(u.requests, 0);
|
|
assert_eq!(u.total_tokens, 0);
|
|
}
|
|
}
|