Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cbc24a4b19 | ||
|
|
dd74a1ef74 | ||
|
|
9d5107fb4a | ||
|
|
8bdceda883 |
@@ -77,8 +77,10 @@ const KIMI_MODEL = process.env.LH_AUDITOR_KIMI_MODEL ?? "moonshotai/kimi-k2.6";
|
|||||||
// Override the alternation entirely with LH_AUDITOR_KIMI_MODEL
|
// Override the alternation entirely with LH_AUDITOR_KIMI_MODEL
|
||||||
// (forces one model regardless of audit count); set
|
// (forces one model regardless of audit count); set
|
||||||
// LH_AUDITOR_KIMI_ALT_MODEL to the alternate.
|
// LH_AUDITOR_KIMI_ALT_MODEL to the alternate.
|
||||||
const ALT_MODEL = process.env.LH_AUDITOR_KIMI_ALT_MODEL ?? "claude-haiku-4-5";
|
// 2026-09-02: opencode lane 401s (CreditsError, balance drained) — alt
|
||||||
const ALT_PROVIDER = process.env.LH_AUDITOR_KIMI_ALT_PROVIDER ?? "opencode";
|
// lane joins the OpenRouter sweep. Same Anthropic lineage, probed live.
|
||||||
|
const ALT_MODEL = process.env.LH_AUDITOR_KIMI_ALT_MODEL ?? "anthropic/claude-haiku-4.5";
|
||||||
|
const ALT_PROVIDER = process.env.LH_AUDITOR_KIMI_ALT_PROVIDER ?? "openrouter";
|
||||||
const FORCE_DEFAULT = process.env.LH_AUDITOR_KIMI_MODEL !== undefined && process.env.LH_AUDITOR_KIMI_MODEL !== "";
|
const FORCE_DEFAULT = process.env.LH_AUDITOR_KIMI_MODEL !== undefined && process.env.LH_AUDITOR_KIMI_MODEL !== "";
|
||||||
|
|
||||||
function selectModel(diffLen: number, auditIndex: number = 0): { provider: string; model: string; promoted: boolean } {
|
function selectModel(diffLen: number, auditIndex: number = 0): { provider: string; model: string; promoted: boolean } {
|
||||||
@@ -112,7 +114,7 @@ const MAX_TOKENS_OVERRIDE = Number(process.env.LH_AUDITOR_KIMI_MAX_TOKENS) || 0;
|
|||||||
function maxTokensFor(model: string): number {
|
function maxTokensFor(model: string): number {
|
||||||
if (MAX_TOKENS_OVERRIDE > 0) return MAX_TOKENS_OVERRIDE;
|
if (MAX_TOKENS_OVERRIDE > 0) return MAX_TOKENS_OVERRIDE;
|
||||||
if (model.startsWith("claude-opus")) return 32_000;
|
if (model.startsWith("claude-opus")) return 32_000;
|
||||||
if (model.startsWith("claude-haiku") || model.startsWith("claude-sonnet")) return 8_192;
|
if (model.startsWith("claude-haiku") || model.startsWith("claude-sonnet") || model.includes("/claude-haiku") || model.includes("/claude-sonnet")) return 8_192;
|
||||||
if (model.startsWith("kimi-") || model.includes("/kimi-")) return 128_000;
|
if (model.startsWith("kimi-") || model.includes("/kimi-")) return 128_000;
|
||||||
if (model.startsWith("gpt-5") || model.startsWith("o1") || model.startsWith("o3") || model.startsWith("o4")) return 32_000;
|
if (model.startsWith("gpt-5") || model.startsWith("o1") || model.startsWith("o3") || model.startsWith("o4")) return 32_000;
|
||||||
return 16_000;
|
return 16_000;
|
||||||
|
|||||||
@@ -453,6 +453,8 @@ impl ExecutionLoop {
|
|||||||
stream: Some(false),
|
stream: Some(false),
|
||||||
think: Some(think),
|
think: Some(think),
|
||||||
provider: Some("ollama_cloud".into()),
|
provider: Some("ollama_cloud".into()),
|
||||||
|
tools: None,
|
||||||
|
tool_choice: None,
|
||||||
};
|
};
|
||||||
let resp = crate::v1::ollama_cloud::chat(key, &req).await
|
let resp = crate::v1::ollama_cloud::chat(key, &req).await
|
||||||
.map_err(|e| format!("ollama_cloud: {e}"))?;
|
.map_err(|e| format!("ollama_cloud: {e}"))?;
|
||||||
@@ -693,6 +695,8 @@ impl ExecutionLoop {
|
|||||||
stream: Some(false),
|
stream: Some(false),
|
||||||
think: Some(false),
|
think: Some(false),
|
||||||
provider: Some("ollama".into()),
|
provider: Some("ollama".into()),
|
||||||
|
tools: None,
|
||||||
|
tool_choice: None,
|
||||||
};
|
};
|
||||||
let resp = crate::v1::ollama::chat(&self.state.ai_client, &chat_req).await
|
let resp = crate::v1::ollama::chat(&self.state.ai_client, &chat_req).await
|
||||||
.map_err(|e| format!("ollama overseer: {e}"))?;
|
.map_err(|e| format!("ollama overseer: {e}"))?;
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ pub async fn chat(
|
|||||||
model,
|
model,
|
||||||
choices: vec![Choice {
|
choices: vec![Choice {
|
||||||
index: 0,
|
index: 0,
|
||||||
message: Message { role: "assistant".into(), content: serde_json::Value::String(text) },
|
message: Message { role: "assistant".into(), content: serde_json::Value::String(text), ..Default::default() },
|
||||||
finish_reason: choice.finish_reason.unwrap_or_else(|| "stop".into()),
|
finish_reason: choice.finish_reason.unwrap_or_else(|| "stop".into()),
|
||||||
}],
|
}],
|
||||||
usage: UsageBlock {
|
usage: UsageBlock {
|
||||||
|
|||||||
+122
-10
@@ -148,10 +148,22 @@ pub fn router(state: V1State) -> Router {
|
|||||||
/// providers can take it verbatim. `Message::text()` flattens for
|
/// providers can take it verbatim. `Message::text()` flattens for
|
||||||
/// places that need a plain string (Ollama prompt assembly, char
|
/// places that need a plain string (Ollama prompt assembly, char
|
||||||
/// counts, the assistant's own response synthesis).
|
/// counts, the assistant's own response synthesis).
|
||||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
|
||||||
pub struct Message {
|
pub struct Message {
|
||||||
pub role: String,
|
pub role: String,
|
||||||
|
/// `Value::Null` is legal: an assistant turn that only makes tool calls.
|
||||||
|
#[serde(default)]
|
||||||
pub content: serde_json::Value,
|
pub content: serde_json::Value,
|
||||||
|
/// OpenAI tool-calling fields (2026-09-19). Passed through verbatim on the OpenRouter lane;
|
||||||
|
/// omitted from JSON when unset, so plain-text traffic is byte-identical to before. Before
|
||||||
|
/// this, serde silently DROPPED them — tool-using agents routed through :3100 never saw their
|
||||||
|
/// tools and fabricated tool output (autobun KB finding infra.lakehouse_gateway.tools).
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub tool_calls: Option<serde_json::Value>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub tool_call_id: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub name: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Message {
|
impl Message {
|
||||||
@@ -163,6 +175,7 @@ impl Message {
|
|||||||
Self {
|
Self {
|
||||||
role: role.into(),
|
role: role.into(),
|
||||||
content: serde_json::Value::String(body.into()),
|
content: serde_json::Value::String(body.into()),
|
||||||
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/// Flatten content to a plain string. Strings pass through; content-
|
/// Flatten content to a plain string. Strings pass through; content-
|
||||||
@@ -195,9 +208,8 @@ pub struct ChatRequest {
|
|||||||
pub temperature: Option<f64>,
|
pub temperature: Option<f64>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub max_tokens: Option<u32>,
|
pub max_tokens: Option<u32>,
|
||||||
/// Accepted for shape-compat but ignored in the thin slice —
|
/// Real SSE passthrough on the openrouter lane (2026-09-20); other lanes reject
|
||||||
/// Phase 38 returns non-streaming even when the client asked for it.
|
/// stream=true with a 400 rather than silently returning a non-streamed body.
|
||||||
/// Phase 39+ wires real streaming.
|
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub stream: Option<bool>,
|
pub stream: Option<bool>,
|
||||||
/// Non-OpenAI extension. Passes through to the provider's thinking
|
/// Non-OpenAI extension. Passes through to the provider's thinking
|
||||||
@@ -215,6 +227,21 @@ pub struct ChatRequest {
|
|||||||
/// from the model name; Phase 38/39 requires explicit selection.
|
/// from the model name; Phase 38/39 requires explicit selection.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub provider: Option<String>,
|
pub provider: Option<String>,
|
||||||
|
/// OpenAI tool definitions + choice (2026-09-19). Forwarded on the OpenRouter lane only;
|
||||||
|
/// any other provider REJECTS them with 400 instead of silently stripping them.
|
||||||
|
#[serde(default)]
|
||||||
|
pub tools: Option<serde_json::Value>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub tool_choice: Option<serde_json::Value>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChatRequest {
|
||||||
|
/// True if the request uses tool calling anywhere (definitions or tool-call history).
|
||||||
|
pub fn uses_tools(&self) -> bool {
|
||||||
|
self.tools.is_some()
|
||||||
|
|| self.tool_choice.is_some()
|
||||||
|
|| self.messages.iter().any(|m| m.role == "tool" || m.tool_calls.is_some() || m.tool_call_id.is_some())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
@@ -319,9 +346,47 @@ mod resolve_provider_tests {
|
|||||||
stream: None,
|
stream: None,
|
||||||
think: None,
|
think: None,
|
||||||
provider: provider.map(|s| s.to_string()),
|
provider: provider.map(|s| s.to_string()),
|
||||||
|
tools: None,
|
||||||
|
tool_choice: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Streaming is a per-lane capability, and the routing decision must be visible in a test:
|
||||||
|
/// before 2026-09-20 the gateway accepted stream=true on every lane and returned a single
|
||||||
|
/// JSON body, so a client waiting for SSE frames hung until its own timeout.
|
||||||
|
#[test]
|
||||||
|
fn streaming_is_openrouter_only() {
|
||||||
|
let mut r = mk_req(None, "openrouter/z-ai/glm-5.3-flash");
|
||||||
|
r.stream = Some(true);
|
||||||
|
assert_eq!(resolve_provider(&r).0, "openrouter", "openrouter lane streams");
|
||||||
|
|
||||||
|
let mut local = mk_req(None, "qwen3.5:latest");
|
||||||
|
local.stream = Some(true);
|
||||||
|
assert_eq!(
|
||||||
|
resolve_provider(&local).0,
|
||||||
|
"ollama",
|
||||||
|
"a local model resolves to ollama, where the handler rejects stream=true with a 400 \
|
||||||
|
instead of silently returning a non-streamed body"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uses_tools_detects_definitions_and_history() {
|
||||||
|
let mut r = mk_req(None, "openrouter/z-ai/glm-5.3-flash");
|
||||||
|
assert!(!r.uses_tools());
|
||||||
|
r.tools = Some(serde_json::json!([]));
|
||||||
|
assert!(r.uses_tools());
|
||||||
|
let mut r2 = mk_req(None, "qwen3.5:latest");
|
||||||
|
r2.messages.push(Message { role: "tool".into(), content: "ok".into(), tool_call_id: Some("c1".into()), ..Default::default() });
|
||||||
|
assert!(r2.uses_tools());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn plain_message_serialization_unchanged() {
|
||||||
|
let m = Message::new_text("user", "hi");
|
||||||
|
assert_eq!(serde_json::to_string(&m).unwrap(), r#"{"role":"user","content":"hi"}"#);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn explicit_provider_wins() {
|
fn explicit_provider_wins() {
|
||||||
let r = mk_req(Some("openrouter"), "qwen3.5:latest");
|
let r = mk_req(Some("openrouter"), "qwen3.5:latest");
|
||||||
@@ -378,19 +443,27 @@ async fn chat(
|
|||||||
State(state): State<V1State>,
|
State(state): State<V1State>,
|
||||||
headers: axum::http::HeaderMap,
|
headers: axum::http::HeaderMap,
|
||||||
Json(req): Json<ChatRequest>,
|
Json(req): Json<ChatRequest>,
|
||||||
) -> Result<Json<ChatResponse>, (StatusCode, String)> {
|
) -> Result<axum::response::Response, (StatusCode, String)> {
|
||||||
|
use axum::response::IntoResponse as _;
|
||||||
if req.messages.is_empty() {
|
if req.messages.is_empty() {
|
||||||
return Err((StatusCode::BAD_REQUEST, "messages must be non-empty".into()));
|
return Err((StatusCode::BAD_REQUEST, "messages must be non-empty".into()));
|
||||||
}
|
}
|
||||||
if req.stream.unwrap_or(false) {
|
|
||||||
tracing::warn!("/v1/chat: stream=true requested but Phase 38 returns non-streaming");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Provider resolution: explicit `req.provider` wins; otherwise
|
// Provider resolution: explicit `req.provider` wins; otherwise
|
||||||
// infer from a model-name prefix. Phase 39 PRD gate example:
|
// infer from a model-name prefix. Phase 39 PRD gate example:
|
||||||
// `model: "openrouter/openai/gpt-4o-mini"` → provider "openrouter",
|
// `model: "openrouter/openai/gpt-4o-mini"` → provider "openrouter",
|
||||||
// adapter gets the stripped "openai/gpt-4o-mini".
|
// adapter gets the stripped "openai/gpt-4o-mini".
|
||||||
let (provider, effective_model) = resolve_provider(&req);
|
let (provider, effective_model) = resolve_provider(&req);
|
||||||
|
// Tool calling is only wired on the OpenRouter lane. Refuse loudly elsewhere — silently
|
||||||
|
// dropping tools made agents fabricate tool results (2026-09-19).
|
||||||
|
if req.uses_tools() && !matches!(provider.as_str(), "openrouter" | "openrouter_free") {
|
||||||
|
return Err((
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
format!(
|
||||||
|
"tools/tool_calls are only supported on the openrouter lane via this gateway \
|
||||||
|
(resolved provider: {provider}); use an OpenRouter model id (e.g. openrouter/<vendor>/<model>)"
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
let start_time = chrono::Utc::now();
|
let start_time = chrono::Utc::now();
|
||||||
let start_instant = std::time::Instant::now();
|
let start_instant = std::time::Instant::now();
|
||||||
|
|
||||||
@@ -406,6 +479,45 @@ async fn chat(
|
|||||||
std::borrow::Cow::Owned(cloned)
|
std::borrow::Cow::Owned(cloned)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// STREAMING (2026-09-20). Phase 38 accepted `stream: true` and returned a single JSON body;
|
||||||
|
// a client waiting for SSE frames hangs and then reports "no response". Forward the upstream
|
||||||
|
// SSE verbatim on the lane that supports it, and say so plainly on the lanes that do not
|
||||||
|
// instead of pretending. Streamed calls skip usage metering + Langfuse (see chat_stream).
|
||||||
|
if req.stream.unwrap_or(false) {
|
||||||
|
if !matches!(provider.as_str(), "openrouter" | "openrouter_free") {
|
||||||
|
return Err((
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
format!(
|
||||||
|
"stream=true is only supported on the openrouter lane via this gateway \
|
||||||
|
(resolved provider: {provider}); resend with stream=false or use an \
|
||||||
|
OpenRouter model id"
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let key = state.openrouter_key.as_deref().ok_or((
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"OPENROUTER_API_KEY not configured".to_string(),
|
||||||
|
))?;
|
||||||
|
let upstream = openrouter::chat_stream(key, &*req_for_adapter)
|
||||||
|
.await
|
||||||
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("openrouter stream: {e}")))?;
|
||||||
|
tracing::info!(
|
||||||
|
target: "v1.chat",
|
||||||
|
provider = %provider,
|
||||||
|
model = %effective_model,
|
||||||
|
"streaming passthrough — usage metering and Langfuse are skipped for this call",
|
||||||
|
);
|
||||||
|
let body = axum::body::Body::from_stream(upstream.bytes_stream());
|
||||||
|
return Ok((
|
||||||
|
[
|
||||||
|
(axum::http::header::CONTENT_TYPE, "text/event-stream"),
|
||||||
|
(axum::http::header::CACHE_CONTROL, "no-cache"),
|
||||||
|
],
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
.into_response());
|
||||||
|
}
|
||||||
|
|
||||||
let (resp, used_provider) = match provider.as_str() {
|
let (resp, used_provider) = match provider.as_str() {
|
||||||
"ollama" | "local" | "" => {
|
"ollama" | "local" | "" => {
|
||||||
let r = ollama::chat(&state.ai_client, &*req_for_adapter)
|
let r = ollama::chat(&state.ai_client, &*req_for_adapter)
|
||||||
@@ -589,7 +701,7 @@ async fn chat(
|
|||||||
provider_usage.total_tokens += resp.usage.total_tokens as u64;
|
provider_usage.total_tokens += resp.usage.total_tokens as u64;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Json(resp))
|
Ok(Json(resp).into_response())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn usage(State(state): State<V1State>) -> impl IntoResponse {
|
async fn usage(State(state): State<V1State>) -> impl IntoResponse {
|
||||||
|
|||||||
@@ -113,10 +113,10 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn flatten_separates_system_from_turns() {
|
fn flatten_separates_system_from_turns() {
|
||||||
let msgs = vec![
|
let msgs = vec![
|
||||||
Message { role: "system".into(), content: "Rules here.".into() },
|
Message { role: "system".into(), content: "Rules here.".into(), ..Default::default() },
|
||||||
Message { role: "user".into(), content: "Q1".into() },
|
Message { role: "user".into(), content: "Q1".into(), ..Default::default() },
|
||||||
Message { role: "assistant".into(), content: "A1".into() },
|
Message { role: "assistant".into(), content: "A1".into(), ..Default::default() },
|
||||||
Message { role: "user".into(), content: "Q2".into() },
|
Message { role: "user".into(), content: "Q2".into(), ..Default::default() },
|
||||||
];
|
];
|
||||||
let (system, prompt) = flatten_messages(&msgs);
|
let (system, prompt) = flatten_messages(&msgs);
|
||||||
assert_eq!(system, "Rules here.");
|
assert_eq!(system, "Rules here.");
|
||||||
@@ -129,9 +129,9 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn flatten_concatenates_multiple_system_messages() {
|
fn flatten_concatenates_multiple_system_messages() {
|
||||||
let msgs = vec![
|
let msgs = vec![
|
||||||
Message { role: "system".into(), content: "First.".into() },
|
Message { role: "system".into(), content: "First.".into(), ..Default::default() },
|
||||||
Message { role: "system".into(), content: "Second.".into() },
|
Message { role: "system".into(), content: "Second.".into(), ..Default::default() },
|
||||||
Message { role: "user".into(), content: "Hi".into() },
|
Message { role: "user".into(), content: "Hi".into(), ..Default::default() },
|
||||||
];
|
];
|
||||||
let (system, _) = flatten_messages(&msgs);
|
let (system, _) = flatten_messages(&msgs);
|
||||||
assert_eq!(system, "First.\nSecond.");
|
assert_eq!(system, "First.\nSecond.");
|
||||||
@@ -139,7 +139,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn flatten_with_no_system_returns_empty_system() {
|
fn flatten_with_no_system_returns_empty_system() {
|
||||||
let msgs = vec![Message { role: "user".into(), content: "hi".into() }];
|
let msgs = vec![Message { role: "user".into(), content: "hi".into(), ..Default::default() }];
|
||||||
let (system, prompt) = flatten_messages(&msgs);
|
let (system, prompt) = flatten_messages(&msgs);
|
||||||
assert!(system.is_empty());
|
assert!(system.is_empty());
|
||||||
assert!(prompt.contains("user: hi"));
|
assert!(prompt.contains("user: hi"));
|
||||||
@@ -147,11 +147,11 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn estimate_tokens_chars_div_4_ceiling() {
|
fn estimate_tokens_chars_div_4_ceiling() {
|
||||||
let msgs = vec![Message { role: "user".into(), content: "abcdefgh".into() }];
|
let msgs = vec![Message { role: "user".into(), content: "abcdefgh".into(), ..Default::default() }];
|
||||||
// 8 chars / 4 = 2, with ceiling → 2
|
// 8 chars / 4 = 2, with ceiling → 2
|
||||||
assert_eq!(estimate_prompt_tokens(&msgs), 2);
|
assert_eq!(estimate_prompt_tokens(&msgs), 2);
|
||||||
|
|
||||||
let msgs2 = vec![Message { role: "user".into(), content: "abcdefghi".into() }];
|
let msgs2 = vec![Message { role: "user".into(), content: "abcdefghi".into(), ..Default::default() }];
|
||||||
// 9 chars → (9+3)/4 = 3 (ceiling)
|
// 9 chars → (9+3)/4 = 3 (ceiling)
|
||||||
assert_eq!(estimate_prompt_tokens(&msgs2), 3);
|
assert_eq!(estimate_prompt_tokens(&msgs2), 3);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -145,7 +145,7 @@ pub async fn chat(
|
|||||||
model,
|
model,
|
||||||
choices: vec![Choice {
|
choices: vec![Choice {
|
||||||
index: 0,
|
index: 0,
|
||||||
message: Message { role: "assistant".into(), content: serde_json::Value::String(text) },
|
message: Message { role: "assistant".into(), content: serde_json::Value::String(text), ..Default::default() },
|
||||||
finish_reason: choice.finish_reason.unwrap_or_else(|| "stop".into()),
|
finish_reason: choice.finish_reason.unwrap_or_else(|| "stop".into()),
|
||||||
}],
|
}],
|
||||||
usage: UsageBlock {
|
usage: UsageBlock {
|
||||||
|
|||||||
@@ -20,7 +20,19 @@ use serde::{Deserialize, Serialize};
|
|||||||
use super::{ChatRequest, ChatResponse, Choice, Message, UsageBlock};
|
use super::{ChatRequest, ChatResponse, Choice, Message, UsageBlock};
|
||||||
|
|
||||||
const OR_BASE_URL: &str = "https://openrouter.ai/api/v1";
|
const OR_BASE_URL: &str = "https://openrouter.ai/api/v1";
|
||||||
const OR_TIMEOUT_SECS: u64 = 180;
|
// Default request ceiling. 180s starves long-context calls (kimi_architect's
|
||||||
|
// ~180K-char second pass died at exactly this wall on 2026-09-02, surfacing
|
||||||
|
// as a misleading "error decoding response body") — override per host with
|
||||||
|
// OR_TIMEOUT_SECS. The auditor budgets up to 900s client-side.
|
||||||
|
const OR_TIMEOUT_SECS_DEFAULT: u64 = 180;
|
||||||
|
|
||||||
|
fn or_timeout_secs() -> u64 {
|
||||||
|
std::env::var("OR_TIMEOUT_SECS")
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.trim().parse::<u64>().ok())
|
||||||
|
.filter(|&s| s > 0)
|
||||||
|
.unwrap_or(OR_TIMEOUT_SECS_DEFAULT)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn resolve_openrouter_key() -> Option<String> {
|
pub fn resolve_openrouter_key() -> Option<String> {
|
||||||
if let Ok(k) = std::env::var("OPENROUTER_API_KEY") {
|
if let Ok(k) = std::env::var("OPENROUTER_API_KEY") {
|
||||||
@@ -49,6 +61,52 @@ pub fn resolve_openrouter_key() -> Option<String> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Streaming passthrough (2026-09-20). The gateway used to accept `stream: true` and return a
|
||||||
|
/// single non-streamed JSON body — a client waiting for SSE frames hangs until its own timeout
|
||||||
|
/// and reports "no response" (this is what broke openclaw's gateway lane). We now forward the
|
||||||
|
/// upstream SSE byte-for-byte.
|
||||||
|
///
|
||||||
|
/// The caller streams the body straight to its client, so nothing here parses the chunks:
|
||||||
|
/// usage metering and Langfuse tracing are SKIPPED for streamed calls, and the handler logs
|
||||||
|
/// that. Metering streamed responses needs the final usage chunk and belongs in its own change.
|
||||||
|
pub async fn chat_stream(key: &str, req: &ChatRequest) -> Result<reqwest::Response, String> {
|
||||||
|
let model = req.model.strip_prefix("openrouter/").unwrap_or(&req.model).to_string();
|
||||||
|
let body = ORChatBody {
|
||||||
|
model,
|
||||||
|
messages: req.messages.iter().map(|m| ORMessage {
|
||||||
|
role: m.role.clone(),
|
||||||
|
content: m.content.clone(),
|
||||||
|
tool_calls: m.tool_calls.clone(),
|
||||||
|
tool_call_id: m.tool_call_id.clone(),
|
||||||
|
name: m.name.clone(),
|
||||||
|
}).collect(),
|
||||||
|
tools: req.tools.clone(),
|
||||||
|
tool_choice: req.tool_choice.clone(),
|
||||||
|
max_tokens: Some(req.max_tokens.unwrap_or(800)),
|
||||||
|
temperature: req.temperature.unwrap_or(0.3),
|
||||||
|
stream: true,
|
||||||
|
};
|
||||||
|
let client = reqwest::Client::builder()
|
||||||
|
.timeout(Duration::from_secs(or_timeout_secs()))
|
||||||
|
.build()
|
||||||
|
.map_err(|e| format!("build client: {e}"))?;
|
||||||
|
let resp = client
|
||||||
|
.post(format!("{}/chat/completions", OR_BASE_URL))
|
||||||
|
.bearer_auth(key)
|
||||||
|
.header("HTTP-Referer", "https://vcp.devop.live")
|
||||||
|
.header("X-Title", "Lakehouse Scrum")
|
||||||
|
.json(&serde_json::to_value(&body).map_err(|e| format!("serialize body: {e}"))?)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("openrouter.ai unreachable: {e}"))?;
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
let code = resp.status();
|
||||||
|
let text = resp.text().await.unwrap_or_else(|_| "?".into());
|
||||||
|
return Err(format!("openrouter {code}: {}", text.chars().take(300).collect::<String>()));
|
||||||
|
}
|
||||||
|
Ok(resp)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn chat(
|
pub async fn chat(
|
||||||
key: &str,
|
key: &str,
|
||||||
req: &ChatRequest,
|
req: &ChatRequest,
|
||||||
@@ -65,14 +123,19 @@ pub async fn chat(
|
|||||||
messages: req.messages.iter().map(|m| ORMessage {
|
messages: req.messages.iter().map(|m| ORMessage {
|
||||||
role: m.role.clone(),
|
role: m.role.clone(),
|
||||||
content: m.content.clone(),
|
content: m.content.clone(),
|
||||||
|
tool_calls: m.tool_calls.clone(),
|
||||||
|
tool_call_id: m.tool_call_id.clone(),
|
||||||
|
name: m.name.clone(),
|
||||||
}).collect(),
|
}).collect(),
|
||||||
|
tools: req.tools.clone(),
|
||||||
|
tool_choice: req.tool_choice.clone(),
|
||||||
max_tokens: Some(req.max_tokens.unwrap_or(800)),
|
max_tokens: Some(req.max_tokens.unwrap_or(800)),
|
||||||
temperature: req.temperature.unwrap_or(0.3),
|
temperature: req.temperature.unwrap_or(0.3),
|
||||||
stream: false,
|
stream: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
let client = reqwest::Client::builder()
|
let client = reqwest::Client::builder()
|
||||||
.timeout(Duration::from_secs(OR_TIMEOUT_SECS))
|
.timeout(Duration::from_secs(or_timeout_secs()))
|
||||||
.build()
|
.build()
|
||||||
.map_err(|e| format!("build client: {e}"))?;
|
.map_err(|e| format!("build client: {e}"))?;
|
||||||
|
|
||||||
@@ -145,7 +208,10 @@ pub async fn chat(
|
|||||||
let latency_ms = t0.elapsed().as_millis();
|
let latency_ms = t0.elapsed().as_millis();
|
||||||
let choice = parsed.choices.into_iter().next()
|
let choice = parsed.choices.into_iter().next()
|
||||||
.ok_or_else(|| "openrouter returned no choices".to_string())?;
|
.ok_or_else(|| "openrouter returned no choices".to_string())?;
|
||||||
let reasoning_only = choice.message.content.is_none() && choice.message.reasoning.is_some();
|
let tool_calls = choice.message.tool_calls.clone();
|
||||||
|
let reasoning_only = choice.message.content.is_none()
|
||||||
|
&& choice.message.reasoning.is_some()
|
||||||
|
&& tool_calls.is_none();
|
||||||
let text = choice.message.content.unwrap_or_default();
|
let text = choice.message.content.unwrap_or_default();
|
||||||
if reasoning_only {
|
if reasoning_only {
|
||||||
// The model spent its whole budget thinking. Return well-formed empty
|
// The model spent its whole budget thinking. Return well-formed empty
|
||||||
@@ -186,7 +252,17 @@ pub async fn chat(
|
|||||||
model,
|
model,
|
||||||
choices: vec![Choice {
|
choices: vec![Choice {
|
||||||
index: 0,
|
index: 0,
|
||||||
message: Message { role: "assistant".into(), content: serde_json::Value::String(text) },
|
message: Message {
|
||||||
|
role: "assistant".into(),
|
||||||
|
// OpenAI shape: a tool-call-only turn carries `content: null`.
|
||||||
|
content: if tool_calls.is_some() && text.is_empty() {
|
||||||
|
serde_json::Value::Null
|
||||||
|
} else {
|
||||||
|
serde_json::Value::String(text)
|
||||||
|
},
|
||||||
|
tool_calls,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
finish_reason: choice.finish_reason.unwrap_or_else(|| "stop".into()),
|
finish_reason: choice.finish_reason.unwrap_or_else(|| "stop".into()),
|
||||||
}],
|
}],
|
||||||
usage: UsageBlock {
|
usage: UsageBlock {
|
||||||
@@ -207,10 +283,23 @@ struct ORChatBody {
|
|||||||
max_tokens: Option<u32>,
|
max_tokens: Option<u32>,
|
||||||
temperature: f64,
|
temperature: f64,
|
||||||
stream: bool,
|
stream: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
tools: Option<serde_json::Value>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
tool_choice: Option<serde_json::Value>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
struct ORMessage { role: String, content: serde_json::Value }
|
struct ORMessage {
|
||||||
|
role: String,
|
||||||
|
content: serde_json::Value,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
tool_calls: Option<serde_json::Value>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
tool_call_id: Option<String>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
name: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
struct ORChatResponse {
|
struct ORChatResponse {
|
||||||
@@ -238,6 +327,9 @@ struct ORMessageResp {
|
|||||||
content: Option<String>,
|
content: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
reasoning: Option<String>,
|
reasoning: Option<String>,
|
||||||
|
/// OpenAI tool calls requested by the model (passed back to the client verbatim).
|
||||||
|
#[serde(default)]
|
||||||
|
tool_calls: Option<serde_json::Value>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
@@ -274,17 +366,57 @@ mod tests {
|
|||||||
let body = ORChatBody {
|
let body = ORChatBody {
|
||||||
model: "openai/gpt-oss-120b:free".into(),
|
model: "openai/gpt-oss-120b:free".into(),
|
||||||
messages: vec![
|
messages: vec![
|
||||||
ORMessage { role: "user".into(), content: "review this".into() },
|
ORMessage { role: "user".into(), content: "review this".into(),
|
||||||
|
tool_calls: None, tool_call_id: None, name: None },
|
||||||
],
|
],
|
||||||
max_tokens: Some(800),
|
max_tokens: Some(800),
|
||||||
temperature: 0.3,
|
temperature: 0.3,
|
||||||
stream: false,
|
stream: false,
|
||||||
|
tools: None,
|
||||||
|
tool_choice: None,
|
||||||
};
|
};
|
||||||
let json = serde_json::to_string(&body).unwrap();
|
let json = serde_json::to_string(&body).unwrap();
|
||||||
assert!(json.contains("\"model\":\"openai/gpt-oss-120b:free\""));
|
assert!(json.contains("\"model\":\"openai/gpt-oss-120b:free\""));
|
||||||
assert!(json.contains("\"messages\""));
|
assert!(json.contains("\"messages\""));
|
||||||
assert!(json.contains("\"max_tokens\":800"));
|
assert!(json.contains("\"max_tokens\":800"));
|
||||||
assert!(json.contains("\"stream\":false"));
|
assert!(json.contains("\"stream\":false"));
|
||||||
|
// no tool fields on plain traffic (byte-compat with pre-2026-09-19 bodies)
|
||||||
|
assert!(!json.contains("tools") && !json.contains("tool_call"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chat_body_forwards_tools_and_tool_messages() {
|
||||||
|
let tools = serde_json::json!([{"type":"function","function":{"name":"exec","parameters":{"type":"object"}}}]);
|
||||||
|
let body = ORChatBody {
|
||||||
|
model: "z-ai/glm-5.3-flash".into(),
|
||||||
|
messages: vec![
|
||||||
|
ORMessage { role: "assistant".into(), content: serde_json::Value::Null,
|
||||||
|
tool_calls: Some(serde_json::json!([{"id":"c1","type":"function","function":{"name":"exec","arguments":"{}"}}])),
|
||||||
|
tool_call_id: None, name: None },
|
||||||
|
ORMessage { role: "tool".into(), content: "ok".into(), tool_calls: None,
|
||||||
|
tool_call_id: Some("c1".into()), name: Some("exec".into()) },
|
||||||
|
],
|
||||||
|
max_tokens: Some(100),
|
||||||
|
temperature: 0.0,
|
||||||
|
stream: false,
|
||||||
|
tools: Some(tools),
|
||||||
|
tool_choice: Some(serde_json::json!("auto")),
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&body).unwrap();
|
||||||
|
assert!(json.contains("\"tools\":[{"));
|
||||||
|
assert!(json.contains("\"tool_choice\":\"auto\""));
|
||||||
|
assert!(json.contains("\"tool_call_id\":\"c1\""));
|
||||||
|
assert!(json.contains("\"tool_calls\":[{"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn response_with_tool_calls_parses() {
|
||||||
|
let raw = r#"{"choices":[{"message":{"content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"exec","arguments":"{}"}}]},"finish_reason":"tool_calls"}]}"#;
|
||||||
|
let parsed: ORChatResponse = serde_json::from_str(raw).unwrap();
|
||||||
|
let c = &parsed.choices[0];
|
||||||
|
assert!(c.message.content.is_none());
|
||||||
|
assert!(c.message.tool_calls.is_some());
|
||||||
|
assert_eq!(c.finish_reason.as_deref(), Some("tool_calls"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
Reference in New Issue
Block a user