shared: build ModelMatrix — migrate 5 call sites off deprecated estimate_tokens
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
lakehouse/auditor 1 blocking issue: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
The `aibridge::context::estimate_tokens` deprecation has been pointing
at `shared::model_matrix::ModelMatrix::estimate_tokens` for a while,
but that module didn't exist — so the deprecation was aspirational
noise, not actionable guidance.
Built the minimal target: `shared::model_matrix::ModelMatrix` with
an associated `estimate_tokens(text: &str) -> usize` method. Same
chars/4 ceiling heuristic as the deprecated helper. 6 tests cover
empty/3/4/5-char cases, multi-byte UTF-8 (emoji count as 1 char each),
and linear scaling to 400-char inputs.
Migrated 5 call sites:
- aibridge/context.rs:88 — opts.system token count
- aibridge/context.rs:89 — prompt token count
- aibridge/tree_split.rs:22 — import (now uses ModelMatrix)
- aibridge/tree_split.rs:84, 89 — truncate_scratchpad budget loop
- aibridge/tree_split.rs:282 — scratchpad post-truncation assertion
- aibridge/context.rs:183 — system-prompt budget test
Also cleaned up two parallel test warnings:
- aibridge/context.rs legacy estimate_tokens_ceiling_divides_by_four
test deleted (ModelMatrix's tests cover the same behavior now).
- vectord/playbook_memory.rs:1650 unused_mut on e_alive.
Net workspace warning count: 11 → 0 (including --tests build).
The deprecated `estimate_tokens` wrapper stays in aibridge/context.rs
for external callers. Future commits can remove it entirely once no
public API surface still references it.
The applier's warning-count gate now has a floor of 0 — any future
patch that introduces a single warning trips the gate automatically.
Previously a floor of 11 tolerated noise.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -85,8 +85,8 @@ pub fn assert_context_budget(
|
||||
let window = context_window_for(model);
|
||||
let safety = opts.safety_margin.unwrap_or(DEFAULT_SAFETY_MARGIN);
|
||||
let max_tokens = opts.max_tokens.unwrap_or(DEFAULT_MAX_TOKENS);
|
||||
let sys_tokens = opts.system.map(estimate_tokens).unwrap_or(0);
|
||||
let estimated = estimate_tokens(prompt) + sys_tokens + max_tokens;
|
||||
let sys_tokens = opts.system.map(shared::model_matrix::ModelMatrix::estimate_tokens).unwrap_or(0);
|
||||
let estimated = shared::model_matrix::ModelMatrix::estimate_tokens(prompt) + sys_tokens + max_tokens;
|
||||
let remaining = window as i64 - estimated as i64 - safety as i64;
|
||||
let check = BudgetCheck { estimated, window, remaining };
|
||||
if remaining < 0 && !opts.bypass {
|
||||
@@ -110,14 +110,10 @@ pub fn overflow_message(model: &str, check: &BudgetCheck, over_by: usize, safety
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn estimate_tokens_ceiling_divides_by_four() {
|
||||
assert_eq!(estimate_tokens(""), 0);
|
||||
assert_eq!(estimate_tokens("abc"), 1); // 3 → ceil(3/4) = 1
|
||||
assert_eq!(estimate_tokens("abcd"), 1); // 4 → ceil(4/4) = 1
|
||||
assert_eq!(estimate_tokens("abcde"), 2); // 5 → ceil(5/4) = 2
|
||||
assert_eq!(estimate_tokens(&"x".repeat(400)), 100);
|
||||
}
|
||||
// Deprecated-function behavior is now canonically tested in
|
||||
// crates/shared/src/model_matrix.rs. This test was the legacy
|
||||
// pin that preceded the migration; delete when the deprecated
|
||||
// wrapper itself goes (see the #[deprecated] attribute).
|
||||
|
||||
#[test]
|
||||
fn context_window_known_and_fallback() {
|
||||
@@ -180,7 +176,7 @@ mod tests {
|
||||
).unwrap();
|
||||
assert!(with_sys.estimated > without_sys.estimated,
|
||||
"system prompt should raise estimate");
|
||||
assert_eq!(with_sys.estimated - without_sys.estimated, estimate_tokens(&sys));
|
||||
assert_eq!(with_sys.estimated - without_sys.estimated, shared::model_matrix::ModelMatrix::estimate_tokens(&sys));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user