Phase E: Scheduled ingest — the substrate runs itself

Background Scheduler task fires due ingests on interval, records
outcomes, reschedules. Single-flight per schedule_id so a slow run
can't pile up. 10s tick cadence, schedules' own intervals independent.

ScheduleDef persisted as JSON at primary://_schedules/{id}.json,
rebuilt on startup. ScheduleKind supports Mysql and Postgres (both
through existing streaming paths). ScheduleTrigger::Interval is
live; Cron variant defined in the enum but parsing stubbed with a
safe 1h fallback.

next_run_at set to "now" on creation so operators see success or
failure within one tick — no waiting for the first full interval.
run-now endpoint fires even when schedule is disabled (manual
override for testing). Full catalog integration: PII detection,
lineage with redacted DSN, mark-stale + autotune agent trigger.

Verified live: 20s MySQL schedule against MariaDB lh_demo.customers.
Source mutated between runs (added row + updated value). Second
auto-fire picked up both changes (10→11 rows). DataFusion SQL
confirmed mutations in the lakehouse. 6 unit tests pass.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/ingestd/src/service.rs

@@ -1,9 +1,9 @@
 use axum::{
     Json, Router,
-    extract::{Multipart, Query, State},
+    extract::{Multipart, Path, Query, State},
     http::{HeaderMap, StatusCode},
     response::IntoResponse,
-    routing::{get, post},
+    routing::{delete, get, patch, post},
 };
 use bytes::Bytes;
 use object_store::ObjectStore;
@@ -11,7 +11,7 @@ use serde::Deserialize;
 use std::sync::Arc;
 
 use catalogd::registry::Registry;
-use crate::{db_ingest, my_stream, pg_stream, pipeline};
+use crate::{db_ingest, my_stream, pg_stream, pipeline, schedule};
 use shared::arrow_helpers::record_batch_to_parquet;
 use shared::types::{ObjectRef, SchemaFingerprint};
 use storaged::ops;
@@ -30,6 +30,9 @@ pub struct IngestState {
     /// Used to look up which HNSW indexes are attached to the
     /// just-ingested dataset. Each matching index gets one trigger.
     pub index_registry: vectord::index_registry::IndexRegistry,
+    /// Scheduled-ingest registry. The scheduler task runs against this
+    /// store; HTTP CRUD endpoints write through it.
+    pub schedules: schedule::ScheduleStore,
 }
 
 /// Push `DatasetAppended` triggers for every HNSW index bound to this
@@ -86,6 +89,12 @@ pub fn router(state: IngestState) -> Router {
         .route("/postgres/import", post(import_pg_table))
         .route("/db", post(ingest_db_stream))
         .route("/mysql", post(ingest_mysql_stream))
+        // Phase E: scheduled ingest
+        .route("/schedules", get(list_schedules).post(create_schedule))
+        .route("/schedules/{id}", get(get_schedule)
+            .patch(patch_schedule)
+            .delete(delete_schedule))
+        .route("/schedules/{id}/run-now", post(run_schedule_now))
         .with_state(state)
 }
 
@@ -475,3 +484,141 @@ fn redact_dsn(dsn: &str) -> String {
     let colon_idx = scheme_end + colon_offset;
     format!("{}:***{}", &dsn[..colon_idx], &dsn[at_idx..])
 }
+
+// --- Phase E: Scheduled ingest CRUD endpoints ---
+
+#[derive(Deserialize)]
+struct CreateScheduleRequest {
+    /// Stable id used in URLs and on disk. Required so callers can
+    /// PATCH/DELETE deterministically. Sanitized on storage but the
+    /// caller's id is what gets persisted as the canonical key.
+    id: String,
+    kind: schedule::ScheduleKind,
+    trigger: schedule::ScheduleTrigger,
+    #[serde(default = "default_enabled")]
+    enabled: bool,
+    #[serde(default)]
+    created_by: String,
+}
+
+fn default_enabled() -> bool { true }
+
+async fn create_schedule(
+    State(state): State<IngestState>,
+    Json(req): Json<CreateScheduleRequest>,
+) -> impl IntoResponse {
+    if state.schedules.get(&req.id).await.is_some() {
+        return Err((StatusCode::CONFLICT, format!("schedule '{}' already exists", req.id)));
+    }
+    let now = chrono::Utc::now();
+    let def = schedule::ScheduleDef {
+        id: req.id,
+        kind: req.kind,
+        trigger: req.trigger,
+        enabled: req.enabled,
+        created_at: now,
+        created_by: req.created_by,
+        // Fire on the next tick — operators usually want to verify
+        // their schedule works without waiting for the full interval.
+        next_run_at: now,
+        last_run_at: None,
+        last_outcome: None,
+        run_count: 0,
+        failure_count: 0,
+    };
+    match state.schedules.put(def).await {
+        Ok(d) => Ok((StatusCode::CREATED, Json(d))),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+async fn list_schedules(State(state): State<IngestState>) -> impl IntoResponse {
+    Json(state.schedules.list().await)
+}
+
+async fn get_schedule(
+    State(state): State<IngestState>,
+    Path(id): Path<String>,
+) -> impl IntoResponse {
+    match state.schedules.get(&id).await {
+        Some(d) => Ok(Json(d)),
+        None => Err((StatusCode::NOT_FOUND, format!("schedule '{id}' not found"))),
+    }
+}
+
+#[derive(Deserialize)]
+struct PatchScheduleRequest {
+    /// Toggle on/off without changing anything else.
+    #[serde(default)]
+    enabled: Option<bool>,
+    /// Replace the trigger spec — useful for bumping intervals.
+    #[serde(default)]
+    trigger: Option<schedule::ScheduleTrigger>,
+}
+
+async fn patch_schedule(
+    State(state): State<IngestState>,
+    Path(id): Path<String>,
+    Json(req): Json<PatchScheduleRequest>,
+) -> impl IntoResponse {
+    let Some(mut def) = state.schedules.get(&id).await else {
+        return Err((StatusCode::NOT_FOUND, format!("schedule '{id}' not found")));
+    };
+    if let Some(e) = req.enabled { def.enabled = e; }
+    if let Some(t) = req.trigger { def.trigger = t; }
+    match state.schedules.put(def).await {
+        Ok(d) => Ok(Json(d)),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+async fn delete_schedule(
+    State(state): State<IngestState>,
+    Path(id): Path<String>,
+) -> impl IntoResponse {
+    match state.schedules.delete(&id).await {
+        Ok(true) => Ok(Json(serde_json::json!({ "deleted": id }))),
+        Ok(false) => Err((StatusCode::NOT_FOUND, format!("schedule '{id}' not found"))),
+        Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e)),
+    }
+}
+
+/// Force a schedule to fire right now, regardless of its next_run_at.
+/// Records the outcome and reschedules. Useful for "test my new schedule
+/// works without waiting an hour."
+async fn run_schedule_now(
+    State(state): State<IngestState>,
+    Path(id): Path<String>,
+) -> impl IntoResponse {
+    let Some(def) = state.schedules.get(&id).await else {
+        return Err((StatusCode::NOT_FOUND, format!("schedule '{id}' not found")));
+    };
+    let deps = schedule::SchedulerIngestDeps {
+        store: state.store.clone(),
+        registry: state.registry.clone(),
+        buckets: state.buckets.clone(),
+        agent_handle: Some(state.agent_handle.clone()),
+        index_registry: state.index_registry.clone(),
+    };
+
+    let t0 = std::time::Instant::now();
+    let result = schedule::run_schedule(&def, &deps).await;
+    let elapsed = t0.elapsed().as_secs_f32();
+    let outcome = schedule::RunOutcome {
+        at: chrono::Utc::now(),
+        success: result.is_ok(),
+        message: match &result {
+            Ok(s) => s.clone(),
+            Err(e) => e.clone(),
+        },
+        rows: result.as_ref().ok().and_then(|m: &String| {
+            m.split_whitespace()
+                .find_map(|tok| tok.strip_prefix("rows=").and_then(|n| n.parse().ok()))
+        }),
+        duration_secs: elapsed,
+    };
+    if let Err(e) = state.schedules.record_run(&id, outcome.clone()).await {
+        tracing::warn!("schedule {id}: record_run failed: {e}");
+    }
+    Ok(Json(outcome))
+}