Phase 10: Rich catalog v2 — metadata as product

- DatasetManifest expanded: description, owner, sensitivity, columns,
  lineage, freshness contract, tags, row_count
- All new fields use #[serde(default)] for backward compatibility
- PII auto-detection: scans column names for email, phone, SSN, salary,
  address, DOB, medical terms — flags as PII/PHI/Financial
- Column-level metadata: name, type, sensitivity, is_pii flag
- Lineage tracking: source_system, source_file, ingest_job, timestamp
- Ingest pipeline auto-populates: PII scan, column meta, lineage, row count
- PATCH /catalog/datasets/by-name/{name}/metadata — update metadata
- Catalog responses now include all rich fields
- 25 unit tests passing (5 new PII detection tests)

Per ADR-013: datasets without metadata become mystery files.
This makes every ingested file self-describing from day one.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/ingestd/src/pipeline.rs

@@ -94,7 +94,7 @@ pub async fn ingest_file(
     ops::put(store, &storage_key, parquet_bytes).await?;
     tracing::info!("stored {} as {} ({} bytes)", filename, storage_key, parquet_size);
 
-    // 7. Register in catalog
+    // 7. Register in catalog with rich metadata
     let schema_fp = fingerprint_schema(&schema);
     let now = chrono::Utc::now();
     let obj_ref = ObjectRef {
@@ -104,13 +104,49 @@ pub async fn ingest_file(
         created_at: now,
     };
 
-    // Use content hash as fingerprint for dedup tracking
+    // Register base dataset
     registry.register(
         safe_name.clone(),
         SchemaFingerprint(hash.clone()),
         vec![obj_ref],
     ).await?;
 
+    // Auto-populate metadata: PII detection, column info, lineage, row count
+    let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
+    let dataset_sensitivity = shared::pii::detect_dataset_sensitivity(&col_names);
+
+    let column_meta: Vec<shared::types::ColumnMeta> = schema.fields().iter().map(|f| {
+        let sens = shared::pii::detect_sensitivity(f.name());
+        shared::types::ColumnMeta {
+            name: f.name().clone(),
+            data_type: f.data_type().to_string(),
+            sensitivity: sens.clone(),
+            description: String::new(),
+            is_pii: matches!(sens, Some(shared::types::Sensitivity::Pii)),
+        }
+    }).collect();
+
+    let lineage = shared::types::Lineage {
+        source_system: format!("{:?}", file_type).to_lowercase(),
+        source_file: filename.to_string(),
+        ingest_job: format!("ingest-{}", now.timestamp_millis()),
+        ingest_timestamp: now,
+        parent_datasets: vec![],
+    };
+
+    let pii_count = column_meta.iter().filter(|c| c.is_pii).count();
+    if pii_count > 0 {
+        tracing::info!("auto-detected {} PII columns in '{}'", pii_count, safe_name);
+    }
+
+    let _ = registry.update_metadata(&safe_name, catalogd::registry::MetadataUpdate {
+        sensitivity: dataset_sensitivity,
+        columns: Some(column_meta),
+        lineage: Some(lineage),
+        row_count: Some(total_rows as u64),
+        ..Default::default()
+    }).await;
+
     Ok(IngestResult {
         dataset_name: safe_name,
         file_type: format!("{:?}", file_type),