Robust SQL extraction: handles explanations, markdown, prefixes

clean_sql now uses 3 strategies in priority order:
1. Extract from ```sql...``` markdown blocks
2. Find first SELECT/WITH/INSERT statement in text
3. Strip leading "sql" keyword fallback

Tested against 5 real model output patterns:
- Clean SQL ✓
- "sql" prefixed ✓
- Markdown fenced ✓
- Explanation before ```sql block ✓
- Explanation with SELECT buried in text ✓

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-03-27 20:42:11 -05:00
co-authored by Claude Opus 4.6
parent 34c03894ae
commit 0bd753294b
5 changed files with 195 additions and 13 deletions
@@ -0,0 +1,65 @@
{
"id": "3b12ae24-17d4-4325-92bf-f3155982f3bf",
"name": "users",
"schema_fingerprint": "30c0e31f0963e6f4af02131bbb9ea246fbbd068b849b833565a4b28211fbc90b",
"objects": [
{
"bucket": "data",
"key": "datasets/users.parquet",
"size_bytes": 2012,
"created_at": "2026-03-28T01:38:59.904968123Z"
}
],
"created_at": "2026-03-28T01:38:59.904968680Z",
"updated_at": "2026-03-28T01:38:59.905221340Z",
"description": "",
"owner": "",
"sensitivity": null,
"columns": [
{
"name": "id",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "username",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "password_hash",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "role",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "created_at",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
}
],
"lineage": {
"source_system": "postgresql",
"source_file": "127.0.0.1:5432/knowledge_base.users",
"ingest_job": "pg-import-1774661939904",
"ingest_timestamp": "2026-03-28T01:38:59.904968123Z",
"parent_datasets": []
},
"freshness": null,
"tags": [],
"row_count": 1
}
@@ -0,0 +1,93 @@
{
"id": "3c2579b4-f3f3-4875-95fa-58d8b49ad94c",
"name": "meta_runs",
"schema_fingerprint": "68f2c0d7a3ceb0aaa3c17c64900704519c72d213161bc9e5179c42ee53f6d0df",
"objects": [
{
"bucket": "data",
"key": "datasets/meta_runs.parquet",
"size_bytes": 729773,
"created_at": "2026-03-28T01:38:57.380576453Z"
}
],
"created_at": "2026-03-28T01:38:57.380577270Z",
"updated_at": "2026-03-28T01:38:57.380846224Z",
"description": "",
"owner": "",
"sensitivity": null,
"columns": [
{
"name": "id",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "pipeline_id",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "iteration",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "stage_results",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "final_output",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "score",
"data_type": "Float64",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "model_config",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "duration_ms",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "created_at",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
}
],
"lineage": {
"source_system": "postgresql",
"source_file": "127.0.0.1:5432/knowledge_base.meta_runs",
"ingest_job": "pg-import-1774661937380",
"ingest_timestamp": "2026-03-28T01:38:57.380576453Z",
"parent_datasets": []
},
"freshness": null,
"tags": [],
"row_count": 38
}
Binary file not shown.
Binary file not shown.