rootandClaude Opus 4.6
bf033b2d9d
- extract_knowledge.py: LLM processes sampled pages from all conversations - Extracts structured JSON: topics, decisions, insights, open questions - Stores as queryable Parquet tables (knowledge_topics, knowledge_decisions, knowledge_insights, knowledge_questions) - Each finding traces to source conversation + page number - Fixed Unicode chunker crash (byte boundary on multi-byte chars) - Tested on 8 real ChatGPT PDFs, 477 pages: 496 topics, 300 decisions, 100 insights, 200 open questions extracted Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
51 lines
1.2 KiB
JSON
51 lines
1.2 KiB
JSON
{
|
|
"id": "80eaf246-778b-4987-9bef-62a6cccb6021",
|
|
"name": "knowledge_questions",
|
|
"schema_fingerprint": "d86b4ddf7a9694118aeff3d8aa41312e0c3091b0f15cad1079ee9561c29f9c91",
|
|
"objects": [
|
|
{
|
|
"bucket": "data",
|
|
"key": "datasets/knowledge_questions.parquet",
|
|
"size_bytes": 17253,
|
|
"created_at": "2026-03-28T04:06:05.363389625Z"
|
|
}
|
|
],
|
|
"created_at": "2026-03-28T04:06:05.363390263Z",
|
|
"updated_at": "2026-03-28T04:06:05.363439745Z",
|
|
"description": "",
|
|
"owner": "",
|
|
"sensitivity": null,
|
|
"columns": [
|
|
{
|
|
"name": "conversation",
|
|
"data_type": "Utf8",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "page",
|
|
"data_type": "Int64",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "question",
|
|
"data_type": "Utf8",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
}
|
|
],
|
|
"lineage": {
|
|
"source_system": "csv",
|
|
"source_file": "knowledge_questions.csv",
|
|
"ingest_job": "ingest-1774670765363",
|
|
"ingest_timestamp": "2026-03-28T04:06:05.363389625Z",
|
|
"parent_datasets": []
|
|
},
|
|
"freshness": null,
|
|
"tags": [],
|
|
"row_count": 200
|
|
} |