Metacognitive knowledge extraction: 496 topics, 300 decisions, 100 insights

- extract_knowledge.py: LLM processes sampled pages from all conversations - Extracts structured JSON: topics, decisions, insights, open questions - Stores as queryable Parquet tables (knowledge_topics, knowledge_decisions, knowledge_insights, knowledge_questions) - Each finding traces to source conversation + page number - Fixed Unicode chunker crash (byte boundary on multi-byte chars) - Tested on 8 real ChatGPT PDFs, 477 pages: 496 topics, 300 decisions, 100 insights, 200 open questions extracted Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-27 23:06:36 -05:00
parent 0756075c6d
commit bf033b2d9d
9 changed files with 339 additions and 0 deletions
@@ -0,0 +1,51 @@
+{
+  "id": "667478c3-47f9-4076-bab2-8293999d4db2",
+  "name": "knowledge_insights",
+  "schema_fingerprint": "36896987c7df0e78f5282208e3b8bbbaf9c017588d0bb42a20a7dbe97d8d4fa0",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/knowledge_insights.parquet",
+      "size_bytes": 13099,
+      "created_at": "2026-03-28T04:06:05.361963023Z"
+    }
+  ],
+  "created_at": "2026-03-28T04:06:05.361963273Z",
+  "updated_at": "2026-03-28T04:06:05.362127023Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "insight",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "knowledge_insights.csv",
+    "ingest_job": "ingest-1774670765361",
+    "ingest_timestamp": "2026-03-28T04:06:05.361963023Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 100
+}
@@ -0,0 +1,51 @@
+{
+  "id": "799c825c-950e-480e-a38e-4407377e43e3",
+  "name": "knowledge_topics",
+  "schema_fingerprint": "356c37c8fef56ce7f7882c80f1301cc2f70bcbf874719cb0628b42179874494c",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/knowledge_topics.parquet",
+      "size_bytes": 14252,
+      "created_at": "2026-03-28T04:06:05.358571822Z"
+    }
+  ],
+  "created_at": "2026-03-28T04:06:05.358572461Z",
+  "updated_at": "2026-03-28T04:06:05.358833280Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "topic",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "knowledge_topics.csv",
+    "ingest_job": "ingest-1774670765358",
+    "ingest_timestamp": "2026-03-28T04:06:05.358571822Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 496
+}
@@ -0,0 +1,51 @@
+{
+  "id": "80eaf246-778b-4987-9bef-62a6cccb6021",
+  "name": "knowledge_questions",
+  "schema_fingerprint": "d86b4ddf7a9694118aeff3d8aa41312e0c3091b0f15cad1079ee9561c29f9c91",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/knowledge_questions.parquet",
+      "size_bytes": 17253,
+      "created_at": "2026-03-28T04:06:05.363389625Z"
+    }
+  ],
+  "created_at": "2026-03-28T04:06:05.363390263Z",
+  "updated_at": "2026-03-28T04:06:05.363439745Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "question",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "knowledge_questions.csv",
+    "ingest_job": "ingest-1774670765363",
+    "ingest_timestamp": "2026-03-28T04:06:05.363389625Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 200
+}
@@ -0,0 +1,51 @@
+{
+  "id": "fe6b6872-03f8-491a-b2c2-7d25b19cd9bb",
+  "name": "knowledge_decisions",
+  "schema_fingerprint": "be65fb1ba2121bcf43c195c06dbdc0ae892eb835f4a909e9c1c0438ac81a1d0d",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/knowledge_decisions.parquet",
+      "size_bytes": 21274,
+      "created_at": "2026-03-28T04:06:05.360578855Z"
+    }
+  ],
+  "created_at": "2026-03-28T04:06:05.360579258Z",
+  "updated_at": "2026-03-28T04:06:05.360801969Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "decision",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "knowledge_decisions.csv",
+    "ingest_job": "ingest-1774670765360",
+    "ingest_timestamp": "2026-03-28T04:06:05.360578855Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 300
+}