LLM Team integration: extract mode processes 8 conversations

- process_knowledge.py: routes PDF content through devop.live /api/run - Uses extract mode: LLM extracts facts/entities/relationships, verifier checks - 543 extractions (279 facts, 264 entities) from 8 conversations - 60 verified entries from gemma2 cross-checking - All stored as queryable Parquet (processed_extractions, processed_validated) - Streaming SSE parsing for LLM Team responses - This proves the architecture: Lakehouse storage + devop.live processing Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-28 00:22:29 -05:00
parent bd68b662b7
commit 3f19ad5d93
5 changed files with 279 additions and 0 deletions
@@ -0,0 +1,58 @@
+{
+  "id": "24a05512-dc06-4884-9889-2ea978abdaec",
+  "name": "processed_extractions",
+  "schema_fingerprint": "c2853ba4c0cc5a897b37680fefea2708f51d51774226b00b72444e79d594c874",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/processed_extractions.parquet",
+      "size_bytes": 28175,
+      "created_at": "2026-03-28T05:16:57.437041979Z"
+    }
+  ],
+  "created_at": "2026-03-28T05:16:57.437042648Z",
+  "updated_at": "2026-03-28T05:16:57.437403737Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "type",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "processed_extractions.csv",
+    "ingest_job": "ingest-1774675017437",
+    "ingest_timestamp": "2026-03-28T05:16:57.437041979Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 543
+}
@@ -0,0 +1,58 @@
+{
+  "id": "3ead1a7a-810b-49d1-add8-752bab52dfae",
+  "name": "processed_validated",
+  "schema_fingerprint": "8078de214059a2a4582203b32c1172fcc311c2d3c762a3b075c07268308f1841",
+  "objects": [
+    {
+      "bucket": "data",
+      "key": "datasets/processed_validated.parquet",
+      "size_bytes": 15271,
+      "created_at": "2026-03-28T05:16:57.439472748Z"
+    }
+  ],
+  "created_at": "2026-03-28T05:16:57.439473552Z",
+  "updated_at": "2026-03-28T05:16:57.439749766Z",
+  "description": "",
+  "owner": "",
+  "sensitivity": null,
+  "columns": [
+    {
+      "name": "conversation",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "page",
+      "data_type": "Int64",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "type",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    },
+    {
+      "name": "content",
+      "data_type": "Utf8",
+      "sensitivity": null,
+      "description": "",
+      "is_pii": false
+    }
+  ],
+  "lineage": {
+    "source_system": "csv",
+    "source_file": "processed_validated.csv",
+    "ingest_job": "ingest-1774675017439",
+    "ingest_timestamp": "2026-03-28T05:16:57.439472748Z",
+    "parent_datasets": []
+  },
+  "freshness": null,
+  "tags": [],
+  "row_count": 60
+}