profit/lakehouse
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Database connector: PostgreSQL → Parquet import

Browse Source 
- POST /ingest/postgres/tables — list all tables in a database
- POST /ingest/postgres/import — import table → Parquet → catalog → queryable
- Auto type mapping: int2/4/8 → Int, float4/8 → Float64, bool → Boolean,
  text/varchar/jsonb/timestamp → Utf8 (safe default per ADR-010)
- Auto PII detection + lineage on import
- Empty password support for trust auth
- Tested: imported lab_trials (40 rows, 10 cols) and threat_intel (20 rows, 30 cols)
  from local knowledge_base Postgres database — immediately queryable

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-03-27 20:14:16 -05:00
co-authored by Claude Opus 4.6
parent 294f3f6a49
commit 9992b5f135
10 changed files with 880 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/_catalog/manifests/e2a8f88a-59f6-40c7-a45b-e23d8f3533b6.json
+100
View File
@@ -0,0 +1,100 @@
{
"id": "e2a8f88a-59f6-40c7-a45b-e23d8f3533b6",
"name": "lab_trials",
"schema_fingerprint": "1d5782349402439a7e44efd0ccab9ae64ac3044221adef9e828b60b8bbb44dd5",
"objects": [
{
"bucket": "data",
"key": "datasets/lab_trials.parquet",
"size_bytes": 64646,
"created_at": "2026-03-28T01:14:03.026116573Z"
}
],
"created_at": "2026-03-28T01:14:03.026117277Z",
"updated_at": "2026-03-28T01:14:03.026247826Z",
"description": "",
"owner": "",
"sensitivity": null,
"columns": [
{
"name": "id",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "experiment_id",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "trial_num",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "config_diff",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "config_snapshot",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "scores",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "avg_score",
"data_type": "Float64",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "improved",
"data_type": "Boolean",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "duration_ms",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "created_at",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
}
],
"lineage": {
"source_system": "postgresql",
"source_file": "127.0.0.1:5432/knowledge_base.lab_trials",
"ingest_job": "pg-import-1774660443026",
"ingest_timestamp": "2026-03-28T01:14:03.026116573Z",
"parent_datasets": []
},
"freshness": null,
"tags": [],
"row_count": 40
}
data/_catalog/manifests/e7304f05-5278-4e17-961a-51f2588fd2aa.json
+240
View File
@@ -0,0 +1,240 @@
{
"id": "e7304f05-5278-4e17-961a-51f2588fd2aa",
"name": "threat_intel",
"schema_fingerprint": "df1e126046147b3de42086880e10c3501a3a615ecddf336bc24957a24c321241",
"objects": [
{
"bucket": "data",
"key": "datasets/threat_intel.parquet",
"size_bytes": 111130,
"created_at": "2026-03-28T01:14:03.054140697Z"
}
],
"created_at": "2026-03-28T01:14:03.054141294Z",
"updated_at": "2026-03-28T01:14:03.054427047Z",
"description": "",
"owner": "",
"sensitivity": null,
"columns": [
{
"name": "id",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "ip",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "threat_level",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "classification",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "confidence",
"data_type": "Float64",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "summary",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "indicators",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "recommendation",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "pattern",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "attack_type",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "likely_automated",
"data_type": "Boolean",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "country",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "country_code",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "city",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "isp",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "org",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "asn",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "is_proxy",
"data_type": "Boolean",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "is_hosting",
"data_type": "Boolean",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "open_ports",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "blocklist_count",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "blocklist_total",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "blocklists_blocked",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "reverse_dns",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "traceroute",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "log_count",
"data_type": "Int32",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "banned",
"data_type": "Boolean",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "enriched_at",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "updated_at",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
},
{
"name": "raw_data",
"data_type": "Utf8",
"sensitivity": null,
"description": "",
"is_pii": false
}
],
"lineage": {
"source_system": "postgresql",
"source_file": "127.0.0.1:5432/knowledge_base.threat_intel",
"ingest_job": "pg-import-1774660443054",
"ingest_timestamp": "2026-03-28T01:14:03.054140697Z",
"parent_datasets": []
},
"freshness": null,
"tags": [],
"row_count": 20
}
data/datasets/lab_trials.parquet
BIN
View File
Binary file not shown.
data/datasets/threat_intel.parquet
BIN
View File
Binary file not shown.