From bf033b2d9dd74289c4ffea518be914963cb7f478 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 27 Mar 2026 23:06:36 -0500 Subject: [PATCH] Metacognitive knowledge extraction: 496 topics, 300 decisions, 100 insights - extract_knowledge.py: LLM processes sampled pages from all conversations - Extracts structured JSON: topics, decisions, insights, open questions - Stores as queryable Parquet tables (knowledge_topics, knowledge_decisions, knowledge_insights, knowledge_questions) - Each finding traces to source conversation + page number - Fixed Unicode chunker crash (byte boundary on multi-byte chars) - Tested on 8 real ChatGPT PDFs, 477 pages: 496 topics, 300 decisions, 100 insights, 200 open questions extracted Co-Authored-By: Claude Opus 4.6 (1M context) --- .../667478c3-47f9-4076-bab2-8293999d4db2.json | 51 +++++++ .../799c825c-950e-480e-a38e-4407377e43e3.json | 51 +++++++ .../80eaf246-778b-4987-9bef-62a6cccb6021.json | 51 +++++++ .../fe6b6872-03f8-491a-b2c2-7d25b19cd9bb.json | 51 +++++++ data/datasets/knowledge_decisions.parquet | Bin 0 -> 21274 bytes data/datasets/knowledge_insights.parquet | Bin 0 -> 13099 bytes data/datasets/knowledge_questions.parquet | Bin 0 -> 17253 bytes data/datasets/knowledge_topics.parquet | Bin 0 -> 14252 bytes scripts/extract_knowledge.py | 135 ++++++++++++++++++ 9 files changed, 339 insertions(+) create mode 100644 data/_catalog/manifests/667478c3-47f9-4076-bab2-8293999d4db2.json create mode 100644 data/_catalog/manifests/799c825c-950e-480e-a38e-4407377e43e3.json create mode 100644 data/_catalog/manifests/80eaf246-778b-4987-9bef-62a6cccb6021.json create mode 100644 data/_catalog/manifests/fe6b6872-03f8-491a-b2c2-7d25b19cd9bb.json create mode 100644 data/datasets/knowledge_decisions.parquet create mode 100644 data/datasets/knowledge_insights.parquet create mode 100644 data/datasets/knowledge_questions.parquet create mode 100644 data/datasets/knowledge_topics.parquet create mode 100644 scripts/extract_knowledge.py diff --git a/data/_catalog/manifests/667478c3-47f9-4076-bab2-8293999d4db2.json b/data/_catalog/manifests/667478c3-47f9-4076-bab2-8293999d4db2.json new file mode 100644 index 0000000..68d7e90 --- /dev/null +++ b/data/_catalog/manifests/667478c3-47f9-4076-bab2-8293999d4db2.json @@ -0,0 +1,51 @@ +{ + "id": "667478c3-47f9-4076-bab2-8293999d4db2", + "name": "knowledge_insights", + "schema_fingerprint": "36896987c7df0e78f5282208e3b8bbbaf9c017588d0bb42a20a7dbe97d8d4fa0", + "objects": [ + { + "bucket": "data", + "key": "datasets/knowledge_insights.parquet", + "size_bytes": 13099, + "created_at": "2026-03-28T04:06:05.361963023Z" + } + ], + "created_at": "2026-03-28T04:06:05.361963273Z", + "updated_at": "2026-03-28T04:06:05.362127023Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "conversation", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page", + "data_type": "Int64", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "insight", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "csv", + "source_file": "knowledge_insights.csv", + "ingest_job": "ingest-1774670765361", + "ingest_timestamp": "2026-03-28T04:06:05.361963023Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 100 +} \ No newline at end of file diff --git a/data/_catalog/manifests/799c825c-950e-480e-a38e-4407377e43e3.json b/data/_catalog/manifests/799c825c-950e-480e-a38e-4407377e43e3.json new file mode 100644 index 0000000..7aa1852 --- /dev/null +++ b/data/_catalog/manifests/799c825c-950e-480e-a38e-4407377e43e3.json @@ -0,0 +1,51 @@ +{ + "id": "799c825c-950e-480e-a38e-4407377e43e3", + "name": "knowledge_topics", + "schema_fingerprint": "356c37c8fef56ce7f7882c80f1301cc2f70bcbf874719cb0628b42179874494c", + "objects": [ + { + "bucket": "data", + "key": "datasets/knowledge_topics.parquet", + "size_bytes": 14252, + "created_at": "2026-03-28T04:06:05.358571822Z" + } + ], + "created_at": "2026-03-28T04:06:05.358572461Z", + "updated_at": "2026-03-28T04:06:05.358833280Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "conversation", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page", + "data_type": "Int64", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "topic", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "csv", + "source_file": "knowledge_topics.csv", + "ingest_job": "ingest-1774670765358", + "ingest_timestamp": "2026-03-28T04:06:05.358571822Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 496 +} \ No newline at end of file diff --git a/data/_catalog/manifests/80eaf246-778b-4987-9bef-62a6cccb6021.json b/data/_catalog/manifests/80eaf246-778b-4987-9bef-62a6cccb6021.json new file mode 100644 index 0000000..89c6769 --- /dev/null +++ b/data/_catalog/manifests/80eaf246-778b-4987-9bef-62a6cccb6021.json @@ -0,0 +1,51 @@ +{ + "id": "80eaf246-778b-4987-9bef-62a6cccb6021", + "name": "knowledge_questions", + "schema_fingerprint": "d86b4ddf7a9694118aeff3d8aa41312e0c3091b0f15cad1079ee9561c29f9c91", + "objects": [ + { + "bucket": "data", + "key": "datasets/knowledge_questions.parquet", + "size_bytes": 17253, + "created_at": "2026-03-28T04:06:05.363389625Z" + } + ], + "created_at": "2026-03-28T04:06:05.363390263Z", + "updated_at": "2026-03-28T04:06:05.363439745Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "conversation", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page", + "data_type": "Int64", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "question", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "csv", + "source_file": "knowledge_questions.csv", + "ingest_job": "ingest-1774670765363", + "ingest_timestamp": "2026-03-28T04:06:05.363389625Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 200 +} \ No newline at end of file diff --git a/data/_catalog/manifests/fe6b6872-03f8-491a-b2c2-7d25b19cd9bb.json b/data/_catalog/manifests/fe6b6872-03f8-491a-b2c2-7d25b19cd9bb.json new file mode 100644 index 0000000..1b81976 --- /dev/null +++ b/data/_catalog/manifests/fe6b6872-03f8-491a-b2c2-7d25b19cd9bb.json @@ -0,0 +1,51 @@ +{ + "id": "fe6b6872-03f8-491a-b2c2-7d25b19cd9bb", + "name": "knowledge_decisions", + "schema_fingerprint": "be65fb1ba2121bcf43c195c06dbdc0ae892eb835f4a909e9c1c0438ac81a1d0d", + "objects": [ + { + "bucket": "data", + "key": "datasets/knowledge_decisions.parquet", + "size_bytes": 21274, + "created_at": "2026-03-28T04:06:05.360578855Z" + } + ], + "created_at": "2026-03-28T04:06:05.360579258Z", + "updated_at": "2026-03-28T04:06:05.360801969Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "conversation", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page", + "data_type": "Int64", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "decision", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "csv", + "source_file": "knowledge_decisions.csv", + "ingest_job": "ingest-1774670765360", + "ingest_timestamp": "2026-03-28T04:06:05.360578855Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 300 +} \ No newline at end of file diff --git a/data/datasets/knowledge_decisions.parquet b/data/datasets/knowledge_decisions.parquet new file mode 100644 index 0000000000000000000000000000000000000000..20ec9b82945ea794beccf00bbabfff439be13e84 GIT binary patch literal 21274 zcmd6vYmj7TdEZ;Ri@cIWEP-a2MVt5-iQSdO^ax#S0V3Ysncmr!=C(7lxlw1D(NU2ZpHnrVbtnu@$1&^RAuUxsaxJM_3DwS z>!+@sy88GpRZdM`c`Y+<_$edba05dh=D$~uTy^+j<>FNr4_!Qb@d&$o@iqKintH%? zc@=l)&(F9^<+c8*J?hUfcd7imzrNXDZ}Zo;`s*+I>pT5*-d`L3y6UeN{PkV_8v5&w zzxMn!^VcK(dab`+@2@%?{Ws|i(c7K&b^yfA1Qn|)of7V}r&R-=x z{n4T7kIqnkZgQ8(-}l#F@YkFD^`yUYgg-jW-^oKfns007XfF8ibGmxDQaMr!j=psJ zbw_G9*29_az32GR$8TvobLu~Q{VmtMeAn_{-t(vb>)z{+)X(42_~BiD)0@|BoH|mw zaN+Ubp(pRY?&Z-xI#PRd=7AgQPexCKolor@|Nb|6M;{*?|HLhg|1zpn{zf-neENU? z)2-`|1V^6;PaS>y)QA7_p1=3<>5sJ^e)34|sV^Kny>tG?Qy>09{MgJBFT8+wP{D6J zb4dR!NyBApr}=fgzZ3VPENCY|KN$weem@v?qCn;qG)Kc=5DnvDob-c+v%+9INrP^* z!!LVb>!7MJHe|0sGKkV}$WS4FgJw8vb%HF}i_$3G=?DAquoLv+R+`MZkHTh_bVtKD z>s1kH<*ZCSh_fIYrcv0N?T5p7FABnb`;viMNgrz-C8KO?89c?*0W0dTo2t}*lU=mf zMK{@Db1X~SYe%iP9R>THn8n#1nKAASK!~u3!;H13!EQgopnAVs zdpGJuY;Pb0WQj>Li_$#=%&`uWpp|sHtS0D1!_Z`+vtqiS)d{5?JNemH=NRrn76rA2 zo@Viv5l%M)$fAeqddLfJ)pAhW|VbKsPEFo(c)oU~e_)ZraUH9#@3|AZ3{wTX5)j*virG$DL8VlZRl z#r;-y1T>`>V^bQvGYY|?VTADR;OTi5jOv6zKibDm(EdL9lMT^kGm*y<&>-C2j{EWO zz!0j&Q{@S@l39;x?FhVOdWf7QCN9`Ns0 zGQth?cV@YBQ8o~SBODTUgE3v4pR-t#8=5kvA3SizoKigQ37uc|qHJ-`0&wD{u!S4C z0J&f@5|vUrevq$On{9Gz!8U5+*pc@`SH72>wC`q`wKN$dnT{qDKVkfCf;!LC;Bshts%6Wrf5)jN_$qE4Ab5bV=-!0?;EYEEj-ZJ zSY3a3YxRNp`g&ubu03K#!pEFGWFsKn%rAS}xR`^p>-G90^@&HliIk_M75IT}=V8&6 z#>xZrjm^elZL{&fW{&Y1EN&N0JEHHWso;7NwK+N>LV`~ko> zjJvR$KHAa7`N{K|Q$MYTBfJb|suK^gk>koO%ri>k(4ZH$G#E&E*SH3}7O**l22=Yn$s!92`;_nctd-~rqi#?V7#jKh?(XO|Fuhd++( zNJDBXE5RO$3kqiF6GT%&JhObq-LrAl!G11+1NOmvOjaNx)^l>vh(nT|7S9YOS<)$V9pNyhY(WU`fnsSeX=kn|$hWT@bdxjTbm zFcu|Ux8sQX(#T+FBOAt}nJnMRxRJ?i?}*b*42Aei62aI|=01y{C=B+aXcx9Z02jmB zK_09lHCV%p00;@hhkZHL4K3@kpE9N4RaRB(upZ5_Cq;Ojc&9T<2n7`m(ow&eB)jsU z{b5IJI0+&bKf(1!41Qp&Q+keMXPcnFE>DR`N+&fZ<{)izxtOG1{S{_gGvXr00UsyN z0?@iGptTXX0rr3{gh6M-k%)b-+XkbwCe^5}OQ6mzY>ya(({3gg@p3+N)O4VOxtEcA zoGBXUvgJZ$#~lgz3HqXUdIwY8(86FKZpUqx7!lsj+8EJ0+;I?quqH4J>P3d6$4`64 z?K98n(&#-S=)T`kRc>?v-YQWXXs5ZqK$hS;5)eb}5YQIkf1LGC4y$K)u-5Gc`*M6d z#*jN^*F>KrqWuy=yr{I@$OC683<`uGHIut)`7cJr0ei8b4n+9dmNq#F@Q zqwpdeu}7v)q>jXjFn&xC4HF7CFH0}yUQ^a1xSj0~jM!%Lc;~h;ccPrHWA(Yzg5?L+ zoRSQa-KamWrHEe#qd~Ba-bCr#y7M~xgwFK)TRsX z@)L?JFltyRv6z?iR&fjn-eue@LBlds1B$9DjW~KU#y}W9#K8jgYz1GV2;;IO zBl$;68(U5G9Udz8GQxFYb>&z_zI+O#5NVl+l-(|?lDV#mD3{h!q-Tx7*$H^Cm&_noF;v@y#&)h6#_NSl*n&S7ey;(*Iz~)I zwA5Kd5e0LzCBAQwU}ayV9g1AHG85b&!bRjDr&v|Zr)JE zm9;n*Q0n)X?HjY-MiUa{b+5qYFn&+nEFkdflpX-`$O~DgN?!i?57T;FL6eN=zU!slviy1WaSm!P>&vA`W&u+65i+op89gwkhc0 z7Ull6$-B_vjT22Y;X(LS9IOC&V(_JA6avGGB(uho#k zmtS=J(30Opn#4zI9nw0Ww=gmZ<{QB&g?WmEVo2A)h-_lDN+NIHlrAIO*XEBs*VB zarFXAe`i(Yy04&7miZ;R&wP3^Qv`_#;jqvV#44$0%!j#?cZdX5u5dua?lykDZD~x5 zc{%RxiBWIoL2qRB*ODtp`ofYvBa_#VrsWYfITIYOAOTlu@q_qzUVpqKlJdup$J9i# ziK;ke_uDS|P|Agk$o4izsK3PMf2UcL`75#Oq#sGH^rRU6s;jF^1V!@9Md)VvSqoR> zG>9gAMM>gnbw+&`3GTRb4%xv2!C$FK!~Mi(rUVo&Y0e?~AkZ|U)P^DmQz|LamTJheZFYp^%3VN_nddLIsTqY76R8$7IvMSBj3)t^V6vktELnMO zY110vDtp{GzhpBP!?=yusd1;c69ofAgE6A z%kNXQ5<7Q#>ervjur05k|MxSuWg%bHlTLYMxh)3*&>3HIwq=API1{;zlPrbUUGXk~=vo&qV{7QqV0IRchP^2o`{ZTZ)CQ11UdnugbYW%S~FB$i(%DYE;H0tlg zsHDyv&PEWR&_``N5+U)BgfDT3j-~COvEYv0O0f8`K`sD4y_LY1kP9xr4wS5f=VEy> zQTT~Ylt;%_v~7tqv%1fir1`9@N=b#w&wOsibj+6$4NZ2S4$8)u4yeu`cQ3$}_Ci)n zgYsNe8qF%BJ0zTGIp2<~!0woe>xf|4=O4g6mt*GE6qysoY75dFTw7*Y?MudSr7Y1O z723`qW3=1FU1R(;8g$S_4JjB^_|vZH2-lU?B|iwFN>Lbi@g?FJQ(}uqIJS*U#ymSKCS5XDz+By+A!g6i6aM z=wWTIMQ>bWPg)s$2xT&RDb|GM7Tqu&Uh+-n0@Xs0jzKp})w*;rQYdW}7pKQf)?ETn zfCa9G=`-zEq$40ul)jh7EbNlX6OMA!>p6~5RMTWoumm&MO?=P>K9XfVs9%%k0$Ezv zm4$RDgA#YSYIUPjG+^C>k}AsHd`LWXVR~f1QN$+Nqu605Wzo*pOBzlFyy-}^3n(NgG823E2vw*6Rx>!WC_=Pkf-NqVW5^-JZ$Gz6cdWt zfjP8G?AP_5xRSGtOuX+^#4=J|64Z5Q1u9gHMG;Rq4I@Y;TS04)FD-hTr(wgnBp)VO z7T43Gu^J`YF^yKELyR2{lteNcc23B}IO6j#vz+!IZI$D68pakGc*4LdZzEt8x|82G9^XVNyDBnnw<7Tg$j_xp4E26DOv9T+z#p z4>6gm1F)?fYd~ffuE{FQzTM6)g?YFbASxZH&aPcxR zzqr-92NiSDX*RpIz7TlDFGD*ilS9GzbG4Pl`qp}#{*R5#jcN~psZe74SAd+04lH%a zKXA>K$@6HCW31x{sDo2gt8SUV{512R)M2Cd@FW1nPf%G6Xrf#YiGS*8EI7Ql#e|l@ zlnpu$aIdj4y8m*yVydW#F`TygQeZ}64sa2gqHwNsh5FQvTy?7AAy!eO z6!*h}tom;DDFA$w&RpH4y3K6Xd{hTyT4Zy>#_33b5FU_=(_JpCeiV{6+YRkz(>8M{ zbFSkHScX)IqArI68wCj)pEYtLhIU{D1@XC`w+3QgC1qp5XR5IGL5V-YZOyccoHo?#(S zN-L0rJ4zPN9_-s95O98sMp842B>+OwNY2|DqUZNO3smMsm7NRIxY*ozmO{&izw`39IF3gu$*Md!Kxjk160Rxx_Qs7E2v&r@>7(t<;1dSr|$ZF;` zR~J@;+YGJ-rdc>a*RxT*ftBWi7V7DiubQ2xJgklI$gpda!l?>Hb>WK7M7tIFVvrQq zSFhlKwl>RWEGY}8bLV`;N#0>)C;A zN{qm`2!vBtovS;(oqT0zlF_luXp&Ew$*%H_Rx#r--FJY#&^UW`YvaQ5a*eL4&HD1% zQf;%2xGhgsBx`MR>VzOYx~pDGjyfBTn$)Vyhvr&p?LuU_aYYCY1d+FETll&jSq;^! z+CEnTRjGHY*jYT5(M=>sw0r2(?Wk8Do3lKdWIUN0Ho2VJTwanilunkmNxJzaUFbQM zf>FWD_asf;!vJKWf?cKMJvwA+&RTE*D(f|W$=J=J^RPcuopZwy$Ol1fvA(jowSHl# z?p!Ok`f%2rb>6M45*p1x@Tx95%~M{3cwotCv7J?QSx+Puv@adr1k))3!?FD-l&=>k zYHhw~1)KB17Zitym9NgrWPWGD>nrTQEnT4LwLP;fyWoz%TNT=!B(Hu8@WDa+g@w-d zAS=@I!|b)JYRwZLwUb#XcN1`z8hGzo@B~WVlx^@{xI~?$$y-osrsYcuFhrcS;94s% zN?=~zXSjN(ot~f;fr>JotlsEkW`=g(s1Jq%!`7{6ETkxXnIiI}JLe*L)l&0R3*))= zLMw`)lN@8A)|%9-#7L+ha0K499-WxFBq!|L5y6udQYstN?}v0Wi4xH8q)gBx;~cAj zsXd@XUdxn^^%AbS1l5L;A8>Nq9xKb=yg znbTYYhO-!~aub{<2^jXIqer_Ce+vI$&m<-6Wf}IRt`=||^m3|1c2Yf%H!=5v@{^{M zg+t9@P1XPU#eu8@_*cyC_n;~PiYf<-%#_}iQWh(+cJN*9ftbkUdUrm%w%;gOpqg~k@_h^2R~g5#rr zs+Ll-Q~?^FjwC(f92Dr>7r4%c*Gb|U`I?=bBz4zfTYIV@Mpky6_J*r>yUeHJpWE<+l)ij*9u>8rQA))vI4*gY z#a9M@u4UcvgTfJ=-P~Yl6!mPf62tTw3#1j=ZEI(H(qZ#%zBZ^}+vRJo@YMJ!CswFX zgup*f7y0mVd(^kYH~Iw^vE;qr?w3zT2e#yzzpY%cNdFFcs@BF?;!wktBFfZ((Jo6Z z05Ol*OUbNwzq@G z=^!MHU`XieXmq0}HeKx~yM^05s>(E!ZL)hge~Xt1-ZFJ2=MJsxSaTgV+@)3& ztyTr2{aN+m0#D)ric_?9<4Sd0XL4|9`+^;TAb^7Dfr`#WnOOp0aD=>YG}dzHbXaQTT7Mfx z{VeX0oV*8e$&~z^2Mt6^0T}MO^VpodrKDthGm=6UHFt}@e9C4r?J7x{{F;Itq_1)S zAn|zHB%%Ib$9LBIcDp=fjtE-28piB`4Wt;fObFxHxHCAl9QPZm zx8_CVOVbO59IV$z;X*0BkVckZy=X?v!>*;J7@Trq=5NQG4q&F-GsnSo9*JO1%FRXeA5sY%2c~MdwnB}G}LmI)O5|zX- z@aG=tX@ySPg#kLUl;2%Iw7gjb5L~X3j$}GdTDCCV%u^d=0L5wYt+^VQ<=S%PRLi>S zd1TpaO$*{SwZP{DSf|H3pM*F#+``JoMzx*{#}D+HhrQtfXHVg)eUf($$(_~H^o!-0 ztvfxh>SY4A06(HV08Q-jij|MpX=1|{)2mPA+1#qJYXF*t40gyX_=WZx40ygV5=sE; zMV`$!g_lj*3umN7Q6Fnj#nxqiixy)epylC7K0TFm%dOw$Hz|#G2e6pTIiFUzxGP93 zceVydNZjQwrAH=yVN|B3vSq%j#2tx~rE-7nX7%rC(n~uf)N~W2Qf^i94MT)+%A~kc zGR+cM>du*dl_fW&JEh!uuP#>@S>4I=ddVJdt6Z1mxV9r@PEEQ#SvChyZ3~GF&eS}K zWX(9`GfNS=UX)zn^NBd|w+F7VNh|lKw@t`quP+zoXKNpC12VwOs+&CIiK!LB8iG#0 z?InAKltb>(PKavmsK(=1Mae}ab*fl8+u~hYVt<+Yk}wSraoZgiC}WzY)6$POhAS{> zcAiJZe7^T(&Q<3xebPh3gytK0VW{hZ9zu9I0r|5$1wduly}qj#YS-kR=#+&_#e`B= zq61TbgMV*PJs|2&GSBK#{o?(MSiU3W8|4%nZ5X>y7ml{i+l|!d>?K_9CiwN(YiIS` zWi3gLQe5Ps&M~x50NfC3pUeV+XnffYnRx+w-Rd{4_~Mc&yb_^Q0jV4lyg{ye5s>r| zeHodBaGNC~uoR5mrs69$#^YmHf%OBFc*TCo7A(pf6!hGeU~GzgAMA>^tWzt#P&C0rYFhe0$MXXs>FKq{F?hRI)mh}V+-+6x*O3h=c=NO)_7InpC@kx87+lEuFRRM%X8t#p?>3noGm+ zyp^T(#^Vle?i5kA8wZTB`)7IGMbA`!k<+2fQM}`3<0+icgnsTq4E_}&YB5&&Q3}QN zvGVG*^vnwBrbTw1W%=Kg98}2@#yqm2{w4{-e$Y(gro}teGGQ|jL_m(;z-H_g-`}E$ zK=+wgRfy@Uz$EQ1F-L7}?1N?<`uLfsc#l?Z7wCvMb7Pevad{hC+4Ng|h|zqCc5`#7 zd~Hr+U7_8692PaMz5T#O#}3?zr-^*RU3yoe-M z$I@_v>BtgFRldmSk6**z$uDxc@9!>6HM8({(aJFXHB#3sR}LMWIsKXU{p#VPuRHb7 zw?6Z$@Bf+W-*oP&)1Q6(*M4^9j?Sy!`t18Z@N>bL56;w{`R5;c{i*fm9{Tn(zxI(g z+}?cUsoLkB`1PCaNq_w6w?FrRkN*7W$B!O6^XvzI^X79OJN58O&wl9RC)b`i_sKJ# z|L||!_R!ZmfA-SnKk|vWM_>BjvH36ji%;I!`QdX9zx;(?|MV{p{^FHS&VTV={?0ot z{^ZAh_VO1$`n$jKzC+i%apAe&`0V@#Uw7*x-+AsgKezZBZ)$vM;rWk!{{E-#h=25* z=Rf|%)z6&y&>QP7{Hy0To?CzZk?+3nTVHzc#pVw_RsYhHUwPLz)4%%Bcfa(BuWo(o z@oR26`{k#;)_Ub*x4!FpU;gCp@BGJS8lOJZ~i(qQ+yrRrT3iv+p5H|nbUtu z4gH1jH65RRnlJ3SY8n|#zw&71*zxISubKMB5&o^8zwzRssV9%xhyVGbhbla{bnd#4 zS9SM!nO?v-b#-Oxx;G!1xsLcme&3Am&6Op2{;4B;aAbfpboU(1-#hT1PrrKP*zsfc zo~Rt3y6)KBm8-8mHaW~xg*&GI6vJbmcI(f`Nq$~*QJzLt!oSNPxcT<&F`?$9dJ zfBZ*>rh@4|`R1Xi$FHhrAxHc|Y}sxBPh30w#I+}$z2?M!I()o5*Ya!LrYon4(=LCD z)m}F7+e}p|)BlZGXQu!34-ZY9vq|&a7Xoq<|Kx>3)BJaWLpbrH*J}6wKVs8myPQT- za{2x>JH2K4$$xWb>bXOe>F<2=5GQ)#sD8Ln`ZaT1ZGC<9!M8hIn)#zcbNX}J`5J%h ze_Zdc)%I)iJGJ}tE3UQL8GX#3^ZdJ3Yv@OQm0$VSGXL=je^%`O8fw0!q0idC_wE0+ zTK(*T|B26+^|?K@xZZvEj_!NI2j4cbkCz|glY1W4&o+sEINxaL!u?FfI`-X1`Jx$H zAG@WVJJWgOjtlm0etSW`T-f2eZrm0dem^#Tv7Yl6%AvTAM{Gd*d~s#3*?*+l>aTR# vi`~7ZWKyfWvodpx_=#v5=9%RrJh=Pr>K)bFkKKzwy1$k09){NOt(gB8-=u6< literal 0 HcmV?d00001 diff --git a/data/datasets/knowledge_insights.parquet b/data/datasets/knowledge_insights.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bc35aecae1ea0e6d2d88ca616afcf743339f2658 GIT binary patch literal 13099 zcmd5@ZLA#ES>CmsIB5+ez~0_V3{1hXZ%FPWwHio5Qe|z&u~Vn9osg6Qy`7yoyF0$K zGn@Ii_pX0H2x$pbp{0DK6)F*pAQccIBv8VK00ANhg!%)al3z&ug%A?IK=6y_dCz-h z&aQ2KfP}cZzGrvl%sKDp^M0K1#o^@-3=R$ct&iUegZB)2-`wl1^?JQ(9+f-WyVdqM zOY@!WG&UnsPQC?qri)RTCfiZEokVrCokcrl&|OaR#O$Z}WVxV zHL9w#s-rwM*r!)ERlS|&+qH?O+qfL>40?lOgJbs$-sul+8{9m&`RModHrK9C%li9n zu=4Gb9Y3Js_v!eEj;D1zuj5lX{(z1@q+_JxM8}zqm5zsWe2{kZ(h)C6%+njlHXV^bQ~VUVS! z3er577JGroCuuHUYg5ikQwLQZ&8ulqS0x@F=D~UE2tgf}nz%Me5XFl)OXFZtM)PSQpTw?D zI}(Bjlw+s78W-gZi;l};7RpcMLcO z{Z8}P*r_*wU*1wuhcvj_$PIVM|04&JzJF{(@nRaJA)KE_6^q%7Yp zvl&c@Y&Jl~GF$K?bw%NV0HH?i&r&N6T!D5YN-U)MDRV1XSR15BT59~`LUEB3PfZR( z6}z6B(q4c@NoBNKq>0R&M$}sXv#7D4W$#2~ytbTSNJ&(tgj8R?rI_6`p@(mYN&oor!~&9v zY6t`pjqp34n_RVvHS#zG^TL?Ale2kHVe$$F1;48!weXTo4&10Z6VI|@k8+-#8l}d4+JW5XiX4}UW;2Kw3a8z|S4}iiWmVxZB`$(55re>coSCS^wspEy zjOhG;scKv2sgiwyTIe960MP^hEXazgVD?zp?1RuWcd#*91aUNn1gN1jpEq^&A)oh1cnKVS;=0!CAv&&2n+-#kXisiG&$8o2ns)r?JgULcM(+StjWaE85rZT ziBm8@bZA#21?3EmU(GBHP?`lO=dCh-318N+!|7r+vXJN8M*?v5$z`E?up`ohy%UFn zF+{jkMF^>~s~0N>;z({qzQ7<@5TMj#<1N4`y_)*UiUJ5lk*NmM`ZFvaTU-r)0fbyE z3m3mlM2r9qHbAoo4^O!8yg&-Dz$%bQEZAySElbE~ColG}BEmv7jl?qPLB157rm2?B z6`(p73Y7ugM>MICnOEUYQ9=R2KL9o`vyn*zC;)h*b_!3XWSyYkpp8I-xH!Hs*g%(1 zzUssTqXH5|ZUh|Og?t;L8CKbXd(*oJ6Qod^($ zK^3GYfgF*>tVCrAhrFK)5T$ujAn|Y}DYS^GT3d8kkH;GnotizX(+rVJGPI^H@?yr7 znmK^o+Jpqq;KZ~+N){k#8rLUQxo9d(jRIR2F_QR#6d=8D%xAR`VO=e@qN9 zCxY2Sqxd`yB8UPR2-QlvRp*d|4QU~gD;tHc!3ug9BDy0UTsJTfU)V-W9mFI^94Fz^ z%l2Sq^J!E`ng?aAWOcMK4@4O&3SfH)$m)W!rQquFvZ5eBQ!PPVS(4>yd|BeoF~+-K zU2+6Uat>s~%oS|(a7309_Cu=KGjik!Y20M6D@6CKn7~%DOcH&fEuV9-w*r$aE$?|uBY4oiQI3Sp50Rn`4 z$*E}KOct}$MQYw7+M4~EhG}mL@(4X(#b(NIBz!hZBauo-h)e6RP@?K$2jLc}n1SFu zfg=$Y?wC3tSELVg_}4lztP7SLp)y3=j9InFB7}Nskm7__Ta{V3wc67J&t_=XDN{PD z%?m9Qm(xYsHAqA1C0;DarYkFO9hy?#vh)ym2*?;nB(Q&XSt-H=thlAN^;EcMVRO`= zYzQ@!5`EEZwe?MF8B6|VVq{TTiP1ozIyhyUgwXh)x2M@n@`VdIBN3!TZAe94o51@E zq`U}_4nX3miFf9x763t7m530OLq1+d6>=~3T0EEYtxqwD3{{R;x4wdmDpjL;Mc?rj z(`b!YZXrp;f*^?e8YZo>YL}k5$lqz#%NANHAQk6mmBf>wiyy9mcD+E7p|yq*|K3r7 z8av9LQ$kNnvSI|q%cH^FNbJf=a!0vvBA7=tvMway^gbvFMO^~-q2VIx_#zatxDF4+@*iGSi5LGh>Q+ccnao6UiyYJos<^DNRnb12Rq)fA z=L5Rjj0sE3l%9=~h^AgsN^QC-$u9tX3zT_7g=5IEw5#T$qMZaAeVR?|W01VK*F9?Rnr?NgbCN6-LC$I@zm zuX{3w0++ebh=wVyG5{mHXjX+ku`F_Fgh<>5$U6HHm5S-JK!-wGp^y;GIeo}vS;u}d zbSkip5>msj&Adyq#hSftOXhG2GS0hB1XnflDKh7h`l3t6@FPSfeTXG1(<+zAXGK2Q zLK}>YtP;pHarFtGH97cU=SyKB{54NAbYGX4wWcdFf*-&{mN-TK&*s-{%#f<%`tV}G z+Vy2<_hP8d!8-kTjUY^cN$dgexlt9e#YrQ4Iz_c#2G6kookzk8dQyOIA~+smv4y@$ zR3h_WK@T@W-Rj~18eDTUqHt3bse!Bmrqy&uO@oSN*Hb!iR2Ip5dEIS#+a(m028-aG*L=L z5JjS_MI0>vF$zT6t>{RW~|!et7|Se3ro-mlMaex493<7==`>^95HB|P8yrqiOacS zvNivYDt8c+G+jNAZSnA0O*xzR??WnkBeYkP!;(J%hrqSZAQ2AIfpeAVq(Jwo3YdBB z7$Y(gU+;1=PZV981>sDIMrIoA0*pmw&!YgLQK6(nmnTdekG2rE;$vFQ8W1@KLzIVC z*anwC7TOlexJ>(VxFH*Pn2*c`;kOFsj%lk-uecdAkb5m`w9Qd)X+O2ieq0B8)qp7r zs|eI|Xo}k^S9Be|h&|6+FtlxIC>~{CN;c5U))I(~I6|K!|7{0EY@T3>G+&QL(2tE4 zVlK80Mh}5$4|6n90T5)<4c-K(>5BgFigJ$JkkJWHpBOe_1vv8{XTty{fY#VjNh4nG z6?2|U#jl)MyOdNIufm^mO0{_}umSP}Z&L$1fO46*loQBGT3w_YSu!O;=L4iqa|!^@-=3eME98mB12gP6RKtn8bp!ay$iVU~z;T11W^t*}4#RwDNHjO20=o z>I7=JTJ57ao5SktZq8s}I3tU@G{NrRrv>7|t`nkAO?$AX3l3Lp@S~gAHYwbQhg#l& z4T4H|S>(;m`7%eeAyu6xSqYxD#X}LT*d6xL^6^t@R~IjzX`KoTicRprPLA{mob#|w zC)CR8OLocMmejkw2~0t7gHqY$r%NG z)HW|G16{~d`v{Y$`Jk20v@F16p`b~@GE$ZioMB4@SXyq4`9K1I5LZM~${voPoGcqC z8ELm;3!Ko{zMQFg7 z)egXtSk#BK*1?+0L3zqd3gkr3mf(y+D@rNa`eWSa+YDUVg&g#-o>DVZ8`tATO7aBM zk=pmAijr=QtOG6zLx=TNg3A$$ZpnP9$g)Qfy)zRng#}*W&XSRnG`LI`1@& zQ0UHOM8Rz?fH!kI{OW{3+4@b+(7oTGEJ~Zs3a~vq^h>VWcooM@&GH4N8q6}6J z;@bB+fCDSe8}b!_XF8zs{IF}FYH7E!8(RVr`J_)|D+}@nvQbIfmr8P>Ua(%XF;5n8 zj8g*F#|powgqVGtTEfD8N9om7Liok+!U1 z-rS|#;ZUj5W&DI68L?!?&0JK>9F)Bz>cO4rp+lC=>+08ak!yO}Y;sbst**tfW@c6( z&TOJum-Z*{y*78rCT;AoR z$FH=zJZ*T)_J|_=+|@|M~Cs2cJJA zH^2Jjeh&{`ICmt%8xVWj6*|qp>Hj2hw{clIvu)$YL|+&DBx$!g~u z*0UnLz4lIjg*7-F(}|@+yNKW~}|qLHr*7(t+c*{10EG-?#^FQiJjy z{@jM)VX$eH(Odh^KkN^JwZDJ6KX|dYvF90LYSFgTb1;pPMg5Q2dOk{6x=PZYN*9;osQR8hrCAn<+XzjXJ` z$V%8+xp#MF`al1z(-b{LY!3YqhnRwH<3a*JiQH*RI0%$@*im%Ne|bUvE*D+S~M%d*s)Gy3}se z*PHb9R(-ugU*E5<59#YseO=bqRee3DuOHG^Phb1`I@H%xUuX68T7BK4ue1)o_Nq(m zz504UUk~c*-TL}~zEUInx=LMY*XZk8_4V!gO6~EBM&%c6$gg*)OYPnI`W}6KpT0Kq z6()Rr2fnLD+-0^kzso4TY^SU5s_mJ5=JwsQr>}qZ;@({^?0fDDx9$GhyFdA#5AL~s z>4m#bUE$Tkr<-f%o_X@ZS4SV2{e-vlDgW75g0Fvmxa%tym%owM-u%^n8UOPQv(Egp z&p6Nh>wUZTU3hWUd+FaUTz}xD&p!Lg=ioHYzA?jpC+U-nj)$?&6Y-YQjh$}L>-mWv zWlrV~M{(kX4jeomj(pD2XW@~6K^y~MHE?w3xr;R^|*UrLFyjgE?44*{wAToNjot#fF(Sr>tgOFV<(0V zhr!dbpyNedY&*e5ar2JvKDJ%DQJldF0rWpgf=y__>BM+r0wZh%*#N(VAxuH3(j2<% z`k5Dmeiz%IPBKU`^s{8_-e>L{DBX?nWE7{qV|RQp_qw=U7rG51s2A_UQkj1#a|r}= zjIAIH9jlox?k(nO`wnc{<*N8(FVAx9c{G4>Qn8t++j;=)-^U_!jmY=A>Qgjl$BW_! zrr=)Vr|4k%CIle6KHr&-pnQC=8+Y;{^rMui)vN{R`kQ_jkNC2C!ff97<94H`S)5RZ zQH~vMdSMO$>3wKKC#7BpjS$24d4VlC&(OV zwe54IjyIy7x`jhp4Z&B$8M7ql_x%L!GxWOwHtaym(TJb+I&iEmf68JSo)PGDk|6Vw zz;kaei&!{ghJfyJeJ5_GezFM#B03TW{^+|)MnzT(e>3R%bYd-s_M5>t6&qFj7M*b@zpW#5Sgl}~zen0mT55M?M0+S_a&0D1V7I6L-%-DO+8A4#wUup)K~kLu(h9mf5D*(HKv5JMT_(llGDghBlH z$q;Y^=N}G%MJj01fj5FTGxJrw+1N@P3xpwZ+E4=&%MkLNJ{24(D1evx4sapN#a&&d zO$E0QlW$HHoT*rHtm2^C4pj?}@|>QEZUo3Mmf(Yvb`t;}BcDJJM=tYXRmyA(>3M^Q zKobT}LqY*>imm+78;#%{P!WNmY(I--__&GvR8|u`Nz5zT-|gO8-qM+s=GvJPt7o0{ z)pKW;SB?W5pj!f=AlK=0OY0Y$rIq8$D@$hz;Ir-*`d$*z!$+aV>_Ab2XfqD!w}Ma= z_0kb4E~)d@DJ7i;z)QZPNAIUPZJdgC`j?m|0bz*MU@>kJpz)D( z3V@m5LITahE<@0SR|k7S6nOCNHYl=_217C>;4HZt20JAs_fdmQQcsc+Y*P(|>`gQzE zekT{HlRSjINfR$`oJ>Ga3-37%n3X2)!Wa7qc_R2@12D*tpn6MC)5s$Uga(tNr?Nod zghkY=0Cpe$HslZwAprvQ?7~~Y+nCYf4#FKErc|swukgjxZE4B6vBqYt7G^EtC}21N zD1j;|YDaAm>OB%91>+@JC($HWfFL52km@nvRt#E&f8m&mDRP%$nt&0oCNTqJQ==RH z=$6Es~)5(MF+QM?7(4E-H}{@hbqUh8#@EXhIgX*M(~ ze5f8gRRw~foVpaF0~Xdax$iJ;D53goxCU;}7$vaZ#g?EGqLct7wH*D_3RU&f$lnsI z6?)-wS1DLV21i0TD7B$&0>=foG52_6y{YRO$*i+xfxp;Bdf0FTW<>dtHkef5mT1?2 zI^eT$C}<>Hu7n=z;H&5&AakJ7|+As7CNqH`-c|#8jv*vBV zR*{~CXbQWs?yWJ-g1BpnDY``0$8J#j!cLvg3s97U1nNAf1)6lKGZULM$=MVvBOiJK z_kcZ^YEW5{;FDJ9oBG5xrNI<%l==O!`^3a;O+8a}1uxIyAwJu2Hq(Q870R#6N~}N( z3!*&EQ%kHY+G-|Xz|1-I*qRVDCT844bZ7l_D^50gh zWipJ7T)S(NW+JP@5L0roC6EISe&&uWx+kRIV@P>sqMHf@|$ixm!MWVcc z{n97^-&T=9ZWlmx!2ci`femf7qZI1$#Hcr6Rkb3$cIDNQL(IY@7EsI$cq7TFhj4QIP%SS+R4@udU zM9gp?*h~-BO2#^=dr#Tv7@+`5mC=g`K7>0Ou3!>ck)|xnl4Q5zctZ=5N9q@oa>a_; zQw7Zn&^8F_esDrjhD=s`lxR=j6%C}q8vJB21vwWwYJ#YO1^$S@%?bo%Jd^DwmNK~E*8)*se+K)kz@Hf?E76Ps+D zLSm$2X(XYdkTxt^AK->;9bpLsuN2_ZNr+8SOz{#T(hb$*QF&-Neq46`OIp$T&`u)U z+yYX$aB%OOu+sLFMIWJ8h*Pw~xU-SkEyJ-I$5^JbnmBmyAnFs{X|tx_2BiL!l1U&gh`O!}go=VXv;*)m@^vJoGNM7UO^Fy;ik&T_)L_GRPIsBnncPnW;bxv% zsx7%@9qr2W4Ei_H+(%-DL!|0go45#IBrWhK!FSMru5|xVj3_+Updx0d;&MaZOiJE{ z(z0nXXgBWq!`S73>%5E@s40qXin9Z2ahkCnoh)W;g)F%!#>_ydTRHm{8BS{(s*?^@ z$v4a8AGAvU(!?2E#X!+`&CU2)wiPd`avMRX>pa8a*$AyFz{)uCGSEmF2B0=EI#A&R z9tZR-ZZ|-S`)N=kR54>I;wv87TwAtINly?aq&~bz9)vVY=7I!8+KQEQxL5n(vQ5l) zGBBRR6o{1)VQqPNdwrlTKqIfyN2rYZ`iB&t&PVzM5O@fm& z^f(Mh77`kBeXMF+5?{~D=su?C~PrUQ!O0SuaVPS#z#De z$91H4q^E(ZafNmaXdIXcRz1KqD|Tq2*sGdii)#0iCAqK)1^^k94LnhID-@dv^Wox!GQ?0#!TC`2pH8;Glehu z;tp{Hi8J=Tdy0mUEPxeDevemd#{$}>3dap$wP`Pkv zrK8k`oYb1mI@+_;AlhY7Uva$R1b_}*RaDa7bj^f7V1$i3^N=JEtF{zUiZT`6o0JJU z=ZOrQSV7%vE@I4sB7-5{sYavi(b&d^W?mXUw)e`~3V>DHvBb%Anyzw)qDRZgnVPkj zjySguJqi4y1!};R5~^V`^1y%@K;2SU+(iA_#v5#^l?apjFe5RSjz=+S2RWnRse>qb zNC2neNUjQ=MWtru;+VH(0p{-E(LhP zn8(sC?H>}aHpUypu@A98155yVEctHb_=&EcH-lI;5EW6e)usd=AOzdN5Y0V0;r7!q zv})zhbqiT^>mB)Mqv1|WL(pxEWn~+1KtjVxIq?LF02J5;4?~=vsjh^e%#?x@7N&VG!$sHDk6{KX<~X~0Y?bp+Cd3MWra8{HV1sxY0t1Iv zq^TlsT9n7tmaM^di6HkTV_xl9rjlTD4c%W|3qX{KrEQP5azidnnyEp&RHTdbs6Q4n zxS`9#ih|vGX*QBmGy|uwmvRE-3M7GOz+e%@E}O@fR?fE8&z)S7vGHMZXWO%rI^Trh zcEX={5C~~qvW^1g)Fnf&sJgj)NEglARB;M}TQcPch7*=;QtLbgp+@0!2EJ3TRXq^K z7nG-HG&HSvIJKT$lbkDuC}18Df;N81djwy|D0oA(TNXL;V$}%-jEss~fMnr{SA}#H z$Y~WtWLJH90b#))mDU<^5Kj_U13ir-g?s36SB^6QXxcVxS5jLPFpaDzRHTRFHZm%J z5NVFXQ|M5@^QB;CVZ!r=l5=f|#Djbl15`9LF*4<*Xl2Q&rn_kIHIFzq?!aSExrfF* zrjT2-$cdh|PSy#}27g05iEJ~9Jd9}Bnc+|oWLGvFW5(4VC=BI@oP5I`nX+1*mKXVX zYd~FoaIwka_w3n|7&iG{geExcx_=tef=o;7(+xOG7wL95w_23SY=d*?g+9~dn8J|UQY=k?gx^VWo(_bN z0syQYt-}knltj~Utd6;Q!#F45fTEFLN&xIgUnU_E7fLF!cVRHa3l{*SXGDo-5K)dnn5Iqg~wcX08 zO87-3%vFomw!Ux}TF&8Cr(AUVau5}q%WSJ$R52##X{u*{Q=}2{Kbzg`q3DMa=x!Cx zWKX7GqBCpwvQAm1%W0<~RY<_QhQJTy*=(xQ)`suG ztgmS2-MohB+I0Q0>hzi&Q_}` zRMZh{=3RxV0HQdt{d|#uo`92i7;fTxq{I|9Bp9xVQZWz{?MQYI7!W4k^q+-ZAt zrcnnouuyQCmHn>g83TVrwKxMP>A-Kf|2WfcpDM=-8Y>g$o zL1WEH6f6ef+t8#`DaM$hdg?s3u9~J)9yss_LIXEdh32$IGl*FvhgC?x*)ou6WYs;k zjeoY#8MO^S9tEpJs>Up>T5fJTs~Pq6gaQiS9#ybseK%aHe0H}^vfwcCG z0c~0VDnT2-FpRi){tcrQQ5DK+{b&-r0S*mEfu>kqg6Q!v4%O!8Au84Uhc596Hdw=a z7^U{KpjCX;(=}R24FrZC$zexsUuP8MbKtMWXXSweiG(Q>PVf{EA*bK)=!i8S9_5l` zgrWs`8(>sIP=Ru|2+4?rr99%XAp4w9)h!fI*~8DNkJ^fpZX3=;BXhfudRtItTT&t@ zU&nScaRYO15I^!HCUmMRoKkii?A3c>;c49yx%f9>sh+|q)ppi#?GR6J^2@&7~;bmmdn}We~Of`&14P2 zed1imV&wPnNQpx8GGf5tIK#uC8pv+jwNsA9w+Zl$6E?kv9$^2XLln z36%9+(ILQso3X}r%urfZ*ZEfbTB{!rIHe08=d#O_kH_r z+TS>E^DVdDws`Q+?RVUH*WLHr`~Lgxf8YZTKJ-ENaP!E~V@nSoKe7DCqbENNwa)&>(cn?kAC0BzW)b4{)uOP z@P~f**&q4QAA9b{f8r-U`BTq->V=>FnV)^}=YIYdUi!sP|I*99{41|~=2u_+wa@wo?ifBBWK{?%W9?duEI?O3?% zW8(|+mwC{+u!pYsCr@kN`@=neDB=LJL+GYL0@%Y z?zO|Ug}rmHUQ>T*C;P1U`3*eB{~xpR;@AFhriKrmJh9uuar3Pf4_XYp`qj1i?wfYZ z?*>{4$7f*NR6EJ~vp$O#W=E+1)O&V;`$Dg0XX?+))F{ZT z7DU8W0=%(n?u}jhU%h7k-|gI67^ryce6DuDnyvUN9DZWq`^Dz9xo=_B`MH1i@0t1u zS=8*_Do8E->3^S@!@vD7!v5EHbN5#=>~|QE69Ue`$VN3U{_jkE-`uPJGgJT4Ol|HP z-<*Ll_V3~k*V8rTcQ@D9SI@cJpfE}uH_@EuP) zzH+;F{{CEEKJ^q{x$gphmPPo(>E#YDcs~|{99!xoBQ);KmzAg|jtri-^PK#Szn8JyyGtU5Dm~(e7w9*6z$Q zvnzRX8VG4H5DqzzCJv;jfnZ#cQksVHs#^+ylcwP`B+&9GB{;lWXbz7SI5~yW@B7_5 zkCo(qINc-rYG>};`+I)B#~tO(!oKu|^e;Q`GnRfqI(1lfih$t&Blf#)}P z)d~V9XjpE^#y%;(88l`cceY`d7G`m|w3tq%cc*vnN}t`F-kRQ&-qiPs)Xwg;Y3aG> z94l|zhNaiz-`UL_>p$C(dc33K@s4$mcdUQBV*^-ycq4ws(uaiA4ou@`vl>z_)FTijV70J)VYDit8#~17IrFy(f zkI>`u8}ZYw$)&-0J(sH}v7^ z=v_N<5A3|-fn8sGWEWesK*R%5Tq3?a1A=WBe;ujsFhojDPo4=UOV2 zo!XuTEz>hIGq-^@`^br_vH2f;e)AJG5Znp6 z?*&T|l;hK0ii8CI zGE7LIMGOG|jJ)raY`C>sF|uyOaqWOcTExRO%g8Qy)n>!-TpE^M%gErdswZGFZZ|B# zh%s)tR>iK_ZiA*L3uipfFFURkK3;UKda&R%j1k|eF9bB%nZWjqoZG--wAB1UD{x9y z)tIa|oSJj9Jo+Nsh#uwKYSpROKp;sUCvlN!ows%iTzBYVs)i2}j7>p(yt;8LznrBRv{-)XcAkY;2&m*5VP zT9j(Rw#oyOZnb4h&OuE}LP@Y4cjE10v5-^B7@XU_UdHCdRVv`_8EC=C)#?;}$m2-U zfodZAYnEu0Mh4{BzGG2k5L(hOxKLy<lAFjD1(5P)W;AQ$GTFgwVT6GwGdVF`m>e_Ai6LVwYZfN(sF5Et@c^B7 zIy;sf&rV|%PAK5pA#);=<&z?~!4*XdkvGXi%>mbhs-O+usya8@Wy5SX z7T^($7X76STFg19z|ap~fxBjT$w3m-GD4*f!r00Nv>{esHbP4Y`Z0UVwtN>xW#p?C zz@TUYdf~vvJaiblBzH}yy5CUxgC910Pe96a8_s|?ca)l0vIFu~bZX72_~M>|U9Va# zfk6d!f!Fkbb;Fk91I7adc9i`j>W>@MFB;BjXMnT0DYcro4#~lXV+cJ>L*O3G}vO&b(b}l>o5u8alg}eKaX(pf8cDA!(W;FX`U4bD1qwqX3h@;VReXk7Sj>>kpn>0n z00_U@aO}XC2hHGA&!P{7>o8~uFlgcB+$9T`2w0&Ql4QnVN2@4S5U0l9jLBxB-fYOK zq{jy?Izi1S5E=}%w=yD_C2S9vjxdpRL1amQhkTAoDPf3{jiHWUmYb z0sVWcQSN{kRN3KIdnVGP;}){^7-q%`3dMFbxQhKQ?0yE;FR z@qAy&PfR}mvfIR12vm{?@}dMCtC~0DVT7pg;VROHfYz*buSFS**lpbdBP}{}*7u>A zdbEk$UUa?Xs$E735iCF#ZiUf81W{v#5n6wsQj$c{U?q|U6(2*1=oE$AXKQnI8M#vh z5Jcdpw75dX3mQfdHCOD6ye#Ao3|l46A9n0&IiU645K&k}7Ll}jZcC2=nAZqtIo5n? z1E|U=HDHE;&ifce=$ek(giaZ=i5HC8MuzT0rABC{%u{S?+6opytBP#}>`&~1rVo4m z;(XOxPPic@2PF1tm{pF0>A1BBh8X$Ak;yWriuPf)VwREWh&Qr!QaTF#O*EvyH1ahv zUBrWP-mw^gNqrG{F8C`Ucj7h`a-6U}of#346`Hf;3QblpDi-vlAhj`V9rIkTCLta( zW1H9VAtdF+yb@9{m>cA%d`$5X z9V&mNX^~>6EMAttflS@=VV%V8NSUNY{JmfUQ^Z$EkPdw`BB{5G;?$T>+MZjUOV$wv zF}aPYPzD{QnaVS`go&=qJo4KxQ;~TL+jJcoY7~75&1MC`r2;RL+AD$jycut%Bc`$` zSutYTz{IeUI&3?Yg@#gt3MP_o1WI!us6p;_KpRoMF)#+Gt{N;g0BjB-7|XzEZ_x%C z5jZ(jea#@}sHx@iR;%h+Wden|2O5yaXsOIw&1yrojglHcJjuDud=62oTu17y<|sY^ z=(3X3x-omoE2;B@l@5AMw+xK81F77aWv9W^$f?T1$|7Uw2_lCPEe=4`=NitHD3-)A zB@PzsC9kB-jp10*1lE(rUPEmcg?vc~hhX6>9hD`9wm>dm%5f+;#2~KoDzV(y6{5N| zko0fCE-i}0U`5&hfHles1#ZfAt?o&sXoPhOA3R4&1W|ic@o`1H4(O7+pq@Y)fI2U# zKZpf7dC@5)C6kOnQniYFIJbxbS8CYEej*h|_Mkk5a#Z!O=dYBbgokJ)+X|4s#Th8W zQk7Xy%g9hTRivY|bDaQ50EMh%k^L$k%7uLdLdqfJD=@ht%7W4Y*%?EVwPaQI3_>zC zBC)!oST8$>v$_o{47S|H0t&v8EKJnW%TbA}c>^|e3O1f2O=_uV1s$&94ThVNglab? zj3LKpN)rQYvMmtX1)(4$MGPUbCNY$RzL7~Fe?xwf2l{|km9IuZ2-l={OfP!_bus7Y z_6}SQxk<)EB5JajrWdW?Zv|OwXeuTwU?U+LdXYI954nrVz=?NxBvXJxz%wNpQ6dS7 z?NgA14uYUAGB}M^XdxBxb#IE!py?s0LrzdacI3Bc2toxi4iKRlS=6O2_PGuc3W+v? zCpv6!Ne=g`pDO;s5TMv#t#P1fs3{ueFjEwVs-LsPd}oEqH2s?`&0iMkaL#$Xf0 zM1z(oY)m`gie33gc`JH3NEfw=B|e-(hJm4&i<*tXi|i2Fm{xnSS*sz;DPxVw87(9w zaBXl@M^U-bfP`Yhgna^tcq7X2ljSt6C|gHMnKIM{rb;6zz;XlXULM=HBqtQZU^-D1Doc^OkO>XZvaDe>D^bIdrCp}#HMyng8*}M$*hfUZ z5PCkHMGS=KP2ZenAC0yx*qBhL>>u^aDc4YZ+T$gp=r}#L%1(%mVKYG+9PLlQ2q{IP zawKOeSs6f`gX$Ln3W=3hCwr8cbXa507Q&{Q=zBFdIy#EI z6=WHaHw>ZmZk zVDgQIo$Ss?v4*XO&I_MF0Ska&I+8$U)eKLD65&@YhkL|D4uxH4!jEg{+o5(b%$!b$ z6B3TfSbC_=t0GRh-U{*aSS>Dbr+L7j;dGS{s9BYqgzb=oeF-J88PJt_%SOHzDppxT7{E!|ip7Pl(Cee%Vv1=- zPD5$5Yd5hcrI--Yq+dbSq1?s5t+fp=_F$=4VxwXmvEIQHkEb+7Q8?oGYXuyRZDRZMUFBSg7@nFs+7G9v%_dDMqr+3mI(Gh zSxH2s!4Z8JhLk6$?ofN@Fy3+6$H@SCX@GgbiMIBnGIK7qLL-Sy;>=L8CvrSYIxijVpx{j%+==;QJ1KpIy}gjvAmK|r`A zJsyTlFj8ws`?oC&#iq?{6)ehv67?wp0Rbi}ZGIsugpo8}f!oE{9-fto6VYy>0452E z>IgFZ5~BF*_@VrYQ$x)Fl$|^;jU|TExz?l=F>=v2!2mi+fEDx;YF0hDs}LO%5!%cZ z@|5G292{2$hBt4Fqg9TGWE?_Gj5IY|awo{UoLXt4MHG4n@e7#+XduBrQxRJ%7#g)) zYB%cGJjJ=h2E;aWUK$^n5}_hhP@Q2MjDT7aN+BkM`%ry?YWloiIWHfgtP?5;r6I|( z5F%P3Sjr(j&WMb0ENFP)Nk&29+Fu}{|S2&I!eDh=h>8#qDoaA8skQPqh z5)_DESCOEq*=?Go>X23WO&%6Y(dGlQY^88X@)gSJ>Q>iQ19)l_O)TA%T7&4dXqL3p z7;d^!8%Rb)3)YEKzUd=g+0<7|kZ49{BWfPGj9z{aS~dH$kI>Cy`(XY6@=%G&`86Q< zoN|p6zS_}PTom_*{UKsO()a1!LTKoh6lz=XuQ*@`i<`zG*FiKKkrSgqo=o!Ab`yqy zuy7GKp!xA)v{iWI5jjQ7iKJ>#kCX&zsjkfI$cWfAibHAM!Z1T0k?Y9^+8EWlBZX9| zP&t3R3N2p56l;f;83TS>78CFVumX~vbr?syP53|G!lAR|^s)@k%?vp&<1z_y+ZrF6 zaZh)UY1B$jQ72ouwW?w|aT@@`^BsA!M4D{`1YV$Vpf}2PW?4?a<3?gAolu7akuj@f z`%FC&-0>|x14n79J;9q$92oRl10mJ> zTLh%C1qE1>q$loyp{ZI|chiF^l+Uah%+s@jSf_QzJ7iplfZM)6jYWF=_t)dN~zxGl=3jS1}c`I zLFXR{`;;)n@Ck=NGpK@b563TnGNWmK5lEd_ha#qC zia62;iAts@yf=&k9&dS|StpjnIk_}yViyc?G!YJTdp113X08=iI*e+P@z`@X3`fY6 z#1T%SfPex?RK{3cVGz-f)m``iPQ{2L#ggyTlf4hlQ#EZ=n@|p+Tt)ROVll~uVd_^< zq~erBPrqvsXAP8gN*YVq_p%qCRE)2J!b1Gz!w`IaE zyzO%TGPrk$UU-mtJ>k=Zohyy#2!diT96wU~c0(x_4iH&8=)j>|E&|3>|T2A>_2_5a>4tqd*g3YFW9lL z^#g;)&wug3Qw!TZIDF@&<{hW9U*2)>o?SORdiR6xIz4{>f!6aIuDJWdQ>PBT_rP1O zxaXmnkKFj_n>Sv0;v+}?=IA%>sGoOu_f2PxIsftfA7AwF>mNP7?aKL|^gL32;>5ni zAN=HV`xl-*HNH6fm-l^q@#~Ldg9mCCf6{&CGjBfcGPm#3{tuq~zzu)yUUFvXxo1B2 zj?27D|NiD*KKE?#-pOxY_r~A+?{C)kEEt9f7h|S&pvwmf|pL*U;NqJWAD7=wa5F8{Ji|cJp;3k-GAg4^H2WSwQt&V z$&LU0*H6DCcke51dA{R6AANAzwqn&cY3$2>pkb6wDk6zw3FWm-pjOT~e|@^mi4#J<%UqtemsjZhL#y zu1lO0P#t4S#4Y;wPZdGhGXJZ4$VI_UD`|Rf4zTRv4Q+?^K z-m6oaw)VEKkxpSo_gB~Br~l--{?6a=?}d7I;jbV-cMAV)#c&O|RANnaKk-yY`X$|W zJ=u}Its_NBHfTzOXhp&4uI|%a{U6=Z|IGTnSn~0oxpb#?M*4`a0xwqW{Ay8Vs{1F{ zwWs^;FLk6xWz&#-B_6fwouBFG#=m~ZzyJ2FX%;H{jumBT zMJm1x7I;y8uqvj=0&@Sz%U|6r*ABHxFCX4(9e#CF&ARs5R8KE*eH4t10Y5O9bh2A{ Z_0_vyzI$))HSi7flgh70N`Sx9`#(=hY90Up literal 0 HcmV?d00001 diff --git a/scripts/extract_knowledge.py b/scripts/extract_knowledge.py new file mode 100644 index 0000000..de66265 --- /dev/null +++ b/scripts/extract_knowledge.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +""" +Knowledge extraction pipeline: process all conversations through LLM, +extract structured findings, build knowledge graph as Parquet tables. +""" + +import json, urllib.request, time, re + +API = "http://localhost:3100" + +def sql(query): + req = urllib.request.Request(f"{API}/query/sql", + data=json.dumps({"sql": query}).encode(), + headers={"Content-Type": "application/json"}, method="POST") + return json.loads(urllib.request.urlopen(req, timeout=30).read()) + +def generate(prompt, max_tokens=512): + req = urllib.request.Request(f"{API}/ai/generate", + data=json.dumps({"prompt": prompt, "max_tokens": max_tokens, "temperature": 0.1}).encode(), + headers={"Content-Type": "application/json"}, method="POST") + return json.loads(urllib.request.urlopen(req, timeout=120).read()) + +def ingest_csv(name, csv_text): + boundary = "----kb" + str(int(time.time())) + body = f"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{name}.csv\"\r\nContent-Type: text/csv\r\n\r\n{csv_text}\r\n--{boundary}--\r\n" + req = urllib.request.Request(f"{API}/ingest/file?name={name}", + data=body.encode(), headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, method="POST") + try: return urllib.request.urlopen(req, timeout=15).read().decode() + except: return "error" + +# Get all conversation datasets +old = ['call_log','placements','clients','threat_intel','timesheets','meta_runs','candidates', + 'job_orders','lab_trials','users','email_log','new_candidates','schema_test_v1', + 'schema_test_v2','stress_test_ingest','stress_ingest','knowledge_topics', + 'knowledge_decisions','knowledge_insights'] +ds = json.loads(urllib.request.urlopen(f"{API}/catalog/datasets").read()) +convs = [d for d in ds if d['name'] not in old] + +print(f"Processing {len(convs)} conversations...") + +# Knowledge tables +all_topics = [] # (conversation, topic) +all_decisions = [] # (conversation, decision, context) +all_insights = [] # (conversation, page, insight, reasoning) +all_questions = [] # (conversation, question) + +for conv in convs: + name = conv['name'] + pages = conv.get('row_count', 0) or 0 + if pages == 0: continue + + print(f"\n{'='*50}") + print(f"Extracting from: {name} ({pages} pages)") + + # Sample pages (every 5th page for large convos, all for small) + step = max(1, pages // 20) # ~20 samples max per conversation + r = sql(f"SELECT turn_number, content FROM {name} WHERE turn_number % {step} = 1 ORDER BY turn_number LIMIT 20") + + for row in r['rows']: + page_num = row.get('turn_number', 0) + content = (row.get('content', '') or '')[:2000] + if len(content) < 100: continue + + prompt = f"""Analyze this excerpt from a technical AI conversation. Extract structured knowledge. + +TEXT: +{content} + +Output a JSON object with: +- topics: list of technology/concept topics (max 5) +- decisions: list of decisions made or recommended (max 3) +- key_insight: single most important takeaway (1 sentence) +- open_questions: unresolved questions (max 2) + +Output ONLY valid JSON.""" + + try: + resp = generate(prompt, 300) + text = resp['text'].strip() + # Extract JSON from response + if '```' in text: + text = text.split('```')[1].replace('json','',1).strip() + # Find JSON object + start = text.find('{') + end = text.rfind('}') + 1 + if start >= 0 and end > start: + data = json.loads(text[start:end]) + + for topic in data.get('topics', [])[:5]: + all_topics.append(f"{name},{page_num},{topic}") + + for decision in data.get('decisions', [])[:3]: + all_decisions.append(f"{name},{page_num},{decision}") + + insight = data.get('key_insight', '') + if insight: + all_insights.append(f"{name},{page_num},{insight}") + + for q in data.get('open_questions', [])[:2]: + all_questions.append(f"{name},{page_num},{q}") + + print(f" p{page_num}: {len(data.get('topics',[]))} topics, {len(data.get('decisions',[]))} decisions") + else: + print(f" p{page_num}: no valid JSON") + except Exception as e: + print(f" p{page_num}: error — {str(e)[:50]}") + +# Build and ingest knowledge tables +print(f"\n{'='*50}") +print(f"Extracted: {len(all_topics)} topics, {len(all_decisions)} decisions, {len(all_insights)} insights, {len(all_questions)} questions") + +if all_topics: + csv = "conversation,page,topic\n" + "\n".join(all_topics) + print(f"Ingesting knowledge_topics ({len(all_topics)} rows)...", end=" ") + print(ingest_csv("knowledge_topics", csv)) + +if all_decisions: + csv = "conversation,page,decision\n" + "\n".join(all_decisions) + print(f"Ingesting knowledge_decisions ({len(all_decisions)} rows)...", end=" ") + print(ingest_csv("knowledge_decisions", csv)) + +if all_insights: + csv = "conversation,page,insight\n" + "\n".join(all_insights) + print(f"Ingesting knowledge_insights ({len(all_insights)} rows)...", end=" ") + print(ingest_csv("knowledge_insights", csv)) + +if all_questions: + csv = "conversation,page,question\n" + "\n".join(all_questions) + print(f"Ingesting knowledge_questions ({len(all_questions)} rows)...", end=" ") + print(ingest_csv("knowledge_questions", csv)) + +print(f"\nDone. Query with:") +print(f" SELECT topic, COUNT(*) FROM knowledge_topics GROUP BY topic ORDER BY COUNT(*) DESC") +print(f" SELECT * FROM knowledge_decisions") +print(f" SELECT * FROM knowledge_insights")