From 3f19ad5d93031afac3572e7cf0f182ed036255a7 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 28 Mar 2026 00:22:29 -0500 Subject: [PATCH] LLM Team integration: extract mode processes 8 conversations - process_knowledge.py: routes PDF content through devop.live /api/run - Uses extract mode: LLM extracts facts/entities/relationships, verifier checks - 543 extractions (279 facts, 264 entities) from 8 conversations - 60 verified entries from gemma2 cross-checking - All stored as queryable Parquet (processed_extractions, processed_validated) - Streaming SSE parsing for LLM Team responses - This proves the architecture: Lakehouse storage + devop.live processing Co-Authored-By: Claude Opus 4.6 (1M context) --- .../24a05512-dc06-4884-9889-2ea978abdaec.json | 58 +++++++ .../3ead1a7a-810b-49d1-add8-752bab52dfae.json | 58 +++++++ data/datasets/processed_extractions.parquet | Bin 0 -> 28175 bytes data/datasets/processed_validated.parquet | Bin 0 -> 15271 bytes scripts/process_knowledge.py | 163 ++++++++++++++++++ 5 files changed, 279 insertions(+) create mode 100644 data/_catalog/manifests/24a05512-dc06-4884-9889-2ea978abdaec.json create mode 100644 data/_catalog/manifests/3ead1a7a-810b-49d1-add8-752bab52dfae.json create mode 100644 data/datasets/processed_extractions.parquet create mode 100644 data/datasets/processed_validated.parquet create mode 100644 scripts/process_knowledge.py diff --git a/data/_catalog/manifests/24a05512-dc06-4884-9889-2ea978abdaec.json b/data/_catalog/manifests/24a05512-dc06-4884-9889-2ea978abdaec.json new file mode 100644 index 0000000..ebc0cd9 --- /dev/null +++ b/data/_catalog/manifests/24a05512-dc06-4884-9889-2ea978abdaec.json @@ -0,0 +1,58 @@ +{ + "id": "24a05512-dc06-4884-9889-2ea978abdaec", + "name": "processed_extractions", + "schema_fingerprint": "c2853ba4c0cc5a897b37680fefea2708f51d51774226b00b72444e79d594c874", + "objects": [ + { + "bucket": "data", + "key": "datasets/processed_extractions.parquet", + "size_bytes": 28175, + "created_at": "2026-03-28T05:16:57.437041979Z" + } + ], + "created_at": "2026-03-28T05:16:57.437042648Z", + "updated_at": "2026-03-28T05:16:57.437403737Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "conversation", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page", + "data_type": "Int64", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "type", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "content", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "csv", + "source_file": "processed_extractions.csv", + "ingest_job": "ingest-1774675017437", + "ingest_timestamp": "2026-03-28T05:16:57.437041979Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 543 +} \ No newline at end of file diff --git a/data/_catalog/manifests/3ead1a7a-810b-49d1-add8-752bab52dfae.json b/data/_catalog/manifests/3ead1a7a-810b-49d1-add8-752bab52dfae.json new file mode 100644 index 0000000..4ff0533 --- /dev/null +++ b/data/_catalog/manifests/3ead1a7a-810b-49d1-add8-752bab52dfae.json @@ -0,0 +1,58 @@ +{ + "id": "3ead1a7a-810b-49d1-add8-752bab52dfae", + "name": "processed_validated", + "schema_fingerprint": "8078de214059a2a4582203b32c1172fcc311c2d3c762a3b075c07268308f1841", + "objects": [ + { + "bucket": "data", + "key": "datasets/processed_validated.parquet", + "size_bytes": 15271, + "created_at": "2026-03-28T05:16:57.439472748Z" + } + ], + "created_at": "2026-03-28T05:16:57.439473552Z", + "updated_at": "2026-03-28T05:16:57.439749766Z", + "description": "", + "owner": "", + "sensitivity": null, + "columns": [ + { + "name": "conversation", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "page", + "data_type": "Int64", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "type", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + }, + { + "name": "content", + "data_type": "Utf8", + "sensitivity": null, + "description": "", + "is_pii": false + } + ], + "lineage": { + "source_system": "csv", + "source_file": "processed_validated.csv", + "ingest_job": "ingest-1774675017439", + "ingest_timestamp": "2026-03-28T05:16:57.439472748Z", + "parent_datasets": [] + }, + "freshness": null, + "tags": [], + "row_count": 60 +} \ No newline at end of file diff --git a/data/datasets/processed_extractions.parquet b/data/datasets/processed_extractions.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3574aef15a59ea159f1da6cfd5c547d19338ddf0 GIT binary patch literal 28175 zcmcJY32+?OdFR1H5@pjCMFA9P(^gvt0Z70Aq(~i$zNR)k~FVEvm)M zdemr_;%2)VHmoMG(wb#=1Hva$mjxF1M zb4%aXw`_TO%hq4rvhC|zwtsWW6%gVT{hJw>F+yyirG8%LUi$X=*IWGSLI3(w{;3B+{p*l_J>p*{{Og>5eTRRo`qwr8dWCmmPow|_n6UqwazXmj=R7WcyEfiG<3-{wu)kxqN> zj+CIi`*7v1zWy!!Tl=?Nu|2Qw(U|H8eyuI#_6e@Fk-{eAtfx#Ia>A9?=Q$9Mh~ z{|@hdUH|L*uj${_e{KKn-F^Mn?e6a%=-)#R*Z1$GasRF@yl(B^*S~G|_FZq_${YKy z-?sm^AKCxgo!xKhe{=s0{lTuc^xw!A*Ky;P{+s%5-o2G?Zt35~_TM~k^9(PNg6%A7 zIV`uYGhS zjVKA)txkLOXs{SA)uJFNuSWGS7>d^7T1Rp?nuN>I@JLW>gw13%ZU?K#THM+QhT7G7 z#LXd+I})_wT5Sngqvd>vSnAdwX-4H>s1~nO%OPzV+}&K=Nc4$zvtgsU93}0bMvqA_ zw7wd)gXL3$iv=KBT zKXAIl<+v5pq7`OS56c_l^iq^|3Vte>2;1e=pct=3t*E-v2!>{>qVSTg_wySFqpTc&;BFd{wD;S!L+37|n?j$`U4oeP-A#_S-)E1R< z!Blm5nUNdqpdHnlaVx9^Xt7S+sXj@kzd-_PbJiQ1$lG``-(r*6omQ05GmzG-S080= z5WboONnDTGOl>9Dh&#c0sG;H@T#Kuf+^^F`zPqD^Mmwt2sw)v1G?>TYY8Sf{ln`mxh( z#z#meQ?u)#p-l)^?OkYw;Jln=$0>}k+RLK;Efh^wuQoLKqv=4-ZHy0ZcqgOJVI zVB!cvl|jNruoML?>|-mTK@=v{+D7gT^pZ*BMf)dfbEyT`s-@x=n9iCEW-yDntH}za z(wIn`y{UD;FO}S;b}fi~nRd6*-$aYSRs*E59j*jr#3*Ic3wynh z)&+0w1JfLr8atp>z5umMh*-uY=C_MB7p`N|H9u#WvP;iTMtAdE_N&ix)g~)xvisd_ z4WuP&8-gaMn{W)uDoI6Pzd+tH+fNs`4nZ8rC)*jas?p#)vqtRV^p>-tV2gFUO7L>WaH8PUD@1 z>$&$eeC*z>Pu=^)A6XBA0LJI^G0OXl#uu&_$wEIN_>!{u)sU$l9_EGx&nuOvTur2{ z*yiLCjgJh{c&=BiAp{L9#l`*JA)IypT;XJpzwf@n$-?5p+G&X(h_MtdW9YEL+G!D3 zk_odqeGqI!?VPx7HWga7tA#qiG@{lREnw;8sMW|lLMyk9L%|Op`*cvq`WRZ24Gkte zV-E+-POBLw(a|8^Y(^-CM%=20wd$itu!bLTO)xg^$Ia49Ic?L~Y3`Q1K zZ+K!JIvQlOPerQ7s-n)LmdV*jkZ;HJYMF)986?T&>GJ{XWeX0{=J@`P_N+(?x2caB z=SjI$I`ySsX#)mLu$Vb9MLK>We#z{9auc7{*(H>85{)M94TdSlohEkFR0(cCyIqR} zVlh!hW$A%XXrf}!0l~0s%6b7-*?tBMX+p1i%}3~~9SatsW(`wg79XwxC$xfOBLR`*_R_=U z+6wbe#vRDn5`gWnSJUUCvY|!nW_wk{PdOvFEy&2~2b2;WJY)84c`!(|L3cEfWNumB zBj&eP{LzMt7ffTvK|g4Y5n8gypa%`0X~DD3m-PQ!%D@;e;d&QmC{}sE6IA4KI`eR(}Zgc1p&;Y@LGURCbVJtdyK~5R8M0w?ZA}` z5FwccYOxU+9Fw0kId^h#VQ!{0{m|m{$zoydWHDE-h+pSVW0&k*`Vck&5N_a_*Q76{2!VU7PvGCi&_*@y zK7u3vlUmB{UD{|zNeK;Ed6h12qTyQDs!B9orCE+<)n!qa{Wcp?i3H&yb`vq^fCw^c zR5JT3{jtz(UPOb1wZy0x@vaT@lnuErtkwjp1&i$4Lc>j`Ej{3r|J_o_CoGl`079l@ zPG*Q@h(=41OX7$9x@E`qxEYG3A(dbP+V-<4`&_g~*Z~|&!&zLO$SAs4BesxRkYB$QR^ zQgG)5D^lzsCHbUe#%)RJ(*RZFs?jKFwc=JuM$de%QgmKwh>gAQi{;Lvk8X(5q*s^X znCKUXUA}Ch!SR4FgzobpJ_|FJj2ZBLA!}3o1H+&|;*Q)(GlijT>)J>~t7m@) zT0Nblb9+lb>*#`+0$C9?diIQZHY}H;CakKCTG6{q_vn3Ome5^?OcE?zUUqtgw^~T2 zAwRWoHm7jdMYIf~c`3t*K`02qD|$R?KDhz8#MQ$Yi?&E~ze8qYC@4 zey}g-ji~p1rL;uP3Ah4n-inrCa|c03g3}8#BNptzLujwUB*X_m6VGm1OH^NqD%!eB zF2}yL&8iWv%B|!^fToQkDmmm^E3Ms5l*zR+3L56axdcT* z2-R|Q621cYn=6)g>uoc^b~(qmVH?kFsRMfOYhl^wTE;I)S;K&(dO|xB5vdFLIT!fQO6e&Y_DZnHUd?;u{UAk+p>|d(ThqOBG6(vWwoAeytLj;g!SaS^`v5CFyF<5Cd#IB2w zjx@47>N@v~DA#~g-VJrP26Zxk**juY;_7f9Mw^gUPa~qIpe-CN0G$u>u9FK{Z6|jzM#1$&?9AW< z_6PPbe!_TKdLUTCU7%CFW{PcC8BKpL(PB1+-od$$*9)|K7y$t(vSciUq3O4k;WKnl z>@-P=h#PIv*=9O30{#H{5xdqxTrHeFYVofT8O_Rade1~V> z!j$v<1elcbaHNNfF6WT~yUns-<5-@&v`K1Eer+WBKKjW^G~t!=aU-sizsuN?sA7^S zzQiIjQOmWK&@DI)@|To(P|PN6fmbt7bvvo^Qpzn{U^Z%`+F4xM`Tlne7)Sd_R4pyNI5*NFlO}%*HkIc)Griu7DV_ z;F!Soc}sb*bfRg@jSv8!6cmwNVe<+4VDpJxV=qbQsfW%I((AAp(FP#Quq5Cgw>1)> zOKgTz;e`~orbydLCu~)^H0s7-#>F{fX%=BDYB29|BnBIEc5cksaB0j0MqY}~I|J){ zG*&7uOU+Qgwz5Zx@d~0=SI|wV>MS|_%58*oT#dEy+`-|Xr(HR0hGxszoZdDHD2dzD zlXTG7Bo_pEU1Zu^>TrpfSw!bd+^Ue*Rj{L@2mt?LMcg*f6rfcZD8jh7$GJM)gfEK3ZWm`boho9hzdpg0!R7fXrNTt+~N zwRv}6G!c|?o{1F{M5N;#)Q}r3X;l~&PQ{(46S77lfocw%*ah#3m$;C{LH(DVCU3Mp z=n+=x)vAbx7=XVW4?8sDCVslW+>{yCw6ziunY1rq8w*WIZ#W8!2{u6wPY8lBN;)G! z2F1zUN+0IdY^Quvg7~an(GyQC4TAH1j&vXW>T9N&#;T2c^0ZxE!^W}bfoWSy%4i!2 zVv=kkdruGMd$6ede8I9a8AnMzwUVJp3>0CE7CN2%MFNsll9{ul0aq-oJ1n8|Tim-q zMU{+b4c-hwB?|^gN?i0@MN@-Xn`lC-lCcstbCiz#s`8XjUiblQfz+;EU4y zBf%m{Qh5Z*S1brILl1c1a^8D*G`2Q4+2G_7Fn|z3UNYp415p^&9YhY z<>tW)wamF9l9`%lmn#e8l?Q`x64$K=X3+0VT3+L_`-yW5Bxt$&A?%_=rzL~USeh3w z7m7mGp04mUoQU{vZrCLU_qc3>1y~C%>dNc{bLa>K&`<&udCVv{B5_Pz!8DGQQ`X^< zJEqjlt8JHJpIP78bfmKV{s>X^?2v$_R-N&LcqnuDGbv2)Va!ZRL9{o<@J6d;0ZrYD z=imTUQ@hgT2nYO;sOUV|+opn4+C$k*SetyiLa>XXE=Y_rljSHvmK0cUwJ^}d1mA`9 zP3MTIPG|*w$WiB4H#dFu0eYvj$~352$3^&FResRLma7PvGNSZHFB~X5OFaO^@H1qAfg15BzHUaPesdN zr`A5IAeVHtT+gBW@zK+TVcSfjiUqU@q_9ei)2-a5*7QCm){(*MT9>qZpv}b4V7VdR z8n*{8C(|UGN&?chGQXRay@Lzsa!sRYVwPdM)c8_`wv5d~sr+inm!SRdvCl$fq`YO5 zBiR~Epvi=qq@_J6H6LvTn%VraoVu8aR;WnRZO)9Q8D$4W31c@B#+8OPuJsvn^hXPQ zn&c#i$~#(JS7;?t8{$sPpOO`hPL@nzE=;p%09eCml0!X+eK>cPz6us>bXrRDrD#sm z^RDNRgA60V)C3DtjDrdWtPI~b%IMRnNy;HqzNh`QdNVB+4563`1q1Q07_vJDeexqP zqpf_(7xepdL0g2XD8lm1n{=J%*W|+Kseni=VGhH`;yR2J_|8)kh=FxWf_KRe z3mvra(nh+vTC=I0Q$(%WmLptjt2)KR?1*PN)68W<*msTJvZzKbd3IZHx?untJmoZ4 zFpt%g!jcI1+!q*A&L_BrjcZ0R=-6{LQpr1{f^AKT^qcY4q?6KM66u9}-oL}G0zsJN zA!q`e!e^9&o2YgSUFNVL=2x*k&nlv8ms;X~PZ+n4A})+*(xp`RtFKIvi_6SWUg_Wr z)PMx&Tp4n_p)N34uL6cp7Di0P!qCvzIszkv@GK3eXit)RyZXr|Xhv90_{T7W5KJLO zhprD7a7emEd_CoT)*mckiqd8NQJvvN?g+ipQEHpLl(>+EECCEsO@zzsk^?d)7H4O2 z6>eO#NRgJ+UXPVkWhY@O&(q4r_EZhJJOPLnK$rQ1LS%z7C#I`h$PJGaDBy8#ZiR7F z;DF0@P(d{;kpq>2LlqyUQ#ezVUKKPZ4Fz&#NQkJE33xcg4-7W7V-U@mIsDIBto*Rh zL^~&5*tBDElQxrHiZGi^zGrzoUmOa0zfd@TQj@x$NqvA1GlrdSx8kV#T9m!6#}wPH z5M>I64A|h22(V`ojDtq4lw*)2SS4dBA<57kU5hg^OyqVAM^r|fELFv2Y<)BJ%(a}n z;z>PM7z)Gs`bUkSL4U$1P`M_CSJY5JYVR=M-;mE@&(YB*m`A3g#fV-x zQvIWnf!ja_L~}DHR1YAt4NgmB4&N&3OyZhbw>4!p)!`1*xM^6KbmBJ(#6Ub^QkAH3 zPcRjMK4jYwQ}60f&calQR*D*CItCr%KuxJj56odv1=xIeI%a6Ad7M@CLX(_Ur-3Xa z)`=>-4UsD4Vm)GP6fqcOYlWuLs$dgu*&aNHYMVFMwainnfVmcsPr9+vBuj@rBef-BZp}k7TW`$&SBPiZ(mXo zoHJZA)DjbRM_w{;Zf>eDd|mRQJQubCyR52dOhc~iSM0#7^&qq1SyRf<${(rZ5ftzl`jt#!50GGAU`a@F*34g2`G3DoNpOi?vb%qoW<&^(Z9e zV)rPhDr7V_skg18Q|{f6;kY7yEvD!!>jlCkxoQyR2N23J2Gs(Jun06Jm*c)q!yaS>H3}0(ba50tLsEu@%Mq_|z-Q$cz0LZNGtjmbqEePy zVMbL+Cjoq>+Ose-zg_YwrM z`E;}znXG2IPT>RHz_bfL9m0tWrPN8s#)-*@bSq;YC$#t37vi<}7uX;)R+n5uBP$hbdBbWjh6JsKnJfExD! zr%baYYh})DI*vdkC@f$#YQ&a^+kS+b#UO(&GHW@T!Jv3*#?8uYz6*v>;$|^j)UO3c z7_6kqAri&ZGr}JoSESUv29FrrbJCCx@&QR*z=fFkcIw#06x z345n6g9b(I5);Xi#lfsmH%R0;ixUAZV47C;C$f8Pk~oAHbc%r5aa5UcSB2F846_jJ zk2!#j_O>oMsm7>xSBO9l1p+?9NEyC5DgvMWMy2_ow9CCDg$4iriqJ^>0lIpe2I zc|KpsxZ&F?GeDvdU)!W#yXpc$7TQb*Urm7cAEj=7iuE8r6sr&{l>=-U@ljJ9injNL z+FX;R4gg)j&Sl1%^rT==UxFj+OGk?4H&w`+I&Q9et5XATV9A)2qVrfLh0{bdM^5rE zz7MZ{D!HH2qS0Y;kjre;;0k%LF^*7ScgB`%C4+|5#Fbs&&1|W5B+~)w;1_Ti7)==o z=gnnj_&7|Z*dYMW0Vpf`b0kn8M>xTH%)N=}cu|sJb>U*)x~ep}rAJBL6MQKxb>u~B zw`~W+0FL(CcnA3fhP#Yx=qW3=b=Yn^WipU|m zX}zvFZ|J_Rkrg2F_|r1qImJiv4il+z&fPz1Q7&k=FqLI9HhUmUNQ@Hd)3oryYZ#+iZcV*>gC0*rEW?1;!F!0PjZ* zzgtNk$ES|N5Tpl;BODt!BIl%p6)ihabauLws3d^heDIr zhH3~`Cca3WkU>-%u6s?|SD`LTx4B3#1`^+@Eh?%cPV%&H`s89^cDj^5K7DerG(R^} zn0%P;L@ZxN;ymvx`3DKS(}geEg98zTJKwb_?d}5jXU)tSfiI$&tluPeLsR z65{lk^>>H$ac9=YU9OKivOZiOz(<*S@4Q=g9vn~aJb2LEX(wCVIWb$vf_L;)bD@w> zPZtMm83$R02t#LwT^AA(M=`0p%MeUiipdlClgFn^3)81gPZt+;h(e_|Q`3`$^n}G2 zpL?P_a6y`C7DhHLO0-@eq$Mv)ET)u%fklDvLZy*2@eHMdVu5 zSv7)4l+_2L$Z;NK8!qH)Bf^PikZINpg{` zI@9%5#HCbLsx^2ElI{timMX`A8wasBYl@Z8KIsueK$xcU;bTJp3tj@6J z#9G|ub#xC8h`g>uHe-jLRpxHsKASzRAC?zzpgvL7I0$V!kCs!I|KcX(F?^Y*lx*&% zOE$6uht=-LIVqpZ4XPQ(uNDi7rx!Vg#fhxxN%wAkVQz75a&87aGJ23-GW1jV#e8Y8 zFuhPL*9c;m|eVo0}ptd>x6YeAmQj5T- znp3*aZ7k;7o+Ho|qcM4(>@s_t&dKSBTu=ZMd1+J3#Z@?saCj+dc2UXZyK>4ekV?c0 zwBR@zfQ*~C%3QD{Nb|2aNQf0*XIdEUFkNZ-H}?+uL1kI-K92(+iK0yS(cnS|ra*2G z8UPE1ocSNl1t+D-UD{RFhgJJxG`T!uaWk=F!#|lJbe1x|FnOY|I6b*|dSSXW_u%xx zgN5lcY>%MC<@v;R{z4`Z=bRBSDEpS4^yu00uTF@%)=9aGw4j7_!yKkM9DrCjJp(t+ z(Vb@+r^}rXIbDXbdT^dIvm9P9JT7SHVguqB8~`bzrc5?k#nQ>Nw<`ADA>cxp5S%0d z66eWVtV(!N66=`E)eei!p1U|+qRN<@%o*pleUTp(F_eU@Fd^Gl?vJsExQ=wzW3N%K zM20q)7iz^L2l+kT3g@J}HcwJ1v4yDIqCAIv7e4oi?$80Cs6BL13n6UBe2~qdS_v6J zSb>>tZ|FVgT`xuETBTV+>M4Z$u`D`?BM*m~^kBYF7#`l-1>8@3NZl;Jndno*$%%zR zDVkf+YG$l5%9{dfCvZ)E%&Mliv(Z-KCYzgr?S9UBTXYA41f<+B@@3ig}EP~C_ z7`O|^5Wg|nhAA$W?=N3SZdFdYs`nOyMPMg*K@gW)#YQQg;vXuw(R%~Vib6P$SUGN= z7^ChT#sMN~b%H~Ya%M)_IypUmj|iD7UC>`z@X~YKuv)KoTn$^bL7_9;qz5jb(_-Qd z7M^z8ARUGfI3>~Dvz!pTrtIA>hjZWvShJV%E|Tf@Px?$dYRAAbPctQ(g#RMm5Z7+L zTN;Fc#pGfQn9(zELIrfBI)U!DFo7Lgvj8n38808qz7OePRh5=H5U~=gu1v)yZeKSG z%B;J|BTxX9xs03J7&2k21{H@>-xH?`GiV(x7TDMEVoolb5$}bva^{8pwpn$>$o)6 zJDkuCH_N`zz5JS<9R>!=ulMyBVxg)C+$~1@5$7EFG=mPP({OL81m53;Kp~5s3b=ZmE zM5o#{>WH?O#$UQ$ z8S%`ebgB-G>*&T-TcJW2IG5FkWogLVG3MzKk_Uq!4XWXo+OXR<6{aZQ3Wab6FQbH{ zM|*CSc+aK26w2JXmk;XwW&12GTr{Qb1V5GSMK-_=QF}-NdG%lm;&FEHOvZ~8Yg=h7 zseX&`S|sz#iRp#uF+C|oMm2w#+qun5hPzLp(bQOFHOY4Bh(~)-&Su4V)inz9h97v9 zPVKo#s~uIz5@f{6%`Hq#F9Z`0TQp4!$;leUAEj??IN(heg92q2enX=kZ3(2OryCn<; zx=s-_>ILojRY-uHiS|$^R<#AE8JU5Nq_uDY(LEP}M<3d;aS&81(pDFgeR_C=i_$@s;&N!zWM*B!9&7`|WgUmu69lv3D#g_v z_7pDfy$_ZdB#32|3p_1inUnS|k%-23g-h!2j&c5jt*AuT!qCQ*V6J!~ukW((x=sW0 z7@e?{8+`A@6HHO1f4rMn=`I3*0E+e6m!|I0uN+j83_8+Ui+6FH(uXx%Kwpojk>lc# zIb|WahM$7RS}ClM3QCW>S=vwfXsJppz$NodaYHp7>vktdYbr-^XVLMCGS%rlDtnxc zZM~H-iOQ}`mO9QDfRVlG_l3?W?iG6L*-DuX+G=Ip%wkSfXiC)rr*y(yT4aJ*OY4C$ zm0a`+W6D3|Bv{l`6=m!``oLWii%nANjRGzT(p!kkjw~tvp$cqG3``bP=4ra%ziMH= zg@q(udPa(QoKgy@_)xWipN=WEta{CgfZz|0>ba*aW0EY<=W)P2tDNVl2C|i?Y;8l* z^cX@PdNDkVp`(g66Qirei8c-CuCEBYs`zuymj0j^Y1yYEG2Kb+a}LUYN25Dkf~P8 ziRbflr(yuBM-th^3Lf(4qvI;}hy^X_K_+k;M#|>qqD1F=Lz80U!B?B!phOJT5`3Iy z?DQ})2?R?(I>WBKgc7ELY@ss9qqzOB0etJ85Gh+AOCpvekVQvRB)JGLWz3ghLFcBp z13C8GH8N6mM~|3hcV!1j$TDoGp_H6Sri6ITP0=uqHqebJ&&*jWrij(Y!^--TuI8L9 z=F3?$=}Ul*0@jzig-z3wWOVLG8H2D~Y!yQ?TKR&-99C0!V2P7;lB>s?f;$(oiw~kqo zak}*-Tr_35Is_ZmbTEe(MT^q~svDFZ~56SvRo2G?_jO zq{1`uYs@WTnbIpj&g2HJqfe@;-~u@wrBa&@WA^(fDiQ^BBx~XWtVF6fPz3GAOg)Se zZf?ML(I{aDsoP-i=-uRX1{IieeG5O~HU0D;*T?nc_~1yO4+mY#cWT32Qy%hBB>`&M z+@5hH*ZOEM%j5IrN(*yyi>1@W=>;M{-3#;iV(|>HO+i)HiJlS*ZygOr-FocF&PQ#! zvRh@$%sHHmBodoGLNQOHEh=XFL78Cn1Q+IJ^7HJWd%|nFIy7)i8>_umwT=myDLvCS zqrSAehse1mefdWNIS0hd07jPQf&?j}-i9PuU_IMj_cSPCf3t^<1Y<4_Z{~X#W+e%W zf4wR1oXB(cKreTeF(>0XPY@9!E^&a4ZS&kL4ebZZ)*NiOP=znu!D#?563T?3{R0tE zDMU)Kd<<((l(Qs(J&_Llbzy%hnjkcbswx5(r=~d$kL2Ppl24Vip-7Dx)5BQ3#Bj-d zr!IqJq}B_)X8`eA?~)TGb}!8U zI*;Rzew)>#KDQ;r3xrk8jd$$)L*C5vz}Z%BjK##UnScPk&NL;1JSVSuoTADW73$fB zUR7~l`j8^Wq&-L6Q6Rpm-5LA~6%L9SFa_9`xAdS@8Tw07|2T#zFIr_F7dC5iY^)`t za2bT}T{knXMsHI;&c#I-iFw!qn&dA8zOdy{juC?Qglx!ushbM3DP~hod&*L{%kmkV z3*};B>LD#s?3Ogjb!k=|GSgv5ZbIC3SP0Xvr!`ouu&2Isj3a>5xLtrY@B?)lJ3ZkBF)3zrE`_wJQDWD*P&XN!WW*n5lF=`jJ8z>%OcigdC? zC){?sysJIw!Nox|Wp*5PP$W?&58XpUx;a~ahRrcl6Q97v_{5P0l68o|ER=r`idE3O z>ek33y5!CX9vRtw;wjjm=oU4sn0haPhbVtn-k6mv!VgxsoVFGAW1~b8Z%s(e;Pwzk zuw(XJJ+Kr}W5zQ=Zo23Wdo&{yQ&b>(evKxvxnuW00iE^2al9$5_`_%gcZsq z6I`#O?I=6I|DdXB8SUqQJy=1>7`DL7kOB!SajYB+fm}>yPnvC+UMvp?Bn3+P{!CUc}{Jw4qXWAm`0*2M#0yx z@|3hcAivJx5W`6OMAKq=6mYf7k?vEh?Hqk%ji^;H9!GWhg{2R;=T@F0gmLv5Vf+Aw z$kNC(@2wej=X71#Aw?SQn&7a7uIbPn6cHA>yw3O9xvN&ic-p^g&dU7T^LCkZ!R3N&99E3~Rlv}ty4Iw{Ls!T;KNov(IhWar3j!ZQXfz{;_S>9NYD?+pn9S&u`iDj>qy> zzM=M9{;D^>r*C4%jUPQWarJ?xADej1tzUU=;?;0@=#`oag^7(aLI2lpqxcj|+0T7Tuq55D=`KYrnZ zH@vrR_lJV_?OOQITRs?k@pph)rTOB=j(q3rr#^Pa51KE1 z?9M-Y?7ENN^~X;XKYsV0J^QJTAN}*MzVz{X{^GgoK5_3~{a*1CKlQh-eCiW#{rexk z^oh6qufF~#j%~Yk@rnFZZ~62S6R$n^@)MKSX5`zWN*29)Huu zU%!wK9>4j;6F2{S<;B9#m%jMo{Ucxd(Tfj^f9vL#W)A;;<)zuX{^W}Sx~lm%sm1>AhR_ zeD|H_ufAh%`1roHy-SbW^5yR?ee}qmez*L@^nveHK7D%md(mg3FMn_O>CT^iZ{^v? z4}5?1i$A~o{p!E?(wD#guCILUPrrZetKT~CgW7NW{_+p%SG@9>U%LLP|N3v<82^X= z`9I#+{KnOH?tkQ4`_}iju0QqZr(4e-{La(K@8+-nT>GU{XFk{YPWaU4)_(BFcRsiN z+CTi+>py?~kDomA`HesO=TCk9(LevS?|lBRB4fAY6~^wcx|^zZ-j zJI}oLe{I?O>|@(@KNR%syLRio%l>hneS0tS=YR&b?%UJH<88ja!*!7T8Jc~2v%gJq zVH^MVT=wm?Kkstkw--p$zP-o(FJ*P5cZB_WUfw$JpSIxP?A!Bie{DJ7a$~B4+nPi4eY#S z>)xGUxGY|COW%wP^uTt$*sjxwuFZBmS2X=jH(e#9j*;@$_I!QEzWw`V4)pCG*tzdW z-<7-eZSHoUj~07Axt)IpUfz0O$N%A9Y1%iUzrr)XYWnzVPh0!$6(QUf_U-xdm$wbv zxaU8AVcWnz+tR0*Z1*#PS~L7@{M9{(64OW^Oda!zPt0lXLoAd7yf0Si@M(P*;~^! zy+ZH7o-coK+rYh>dQZvk2Kel!x9#Eo18n($Z@yk*{C_qQ7f%@$OXvL9uWTE*dCw1- z^EbEj?RohN+t~U8JM;$YxNra7o%w}@xid!{6Yl--)-nC8uTSLj`BVI_p9k{!_59>Y z{(fEJHJ{hF`Z>kte7>N!TuNW{1^-XRy8e)UP3c#DTEC)s`af@7&+0Qj6BDP_-8+7V z`hLBv%kNq`bY%SPXO4{LADH6eU8mK^b;X-gr_=lM`TOmA>%VfMw*GeOeyYGtW5;z9 zlXO4RRPgF&Mg7n|abMY;_Jv=WI+$30_)MjiUcJv2bvQq@?$$w`6T{rlrNP_zfdYomrA9$>qPsz z2k~yx_k#Uhuc7NYYJCZ3+WmUuxx0?H>pF?E>pT1U%lNtxb#`5Sp#8Ah(Lu5sb`viQ z;_}fQUf}9OFKF$?I!U{7Pw491IE)g0>%_4aCr;4Nm_jK^<7C$hb`#xb@8avmez{bh zE6**K56e%M&zH|%`SsHL^ypd6e7P}t;HRF#$dmYgIPuu!(&dTElb0X6eD3lWpn#w9 z-$wbaC}0AY@bz=%P@1*JXYBDgd;Cp%tl8rOdvxrvV~;;?kH2V-^Y(bd9&g&?raiuG zkDfgq*yCgN_yv1>+#a8>$BXv(MSG;Ue9>%t{far1p0&r{u*VDbNW=5>OXg5|+8(dl z3FK7c#BptZ@FRm(`^;=gRHL`HLTZ<%^eVKUjP4 zae43C=XZW*&VTHc_iImVz5WM(JlUoiVJcNt8|6tc)Fpg#pd+v8H5|Gd zCh%nW$-)FK<f+_{&s@@H{fCngAXwHZ5`=KtQ=L3w6pJqy+DOcm9({j zPbQk49xtk%m$X$Qj3V6-r&ql;?o)x*u6AFVntE6V*m|gi&A3CtgS6b_c<>sG`vK>j)2A zx#b{|c&m!Lx}oO%um!!~njrkqSXsJJc;I2Gs_8T+kK>0y>tv}KI*y&Fp9!|QxxKx< zy0fsbthOS^kfe~k+wRAFxFQa8ym!Sss^J96*N&@_P(cYz*v-=c{zJuq3x&0C9mMJ@ zuhYd`R98pMFzOH$aDyV^j5muugkl=WnPssST3CRM2AYmxOF)wHVyG_RTzoOLc9&tV zsRdZ`!QI!_w`;d*)tejZG@^P49|+R~ZV)7xgn|{sO<)PSr;7VA{6&S4g0|m*qu{hR zUKX*2F+<*ojjnR;21A(n?|WT5pxcI)6U`|na2L9f-#V)EE-Kdh7K!U%EzjTnUcv-BK6D#YIxc2VgEi|7u+One7 zY<&0HR1m&^Wj7E^=%^-`Sl0eFot?SflKlz~ws3BFs(EC=DIrgkF76+8osF!EJzJdEXJ5k$_< z^w=D;pb5ZPq70?90@^4)?7=peApQ^vwJJ_bDYc+(g^e`k^yAQ{v%`{dLmuMdv@Y<6 z`R9OaQ9%jOnav`i7}Ym?lB6uz9TRC-?UHr|=nNUQAy$+&0MMK`5JW2%Vn`MfZ1#dA z3f&Z7n+ABV!Ps0}7sUE&DR5fH%TrU=nauK|s`Vd!KG{Xw#z&eB8hvm}A07a&cNEeA z+<~bOp4!;A&^DM7;~JCfzIX%V^ei{$#2Igup};rLeCDXgJnps-XjQGLH9GYM2sGuY zYh#okMk-I_*wzwUB=HKB^49WPJ6s8Hq{e1@vNMaF^bL- z(s_L=aW#N_812Sh3&{i=U{@b9g+q$r6V3G9cTGkT9Fsi36rM!#SadAc4PY zcYNSemI5wis@?&TB|@+ESf?yIEtV#Tc+j>zXporcr=0*;nl{?Z9D8zd7pE81p1Is} z`QCot?`nJ^S=lp++FMq9iWz6{r9@L33=K)sPm%YS64l=3YkQ01%~GUd@}T1mmWjvo z=xev7cf&drlq4yzu#OxbG{q2b!j}*-liqqHFZ(D( z$>S^LVy5BBc(dfFSdMs&2V4+J!|4ZkH0uL6+Hmn1#(&?gY7q93;IaWsGotr~<&Pxh}96%uhUEB9oR#B#c z$H5LaQ{TUH@2cc|eb_*d>0rDppO=q`5eJ1M+ZvMLGf)OOuIr?yfnSwsZK)ACz%sv(>0x+t_7gnOoD#0*Fv} zq=d@=1P1&!FI4faq~B#P3eZK1xUovREZ2k#{r%Y5WmuNHa*2u~qh>7(K6&rkiTdQd zZ-KH)mR;SxvyAWH<#;|y7=*ivXhu;Z3MR7$Hc3hqc}7q;SSFS@bld~9icSS`@x!OqAZRTI{1<`jaY&1(JD)YP6e?*GUdX;L+PuG z?&nB8kc)X(x1&tvqd5au#_hnVtpeJpCruYn078! z0B!a>Flyx8L5^ql3NTF4bKpcCdKji1ao}laWWbUQ16VGA@)3HR0szJ=bSVVfP?seW zqmp((ERfpiD{UB^6w2{1#R#@>JE+^&BVo@fm~WQ{tGhU_Qo1HPZ8Oj_SD&EO;KV_DAYg8x-J_RTk1yJ+VI8IT~(#)9uQO8fL zUia8vFbYUCER>A5%NeVf;ZVo~tE`2hK_P?WOmz*{*Xq6pu#S*0swvhTjFiiR(q?#Z z#zvx=H(ZnK5|qY4Z>hyq**d!mu?EQ;FN^q?T|zE6*-5h}AwwcSvqBmeaB~#qiQ3v; zTke2bBKo%<+})|&S>LU`vVM1GcWZN_w)!T{4M7|}=B*+tXHbI8T}uOccC>{er5(2?R5sZ|<1$7EA#X3J9@MmcS zR%Dsau96|lFfhdXnaLsr;}I({y$5(X7CBSt%aV}|bYs`q==j{Gg2Wd_Ehn%`4t(I*7 z0RFK+AjPrq_N?03T-#*i8P)Vn`NkDOcHzg%GVlqQbS(2n2MIeXLb#D+2x?kVXfYIQ zu>sE;49sMaTe?@Z#704}lO#*}Zis|a>FJ7rXWaK&X06}0%g4(yGQS+>Ap-yy3K}j> zHlh@05Sa{V%d!UQvP82LwlCawsJcQ)Gjt!bltB%!C6;Yz2b~YJMWJDjxNG&sD}P_eAN!s>&2BP3Mprif83yfv-~dI7yjwg=ELG6*(JU)FT!TDmB9 zg2Xu`?SE=fvZ&9oGRkDPU66QWEX?0XKWVy`|@b@&x*T17WYF z^zR!m{Ncp_xo+icS`FXq9E_V;Q|a0CI*li-743Iu7VR#C<$c>m@;H1`-$+ zEtFd@mza26hETc#0CRyFc&g861BSCo!MYj}3rk)D_bAA)i3U|O2#tL@?Q()F2F8d$ z!uMsRvm;dFju&_#_n`@X!3jK+gdJF$bCi!O^7kd~Ye9KYn75R|<{rWr#^+twX~ddl zMH&=KONb`!S6Z?RQdM4h@!GZLD*1tw2`Dp?tnTD}tXL!$`gM1Tf3kitu= z+N?z^=F5YOgyvjmbECSY<~IW_p{gp@$ohscZVkFDSJAU*$pR}=A|VfuHHa(NNw&yf zJEFADU@mQTDUP>`?1FO?I4bVIf~P{7U{S~3DpCb17QDB;UeqH}TF@(pIw3{X9Fq@8 zRhK)s z6V#UZg%}30s;+}7AJh|tGME0PD~yJp>Adoy4Q$)R!ZSG_qaZU`z`+C6%4YM+@e4jU7531b{IHjwdCibjF#|;iJ+E z&lrIa0&Z}UVeQCYzHgz0DDq^vs};0cEr$(1LPAk zTv_zt=C^DwpFW9NTG!;}R#P|M!YFE;D>$qmXyaAGSkZJNSX9~cLnn||&AM!Kbf9GQPKu$fn$d4cY-9-7vkINT7_^XK$hI#w; zB?tRFd%NZbE9DEN@}*}cXD)$R0O4Z6!e>ewS1?@p98R3;g1?w+&S4#1XXkj+HL#WYkt&nY5H*b8|UzE?xV@M3;)d@fXy!Amp%}rB)=7! zEPWM95X+aQKl!JLvYLMM;Y9hpi4v!A&Q3#QkjH;~e){9{bMHSf_xtCr9Qnr4Z=0q| z^8=4L`W7C2?6EJ>+e*{l_{l_hX8O@b6Xjd-$V_t4l|1tM6VuIOzdKo){?>;R(}!~xc)ZLQpSe`s-rjupk_q6MpH8mu)$84? zR;%~$pRd=d)n0YARehCT;aIKmEMNEVxmvC9ExyW*Jc0kKA-{ivf7bY?y3Rj(3*#Tk z(0BM5ubVgT^~^i`b$Gtl;Meceuiv=#*25dus;{l#%db4(MP?|xx%VKuzgm4&&dd1j zZNK-H48K;xO)IbPCQQ=2)>m)Kn-<5x^|3i6cghJX83(x8d-I{|XW!lupubq~gvE8j-e?e|3&CKH0wJ1N6C6uW|O)j9{ YxUqbF`PwXzbMUY9HT=|b3qR5QFAu6Gv;Y7A literal 0 HcmV?d00001 diff --git a/scripts/process_knowledge.py b/scripts/process_knowledge.py new file mode 100644 index 0000000..2805858 --- /dev/null +++ b/scripts/process_knowledge.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +Process uploaded conversations through devop.live LLM Team modes. +Runs multi-pass pipeline: extract → validate → organize. +""" + +import json, urllib.request, time, sys + +LAKEHOUSE = "http://localhost:3100" +LLM_TEAM = "http://localhost:5000" + +def sql(query): + req = urllib.request.Request(f"{LAKEHOUSE}/query/sql", + data=json.dumps({"sql": query}).encode(), + headers={"Content-Type": "application/json"}, method="POST") + return json.loads(urllib.request.urlopen(req, timeout=30).read()) + +def run_mode(mode, prompt, models=None): + """Run a devop.live LLM Team mode and collect all responses.""" + if models is None: + models = ["qwen2.5:latest"] + + config = {"mode": mode, "prompt": prompt, "models": models} + req = urllib.request.Request(f"{LLM_TEAM}/api/run", + data=json.dumps(config).encode(), + headers={"Content-Type": "application/json"}, method="POST") + + resp = urllib.request.urlopen(req, timeout=300) + + # Parse SSE stream + responses = [] + final = None + for line in resp: + line = line.decode().strip() + if line.startswith("data: "): + try: + event = json.loads(line[6:]) + if event.get("type") == "response": + responses.append({ + "model": event.get("model", ""), + "role": event.get("role", ""), + "text": event.get("text", ""), + }) + if event.get("role") == "final": + final = event.get("text", "") + except json.JSONDecodeError: + pass + + return {"responses": responses, "final": final} + +def ingest_csv(name, csv_text): + boundary = "----proc" + str(int(time.time())) + body = f"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{name}.csv\"\r\nContent-Type: text/csv\r\n\r\n{csv_text}\r\n--{boundary}--\r\n" + req = urllib.request.Request(f"{LAKEHOUSE}/ingest/file?name={name}", + data=body.encode(), headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, method="POST") + try: return urllib.request.urlopen(req, timeout=15).read().decode() + except: return "error" + +# Get conversation datasets +old = ['call_log','placements','clients','threat_intel','timesheets','meta_runs','candidates', + 'job_orders','lab_trials','users','email_log','new_candidates','schema_test_v1', + 'schema_test_v2','stress_test_ingest','stress_ingest','knowledge_topics', + 'knowledge_decisions','knowledge_insights','knowledge_questions', + 'processed_extractions','processed_qa','processed_validated'] +ds = json.loads(urllib.request.urlopen(f"{LAKEHOUSE}/catalog/datasets").read()) +convs = [d for d in ds if d['name'] not in old and d.get('row_count',0) and d['row_count'] > 0] + +print(f"{'='*60}") +print(f"KNOWLEDGE PROCESSING PIPELINE") +print(f"Using devop.live LLM Team modes") +print(f"{'='*60}") +print(f"\n{len(convs)} conversations to process") + +all_extractions = [] +all_qa = [] +all_validated = [] + +for conv in convs: + name = conv['name'] + pages = conv.get('row_count', 0) or 0 + if pages == 0: continue + + print(f"\n{'='*60}") + print(f"Processing: {name} ({pages} pages)") + print(f"{'='*60}") + + # Sample pages (every Nth for large docs) + step = max(1, pages // 10) + r = sql(f"SELECT turn_number, content FROM {name} WHERE turn_number % {step} = 1 ORDER BY turn_number LIMIT 10") + + for row in r['rows']: + page_num = row.get('turn_number', 0) + content = (row.get('content', '') or '')[:3000] + if len(content) < 100: continue + + print(f"\n --- Page {page_num} ---") + + # PASS 1: Extract (uses devop.live extract mode) + print(f" [EXTRACT] Running extraction...", end=" ", flush=True) + try: + result = run_mode("extract", content) + final = result.get("final", "") + + # Parse the extraction response + for resp in result["responses"]: + if resp["role"].startswith("extraction"): + text = resp["text"] + # Try to find JSON in the response + if "{" in text: + start = text.find("{") + end = text.rfind("}") + 1 + if end > start: + try: + data = json.loads(text[start:end]) + facts = data.get("facts", []) + entities = data.get("entities", []) + relationships = data.get("relationships", []) + + for fact in facts[:5]: + claim = fact if isinstance(fact, str) else fact.get("claim", str(fact)) + all_extractions.append(f'"{name}",{page_num},"fact","{claim[:200].replace(chr(34), chr(39))}"') + + for ent in entities[:5]: + ent_name = ent if isinstance(ent, str) else ent.get("name", str(ent)) + all_extractions.append(f'"{name}",{page_num},"entity","{ent_name[:200].replace(chr(34), chr(39))}"') + + print(f"{len(facts)} facts, {len(entities)} entities, {len(relationships)} relationships") + except json.JSONDecodeError: + print("no JSON found") + else: + print("no structured output") + + # Check verifier response + if resp["role"] == "verifier": + verified_text = resp["text"][:500] + if verified_text.strip(): + all_validated.append(f'"{name}",{page_num},"verification","{verified_text[:200].replace(chr(34), chr(39))}"') + + except Exception as e: + print(f"error: {str(e)[:60]}") + + # Small delay between pages to not overwhelm + time.sleep(0.5) + +# Store results +print(f"\n{'='*60}") +print(f"RESULTS") +print(f"{'='*60}") +print(f"Extractions: {len(all_extractions)}") +print(f"Validated: {len(all_validated)}") + +if all_extractions: + csv = "conversation,page,type,content\n" + "\n".join(all_extractions) + print(f"\nIngesting processed_extractions...", end=" ") + print(ingest_csv("processed_extractions", csv)) + +if all_validated: + csv = "conversation,page,type,content\n" + "\n".join(all_validated) + print(f"Ingesting processed_validated...", end=" ") + print(ingest_csv("processed_validated", csv)) + +print(f"\nDone. LLM Team modes processed {len(convs)} conversations.") +print(f"Query: SELECT * FROM processed_extractions")