Observer KB: new app for ChatGPT/Claude PDF knowledge extraction
- New branch: observer-kb - PRD: apps/observer-kb/docs/PRD.md - ChatGPT PDF parser: extracts conversation turns (user/assistant) - Upload portal: pure HTML/JS (no WASM), drag & drop, progress bars - Browse: view ingested datasets with preview - Search: semantic search across embedded conversations (HNSW) - Served on :3400 via systemd (observer-kb.service) - Uses Lakehouse core for storage, query, vectors, AI Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -0,0 +1,235 @@
|
||||
/// ChatGPT PDF conversation parser.
|
||||
/// Extracts structured conversation turns from ChatGPT export PDFs.
|
||||
/// Handles: user/assistant turns, timestamps, conversation titles.
|
||||
|
||||
use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
|
||||
use arrow::datatypes::{DataType, Field, Schema};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// A single conversation turn.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ConversationTurn {
|
||||
pub source_file: String,
|
||||
pub conversation_title: String,
|
||||
pub turn_number: i32,
|
||||
pub role: String, // "user" or "assistant"
|
||||
pub content: String,
|
||||
pub page_number: i32,
|
||||
}
|
||||
|
||||
/// Parse a ChatGPT PDF into structured conversation turns.
|
||||
pub fn parse_chatgpt_pdf(content: &[u8], filename: &str) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
|
||||
let doc = lopdf::Document::load_mem(content)
|
||||
.map_err(|e| format!("PDF load error: {e}"))?;
|
||||
|
||||
// Extract all text
|
||||
let pages = doc.get_pages();
|
||||
let mut all_text = String::new();
|
||||
let mut page_texts: Vec<(i32, String)> = Vec::new();
|
||||
|
||||
for (&page_num, _) in pages.iter() {
|
||||
let text = doc.extract_text(&[page_num]).unwrap_or_default();
|
||||
let text = text.trim().to_string();
|
||||
if !text.is_empty() {
|
||||
page_texts.push((page_num as i32, text.clone()));
|
||||
all_text.push_str(&text);
|
||||
all_text.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
if all_text.trim().is_empty() {
|
||||
return Err("PDF contains no extractable text".into());
|
||||
}
|
||||
|
||||
// Try to parse as ChatGPT conversation format
|
||||
let turns = extract_turns(&all_text, filename, &page_texts);
|
||||
|
||||
if turns.is_empty() {
|
||||
// Fall back to page-based extraction if we can't detect conversation structure
|
||||
return parse_as_raw_pages(&page_texts, filename);
|
||||
}
|
||||
|
||||
tracing::info!("parsed ChatGPT PDF '{}': {} turns", filename, turns.len());
|
||||
turns_to_batch(&turns)
|
||||
}
|
||||
|
||||
/// Extract conversation turns from ChatGPT PDF text.
|
||||
/// ChatGPT PDFs typically have patterns like:
|
||||
/// - "You said:" / "ChatGPT said:" (newer format)
|
||||
/// - "User:" / "Assistant:"
|
||||
/// - Or just alternating blocks separated by formatting
|
||||
fn extract_turns(text: &str, filename: &str, page_texts: &[(i32, String)]) -> Vec<ConversationTurn> {
|
||||
let mut turns = Vec::new();
|
||||
let mut turn_num = 0;
|
||||
|
||||
// Try to extract conversation title from first line
|
||||
let title = text.lines().next()
|
||||
.map(|l| l.trim().to_string())
|
||||
.filter(|l| l.len() < 200 && !l.is_empty())
|
||||
.unwrap_or_else(|| filename.replace(".pdf", ""));
|
||||
|
||||
// Strategy 1: Look for "You said:" / "ChatGPT said:" markers (common format)
|
||||
let you_markers = ["You said:", "You:", "User:", "Human:"];
|
||||
let ai_markers = ["ChatGPT said:", "ChatGPT:", "Assistant:", "AI:"];
|
||||
|
||||
let has_markers = you_markers.iter().any(|m| text.contains(m))
|
||||
&& ai_markers.iter().any(|m| text.contains(m));
|
||||
|
||||
if has_markers {
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
let mut current_role = String::new();
|
||||
let mut current_content = String::new();
|
||||
let mut current_page = 1i32;
|
||||
|
||||
for line in &lines {
|
||||
let trimmed = line.trim();
|
||||
|
||||
// Check for role markers
|
||||
let mut found_marker = false;
|
||||
for marker in &you_markers {
|
||||
if trimmed.starts_with(marker) || trimmed == *marker {
|
||||
// Save previous turn
|
||||
if !current_content.trim().is_empty() && !current_role.is_empty() {
|
||||
turn_num += 1;
|
||||
turns.push(ConversationTurn {
|
||||
source_file: filename.to_string(),
|
||||
conversation_title: title.clone(),
|
||||
turn_number: turn_num,
|
||||
role: current_role.clone(),
|
||||
content: current_content.trim().to_string(),
|
||||
page_number: current_page,
|
||||
});
|
||||
}
|
||||
current_role = "user".to_string();
|
||||
current_content = trimmed.strip_prefix(marker).unwrap_or("").trim().to_string();
|
||||
current_content.push('\n');
|
||||
found_marker = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !found_marker {
|
||||
for marker in &ai_markers {
|
||||
if trimmed.starts_with(marker) || trimmed == *marker {
|
||||
if !current_content.trim().is_empty() && !current_role.is_empty() {
|
||||
turn_num += 1;
|
||||
turns.push(ConversationTurn {
|
||||
source_file: filename.to_string(),
|
||||
conversation_title: title.clone(),
|
||||
turn_number: turn_num,
|
||||
role: current_role.clone(),
|
||||
content: current_content.trim().to_string(),
|
||||
page_number: current_page,
|
||||
});
|
||||
}
|
||||
current_role = "assistant".to_string();
|
||||
current_content = trimmed.strip_prefix(marker).unwrap_or("").trim().to_string();
|
||||
current_content.push('\n');
|
||||
found_marker = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !found_marker && !current_role.is_empty() {
|
||||
current_content.push_str(trimmed);
|
||||
current_content.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
// Save last turn
|
||||
if !current_content.trim().is_empty() && !current_role.is_empty() {
|
||||
turn_num += 1;
|
||||
turns.push(ConversationTurn {
|
||||
source_file: filename.to_string(),
|
||||
conversation_title: title.clone(),
|
||||
turn_number: turn_num,
|
||||
role: current_role,
|
||||
content: current_content.trim().to_string(),
|
||||
page_number: current_page,
|
||||
});
|
||||
}
|
||||
|
||||
return turns;
|
||||
}
|
||||
|
||||
// Strategy 2: Split by paragraph blocks, alternate user/assistant
|
||||
// This is a heuristic fallback for PDFs without clear markers
|
||||
let paragraphs: Vec<&str> = text.split("\n\n")
|
||||
.map(|p| p.trim())
|
||||
.filter(|p| p.len() > 20) // skip short fragments
|
||||
.collect();
|
||||
|
||||
if paragraphs.len() >= 2 {
|
||||
for (i, para) in paragraphs.iter().enumerate() {
|
||||
turn_num += 1;
|
||||
turns.push(ConversationTurn {
|
||||
source_file: filename.to_string(),
|
||||
conversation_title: title.clone(),
|
||||
turn_number: turn_num,
|
||||
role: if i % 2 == 0 { "user" } else { "assistant" }.to_string(),
|
||||
content: para.to_string(),
|
||||
page_number: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
turns
|
||||
}
|
||||
|
||||
/// Fallback: parse as raw pages with full text.
|
||||
fn parse_as_raw_pages(page_texts: &[(i32, String)], filename: &str) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("source_file", DataType::Utf8, false),
|
||||
Field::new("conversation_title", DataType::Utf8, false),
|
||||
Field::new("turn_number", DataType::Int32, false),
|
||||
Field::new("role", DataType::Utf8, false),
|
||||
Field::new("content", DataType::Utf8, false),
|
||||
Field::new("page_number", DataType::Int32, false),
|
||||
]));
|
||||
|
||||
let sources: Vec<&str> = page_texts.iter().map(|_| filename).collect();
|
||||
let titles: Vec<String> = page_texts.iter().map(|_| filename.replace(".pdf", "")).collect();
|
||||
let turn_nums: Vec<i32> = (1..=page_texts.len() as i32).collect();
|
||||
let roles: Vec<&str> = page_texts.iter().map(|_| "page").collect();
|
||||
let contents: Vec<&str> = page_texts.iter().map(|(_, t)| t.as_str()).collect();
|
||||
let pages: Vec<i32> = page_texts.iter().map(|(p, _)| *p).collect();
|
||||
|
||||
let arrays: Vec<ArrayRef> = vec![
|
||||
Arc::new(StringArray::from(sources)),
|
||||
Arc::new(StringArray::from(titles.iter().map(|s| s.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(Int32Array::from(turn_nums)),
|
||||
Arc::new(StringArray::from(roles)),
|
||||
Arc::new(StringArray::from(contents)),
|
||||
Arc::new(Int32Array::from(pages)),
|
||||
];
|
||||
|
||||
let batch = RecordBatch::try_new(schema.clone(), arrays)
|
||||
.map_err(|e| format!("RecordBatch error: {e}"))?;
|
||||
|
||||
Ok((schema, vec![batch]))
|
||||
}
|
||||
|
||||
fn turns_to_batch(turns: &[ConversationTurn]) -> Result<(Arc<Schema>, Vec<RecordBatch>), String> {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("source_file", DataType::Utf8, false),
|
||||
Field::new("conversation_title", DataType::Utf8, false),
|
||||
Field::new("turn_number", DataType::Int32, false),
|
||||
Field::new("role", DataType::Utf8, false),
|
||||
Field::new("content", DataType::Utf8, false),
|
||||
Field::new("page_number", DataType::Int32, false),
|
||||
]));
|
||||
|
||||
let arrays: Vec<ArrayRef> = vec![
|
||||
Arc::new(StringArray::from(turns.iter().map(|t| t.source_file.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(turns.iter().map(|t| t.conversation_title.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(Int32Array::from(turns.iter().map(|t| t.turn_number).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(turns.iter().map(|t| t.role.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(StringArray::from(turns.iter().map(|t| t.content.as_str()).collect::<Vec<_>>())),
|
||||
Arc::new(Int32Array::from(turns.iter().map(|t| t.page_number).collect::<Vec<_>>())),
|
||||
];
|
||||
|
||||
let batch = RecordBatch::try_new(schema.clone(), arrays)
|
||||
.map_err(|e| format!("RecordBatch error: {e}"))?;
|
||||
|
||||
Ok((schema, vec![batch]))
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod chatgpt_parser;
|
||||
pub mod db_ingest;
|
||||
pub mod detect;
|
||||
pub mod csv_ingest;
|
||||
|
||||
@@ -43,7 +43,13 @@ pub async fn ingest_file(
|
||||
let (schema, batches) = match file_type {
|
||||
FileType::Csv => csv_ingest::parse_csv(content)?,
|
||||
FileType::Json | FileType::NdJson => json_ingest::parse_json(content)?,
|
||||
FileType::Pdf => pdf_ingest::parse_pdf(content, filename)?,
|
||||
FileType::Pdf => {
|
||||
// Try ChatGPT parser first, fall back to generic PDF
|
||||
match crate::chatgpt_parser::parse_chatgpt_pdf(content, filename) {
|
||||
Ok(result) => result,
|
||||
Err(_) => pdf_ingest::parse_pdf(content, filename)?,
|
||||
}
|
||||
}
|
||||
FileType::Text => parse_text_file(content, filename)?,
|
||||
FileType::Unknown => return Err(format!("unknown file type for '{filename}'")),
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user