Phase 3: AI integration with Ollama via Python sidecar

- sidecar: FastAPI app with /embed, /generate, /rerank hitting Ollama
- sidecar: Dockerfile, env var config (EMBED_MODEL, GEN_MODEL, RERANK_MODEL)
- aibridge: reqwest HTTP client with typed request/response structs
- aibridge: Axum proxy endpoints (POST /ai/embed, /ai/generate, /ai/rerank)
- gateway: wires AiClient with SIDECAR_URL env var
- e2e verified: nomic-embed-text returns 768d vectors, qwen2.5 generates text

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

sidecar/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.13-slim
+
+WORKDIR /app
+
+COPY pyproject.toml .
+RUN pip install --no-cache-dir .
+
+COPY sidecar/ sidecar/
+
+ENV OLLAMA_URL=http://host.docker.internal:11434
+ENV EMBED_MODEL=nomic-embed-text
+ENV GEN_MODEL=qwen2.5
+ENV RERANK_MODEL=qwen2.5
+
+EXPOSE 3200
+
+CMD ["uvicorn", "sidecar.main:app", "--host", "0.0.0.0", "--port", "3200"]

sidecar/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "lakehouse-sidecar"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = [
+    "fastapi>=0.115",
+    "uvicorn>=0.34",
+    "httpx>=0.28",
+    "pydantic>=2.0",
+]
+
+[tool.uvicorn]
+host = "0.0.0.0"
+port = 3200

sidecar/sidecar/embed.py

@@ -0,0 +1,44 @@
+import os
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from .ollama import client
+
+router = APIRouter()
+
+EMBED_MODEL = os.environ.get("EMBED_MODEL", "nomic-embed-text")
+
+
+class EmbedRequest(BaseModel):
+    texts: list[str]
+    model: str | None = None
+
+
+class EmbedResponse(BaseModel):
+    embeddings: list[list[float]]
+    model: str
+    dimensions: int
+
+
+@router.post("", response_model=EmbedResponse)
+async def embed(req: EmbedRequest):
+    model = req.model or EMBED_MODEL
+    embeddings = []
+
+    async with client() as c:
+        for text in req.texts:
+            resp = await c.post("/api/embed", json={"model": model, "input": text})
+            if resp.status_code != 200:
+                raise HTTPException(502, f"Ollama error: {resp.text}")
+            data = resp.json()
+            embeddings.extend(data.get("embeddings", []))
+
+    if not embeddings:
+        raise HTTPException(502, "No embeddings returned")
+
+    return EmbedResponse(
+        embeddings=embeddings,
+        model=model,
+        dimensions=len(embeddings[0]),
+    )

sidecar/sidecar/generate.py

@@ -0,0 +1,55 @@
+import os
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from .ollama import client
+
+router = APIRouter()
+
+GEN_MODEL = os.environ.get("GEN_MODEL", "qwen2.5")
+
+
+class GenerateRequest(BaseModel):
+    prompt: str
+    model: str | None = None
+    system: str | None = None
+    temperature: float = 0.7
+    max_tokens: int = 2048
+
+
+class GenerateResponse(BaseModel):
+    text: str
+    model: str
+    tokens_evaluated: int | None = None
+    tokens_generated: int | None = None
+
+
+@router.post("", response_model=GenerateResponse)
+async def generate(req: GenerateRequest):
+    model = req.model or GEN_MODEL
+
+    payload = {
+        "model": model,
+        "prompt": req.prompt,
+        "stream": False,
+        "options": {
+            "temperature": req.temperature,
+            "num_predict": req.max_tokens,
+        },
+    }
+    if req.system:
+        payload["system"] = req.system
+
+    async with client() as c:
+        resp = await c.post("/api/generate", json=payload)
+        if resp.status_code != 200:
+            raise HTTPException(502, f"Ollama error: {resp.text}")
+        data = resp.json()
+
+    return GenerateResponse(
+        text=data.get("response", ""),
+        model=model,
+        tokens_evaluated=data.get("prompt_eval_count"),
+        tokens_generated=data.get("eval_count"),
+    )

sidecar/sidecar/main.py

@@ -0,0 +1,23 @@
+import os
+
+from fastapi import FastAPI
+
+from .embed import router as embed_router
+from .generate import router as generate_router
+from .rerank import router as rerank_router
+
+app = FastAPI(title="Lakehouse AI Sidecar")
+
+app.include_router(embed_router, prefix="/embed", tags=["embed"])
+app.include_router(generate_router, prefix="/generate", tags=["generate"])
+app.include_router(rerank_router, prefix="/rerank", tags=["rerank"])
+
+
+@app.get("/health")
+async def health():
+    return {
+        "status": "ok",
+        "ollama_url": os.environ.get("OLLAMA_URL", "http://localhost:11434"),
+        "embed_model": os.environ.get("EMBED_MODEL", "nomic-embed-text"),
+        "gen_model": os.environ.get("GEN_MODEL", "qwen2.5"),
+    }

sidecar/sidecar/ollama.py

@@ -0,0 +1,12 @@
+"""Shared Ollama HTTP client."""
+
+import os
+
+import httpx
+
+OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434")
+TIMEOUT = float(os.environ.get("OLLAMA_TIMEOUT", "120"))
+
+
+def client() -> httpx.AsyncClient:
+    return httpx.AsyncClient(base_url=OLLAMA_URL, timeout=TIMEOUT)

sidecar/sidecar/rerank.py

@@ -0,0 +1,70 @@
+import os
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from .ollama import client
+
+router = APIRouter()
+
+RERANK_MODEL = os.environ.get("RERANK_MODEL", "qwen2.5")
+
+
+class RerankRequest(BaseModel):
+    query: str
+    documents: list[str]
+    model: str | None = None
+    top_k: int | None = None
+
+
+class ScoredDocument(BaseModel):
+    index: int
+    text: str
+    score: float
+
+
+class RerankResponse(BaseModel):
+    results: list[ScoredDocument]
+    model: str
+
+
+@router.post("", response_model=RerankResponse)
+async def rerank(req: RerankRequest):
+    """Cross-encoder reranking via Ollama generate.
+
+    Scores each document against the query by asking the model to rate relevance 0-10,
+    then sorts by score descending.
+    """
+    model = req.model or RERANK_MODEL
+    scored = []
+
+    async with client() as c:
+        for i, doc in enumerate(req.documents):
+            prompt = (
+                f"Rate the relevance of the following document to the query on a scale of 0 to 10. "
+                f"Respond with ONLY a number.\n\n"
+                f"Query: {req.query}\n\n"
+                f"Document: {doc}\n\n"
+                f"Score:"
+            )
+            resp = await c.post(
+                "/api/generate",
+                json={"model": model, "prompt": prompt, "stream": False, "options": {"temperature": 0.0, "num_predict": 8}},
+            )
+            if resp.status_code != 200:
+                raise HTTPException(502, f"Ollama error: {resp.text}")
+
+            text = resp.json().get("response", "").strip()
+            try:
+                score = float(text.split()[0])
+                score = max(0.0, min(10.0, score))
+            except (ValueError, IndexError):
+                score = 0.0
+
+            scored.append(ScoredDocument(index=i, text=doc, score=score))
+
+    scored.sort(key=lambda x: x.score, reverse=True)
+    if req.top_k:
+        scored = scored[: req.top_k]
+
+    return RerankResponse(results=scored, model=model)