Document Search

1"""The index: SQLite, one file, no server.
2 
3Design notes worth knowing before you change anything here:
4 
5* **Postings, not stored vectors.** For the sparse embedder the index is an
6  inverted list (`term -> chunk, count`). A query then touches only the chunks
7  containing its terms instead of every chunk in the corpus, which is the
8  difference between a search that stays fast at 100k chunks and one that does
9  not. Dense vectors, when that embedder is used, are stored on the chunk row
10  and scanned — brute force is honest at this scale and there is no ANN index
11  pretending to be exact.
12 
13* **Content hashing.** A document is keyed by the hash of its bytes, so
14  re-ingesting an unchanged file is a no-op and re-ingesting a changed one
15  replaces its chunks rather than duplicating them.
16 
17* **One transaction per ingest.** An interrupted ingest leaves the index exactly
18  as it was, never half-updated.
19"""
20 
21from __future__ import annotations
22 
23import hashlib
24import json
25import sqlite3
26import time
27from dataclasses import dataclass
28from pathlib import Path
29 
30from .chunking import Chunk, ChunkConfig, Document
31from .embed import Embedder, HashingTfidfEmbedder, Vector, tokenize
32 
33SCHEMA_VERSION = 1
34 
35SCHEMA = """
36CREATE TABLE IF NOT EXISTS meta (
37  key   TEXT PRIMARY KEY,
38  value TEXT NOT NULL
39);
40 
41CREATE TABLE IF NOT EXISTS documents (
42  id       INTEGER PRIMARY KEY,
43  path     TEXT NOT NULL UNIQUE,
44  title    TEXT NOT NULL DEFAULT '',
45  kind     TEXT NOT NULL DEFAULT 'text',
46  hash     TEXT NOT NULL,
47  meta     TEXT NOT NULL DEFAULT '{}',
48  added_at REAL NOT NULL
49);
50 
51CREATE TABLE IF NOT EXISTS chunks (
52  id          INTEGER PRIMARY KEY,
53  document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
54  idx         INTEGER NOT NULL,
55  text        TEXT NOT NULL,
56  heading     TEXT NOT NULL DEFAULT '',
57  start       INTEGER NOT NULL,
58  end         INTEGER NOT NULL,
59  tokens      INTEGER NOT NULL,
60  vector      TEXT
61);
62 
63CREATE INDEX IF NOT EXISTS chunks_document ON chunks(document_id);
64 
65CREATE TABLE IF NOT EXISTS postings (
66  dimension INTEGER NOT NULL,
67  chunk_id  INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
68  count     INTEGER NOT NULL
69);
70 
71CREATE INDEX IF NOT EXISTS postings_dimension ON postings(dimension);
72CREATE INDEX IF NOT EXISTS postings_chunk ON postings(chunk_id);
73"""
74 
75 
76@dataclass
77class IngestReport:
78    added: int = 0
79    updated: int = 0
80    unchanged: int = 0
81    chunks: int = 0
82    skipped: list[tuple[str, str]] = None  # type: ignore[assignment]
83    seconds: float = 0.0
84 
85    def __post_init__(self) -> None:
86        if self.skipped is None:
87            self.skipped = []
88 
89    def summary(self) -> str:
90        parts = [
91            f"{self.added} added",
92            f"{self.updated} updated",
93            f"{self.unchanged} unchanged",
94            f"{self.chunks} chunks",
95        ]
96        if self.skipped:
97            parts.append(f"{len(self.skipped)} skipped")
98        return ", ".join(parts) + f" in {self.seconds:.2f}s"
99 
100