Personal Search Tools

# pip install fastembed
from fastembed import TextEmbedding

model = TextEmbedding("BAAI/bge-small-en-v1.5")
embeddings = list(model.embed(["My meeting notes", "Garden project ideas"]))
# Returns list of numpy arrays

# pip install sentence-transformers
from sentence_transformers import SentenceTransformer

model = SentenceTransformer("BAAI/bge-small-en-v1.5")
embeddings = model.encode(["My meeting notes", "Garden project ideas"])

ollama pull nomic-embed-text

import ollama

response = ollama.embeddings(
    model="nomic-embed-text",
    prompt="My meeting notes"
)
vector = response["embedding"]

# pip install lancedb
import lancedb
from lancedb.pydantic import LanceModel, Vector
from fastembed import TextEmbedding

embedder = TextEmbedding("BAAI/bge-small-en-v1.5")

class Note(LanceModel):
    text: str
    path: str
    vector: Vector(384)

db = lancedb.connect("./notes.lance")
table = db.create_table("notes", schema=Note)

# Add notes
notes = [
    {"text": "Meeting with Alex about API design", "path": "notes/meetings/2024-01.md"},
    {"text": "Garden layout sketches", "path": "notes/projects/garden.md"},
]
for note in notes:
    vec = list(embedder.embed([note["text"]]))[0]
    note["vector"] = vec

table.add(notes)

# Search
query_vec = list(embedder.embed(["API discussions"]))[0]
results = table.search(query_vec).limit(5).to_list()

# pip install sqlite-vss sentence-transformers
import sqlite3
import sqlite_vss
from sentence_transformers import SentenceTransformer

model = SentenceTransformer("all-MiniLM-L6-v2")

conn = sqlite3.connect("notes.db")
conn.enable_load_extension(True)
sqlite_vss.load(conn)

conn.execute("""
    CREATE VIRTUAL TABLE IF NOT EXISTS notes_vss USING vss0(embedding(384))
""")
conn.execute("""
    CREATE TABLE IF NOT EXISTS notes (
        id INTEGER PRIMARY KEY,
        text TEXT,
        path TEXT
    )
""")

# Index
def add_note(text, path):
    vec = model.encode(text)
    cur = conn.execute("INSERT INTO notes (text, path) VALUES (?, ?)", (text, path))
    conn.execute("INSERT INTO notes_vss (rowid, embedding) VALUES (?, ?)",
                 (cur.lastrowid, vec.tobytes()))
    conn.commit()

# Search
def search(query, limit=5):
    vec = model.encode(query)
    rows = conn.execute("""
        SELECT notes.text, notes.path, vss.distance
        FROM notes_vss vss
        JOIN notes ON notes.id = vss.rowid
        WHERE vss_search(vss.embedding, ?)
        LIMIT ?
    """, (vec.tobytes(), limit)).fetchall()
    return rows

#!/usr/bin/env python3
"""Index markdown notes for semantic search."""

import os
from pathlib import Path
from fastembed import TextEmbedding
import lancedb
from lancedb.pydantic import LanceModel, Vector

NOTES_DIR = Path.home() / "notes"
DB_PATH = Path.home() / ".local/share/note-search"

class Note(LanceModel):
    text: str
    title: str
    path: str
    vector: Vector(384)

def extract_notes(directory: Path):
    """Yield (text, title, path) from markdown files."""
    for md_file in directory.rglob("*.md"):
        content = md_file.read_text()
        title = md_file.stem
        # Extract title from frontmatter if present
        if content.startswith("---"):
            lines = content.split("\n")
            for line in lines[1:]:
                if line.startswith("title:"):
                    title = line.split(":", 1)[1].strip().strip('"\'')
                    break
                if line == "---":
                    break
        yield content, title, str(md_file)

def build_index():
    embedder = TextEmbedding("BAAI/bge-small-en-v1.5")
    db = lancedb.connect(str(DB_PATH))

    # Drop and recreate
    if "notes" in db.table_names():
        db.drop_table("notes")

    notes = []
    texts = []
    for text, title, path in extract_notes(NOTES_DIR):
        notes.append({"text": text, "title": title, "path": path})
        texts.append(text)

    print(f"Embedding {len(texts)} notes...")
    vectors = list(embedder.embed(texts))

    for note, vec in zip(notes, vectors):
        note["vector"] = vec

    table = db.create_table("notes", data=notes, schema=Note)
    print(f"Indexed {len(notes)} notes to {DB_PATH}")

if __name__ == "__main__":
    build_index()

#!/usr/bin/env python3
"""Search notes from command line."""

import sys
from fastembed import TextEmbedding
import lancedb

DB_PATH = "~/.local/share/note-search"

def search(query: str, limit: int = 10):
    embedder = TextEmbedding("BAAI/bge-small-en-v1.5")
    db = lancedb.connect(DB_PATH)
    table = db.open_table("notes")

    vec = list(embedder.embed([query]))[0]
    results = table.search(vec).limit(limit).to_list()

    for r in results:
        score = 1 - r["_distance"]  # Convert distance to similarity
        print(f"{score:.2f} | {r['title']}")
        print(f"      {r['path']}")
        print()

if __name__ == "__main__":
    search(" ".join(sys.argv[1:]))

$ ./search.py "API design decisions"
0.82 | api-versioning-notes
      /home/user/notes/tech/api-versioning-notes.md

0.71 | meeting-alex-march
      /home/user/notes/meetings/meeting-alex-march.md

import json

results = search(query)
items = [
    {
        "title": r["title"],
        "subtitle": r["path"],
        "arg": r["path"],  # Opens file on select
    }
    for r in results
]
print(json.dumps({"items": items}))

pip install datasette datasette-vss
datasette notes.db --open

def hybrid_search(query: str, limit: int = 10):
    # Vector search
    vec = embedder.embed([query])[0]
    vector_results = table.search(vec).limit(limit * 2).to_list()

    # Keyword filter
    keywords = query.lower().split()
    final = []
    for r in vector_results:
        text_lower = r["text"].lower()
        keyword_hits = sum(1 for k in keywords if k in text_lower)
        combined_score = (1 - r["_distance"]) + (keyword_hits * 0.1)
        final.append((combined_score, r))

    final.sort(reverse=True, key=lambda x: x[0])
    return [r for _, r in final[:limit]]