Merge pull request 'Fix make reindex targets and reindex Obsidian in the vector store CLI' (#88) from fix/reindex-targets into main

`reindex` deleted every chunk before contacting Paperless, so an unreachable
or empty Paperless left the store empty (which happened when Paperless was
crash-looping during its v3 upgrade). It now fetches first, aborts without
deleting if that fails or returns nothing, and indexes the fetched documents
instead of fetching twice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Makefile

@@ -34,21 +34,10 @@ logs:
 
 # Vector store reindexing
 reindex:
-	$(COMPOSE) exec $(RAGGR) python -c "\
-		import asyncio; \
-		from blueprints.rag.logic import delete_all_documents, index_documents, index_obsidian_documents; \
-		delete_all_documents(); \
-		asyncio.run(index_documents()); \
-		asyncio.run(index_obsidian_documents()); \
-		print('Reindex complete')"
+	$(COMPOSE) exec $(RAGGR) python -m scripts.manage_vectorstore reindex
 
 reindex-obsidian:
-	$(COMPOSE) exec $(RAGGR) python -c "\
-		import asyncio; \
-		from blueprints.rag.logic import delete_documents_by_metadata, index_obsidian_documents; \
-		delete_documents_by_metadata('source', 'obsidian'); \
-		asyncio.run(index_obsidian_documents()); \
-		print('Obsidian reindex complete')"
+	$(COMPOSE) exec $(RAGGR) python -m scripts.manage_vectorstore reindex-obsidian
 
 # YNAB mirror: delta sync, or full re-download
 ynab-sync:

blueprints/rag/logic.py

@@ -202,9 +202,13 @@ def _sanitize_documents(documents: list[Document]) -> list[Document]:
     return [doc for doc in documents if doc.page_content]
 
 
-async def index_documents():
-    """Index Paperless-NGX documents into vector store."""
-    documents = await fetch_documents_from_paperless_ngx()
+async def index_documents(documents: list[Document] | None = None):
+    """Index Paperless-NGX documents into vector store.
+
+    Pass already-fetched `documents` to index them without fetching again.
+    """
+    if documents is None:
+        documents = await fetch_documents_from_paperless_ngx()
 
     splits = text_splitter.split_documents(documents)
     splits = _sanitize_documents(splits)

scripts/manage_vectorstore.py

@@ -7,8 +7,11 @@ import sys
 
 from blueprints.rag.logic import (
     delete_all_documents,
+    delete_documents_by_metadata,
+    fetch_documents_from_paperless_ngx,
     get_vector_store_stats,
     index_documents,
+    index_obsidian_documents,
     list_all_documents,
 )
 
@@ -31,11 +34,35 @@ async def index():
 
 
 async def reindex():
-    """Clear and reindex all documents."""
+    """Clear and reindex everything: Paperless documents and the Obsidian vault.
+
+    Paperless is fetched before anything is deleted, so an unreachable or
+    empty Paperless leaves the existing index untouched.
+    """
+    print("Fetching documents from Paperless-NGX...")
+    documents = await fetch_documents_from_paperless_ngx()
+    if not documents:
+        raise RuntimeError(
+            "Paperless returned no documents to index; leaving the index untouched"
+        )
+    print(f"Fetched {len(documents)} documents")
     print("Clearing existing documents...")
     delete_all_documents()
     print("Cleared")
-    await index()
+    await index_documents(documents)
+    print("Paperless indexing complete!")
+    await reindex_obsidian(clear=False)
+
+
+async def reindex_obsidian(clear: bool = True):
+    """Re-embed the Obsidian vault, leaving Paperless documents alone."""
+    if clear:
+        print("Clearing Obsidian notes...")
+        delete_documents_by_metadata("source", "obsidian")
+    print("Indexing Obsidian vault...")
+    await index_obsidian_documents()
+    print("Obsidian indexing complete!")
+    stats()
 
 
 def list_docs(limit: int = 10, show_content: bool = False):
@@ -60,7 +87,8 @@ def main():
 Examples:
   %(prog)s stats                    # Show vector store statistics
   %(prog)s index                    # Index new documents from Paperless-NGX
-  %(prog)s reindex                  # Clear and reindex all documents
+  %(prog)s reindex                  # Clear and reindex Paperless + Obsidian
+  %(prog)s reindex-obsidian         # Clear and reindex only Obsidian notes
   %(prog)s list 10                  # List first 10 documents
   %(prog)s list 20 --show-content   # List 20 documents with content preview
         """,
@@ -75,7 +103,12 @@ Examples:
     subparsers.add_parser("index", help="Index documents from Paperless-NGX")
 
     # Reindex command
-    subparsers.add_parser("reindex", help="Clear and reindex all documents")
+    subparsers.add_parser(
+        "reindex", help="Clear and reindex Paperless documents and Obsidian notes"
+    )
+    subparsers.add_parser(
+        "reindex-obsidian", help="Clear and reindex only Obsidian notes"
+    )
 
     # List command
     list_parser = subparsers.add_parser("list", help="List documents in vector store")
@@ -99,6 +132,8 @@ Examples:
             asyncio.run(index())
         elif args.command == "reindex":
             asyncio.run(reindex())
+        elif args.command == "reindex-obsidian":
+            asyncio.run(reindex_obsidian())
         elif args.command == "list":
             list_docs(limit=args.limit, show_content=args.show_content)
     except KeyboardInterrupt:

tests/unit/test_manage_vectorstore.py

@@ -0,0 +1,66 @@
+import sys
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from scripts import manage_vectorstore as cli
+
+
+@pytest.fixture
+def calls(monkeypatch):
+    calls = []
+    monkeypatch.setattr(cli, "delete_all_documents", lambda: calls.append("delete_all"))
+    monkeypatch.setattr(
+        cli,
+        "delete_documents_by_metadata",
+        lambda k, v: calls.append(f"delete {k}={v}"),
+    )
+    monkeypatch.setattr(
+        cli,
+        "fetch_documents_from_paperless_ngx",
+        AsyncMock(side_effect=lambda: calls.append("fetch") or ["doc"]),
+    )
+    monkeypatch.setattr(
+        cli,
+        "index_documents",
+        AsyncMock(side_effect=lambda *docs: calls.append(f"paperless {list(docs)}")),
+    )
+    monkeypatch.setattr(
+        cli,
+        "index_obsidian_documents",
+        AsyncMock(side_effect=lambda: calls.append("obsidian")),
+    )
+    monkeypatch.setattr(cli, "stats", MagicMock())
+    return calls
+
+
+@pytest.mark.parametrize(
+    "command, expected",
+    [
+        # A full reindex must not drop Obsidian notes after wiping everything.
+        # Fetch before deleting, then index the fetched documents (no refetch).
+        ("reindex", ["fetch", "delete_all", "paperless [['doc']]", "obsidian"]),
+        ("reindex-obsidian", ["delete source=obsidian", "obsidian"]),
+    ],
+)
+def test_reindex_commands(calls, monkeypatch, command, expected):
+    monkeypatch.setattr(sys, "argv", ["manage_vectorstore", command])
+    cli.main()
+    assert calls == expected
+
+
+@pytest.mark.parametrize(
+    "failure",
+    [
+        AsyncMock(side_effect=ConnectionError("Name or service not known")),
+        AsyncMock(return_value=[]),
+    ],
+)
+def test_reindex_leaves_index_untouched_when_paperless_fails(
+    calls, monkeypatch, failure
+):
+    monkeypatch.setattr(cli, "fetch_documents_from_paperless_ngx", failure)
+    monkeypatch.setattr(sys, "argv", ["manage_vectorstore", "reindex"])
+    with pytest.raises(SystemExit):
+        cli.main()
+    assert "delete_all" not in calls