From ec3d4229e3e8e32e1eb02a5515db98408e5a1309 Mon Sep 17 00:00:00 2001 From: Ryan Chen Date: Tue, 13 Oct 2026 08:20:18 -0400 Subject: [PATCH 1/2] Fix make reindex targets and reindex Obsidian in the vector store CLI `make reindex` and `make reindex-obsidian` built a `python -c` string with backslash continuations, which kept the tab indentation inside the code, so Python failed with IndentationError. The targets now call the existing `scripts.manage_vectorstore` CLI. Its `reindex` wiped everything but only re-indexed Paperless, dropping Obsidian notes; it now indexes both, and a new `reindex-obsidian` command re-embeds only the vault. Co-Authored-By: Claude Opus 5.5 --- Makefile | 15 ++-------- scripts/manage_vectorstore.py | 28 ++++++++++++++++-- tests/unit/test_manage_vectorstore.py | 41 +++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 16 deletions(-) create mode 100644 tests/unit/test_manage_vectorstore.py diff --git a/Makefile b/Makefile index 0e5d0cf..73b3532 100644 --- a/Makefile +++ b/Makefile @@ -34,21 +34,10 @@ logs: # Vector store reindexing reindex: - $(COMPOSE) exec $(RAGGR) python -c "\ - import asyncio; \ - from blueprints.rag.logic import delete_all_documents, index_documents, index_obsidian_documents; \ - delete_all_documents(); \ - asyncio.run(index_documents()); \ - asyncio.run(index_obsidian_documents()); \ - print('Reindex complete')" + $(COMPOSE) exec $(RAGGR) python -m scripts.manage_vectorstore reindex reindex-obsidian: - $(COMPOSE) exec $(RAGGR) python -c "\ - import asyncio; \ - from blueprints.rag.logic import delete_documents_by_metadata, index_obsidian_documents; \ - delete_documents_by_metadata('source', 'obsidian'); \ - asyncio.run(index_obsidian_documents()); \ - print('Obsidian reindex complete')" + $(COMPOSE) exec $(RAGGR) python -m scripts.manage_vectorstore reindex-obsidian # YNAB mirror: delta sync, or full re-download ynab-sync: diff --git a/scripts/manage_vectorstore.py b/scripts/manage_vectorstore.py index cb41dd7..ad37ab3 100644 --- a/scripts/manage_vectorstore.py +++ b/scripts/manage_vectorstore.py @@ -7,8 +7,10 @@ import sys from blueprints.rag.logic import ( delete_all_documents, + delete_documents_by_metadata, get_vector_store_stats, index_documents, + index_obsidian_documents, list_all_documents, ) @@ -31,11 +33,23 @@ async def index(): async def reindex(): - """Clear and reindex all documents.""" + """Clear and reindex everything: Paperless documents and the Obsidian vault.""" print("Clearing existing documents...") delete_all_documents() print("Cleared") await index() + await reindex_obsidian(clear=False) + + +async def reindex_obsidian(clear: bool = True): + """Re-embed the Obsidian vault, leaving Paperless documents alone.""" + if clear: + print("Clearing Obsidian notes...") + delete_documents_by_metadata("source", "obsidian") + print("Indexing Obsidian vault...") + await index_obsidian_documents() + print("Obsidian indexing complete!") + stats() def list_docs(limit: int = 10, show_content: bool = False): @@ -60,7 +74,8 @@ def main(): Examples: %(prog)s stats # Show vector store statistics %(prog)s index # Index new documents from Paperless-NGX - %(prog)s reindex # Clear and reindex all documents + %(prog)s reindex # Clear and reindex Paperless + Obsidian + %(prog)s reindex-obsidian # Clear and reindex only Obsidian notes %(prog)s list 10 # List first 10 documents %(prog)s list 20 --show-content # List 20 documents with content preview """, @@ -75,7 +90,12 @@ Examples: subparsers.add_parser("index", help="Index documents from Paperless-NGX") # Reindex command - subparsers.add_parser("reindex", help="Clear and reindex all documents") + subparsers.add_parser( + "reindex", help="Clear and reindex Paperless documents and Obsidian notes" + ) + subparsers.add_parser( + "reindex-obsidian", help="Clear and reindex only Obsidian notes" + ) # List command list_parser = subparsers.add_parser("list", help="List documents in vector store") @@ -99,6 +119,8 @@ Examples: asyncio.run(index()) elif args.command == "reindex": asyncio.run(reindex()) + elif args.command == "reindex-obsidian": + asyncio.run(reindex_obsidian()) elif args.command == "list": list_docs(limit=args.limit, show_content=args.show_content) except KeyboardInterrupt: diff --git a/tests/unit/test_manage_vectorstore.py b/tests/unit/test_manage_vectorstore.py new file mode 100644 index 0000000..cd9324b --- /dev/null +++ b/tests/unit/test_manage_vectorstore.py @@ -0,0 +1,41 @@ +import sys +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from scripts import manage_vectorstore as cli + + +@pytest.fixture +def calls(monkeypatch): + calls = [] + monkeypatch.setattr(cli, "delete_all_documents", lambda: calls.append("delete_all")) + monkeypatch.setattr( + cli, + "delete_documents_by_metadata", + lambda k, v: calls.append(f"delete {k}={v}"), + ) + monkeypatch.setattr( + cli, "index_documents", AsyncMock(side_effect=lambda: calls.append("paperless")) + ) + monkeypatch.setattr( + cli, + "index_obsidian_documents", + AsyncMock(side_effect=lambda: calls.append("obsidian")), + ) + monkeypatch.setattr(cli, "stats", MagicMock()) + return calls + + +@pytest.mark.parametrize( + "command, expected", + [ + # A full reindex must not drop Obsidian notes after wiping everything. + ("reindex", ["delete_all", "paperless", "obsidian"]), + ("reindex-obsidian", ["delete source=obsidian", "obsidian"]), + ], +) +def test_reindex_commands(calls, monkeypatch, command, expected): + monkeypatch.setattr(sys, "argv", ["manage_vectorstore", command]) + cli.main() + assert calls == expected -- 2.52.0 From b8d8bb8a578f467f0545adb16a56c1f16f1cd47c Mon Sep 17 00:00:00 2001 From: Ryan Chen Date: Tue, 13 Oct 2026 08:30:27 -0400 Subject: [PATCH 2/2] Fetch Paperless before wiping the index in reindex `reindex` deleted every chunk before contacting Paperless, so an unreachable or empty Paperless left the store empty (which happened when Paperless was crash-looping during its v3 upgrade). It now fetches first, aborts without deleting if that fails or returns nothing, and indexes the fetched documents instead of fetching twice. Co-Authored-By: Claude Opus 5.5 --- blueprints/rag/logic.py | 10 ++++++--- scripts/manage_vectorstore.py | 17 ++++++++++++++-- tests/unit/test_manage_vectorstore.py | 29 +++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/blueprints/rag/logic.py b/blueprints/rag/logic.py index 665f97b..01e2f5b 100644 --- a/blueprints/rag/logic.py +++ b/blueprints/rag/logic.py @@ -202,9 +202,13 @@ def _sanitize_documents(documents: list[Document]) -> list[Document]: return [doc for doc in documents if doc.page_content] -async def index_documents(): - """Index Paperless-NGX documents into vector store.""" - documents = await fetch_documents_from_paperless_ngx() +async def index_documents(documents: list[Document] | None = None): + """Index Paperless-NGX documents into vector store. + + Pass already-fetched `documents` to index them without fetching again. + """ + if documents is None: + documents = await fetch_documents_from_paperless_ngx() splits = text_splitter.split_documents(documents) splits = _sanitize_documents(splits) diff --git a/scripts/manage_vectorstore.py b/scripts/manage_vectorstore.py index ad37ab3..1abc2ca 100644 --- a/scripts/manage_vectorstore.py +++ b/scripts/manage_vectorstore.py @@ -8,6 +8,7 @@ import sys from blueprints.rag.logic import ( delete_all_documents, delete_documents_by_metadata, + fetch_documents_from_paperless_ngx, get_vector_store_stats, index_documents, index_obsidian_documents, @@ -33,11 +34,23 @@ async def index(): async def reindex(): - """Clear and reindex everything: Paperless documents and the Obsidian vault.""" + """Clear and reindex everything: Paperless documents and the Obsidian vault. + + Paperless is fetched before anything is deleted, so an unreachable or + empty Paperless leaves the existing index untouched. + """ + print("Fetching documents from Paperless-NGX...") + documents = await fetch_documents_from_paperless_ngx() + if not documents: + raise RuntimeError( + "Paperless returned no documents to index; leaving the index untouched" + ) + print(f"Fetched {len(documents)} documents") print("Clearing existing documents...") delete_all_documents() print("Cleared") - await index() + await index_documents(documents) + print("Paperless indexing complete!") await reindex_obsidian(clear=False) diff --git a/tests/unit/test_manage_vectorstore.py b/tests/unit/test_manage_vectorstore.py index cd9324b..9252aed 100644 --- a/tests/unit/test_manage_vectorstore.py +++ b/tests/unit/test_manage_vectorstore.py @@ -16,7 +16,14 @@ def calls(monkeypatch): lambda k, v: calls.append(f"delete {k}={v}"), ) monkeypatch.setattr( - cli, "index_documents", AsyncMock(side_effect=lambda: calls.append("paperless")) + cli, + "fetch_documents_from_paperless_ngx", + AsyncMock(side_effect=lambda: calls.append("fetch") or ["doc"]), + ) + monkeypatch.setattr( + cli, + "index_documents", + AsyncMock(side_effect=lambda *docs: calls.append(f"paperless {list(docs)}")), ) monkeypatch.setattr( cli, @@ -31,7 +38,8 @@ def calls(monkeypatch): "command, expected", [ # A full reindex must not drop Obsidian notes after wiping everything. - ("reindex", ["delete_all", "paperless", "obsidian"]), + # Fetch before deleting, then index the fetched documents (no refetch). + ("reindex", ["fetch", "delete_all", "paperless [['doc']]", "obsidian"]), ("reindex-obsidian", ["delete source=obsidian", "obsidian"]), ], ) @@ -39,3 +47,20 @@ def test_reindex_commands(calls, monkeypatch, command, expected): monkeypatch.setattr(sys, "argv", ["manage_vectorstore", command]) cli.main() assert calls == expected + + +@pytest.mark.parametrize( + "failure", + [ + AsyncMock(side_effect=ConnectionError("Name or service not known")), + AsyncMock(return_value=[]), + ], +) +def test_reindex_leaves_index_untouched_when_paperless_fails( + calls, monkeypatch, failure +): + monkeypatch.setattr(cli, "fetch_documents_from_paperless_ngx", failure) + monkeypatch.setattr(sys, "argv", ["manage_vectorstore", "reindex"]) + with pytest.raises(SystemExit): + cli.main() + assert "delete_all" not in calls -- 2.52.0