diff --git a/blueprints/rag/logic.py b/blueprints/rag/logic.py index 665f97b..01e2f5b 100644 --- a/blueprints/rag/logic.py +++ b/blueprints/rag/logic.py @@ -202,9 +202,13 @@ def _sanitize_documents(documents: list[Document]) -> list[Document]: return [doc for doc in documents if doc.page_content] -async def index_documents(): - """Index Paperless-NGX documents into vector store.""" - documents = await fetch_documents_from_paperless_ngx() +async def index_documents(documents: list[Document] | None = None): + """Index Paperless-NGX documents into vector store. + + Pass already-fetched `documents` to index them without fetching again. + """ + if documents is None: + documents = await fetch_documents_from_paperless_ngx() splits = text_splitter.split_documents(documents) splits = _sanitize_documents(splits) diff --git a/scripts/manage_vectorstore.py b/scripts/manage_vectorstore.py index ad37ab3..1abc2ca 100644 --- a/scripts/manage_vectorstore.py +++ b/scripts/manage_vectorstore.py @@ -8,6 +8,7 @@ import sys from blueprints.rag.logic import ( delete_all_documents, delete_documents_by_metadata, + fetch_documents_from_paperless_ngx, get_vector_store_stats, index_documents, index_obsidian_documents, @@ -33,11 +34,23 @@ async def index(): async def reindex(): - """Clear and reindex everything: Paperless documents and the Obsidian vault.""" + """Clear and reindex everything: Paperless documents and the Obsidian vault. + + Paperless is fetched before anything is deleted, so an unreachable or + empty Paperless leaves the existing index untouched. + """ + print("Fetching documents from Paperless-NGX...") + documents = await fetch_documents_from_paperless_ngx() + if not documents: + raise RuntimeError( + "Paperless returned no documents to index; leaving the index untouched" + ) + print(f"Fetched {len(documents)} documents") print("Clearing existing documents...") delete_all_documents() print("Cleared") - await index() + await index_documents(documents) + print("Paperless indexing complete!") await reindex_obsidian(clear=False) diff --git a/tests/unit/test_manage_vectorstore.py b/tests/unit/test_manage_vectorstore.py index cd9324b..9252aed 100644 --- a/tests/unit/test_manage_vectorstore.py +++ b/tests/unit/test_manage_vectorstore.py @@ -16,7 +16,14 @@ def calls(monkeypatch): lambda k, v: calls.append(f"delete {k}={v}"), ) monkeypatch.setattr( - cli, "index_documents", AsyncMock(side_effect=lambda: calls.append("paperless")) + cli, + "fetch_documents_from_paperless_ngx", + AsyncMock(side_effect=lambda: calls.append("fetch") or ["doc"]), + ) + monkeypatch.setattr( + cli, + "index_documents", + AsyncMock(side_effect=lambda *docs: calls.append(f"paperless {list(docs)}")), ) monkeypatch.setattr( cli, @@ -31,7 +38,8 @@ def calls(monkeypatch): "command, expected", [ # A full reindex must not drop Obsidian notes after wiping everything. - ("reindex", ["delete_all", "paperless", "obsidian"]), + # Fetch before deleting, then index the fetched documents (no refetch). + ("reindex", ["fetch", "delete_all", "paperless [['doc']]", "obsidian"]), ("reindex-obsidian", ["delete source=obsidian", "obsidian"]), ], ) @@ -39,3 +47,20 @@ def test_reindex_commands(calls, monkeypatch, command, expected): monkeypatch.setattr(sys, "argv", ["manage_vectorstore", command]) cli.main() assert calls == expected + + +@pytest.mark.parametrize( + "failure", + [ + AsyncMock(side_effect=ConnectionError("Name or service not known")), + AsyncMock(return_value=[]), + ], +) +def test_reindex_leaves_index_untouched_when_paperless_fails( + calls, monkeypatch, failure +): + monkeypatch.setattr(cli, "fetch_documents_from_paperless_ngx", failure) + monkeypatch.setattr(sys, "argv", ["manage_vectorstore", "reindex"]) + with pytest.raises(SystemExit): + cli.main() + assert "delete_all" not in calls