diff --git a/Makefile b/Makefile index 0e5d0cf..73b3532 100644 --- a/Makefile +++ b/Makefile @@ -34,21 +34,10 @@ logs: # Vector store reindexing reindex: - $(COMPOSE) exec $(RAGGR) python -c "\ - import asyncio; \ - from blueprints.rag.logic import delete_all_documents, index_documents, index_obsidian_documents; \ - delete_all_documents(); \ - asyncio.run(index_documents()); \ - asyncio.run(index_obsidian_documents()); \ - print('Reindex complete')" + $(COMPOSE) exec $(RAGGR) python -m scripts.manage_vectorstore reindex reindex-obsidian: - $(COMPOSE) exec $(RAGGR) python -c "\ - import asyncio; \ - from blueprints.rag.logic import delete_documents_by_metadata, index_obsidian_documents; \ - delete_documents_by_metadata('source', 'obsidian'); \ - asyncio.run(index_obsidian_documents()); \ - print('Obsidian reindex complete')" + $(COMPOSE) exec $(RAGGR) python -m scripts.manage_vectorstore reindex-obsidian # YNAB mirror: delta sync, or full re-download ynab-sync: diff --git a/scripts/manage_vectorstore.py b/scripts/manage_vectorstore.py index cb41dd7..ad37ab3 100644 --- a/scripts/manage_vectorstore.py +++ b/scripts/manage_vectorstore.py @@ -7,8 +7,10 @@ import sys from blueprints.rag.logic import ( delete_all_documents, + delete_documents_by_metadata, get_vector_store_stats, index_documents, + index_obsidian_documents, list_all_documents, ) @@ -31,11 +33,23 @@ async def index(): async def reindex(): - """Clear and reindex all documents.""" + """Clear and reindex everything: Paperless documents and the Obsidian vault.""" print("Clearing existing documents...") delete_all_documents() print("Cleared") await index() + await reindex_obsidian(clear=False) + + +async def reindex_obsidian(clear: bool = True): + """Re-embed the Obsidian vault, leaving Paperless documents alone.""" + if clear: + print("Clearing Obsidian notes...") + delete_documents_by_metadata("source", "obsidian") + print("Indexing Obsidian vault...") + await index_obsidian_documents() + print("Obsidian indexing complete!") + stats() def list_docs(limit: int = 10, show_content: bool = False): @@ -60,7 +74,8 @@ def main(): Examples: %(prog)s stats # Show vector store statistics %(prog)s index # Index new documents from Paperless-NGX - %(prog)s reindex # Clear and reindex all documents + %(prog)s reindex # Clear and reindex Paperless + Obsidian + %(prog)s reindex-obsidian # Clear and reindex only Obsidian notes %(prog)s list 10 # List first 10 documents %(prog)s list 20 --show-content # List 20 documents with content preview """, @@ -75,7 +90,12 @@ Examples: subparsers.add_parser("index", help="Index documents from Paperless-NGX") # Reindex command - subparsers.add_parser("reindex", help="Clear and reindex all documents") + subparsers.add_parser( + "reindex", help="Clear and reindex Paperless documents and Obsidian notes" + ) + subparsers.add_parser( + "reindex-obsidian", help="Clear and reindex only Obsidian notes" + ) # List command list_parser = subparsers.add_parser("list", help="List documents in vector store") @@ -99,6 +119,8 @@ Examples: asyncio.run(index()) elif args.command == "reindex": asyncio.run(reindex()) + elif args.command == "reindex-obsidian": + asyncio.run(reindex_obsidian()) elif args.command == "list": list_docs(limit=args.limit, show_content=args.show_content) except KeyboardInterrupt: diff --git a/tests/unit/test_manage_vectorstore.py b/tests/unit/test_manage_vectorstore.py new file mode 100644 index 0000000..cd9324b --- /dev/null +++ b/tests/unit/test_manage_vectorstore.py @@ -0,0 +1,41 @@ +import sys +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from scripts import manage_vectorstore as cli + + +@pytest.fixture +def calls(monkeypatch): + calls = [] + monkeypatch.setattr(cli, "delete_all_documents", lambda: calls.append("delete_all")) + monkeypatch.setattr( + cli, + "delete_documents_by_metadata", + lambda k, v: calls.append(f"delete {k}={v}"), + ) + monkeypatch.setattr( + cli, "index_documents", AsyncMock(side_effect=lambda: calls.append("paperless")) + ) + monkeypatch.setattr( + cli, + "index_obsidian_documents", + AsyncMock(side_effect=lambda: calls.append("obsidian")), + ) + monkeypatch.setattr(cli, "stats", MagicMock()) + return calls + + +@pytest.mark.parametrize( + "command, expected", + [ + # A full reindex must not drop Obsidian notes after wiping everything. + ("reindex", ["delete_all", "paperless", "obsidian"]), + ("reindex-obsidian", ["delete source=obsidian", "obsidian"]), + ], +) +def test_reindex_commands(calls, monkeypatch, command, expected): + monkeypatch.setattr(sys, "argv", ["manage_vectorstore", command]) + cli.main() + assert calls == expected