Merge branch 'main' into fix/reindex-targets

setup-node's `cache: yarn` needs yarn already on PATH, and the Node it
installs does not ship one, so the frontend job failed at setup on every
run. Install the lockfile's Yarn 1.22.22 explicitly instead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitea/workflows/ci.yml

@@ -34,11 +34,13 @@ jobs:
         working-directory: raggr-frontend
     steps:
       - uses: actions/checkout@v4.2.2
+      # No `cache: yarn` here: setup-node requires yarn to already be on PATH
+      # for that, and the Node it installs doesn't ship one.
       - uses: actions/setup-node@v4.4.0
         with:
           node-version: '22.14.0'
-          cache: yarn
-          cache-dependency-path: raggr-frontend/yarn.lock
+      - name: Install Yarn (lockfile v1)
+        run: npm install --global yarn@1.22.22
       - name: Install locked frontend dependencies
         run: yarn install --frozen-lockfile
       - name: Renderer tests

Makefile

@@ -34,21 +34,10 @@ logs:
 
 # Vector store reindexing
 reindex:
-	$(COMPOSE) exec $(RAGGR) python -c "\
-		import asyncio; \
-		from blueprints.rag.logic import delete_all_documents, index_documents, index_obsidian_documents; \
-		delete_all_documents(); \
-		asyncio.run(index_documents()); \
-		asyncio.run(index_obsidian_documents()); \
-		print('Reindex complete')"
+	$(COMPOSE) exec $(RAGGR) python -m scripts.manage_vectorstore reindex
 
 reindex-obsidian:
-	$(COMPOSE) exec $(RAGGR) python -c "\
-		import asyncio; \
-		from blueprints.rag.logic import delete_documents_by_metadata, index_obsidian_documents; \
-		delete_documents_by_metadata('source', 'obsidian'); \
-		asyncio.run(index_obsidian_documents()); \
-		print('Obsidian reindex complete')"
+	$(COMPOSE) exec $(RAGGR) python -m scripts.manage_vectorstore reindex-obsidian
 
 # YNAB mirror: delta sync, or full re-download
 ynab-sync:

blueprints/rag/logic.py

@@ -202,9 +202,13 @@ def _sanitize_documents(documents: list[Document]) -> list[Document]:
     return [doc for doc in documents if doc.page_content]
 
 
-async def index_documents():
-    """Index Paperless-NGX documents into vector store."""
-    documents = await fetch_documents_from_paperless_ngx()
+async def index_documents(documents: list[Document] | None = None):
+    """Index Paperless-NGX documents into vector store.
+
+    Pass already-fetched `documents` to index them without fetching again.
+    """
+    if documents is None:
+        documents = await fetch_documents_from_paperless_ngx()
 
     splits = text_splitter.split_documents(documents)
     splits = _sanitize_documents(splits)

scripts/manage_vectorstore.py

@@ -7,8 +7,11 @@ import sys
 
 from blueprints.rag.logic import (
     delete_all_documents,
+    delete_documents_by_metadata,
+    fetch_documents_from_paperless_ngx,
     get_vector_store_stats,
     index_documents,
+    index_obsidian_documents,
     list_all_documents,
 )
 
@@ -31,11 +34,35 @@ async def index():
 
 
 async def reindex():
-    """Clear and reindex all documents."""
+    """Clear and reindex everything: Paperless documents and the Obsidian vault.
+
+    Paperless is fetched before anything is deleted, so an unreachable or
+    empty Paperless leaves the existing index untouched.
+    """
+    print("Fetching documents from Paperless-NGX...")
+    documents = await fetch_documents_from_paperless_ngx()
+    if not documents:
+        raise RuntimeError(
+            "Paperless returned no documents to index; leaving the index untouched"
+        )
+    print(f"Fetched {len(documents)} documents")
     print("Clearing existing documents...")
     delete_all_documents()
     print("Cleared")
-    await index()
+    await index_documents(documents)
+    print("Paperless indexing complete!")
+    await reindex_obsidian(clear=False)
+
+
+async def reindex_obsidian(clear: bool = True):
+    """Re-embed the Obsidian vault, leaving Paperless documents alone."""
+    if clear:
+        print("Clearing Obsidian notes...")
+        delete_documents_by_metadata("source", "obsidian")
+    print("Indexing Obsidian vault...")
+    await index_obsidian_documents()
+    print("Obsidian indexing complete!")
+    stats()
 
 
 def list_docs(limit: int = 10, show_content: bool = False):
@@ -60,7 +87,8 @@ def main():
 Examples:
   %(prog)s stats                    # Show vector store statistics
   %(prog)s index                    # Index new documents from Paperless-NGX
-  %(prog)s reindex                  # Clear and reindex all documents
+  %(prog)s reindex                  # Clear and reindex Paperless + Obsidian
+  %(prog)s reindex-obsidian         # Clear and reindex only Obsidian notes
   %(prog)s list 10                  # List first 10 documents
   %(prog)s list 20 --show-content   # List 20 documents with content preview
         """,
@@ -75,7 +103,12 @@ Examples:
     subparsers.add_parser("index", help="Index documents from Paperless-NGX")
 
     # Reindex command
-    subparsers.add_parser("reindex", help="Clear and reindex all documents")
+    subparsers.add_parser(
+        "reindex", help="Clear and reindex Paperless documents and Obsidian notes"
+    )
+    subparsers.add_parser(
+        "reindex-obsidian", help="Clear and reindex only Obsidian notes"
+    )
 
     # List command
     list_parser = subparsers.add_parser("list", help="List documents in vector store")
@@ -99,6 +132,8 @@ Examples:
             asyncio.run(index())
         elif args.command == "reindex":
             asyncio.run(reindex())
+        elif args.command == "reindex-obsidian":
+            asyncio.run(reindex_obsidian())
         elif args.command == "list":
             list_docs(limit=args.limit, show_content=args.show_content)
     except KeyboardInterrupt:

tests/unit/test_manage_vectorstore.py

@@ -0,0 +1,66 @@
+import sys
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from scripts import manage_vectorstore as cli
+
+
+@pytest.fixture
+def calls(monkeypatch):
+    calls = []
+    monkeypatch.setattr(cli, "delete_all_documents", lambda: calls.append("delete_all"))
+    monkeypatch.setattr(
+        cli,
+        "delete_documents_by_metadata",
+        lambda k, v: calls.append(f"delete {k}={v}"),
+    )
+    monkeypatch.setattr(
+        cli,
+        "fetch_documents_from_paperless_ngx",
+        AsyncMock(side_effect=lambda: calls.append("fetch") or ["doc"]),
+    )
+    monkeypatch.setattr(
+        cli,
+        "index_documents",
+        AsyncMock(side_effect=lambda *docs: calls.append(f"paperless {list(docs)}")),
+    )
+    monkeypatch.setattr(
+        cli,
+        "index_obsidian_documents",
+        AsyncMock(side_effect=lambda: calls.append("obsidian")),
+    )
+    monkeypatch.setattr(cli, "stats", MagicMock())
+    return calls
+
+
+@pytest.mark.parametrize(
+    "command, expected",
+    [
+        # A full reindex must not drop Obsidian notes after wiping everything.
+        # Fetch before deleting, then index the fetched documents (no refetch).
+        ("reindex", ["fetch", "delete_all", "paperless [['doc']]", "obsidian"]),
+        ("reindex-obsidian", ["delete source=obsidian", "obsidian"]),
+    ],
+)
+def test_reindex_commands(calls, monkeypatch, command, expected):
+    monkeypatch.setattr(sys, "argv", ["manage_vectorstore", command])
+    cli.main()
+    assert calls == expected
+
+
+@pytest.mark.parametrize(
+    "failure",
+    [
+        AsyncMock(side_effect=ConnectionError("Name or service not known")),
+        AsyncMock(return_value=[]),
+    ],
+)
+def test_reindex_leaves_index_untouched_when_paperless_fails(
+    calls, monkeypatch, failure
+):
+    monkeypatch.setattr(cli, "fetch_documents_from_paperless_ngx", failure)
+    monkeypatch.setattr(sys, "argv", ["manage_vectorstore", "reindex"])
+    with pytest.raises(SystemExit):
+        cli.main()
+    assert "delete_all" not in calls