ryan
ec3d4229e3
`make reindex` and `make reindex-obsidian` built a `python -c` string with backslash continuations, which kept the tab indentation inside the code, so Python failed with IndentationError. The targets now call the existing `scripts.manage_vectorstore` CLI. Its `reindex` wiped everything but only re-indexed Paperless, dropping Obsidian notes; it now indexes both, and a new `reindex-obsidian` command re-embeds only the vault. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
136 lines
4.1 KiB
Python
136 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Management script for vector store operations."""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import sys
|
|
|
|
from blueprints.rag.logic import (
|
|
delete_all_documents,
|
|
delete_documents_by_metadata,
|
|
get_vector_store_stats,
|
|
index_documents,
|
|
index_obsidian_documents,
|
|
list_all_documents,
|
|
)
|
|
|
|
|
|
def stats():
|
|
"""Show vector store statistics."""
|
|
s = get_vector_store_stats()
|
|
print("=== Vector Store Statistics ===")
|
|
print(f"Collection: {s['collection_name']}")
|
|
print(f"Total Documents: {s['total_documents']}")
|
|
|
|
|
|
async def index():
|
|
"""Index documents from Paperless-NGX."""
|
|
print("Starting indexing process...")
|
|
print("Fetching documents from Paperless-NGX...")
|
|
await index_documents()
|
|
print("Indexing complete!")
|
|
stats()
|
|
|
|
|
|
async def reindex():
|
|
"""Clear and reindex everything: Paperless documents and the Obsidian vault."""
|
|
print("Clearing existing documents...")
|
|
delete_all_documents()
|
|
print("Cleared")
|
|
await index()
|
|
await reindex_obsidian(clear=False)
|
|
|
|
|
|
async def reindex_obsidian(clear: bool = True):
|
|
"""Re-embed the Obsidian vault, leaving Paperless documents alone."""
|
|
if clear:
|
|
print("Clearing Obsidian notes...")
|
|
delete_documents_by_metadata("source", "obsidian")
|
|
print("Indexing Obsidian vault...")
|
|
await index_obsidian_documents()
|
|
print("Obsidian indexing complete!")
|
|
stats()
|
|
|
|
|
|
def list_docs(limit: int = 10, show_content: bool = False):
|
|
"""List documents in the vector store."""
|
|
docs = list_all_documents(limit=limit)
|
|
print(f"\n=== Documents (showing {len(docs)}) ===\n")
|
|
|
|
for i, doc in enumerate(docs, 1):
|
|
print(f"Document {i}:")
|
|
print(f" ID: {doc['id']}")
|
|
print(f" Metadata: {doc['metadata']}")
|
|
if show_content:
|
|
print(f" Content: {doc['content_preview']}")
|
|
print()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Manage vector store for RAG system",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
%(prog)s stats # Show vector store statistics
|
|
%(prog)s index # Index new documents from Paperless-NGX
|
|
%(prog)s reindex # Clear and reindex Paperless + Obsidian
|
|
%(prog)s reindex-obsidian # Clear and reindex only Obsidian notes
|
|
%(prog)s list 10 # List first 10 documents
|
|
%(prog)s list 20 --show-content # List 20 documents with content preview
|
|
""",
|
|
)
|
|
|
|
subparsers = parser.add_subparsers(dest="command", help="Command to execute")
|
|
|
|
# Stats command
|
|
subparsers.add_parser("stats", help="Show vector store statistics")
|
|
|
|
# Index command
|
|
subparsers.add_parser("index", help="Index documents from Paperless-NGX")
|
|
|
|
# Reindex command
|
|
subparsers.add_parser(
|
|
"reindex", help="Clear and reindex Paperless documents and Obsidian notes"
|
|
)
|
|
subparsers.add_parser(
|
|
"reindex-obsidian", help="Clear and reindex only Obsidian notes"
|
|
)
|
|
|
|
# List command
|
|
list_parser = subparsers.add_parser("list", help="List documents in vector store")
|
|
list_parser.add_argument(
|
|
"limit", type=int, default=10, nargs="?", help="Number of documents to list"
|
|
)
|
|
list_parser.add_argument(
|
|
"--show-content", action="store_true", help="Show content preview"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if not args.command:
|
|
parser.print_help()
|
|
sys.exit(1)
|
|
|
|
try:
|
|
if args.command == "stats":
|
|
stats()
|
|
elif args.command == "index":
|
|
asyncio.run(index())
|
|
elif args.command == "reindex":
|
|
asyncio.run(reindex())
|
|
elif args.command == "reindex-obsidian":
|
|
asyncio.run(reindex_obsidian())
|
|
elif args.command == "list":
|
|
list_docs(limit=args.limit, show_content=args.show_content)
|
|
except KeyboardInterrupt:
|
|
print("\n\nOperation cancelled by user")
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
print(f"\nError: {e}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|