enabling login btw users

oops
i commit
2025-10-25 09:30:54 -04:00 · 2025-10-23 22:45:14 -04:00 · 2025-10-23 22:39:26 -04:00 · 2025-10-23 22:29:12 -04:00 · 2025-10-16 22:43:14 -04:00 · 2025-10-16 22:36:21 -04:00
35 changed files with 1958 additions and 255 deletions
--- a/3
+++ b/3
@@ -23,6 +23,9 @@ RUN uv pip install --system -e .

 # Copy application code
 COPY *.py ./
+COPY blueprints ./blueprints
+COPY aerich.toml ./
+COPY migrations ./migrations
 COPY startup.sh ./
 RUN chmod +x startup.sh

--- a/MIGRATIONS.md
+++ b/MIGRATIONS.md
@@ -0,0 +1,54 @@
+# Database Migrations with Aerich
+
+## Initial Setup (Run Once)
+
+1. Install dependencies:
+```bash
+uv pip install -e .
+```
+
+2. Initialize Aerich:
+```bash
+aerich init-db
+```
+
+This will:
+- Create a `migrations/` directory
+- Generate the initial migration based on your models
+- Create all tables in the database
+
+## When You Add/Change Models
+
+1. Generate a new migration:
+```bash
+aerich migrate --name "describe_your_changes"
+```
+
+Example:
+```bash
+aerich migrate --name "add_user_profile_model"
+```
+
+2. Apply the migration:
+```bash
+aerich upgrade
+```
+
+## Common Commands
+
+- `aerich init-db` - Initialize database (first time only)
+- `aerich migrate --name "description"` - Generate new migration
+- `aerich upgrade` - Apply pending migrations
+- `aerich downgrade` - Rollback last migration
+- `aerich history` - Show migration history
+- `aerich heads` - Show current migration heads
+
+## Docker Setup
+
+In Docker, migrations run automatically on container startup via the startup script.
+
+## Notes
+
+- Migration files are stored in `migrations/models/`
+- Always commit migration files to version control
+- Don't modify migration files manually after they're created
--- a/add_user.py
+++ b/add_user.py
@@ -0,0 +1,130 @@
+# GENERATED BY CLAUDE
+
+import sys
+import uuid
+import asyncio
+from tortoise import Tortoise
+from blueprints.users.models import User
+
+
+async def add_user(username: str, email: str, password: str):
+    """Add a new user to the database"""
+    await Tortoise.init(
+        db_url="sqlite://raggr.db",
+        modules={
+            "models": [
+                "blueprints.users.models",
+                "blueprints.conversation.models",
+            ]
+        },
+    )
+
+    try:
+        # Check if user already exists
+        existing_user = await User.filter(email=email).first()
+        if existing_user:
+            print(f"Error: User with email '{email}' already exists!")
+            return False
+
+        existing_username = await User.filter(username=username).first()
+        if existing_username:
+            print(f"Error: Username '{username}' is already taken!")
+            return False
+
+        # Create new user
+        user = User(
+            id=uuid.uuid4(),
+            username=username,
+            email=email,
+        )
+        user.set_password(password)
+        await user.save()
+
+        print("✓ User created successfully!")
+        print(f"  Username: {username}")
+        print(f"  Email: {email}")
+        print(f"  ID: {user.id}")
+        return True
+
+    except Exception as e:
+        print(f"Error creating user: {e}")
+        return False
+    finally:
+        await Tortoise.close_connections()
+
+
+async def list_users():
+    """List all users in the database"""
+    await Tortoise.init(
+        db_url="sqlite://raggr.db",
+        modules={
+            "models": [
+                "blueprints.users.models",
+                "blueprints.conversation.models",
+            ]
+        },
+    )
+
+    try:
+        users = await User.all()
+        if not users:
+            print("No users found in database.")
+            return
+
+        print(f"\nFound {len(users)} user(s):")
+        print("-" * 60)
+        for user in users:
+            print(f"Username: {user.username}")
+            print(f"Email: {user.email}")
+            print(f"ID: {user.id}")
+            print(f"Created: {user.created_at}")
+            print("-" * 60)
+
+    except Exception as e:
+        print(f"Error listing users: {e}")
+    finally:
+        await Tortoise.close_connections()
+
+
+def print_usage():
+    """Print usage instructions"""
+    print("Usage:")
+    print("  python add_user.py add <username> <email> <password>")
+    print("  python add_user.py list")
+    print("\nExamples:")
+    print("  python add_user.py add ryan ryan@example.com mypassword123")
+    print("  python add_user.py list")
+
+
+async def main():
+    if len(sys.argv) < 2:
+        print_usage()
+        sys.exit(1)
+
+    command = sys.argv[1].lower()
+
+    if command == "add":
+        if len(sys.argv) != 5:
+            print("Error: Missing arguments for 'add' command")
+            print_usage()
+            sys.exit(1)
+
+        username = sys.argv[2]
+        email = sys.argv[3]
+        password = sys.argv[4]
+
+        success = await add_user(username, email, password)
+        sys.exit(0 if success else 1)
+
+    elif command == "list":
+        await list_users()
+        sys.exit(0)
+
+    else:
+        print(f"Error: Unknown command '{command}'")
+        print_usage()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
--- a/aerich_config.py
+++ b/aerich_config.py
@@ -0,0 +1,15 @@
+import os
+
+TORTOISE_ORM = {
+    "connections": {"default": os.getenv("DATABASE_URL", "sqlite:///app/raggr.db")},
+    "apps": {
+        "models": {
+            "models": [
+                "blueprints.conversation.models",
+                "blueprints.users.models",
+                "aerich.models",
+            ],
+            "default_connection": "default",
+        },
+    },
+}
--- a/app.py
+++ b/app.py
@@ -1,43 +1,127 @@
 import os

-from flask import Flask, request, jsonify, render_template, send_from_directory
+from quart import Quart, request, jsonify, render_template, send_from_directory
+from tortoise.contrib.quart import register_tortoise
+
+from quart_jwt_extended import JWTManager, jwt_refresh_token_required, get_jwt_identity

 from main import consult_simba_oracle

-app = Flask(
+import blueprints.users
+import blueprints.conversation
+import blueprints.conversation.logic
+import blueprints.users.models
+
+app = Quart(
    __name__,
    static_folder="raggr-frontend/dist/static",
    template_folder="raggr-frontend/dist",
 )

+app.config["JWT_SECRET_KEY"] = os.getenv("JWT_SECRET_KEY", "SECRET_KEY")
+jwt = JWTManager(app)
+
+# Register blueprints
+app.register_blueprint(blueprints.users.user_blueprint)
+app.register_blueprint(blueprints.conversation.conversation_blueprint)
+
+
+TORTOISE_CONFIG = {
+    "connections": {"default": "sqlite://raggr.db"},
+    "apps": {
+        "models": {
+            "models": [
+                "blueprints.conversation.models",
+                "blueprints.users.models",
+                "aerich.models",
+            ]
+        },
+    },
+}
+
+# Initialize Tortoise ORM
+register_tortoise(
+    app,
+    config=TORTOISE_CONFIG,
+    generate_schemas=False,  # Disabled - using Aerich for migrations
+)
+

 # Serve React static files
@app.route("/static/<path:filename>")
-def static_files(filename):
-    return send_from_directory(app.static_folder, filename)
+async def static_files(filename):
+    return await send_from_directory(app.static_folder, filename)


 # Serve the React app for all routes (catch-all)
@app.route("/", defaults={"path": ""})
@app.route("/<path:path>")
-def serve_react_app(path):
+async def serve_react_app(path):
    if path and os.path.exists(os.path.join(app.template_folder, path)):
-        return send_from_directory(app.template_folder, path)
-    return render_template("index.html")
+        return await send_from_directory(app.template_folder, path)
+    return await render_template("index.html")


@app.route("/api/query", methods=["POST"])
-def query():
-    data = request.get_json()
+@jwt_refresh_token_required
+async def query():
+    current_user_uuid = get_jwt_identity()
+    user = await blueprints.users.models.User.get(id=current_user_uuid)
+    data = await request.get_json()
    query = data.get("query")
-    return jsonify({"response": consult_simba_oracle(query)})
+    conversation = await blueprints.conversation.logic.get_conversation_for_user(
+        user=user
+    )
+    await blueprints.conversation.logic.add_message_to_conversation(
+        conversation=conversation,
+        message=query,
+        speaker="user",
+        user=user,
+    )
+
+    response = consult_simba_oracle(query)
+    await blueprints.conversation.logic.add_message_to_conversation(
+        conversation=conversation,
+        message=response,
+        speaker="simba",
+        user=user,
+    )
+    return jsonify({"response": response})


-@app.route("/api/ingest", methods=["POST"])
-def webhook():
-    data = request.get_json()
-    print(data)
-    return jsonify({"status": "received"})
+@app.route("/api/messages", methods=["GET"])
+@jwt_refresh_token_required
+async def get_messages():
+    current_user_uuid = get_jwt_identity()
+    user = await blueprints.users.models.User.get(id=current_user_uuid)
+
+    conversation = await blueprints.conversation.logic.get_conversation_for_user(
+        user=user
+    )
+    # Prefetch related messages
+    await conversation.fetch_related("messages")
+
+    # Manually serialize the conversation with messages
+    messages = []
+    for msg in conversation.messages:
+        messages.append(
+            {
+                "id": str(msg.id),
+                "text": msg.text,
+                "speaker": msg.speaker.value,
+                "created_at": msg.created_at.isoformat(),
+            }
+        )
+
+    return jsonify(
+        {
+            "id": str(conversation.id),
+            "name": conversation.name,
+            "messages": messages,
+            "created_at": conversation.created_at.isoformat(),
+            "updated_at": conversation.updated_at.isoformat(),
+        }
+    )


 if __name__ == "__main__":
--- a/blueprints/init.py
+++ b/blueprints/init.py
@@ -0,0 +1 @@
+# Blueprints package
--- a/blueprints/conversation/init.py
+++ b/blueprints/conversation/init.py
@@ -0,0 +1,17 @@
+from quart import Blueprint, jsonify
+from .models import (
+    Conversation,
+    PydConversation,
+)
+
+conversation_blueprint = Blueprint(
+    "conversation_api", __name__, url_prefix="/api/conversation"
+)
+
+
+@conversation_blueprint.route("/<conversation_id>")
+async def get_conversation(conversation_id: str):
+    conversation = await Conversation.get(id=conversation_id)
+    serialized_conversation = await PydConversation.from_tortoise_orm(conversation)
+
+    return jsonify(serialized_conversation.model_dump_json())
--- a/blueprints/conversation/logic.py
+++ b/blueprints/conversation/logic.py
@@ -0,0 +1,46 @@
+import tortoise.exceptions
+
+from .models import Conversation, ConversationMessage
+
+import blueprints.users.models
+
+
+async def create_conversation(name: str = "") -> Conversation:
+    conversation = await Conversation.create(name=name)
+    return conversation
+
+
+async def add_message_to_conversation(
+    conversation: Conversation,
+    message: str,
+    speaker: str,
+    user: blueprints.users.models.User,
+) -> ConversationMessage:
+    print(conversation, message, speaker)
+    message = await ConversationMessage.create(
+        text=message,
+        speaker=speaker,
+        conversation=conversation,
+    )
+
+    return message
+
+
+async def get_the_only_conversation() -> Conversation:
+    try:
+        conversation = await Conversation.all().first()
+        if conversation is None:
+            conversation = await Conversation.create(name="simba_chat")
+    except Exception as _e:
+        conversation = await Conversation.create(name="simba_chat")
+
+    return conversation
+
+
+async def get_conversation_for_user(user: blueprints.users.models.User) -> Conversation:
+    try:
+        return await Conversation.get(user=user)
+    except tortoise.exceptions.DoesNotExist:
+        await Conversation.get_or_create(name=f"{user.username}'s chat", user=user)
+
+        return await Conversation.get(user=user)
--- a/blueprints/conversation/models.py
+++ b/blueprints/conversation/models.py
@@ -0,0 +1,44 @@
+import enum
+
+from tortoise.models import Model
+from tortoise import fields
+from tortoise.contrib.pydantic import (
+    pydantic_queryset_creator,
+    pydantic_model_creator,
+)
+
+
+class Speaker(enum.Enum):
+    USER = "user"
+    SIMBA = "simba"
+
+
+class Conversation(Model):
+    id = fields.UUIDField(primary_key=True)
+    name = fields.CharField(max_length=255)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+    user: fields.ForeignKeyRelation = fields.ForeignKeyField(
+        "models.User", related_name="conversations", null=True
+    )
+
+    class Meta:
+        table = "conversations"
+
+
+class ConversationMessage(Model):
+    id = fields.UUIDField(primary_key=True)
+    text = fields.TextField()
+    conversation = fields.ForeignKeyField(
+        "models.Conversation", related_name="messages"
+    )
+    created_at = fields.DatetimeField(auto_now_add=True)
+    speaker = fields.CharEnumField(enum_type=Speaker, max_length=10)
+
+    class Meta:
+        table = "conversation_messages"
+
+
+PydConversationMessage = pydantic_model_creator(ConversationMessage)
+PydConversation = pydantic_model_creator(Conversation, name="Conversation")
+PydListConversationMessage = pydantic_queryset_creator(ConversationMessage)
--- a/blueprints/users/init.py
+++ b/blueprints/users/init.py
@@ -0,0 +1,40 @@
+from quart import Blueprint, jsonify, request
+from quart_jwt_extended import (
+    create_access_token,
+    create_refresh_token,
+    jwt_refresh_token_required,
+    get_jwt_identity,
+)
+from .models import User
+
+
+user_blueprint = Blueprint("user_api", __name__, url_prefix="/api/user")
+
+
+@user_blueprint.route("/login", methods=["POST"])
+async def login():
+    data = await request.get_json()
+    username = data.get("username")
+    password = data.get("password")
+
+    user = await User.filter(username=username).first()
+
+    if not user or not user.verify_password(password):
+        return jsonify({"msg": "Invalid credentials"}), 401
+
+    access_token = create_access_token(identity=str(user.id))
+    refresh_token = create_refresh_token(identity=str(user.id))
+
+    return jsonify(
+        access_token=access_token,
+        refresh_token=refresh_token,
+        user={"id": user.id, "username": user.username},
+    )
+
+
+@user_blueprint.route("/refresh", methods=["POST"])
+@jwt_refresh_token_required
+async def refresh():
+    user_id = get_jwt_identity()
+    new_token = create_access_token(identity=user_id)
+    return jsonify(access_token=new_token)
--- a/blueprints/users/models.py
+++ b/blueprints/users/models.py
@@ -0,0 +1,26 @@
+from tortoise.models import Model
+from tortoise import fields
+
+
+import bcrypt
+
+
+class User(Model):
+    id = fields.UUIDField(primary_key=True)
+    username = fields.CharField(max_length=255)
+    password = fields.BinaryField()  # Hashed
+    email = fields.CharField(max_length=100, unique=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    class Meta:
+        table = "users"
+
+    def set_password(self, plain_password: str):
+        self.password = bcrypt.hashpw(
+            plain_password.encode("utf-8"),
+            bcrypt.gensalt(),
+        )
+
+    def verify_password(self, plain_password: str):
+        return bcrypt.checkpw(plain_password.encode("utf-8"), self.password)
--- a/chunker.py
+++ b/chunker.py
@@ -3,15 +3,20 @@ from math import ceil
 import re
 from typing import Union
 from uuid import UUID, uuid4
-
+from ollama import Client
 from chromadb.utils.embedding_functions.openai_embedding_function import (
    OpenAIEmbeddingFunction,
 )
 from dotenv import load_dotenv
+from llm import LLMClient


 load_dotenv()

+ollama_client = Client(
+    host=os.getenv("OLLAMA_HOST", "http://localhost:11434"), timeout=10.0
+)
+

 def remove_headers_footers(text, header_patterns=None, footer_patterns=None):
    if header_patterns is None:
@@ -80,13 +85,16 @@ class Chunk:


 class Chunker:
-    embedding_fx = OpenAIEmbeddingFunction(
+    def __init__(self, collection) -> None:
+        self.collection = collection
+        self.llm_client = LLMClient()
+
+    def embedding_fx(self, inputs):
+        openai_embedding_fx = OpenAIEmbeddingFunction(
            api_key=os.getenv("OPENAI_API_KEY"),
            model_name="text-embedding-3-small",
        )
-
-    def __init__(self, collection) -> None:
-        self.collection = collection
+        return openai_embedding_fx(inputs)

    def chunk_document(
        self,
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,4 +1,4 @@
-version: '3.8'
+version: "3.8"

 services:
  raggr:
--- a/image_process.py
+++ b/image_process.py
@@ -25,16 +25,18 @@ parser.add_argument("filepath")

 client = Client(host=os.getenv("OLLAMA_HOST", "http://localhost:11434"))

+
 class SimbaImageDescription(BaseModel):
    image_date: str
    description: str

+
 def describe_simba_image(input):
    logging.info("Opening image of Simba ...")
    if "heic" in input.lower() or "heif" in input.lower():
        new_filepath = input.split(".")[0] + ".jpg"
        img = Image.open(input)
-        img.save(new_filepath, 'JPEG')
+        img.save(new_filepath, "JPEG")
        logging.info("Extracting EXIF...")
        exif = {
            ExifTags.TAGS[k]: v for k, v in img.getexif().items() if k in ExifTags.TAGS
@@ -66,7 +68,7 @@ def describe_simba_image(input):
            },
            {"role": "user", "content": prompt, "images": [input]},
        ],
-        format=SimbaImageDescription.model_json_schema()
+        format=SimbaImageDescription.model_json_schema(),
    )

    result = SimbaImageDescription.model_validate_json(response["message"]["content"])
--- a/index_immich.py
+++ b/index_immich.py
@@ -6,6 +6,7 @@ import tempfile

 from image_process import describe_simba_image
 from request import PaperlessNGXService
+import sqlite3

 logging.basicConfig(level=logging.INFO)

@@ -23,8 +24,16 @@ DOWNLOAD_DIR = os.getenv("DOWNLOAD_DIR", "./simba_photos")
 # Set up headers
 headers = {"x-api-key": API_KEY, "Content-Type": "application/json"}

+VISITED = {}

 if __name__ == "__main__":
+    conn = sqlite3.connect("./visited.db")
+    c = conn.cursor()
+    c.execute("select immich_id from visited")
+    rows = c.fetchall()
+    for row in rows:
+        VISITED.add(row[0])
+
    ppngx = PaperlessNGXService()
    people_url = f"{IMMICH_URL}/api/search/person?name=Simba"
    people = httpx.get(people_url, headers=headers).json()
@@ -39,7 +48,7 @@ if __name__ == "__main__":

    assets = results.json()["assets"]
    for asset in assets["items"]:
-        if asset["type"] == "IMAGE":
+        if asset["type"] == "IMAGE" and asset["id"] not in VISITED:
            ids[asset["id"]] = asset.get("originalFileName")
    nextPage = assets.get("nextPage")

@@ -58,13 +67,12 @@ if __name__ == "__main__":
    asset_search = f"{IMMICH_URL}/api/search/smart"
    request_body = {"query": "simba"}
    results = httpx.post(asset_search, headers=headers, json=request_body)
-    print(results.json()["assets"]["total"])
    for asset in results.json()["assets"]["items"]:
        if asset["type"] == "IMAGE":
            ids[asset["id"]] = asset.get("originalFileName")

-    immich_asset_id = list(ids.keys())[1]
-    immich_filename = ids.get(immich_asset_id)
+    for immich_asset_id, immich_filename in ids.items():
+        try:
            response = httpx.get(
                f"{IMMICH_URL}/api/assets/{immich_asset_id}/original", headers=headers
            )
@@ -80,19 +88,28 @@ if __name__ == "__main__":
            image_description = description.description
            image_date = description.image_date

-    description_filepath = os.path.join("/Users/ryanchen/Programs/raggr", f"SIMBA_DESCRIBE_001.txt")
+            description_filepath = os.path.join(
+                "/Users/ryanchen/Programs/raggr", f"SIMBA_DESCRIBE_001.txt"
+            )
            file = open(description_filepath, "w+")
            file.write(image_description)
            file.close()

-    file = open(description_filepath, 'rb')
-
-    ppngx.upload_description(description_filepath=description_filepath, file=file, title="SIMBA_DESCRIBE_001.txt", exif_date=image_date)
-    
-
+            file = open(description_filepath, "rb")
+            ppngx.upload_description(
+                description_filepath=description_filepath,
+                file=file,
+                title="SIMBA_DESCRIBE_001.txt",
+                exif_date=image_date,
+            )
            file.close()

-    
-
+            c.execute("INSERT INTO visited (immich_id) values (?)", (immich_asset_id,))
+            conn.commit()
            logging.info("Processing complete. Deleting file.")
            os.remove(file.name)
+        except Exception as e:
+            logging.info(f"something went wrong for {immich_filename}")
+            logging.info(e)
+
+    conn.close()
--- a/llm.py
+++ b/llm.py
@@ -0,0 +1,64 @@
+import os
+
+from ollama import Client
+from openai import OpenAI
+
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+
+class LLMClient:
+    def __init__(self):
+        try:
+            self.ollama_client = Client(
+                host=os.getenv("OLLAMA_URL", "http://localhost:11434"), timeout=10.0
+            )
+            self.ollama_client.chat(
+                model="gemma3:4b", messages=[{"role": "system", "content": "test"}]
+            )
+            self.PROVIDER = "ollama"
+            logging.info("Using Ollama as LLM backend")
+        except Exception as e:
+            print(e)
+            self.openai_client = OpenAI()
+            self.PROVIDER = "openai"
+            logging.info("Using OpenAI as LLM backend")
+
+    def chat(
+        self,
+        prompt: str,
+        system_prompt: str,
+    ):
+        if self.PROVIDER == "ollama":
+            response = self.ollama_client.chat(
+                model="gemma3:4b",
+                messages=[
+                    {
+                        "role": "system",
+                        "content": system_prompt,
+                    },
+                    {"role": "user", "content": prompt},
+                ],
+            )
+            print(response)
+            output = response.message.content
+        elif self.PROVIDER == "openai":
+            response = self.openai_client.responses.create(
+                model="gpt-4o-mini",
+                input=[
+                    {
+                        "role": "system",
+                        "content": system_prompt,
+                    },
+                    {"role": "user", "content": prompt},
+                ],
+            )
+            output = response.output_text
+
+        return output
+
+
+if __name__ == "__main__":
+    client = Client()
+    client.chat(model="gemma3:4b", messages=[{"role": "system", "promp": "hack"}])
--- a/main.py
+++ b/main.py
@@ -1,28 +1,31 @@
 import datetime
 import logging
 import os
-from typing import Any, Union
+import sqlite3

 import argparse
 import chromadb
 import ollama
-from openai import OpenAI


 from request import PaperlessNGXService
 from chunker import Chunker
-from query import QueryGenerator
 from cleaner import pdf_to_image, summarize_pdf_image
+from llm import LLMClient
+from query import QueryGenerator
+

 from dotenv import load_dotenv

-load_dotenv()
+_dotenv_loaded = load_dotenv()

 # Configure ollama client with URL from environment or default to localhost
-ollama_client = ollama.Client(host=os.getenv("OLLAMA_URL", "http://localhost:11434"))
+ollama_client = ollama.Client(
+    host=os.getenv("OLLAMA_URL", "http://localhost:11434"), timeout=10.0
+)

 client = chromadb.PersistentClient(path=os.getenv("CHROMADB_PATH", ""))
-simba_docs = client.get_or_create_collection(name="simba_docs")
+simba_docs = client.get_or_create_collection(name="simba_docs2")
 feline_vet_lookup = client.get_or_create_collection(name="feline_vet_lookup")

 parser = argparse.ArgumentParser(
@@ -37,25 +40,25 @@ parser.add_argument("--index", help="index a file")

 ppngx = PaperlessNGXService()

-openai_client = OpenAI()
+llm_client = LLMClient()


-def index_using_pdf_llm():
+def index_using_pdf_llm(doctypes):
+    logging.info("reindex data...")
    files = ppngx.get_data()
    for file in files:
-        document_id = file["id"]
+        document_id: int = file["id"]
        pdf_path = ppngx.download_pdf_from_id(id=document_id)
        image_paths = pdf_to_image(filepath=pdf_path)
-        print(f"summarizing {file}")
+        logging.info(f"summarizing {file}")
        generated_summary = summarize_pdf_image(filepaths=image_paths)
        file["content"] = generated_summary

-    chunk_data(files, simba_docs)
+    chunk_data(files, simba_docs, doctypes=doctypes)


 def date_to_epoch(date_str: str) -> float:
    split_date = date_str.split("-")
-    print(split_date)
    date = datetime.datetime(
        int(split_date[0]),
        int(split_date[1]),
@@ -68,23 +71,35 @@ def date_to_epoch(date_str: str) -> float:
    return date.timestamp()


-def chunk_data(docs: list[dict[str, Union[str, Any]]], collection):
+def chunk_data(docs, collection, doctypes):
    # Step 2: Create chunks
    chunker = Chunker(collection)

-    print(f"chunking {len(docs)} documents")
-    print(docs)
+    logging.info(f"chunking {len(docs)} documents")
    texts: list[str] = [doc["content"] for doc in docs]
+    with sqlite3.connect("visited.db") as conn:
+        to_insert = []
+        c = conn.cursor()
        for index, text in enumerate(texts):
-        print(docs[index]["original_file_name"])
            metadata = {
                "created_date": date_to_epoch(docs[index]["created_date"]),
                "filename": docs[index]["original_file_name"],
+                "document_type": doctypes.get(docs[index]["document_type"], ""),
            }
+
+            if doctypes:
+                metadata["type"] = doctypes.get(docs[index]["document_type"])
+
            chunker.chunk_document(
                document=text,
                metadata=metadata,
            )
+            to_insert.append((docs[index]["id"],))
+
+        c.executemany(
+            "INSERT INTO indexed_documents (paperless_id) values (?)", to_insert
+        )
+        conn.commit()


 def chunk_text(texts: list[str], collection):
@@ -99,64 +114,54 @@ def chunk_text(texts: list[str], collection):


 def consult_oracle(input: str, collection):
-    print(input)
    import time

+    chunker = Chunker(collection)
+
    start_time = time.time()

    # Ask
-    # print("Starting query generation")
-    # qg_start = time.time()
-    # qg = QueryGenerator()
+    logging.info("Starting query generation")
+    qg_start = time.time()
+    qg = QueryGenerator()
+    doctype_query = qg.get_doctype_query(input=input)
    # metadata_filter = qg.get_query(input)
-    # qg_end = time.time()
-    # print(f"Query generation took {qg_end - qg_start:.2f} seconds")
-    # print(metadata_filter)
+    metadata_filter = {**doctype_query}
+    logging.info(metadata_filter)
+    qg_end = time.time()
+    logging.info(f"Query generation took {qg_end - qg_start:.2f} seconds")

-    print("Starting embedding generation")
+    logging.info("Starting embedding generation")
    embedding_start = time.time()
-    embeddings = Chunker.embedding_fx(input=[input])
+    embeddings = chunker.embedding_fx(inputs=[input])
    embedding_end = time.time()
-    print(f"Embedding generation took {embedding_end - embedding_start:.2f} seconds")
+    logging.info(
+        f"Embedding generation took {embedding_end - embedding_start:.2f} seconds"
+    )

-    print("Starting collection query")
+    logging.info("Starting collection query")
    query_start = time.time()
    results = collection.query(
        query_texts=[input],
        query_embeddings=embeddings,
-        # where=metadata_filter,
+        where=metadata_filter,
    )
-    print(results)
    query_end = time.time()
-    print(f"Collection query took {query_end - query_start:.2f} seconds")
+    logging.info(f"Collection query took {query_end - query_start:.2f} seconds")

    # Generate
-    print("Starting LLM generation")
+    logging.info("Starting LLM generation")
    llm_start = time.time()
-    # output = ollama_client.generate(
-    # model="gemma3n:e4b",
-    # prompt=f"You are a helpful assistant that understandings veterinary terms. Using the following data, help answer the user's query by providing as many details as possible.  Using this data: {results}. Respond to this prompt: {input}",
-    # )
-    response = openai_client.chat.completions.create(
-        model="gpt-4o-mini",
-        messages=[
-            {
-                "role": "system",
-                "content": "You are a helpful assistant that understands veterinary terms.",
-            },
-            {
-                "role": "user",
-                "content": f"Using the following data, help answer the user's query by providing as many details as possible. Using this data: {results}. Respond to this prompt: {input}",
-            },
-        ],
-    )
+    system_prompt = "You are a helpful assistant that understands veterinary terms."
+    prompt = f"Using the following data, help answer the user's query by providing as many details as possible. Using this data: {results}. Respond to this prompt: {input}"
+    output = llm_client.chat(prompt=prompt, system_prompt=system_prompt)
    llm_end = time.time()
-    print(f"LLM generation took {llm_end - llm_start:.2f} seconds")
+    logging.info(f"LLM generation took {llm_end - llm_start:.2f} seconds")

    total_time = time.time() - start_time
-    print(f"Total consult_oracle execution took {total_time:.2f} seconds")
+    logging.info(f"Total consult_oracle execution took {total_time:.2f} seconds")

-    return response.choices[0].message.content
+    return output


 def paperless_workflow(input):
@@ -175,34 +180,59 @@ def consult_simba_oracle(input: str):
    )


+def filter_indexed_files(docs):
+    with sqlite3.connect("visited.db") as conn:
+        c = conn.cursor()
+        c.execute(
+            "CREATE TABLE IF NOT EXISTS indexed_documents (id INTEGER PRIMARY KEY AUTOINCREMENT, paperless_id INTEGER)"
+        )
+        c.execute("SELECT paperless_id FROM indexed_documents")
+        rows = c.fetchall()
+        conn.commit()
+
+    visited = {row[0] for row in rows}
+    return [doc for doc in docs if doc["id"] not in visited]
+
+
 if __name__ == "__main__":
    args = parser.parse_args()
    if args.reindex:
-        print("Fetching documents from Paperless-NGX")
+        with sqlite3.connect("./visited.db") as conn:
+            c = conn.cursor()
+            c.execute("DELETE FROM indexed_documents")
+
+        logging.info("Fetching documents from Paperless-NGX")
        ppngx = PaperlessNGXService()
        docs = ppngx.get_data()
-        print(docs)
-        print(f"Fetched {len(docs)} documents")
-        #
-        print("Chunking documents now ...")
-        chunk_data(docs, collection=simba_docs)
-        print("Done chunking documents")
-        # index_using_pdf_llm()
+        docs = filter_indexed_files(docs)
+        logging.info(f"Fetched {len(docs)} documents")

-    if args.index:
-        with open(args.index) as file:
-            extension = args.index.split(".")[-1]
+        # Delete all chromadb data
+        ids = simba_docs.get(ids=None, limit=None, offset=0)
+        all_ids = ids["ids"]
+        if len(all_ids) > 0:
+            simba_docs.delete(ids=all_ids)

-            if extension == "pdf":
-                pdf_path = ppngx.download_pdf_from_id(id=document_id)
-                image_paths = pdf_to_image(filepath=pdf_path)
-                print(f"summarizing {file}")
-                generated_summary = summarize_pdf_image(filepaths=image_paths)
-            elif extension in [".md", ".txt"]:
-                chunk_text(texts=[file.readall()], collection=simba_docs)
+        # Chunk documents
+        logging.info("Chunking documents now ...")
+        tag_lookup = ppngx.get_tags()
+        doctype_lookup = ppngx.get_doctypes()
+        chunk_data(docs, collection=simba_docs, doctypes=doctype_lookup)
+        logging.info("Done chunking documents")
+
+    # if args.index:
+    # with open(args.index) as file:
+    #     extension = args.index.split(".")[-1]
+    #     if extension == "pdf":
+    #         pdf_path = ppngx.download_pdf_from_id(id=document_id)
+    #         image_paths = pdf_to_image(filepath=pdf_path)
+    #         print(f"summarizing {file}")
+    #         generated_summary = summarize_pdf_image(filepaths=image_paths)
+    #     elif extension in [".md", ".txt"]:
+    #         chunk_text(texts=[file.readall()], collection=simba_docs)

    if args.query:
-        print("Consulting oracle ...")
+        logging.info("Consulting oracle ...")
        print(
            consult_oracle(
                input=args.query,
@@ -210,4 +240,4 @@ if __name__ == "__main__":
            )
        )
    else:
-        print("please provide a query")
+        logging.info("please provide a query")
--- a/migrations/models/0_20251025081744_init.py
+++ b/migrations/models/0_20251025081744_init.py
@@ -0,0 +1,63 @@
+from tortoise import BaseDBAsyncClient
+
+RUN_IN_TRANSACTION = True
+
+
+async def upgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        CREATE TABLE IF NOT EXISTS "conversations" (
+    "id" CHAR(36) NOT NULL PRIMARY KEY,
+    "name" VARCHAR(255) NOT NULL,
+    "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE TABLE IF NOT EXISTS "conversation_messages" (
+    "id" CHAR(36) NOT NULL PRIMARY KEY,
+    "text" TEXT NOT NULL,
+    "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "speaker" VARCHAR(10) NOT NULL /* USER: user\nSIMBA: simba */,
+    "conversation_id" CHAR(36) NOT NULL REFERENCES "conversations" ("id") ON DELETE CASCADE
+);
+CREATE TABLE IF NOT EXISTS "users" (
+    "id" CHAR(36) NOT NULL PRIMARY KEY,
+    "username" VARCHAR(255) NOT NULL,
+    "password" BLOB NOT NULL,
+    "email" VARCHAR(100) NOT NULL UNIQUE,
+    "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE TABLE IF NOT EXISTS "aerich" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "version" VARCHAR(255) NOT NULL,
+    "app" VARCHAR(100) NOT NULL,
+    "content" JSON NOT NULL
+);"""
+
+
+async def downgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        """
+
+
+MODELS_STATE = (
+    "eJztmG1v4jgQx79KlFddaa9q2W53VZ1OCpTecrvACcLdPtwqMskAVhMnazvboorvfrbJE4"
+    "kJpWq3UPGmhRkPtn8ztv/2nRmEHvjsuBWSn0AZ4jgk5oVxZxIUgPig9b82TBRFuVcaOBr7"
+    "KsAttFQeNGacIpcL5wT5DITJA+ZSHCWdkdj3pTF0RUNMprkpJvhHDA4Pp8BnQIXj23dhxs"
+    "SDW2Dp1+jamWDwvZVxY0/2rewOn0fKNhp1Lq9US9nd2HFDPw5I3jqa81lIsuZxjL1jGSN9"
+    "UyBAEQevMA05ymTaqWk5YmHgNIZsqF5u8GCCYl/CMH+fxMSVDAzVk/xz9oe5BR6BWqLFhE"
+    "sWd4vlrPI5K6spu2p9sAZHb85fqVmGjE+pcioi5kIFIo6WoYprDlL9r6BszRDVo0zbl2CK"
+    "gT4EY2rIOeY1lIJMAT2MmhmgW8cHMuUz8bXx9m0Nxn+sgSIpWimUoajrZdX3Eldj6ZNIc4"
+    "QuBTllB/EqyEvh4TgAPczVyBJSLwk9Tj/sKGAxB69P/HmyCGr42p1ue2hb3b/lTALGfvgK"
+    "kWW3paehrPOS9ei8lIrsR4x/O/YHQ341vvZ77XLtZ+3sr6YcE4p56JDwxkFeYb2m1hTMSm"
+    "LjyHtgYlcjD4l91sSqwcuTZHJd2AKlYYzc6xtEPWfFUzgdgTE0BVZNfzOJvPo4AD87NkuJ"
+    "1hyu3eUv7mbGF2kZp9YivLARrqNXdQWNoGxBRMzbS/qWPdXQ2aBQChDvJ1ScYiIPgmWvBQ"
+    "uHW812bAurHmXafl8ES9022/5sr+ywqSw56lqfX63ssp/6vT/T5gUZ0/rUbx7Uy0s85Krq"
+    "hUWAroHqxX2bxIHKakfgQMSFSnYL4c+8dMzRsD24MGIG9D8y7HSb1oXBcDBG5gNuAKcn97"
+    "gAnJ6s1f/SVVpAxYNmu21eE/qYe/6zblYbtviKHtMDrdK8CingKfkI80r9bpZfO02xoruE"
+    "maKbTEzoykV8EJMEvlzY1rBlXbbNxXpt+5RKbsSUJKpIN2Wv1WpyaR+02f5rM5nHbR+Uij"
+    "H7otF+waNShBi7CammMpuYIDrXwyxGlWCO53x5/9k9nDX0mlKwFvWWYNbs9KzBF73mTdsX"
+    "C7f5xW5bJbwQIOxvU6ZZwOPU6OYl/5gVenpyP9VTJ3uquudwcXiZF4fDs+eLSOy2z55PKQ"
+    "0toNid6cRh4qmVhyhvszP6sEPWvDdp5aHU9KVqTxL2rIeEemr9rXF69u7s/Zvzs/eiiRpJ"
+    "ZnlXU/2dnr1BDsrLivYOt/6YLYQcxGAGUi6NLSAmzfcT4NNolZBwIJrz7K9hv7f2bSYNKY"
+    "EcETHBbx52+WvDx4x/302sNRTlrOsfkstvxqXDSP5AU/eK8yuPl8X/Etg7Fw=="
+)
--- a/migrations/models/1_20251025091926_update.py
+++ b/migrations/models/1_20251025091926_update.py
@@ -0,0 +1,60 @@
+from tortoise import BaseDBAsyncClient
+
+RUN_IN_TRANSACTION = True
+
+
+async def upgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        -- SQLite doesn't support ADD CONSTRAINT, so we need to recreate the table
+        CREATE TABLE "conversations_new" (
+            "id" CHAR(36) NOT NULL PRIMARY KEY,
+            "name" VARCHAR(255) NOT NULL,
+            "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+            "updated_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+            "user_id" CHAR(36),
+            FOREIGN KEY ("user_id") REFERENCES "users" ("id") ON DELETE CASCADE
+        );
+        INSERT INTO "conversations_new" ("id", "name", "created_at", "updated_at")
+        SELECT "id", "name", "created_at", "updated_at" FROM "conversations";
+        DROP TABLE "conversations";
+        ALTER TABLE "conversations_new" RENAME TO "conversations";"""
+
+
+async def downgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        -- Recreate table without user_id column
+        CREATE TABLE "conversations_new" (
+            "id" CHAR(36) NOT NULL PRIMARY KEY,
+            "name" VARCHAR(255) NOT NULL,
+            "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+            "updated_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+        );
+        INSERT INTO "conversations_new" ("id", "name", "created_at", "updated_at")
+        SELECT "id", "name", "created_at", "updated_at" FROM "conversations";
+        DROP TABLE "conversations";
+        ALTER TABLE "conversations_new" RENAME TO "conversations";"""
+
+
+MODELS_STATE = (
+    "eJztmWtP2zAUhv9KlE8gbQg6xhCaJqWlbB20ndp0F9gUuYnbWiROiJ1Bhfjvs91cnMRNKe"
+    "PSon6B9vic2H5s57w+vdU934Eu2Wn4+C8MCaDIx/qRdqtj4EH2Qdn+RtNBEGSt3EDB0BUB"
+    "tuQpWsCQ0BDYlDWOgEsgMzmQ2CEK4s5w5Lrc6NvMEeFxZoowuoqgRf0xpBMYsoaLP8yMsA"
+    "NvIEm+BpfWCEHXyY0bObxvYbfoNBC2waB1fCI8eXdDy/bdyMOZdzClEx+n7lGEnB0ew9vG"
+    "EMMQUOhI0+CjjKedmGYjZgYaRjAdqpMZHDgCkcth6B9HEbY5A030xP/sf9KXwMNQc7QIU8"
+    "7i9m42q2zOwqrzrhpfjN7Wu4NtMUuf0HEoGgUR/U4EAgpmoYJrBlL8L6FsTECoRpn4F2Cy"
+    "gT4EY2LIOGZ7KAGZAHoYNd0DN5YL8ZhO2Nfa+/cVGL8bPUGSeQmUPtvXs13fiZtqszaONE"
+    "Noh5BP2QK0DPKYtVDkQTXMfGQBqROH7iQfVhQwm4PTxe40PgQVfM1Wu9k3jfY3PhOPkCtX"
+    "IDLMJm+pCeu0YN06KCxF+hDtR8v8ovGv2nm30yzu/dTPPNf5mEBEfQv71xZwpPOaWBMwuY"
+    "WNAueBC5uP3Czsiy5sPHhpXQkMreUyiBTyH2kkHtszLuLDkwZPvaNLZc7gMMrwTvwQojE+"
+    "hVOBsMXGAbCtShax6BjEj1lVaJk1G0UIrlM1Im8KNjs2J0hn2dPoN4zjpi4YDoF9eQ1Cx5"
+    "oD04OEgDEkZaD1OPLktAfdVJqpWcoCrj174mq+VeaxFaz8mi8xytErN3k1r2gBmM3bifvm"
+    "PVXQWaCCJYj3E8OWvJAbUbzWopjCG0XKN5lVjTLxXxdRXJXKmz/NXBZPpO9W2/i5ncvkZ9"
+    "3O58RdksqNs259o5Bfo5AqK2QSQHCpEgP8AtnEkVeSArnVlcJf+Ojog36zd6TxjP4b91vt"
+    "unGkEeQNgX6/Jc7dMvd273HJ3Nude8fkTYUDJCea5V7zitDHfOevqYS1CwWv/5SyxfrZyl"
+    "JcqGkV22VZbfuUSk7cGRTSLblLzNdq/GhvtNn6azO+jssWLeWYddFoz1C4DAAh136o2Jl1"
+    "hEE4VcOUowowh1M6u/+sHs4KenUuWGW9xZjVWx2j90uteRN/eePWf5lNo4AXegC5y2zTNO"
+    "Bx9ujiI/+YO3Rv936qp0r2lHXP5uLwOi8Om9L6q1jYtHJXEoCLyp6l35Efp/a5VvXkJ615"
+    "GjBE9kRXaOW4pVItg8xnZeRyC88pvynVMsdc2Azxyr9ozhSV57e1vf0P+4fvDvYPmYsYSW"
+    "r5UPEyaHXMBeqYHwTllXa+6pBCNto4BcmPxhIQY/f1BPg00s3HFGJFev/a73bmlqqSkALI"
+    "AWYTvHCQTd9oLiL0z2piraDIZ11dVy+W0Au5mT+gripqPWch5u4f/FVgYA=="
+)
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,16 +4,9 @@ version = "0.1.0"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.13"
-dependencies = [
-    "chromadb>=1.1.0",
-    "python-dotenv>=1.0.0",
-    "flask>=3.1.2",
-    "httpx>=0.28.1",
-    "ollama>=0.6.0",
-    "openai>=2.0.1",
-    "pydantic>=2.11.9",
-    "pillow>=10.0.0",
-    "pymupdf>=1.24.0",
-    "black>=25.9.0",
-    "pillow-heif>=1.1.1",
-]
+dependencies = ["chromadb>=1.1.0", "python-dotenv>=1.0.0", "flask>=3.1.2", "httpx>=0.28.1", "ollama>=0.6.0", "openai>=2.0.1", "pydantic>=2.11.9", "pillow>=10.0.0", "pymupdf>=1.24.0", "black>=25.9.0", "pillow-heif>=1.1.1", "flask-jwt-extended>=4.7.1", "bcrypt>=5.0.0", "pony>=0.7.19", "flask-login>=0.6.3", "quart>=0.20.0", "tortoise-orm>=0.25.1", "quart-jwt-extended>=0.1.0", "pre-commit>=4.3.0", "tortoise-orm-stubs>=1.0.2", "aerich>=0.8.0", "tomlkit>=0.13.3"]
+
+[tool.aerich]
+tortoise_orm = "app.TORTOISE_CONFIG"
+location = "./migrations"
+src_folder = "./."
--- a/query.py
+++ b/query.py
@@ -2,14 +2,16 @@ import json
 import os
 from typing import Literal
 import datetime
-from ollama import chat, ChatResponse, Client
+from ollama import Client

 from openai import OpenAI

 from pydantic import BaseModel, Field

 # Configure ollama client with URL from environment or default to localhost
-ollama_client = Client(host=os.getenv("OLLAMA_URL", "http://localhost:11434"))
+ollama_client = Client(
+    host=os.getenv("OLLAMA_URL", "http://localhost:11434"), timeout=10.0
+)

 # This uses inferred filters — which means using LLM to create the metadata filters

@@ -38,6 +40,20 @@ class Time(BaseModel):
    time: int


+DOCTYPE_OPTIONS = [
+    "Bill",
+    "Image Description",
+    "Insurance",
+    "Medical Record",
+    "Documentation",
+    "Letter",
+]
+
+
+class DocumentType(BaseModel):
+    type: list[str] = Field(description="type of document", enum=DOCTYPE_OPTIONS)
+
+
 PROMPT = """
 You are an information specialist that processes user queries. The current year is 2025. The user queries are all about
 a cat, Simba, and its records. The types of records are listed below. Using the query, extract the
@@ -45,7 +61,6 @@ the date range the user is trying to query. You should return it as a JSON. The

 If the created_date cannot be ascertained, set it to epoch time start.

-
 You have several operators at your disposal:
 - $gt: greater than
 - $gte: greater than or equal
@@ -84,6 +99,19 @@ Only return the extracted metadata fields. Make sure the extracted metadata fiel
 """


+DOCTYPE_PROMPT = f"""You are an information specialist that processes user queries. A query can have two tags attached from the following options. Based on the query, determine which of the following options is most appropriate: {",".join(DOCTYPE_OPTIONS)}
+
+### Example 1
+Query: "Who is Simba's current vet?"
+Tags: ["Bill", "Medical Record"]
+
+
+### Example 2
+Query: "Who does Simba know?"
+Tags: ["Letter", "Documentation"]
+"""
+
+
 class QueryGenerator:
    def __init__(self) -> None:
        pass
@@ -101,20 +129,43 @@ class QueryGenerator:

        return date.timestamp()

+    def get_doctype_query(self, input: str):
+        client = OpenAI()
+        response = client.chat.completions.create(
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You are an information specialist that is really good at deciding what tags a query should have",
+                },
+                {"role": "user", "content": DOCTYPE_PROMPT + " " + input},
+            ],
+            model="gpt-4o",
+            response_format={
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "document_type",
+                    "schema": DocumentType.model_json_schema(),
+                },
+            },
+        )
+
+        response_json_str = response.choices[0].message.content
+        type_data = json.loads(response_json_str)
+        metadata_query = {"document_type": {"$in": type_data["type"]}}
+        return metadata_query
+
    def get_query(self, input: str):
        client = OpenAI()
-        print(input)
        response = client.responses.parse(
            model="gpt-4o",
            input=[
                {"role": "system", "content": PROMPT},
                {"role": "user", "content": input},
            ],
-            text_format=Time,
+            text_format=GeneratedQuery,
        )
-        print(response)
+        print(response.output)
        query = json.loads(response.output_parsed.extracted_metadata_fields)
-
        # response: ChatResponse = ollama_client.chat(
        # model="gemma3n:e4b",
        # messages=[
@@ -127,17 +178,17 @@ class QueryGenerator:
        # query = json.loads(
        # json.loads(response["message"]["content"])["extracted_metadata_fields"]
        # )
-        date_key = list(query["created_date"].keys())[0]
-        query["created_date"][date_key] = self.date_to_epoch(
-            query["created_date"][date_key]
-        )
+        # date_key = list(query["created_date"].keys())[0]
+        # query["created_date"][date_key] = self.date_to_epoch(
+        # query["created_date"][date_key]
+        # )

-        if "$" not in date_key:
-            query["created_date"]["$" + date_key] = query["created_date"][date_key]
+        # if "$" not in date_key:
+        # query["created_date"]["$" + date_key] = query["created_date"][date_key]

        return query


 if __name__ == "__main__":
    qg = QueryGenerator()
-    print(qg.get_query("How heavy is Simba?"))
+    print(qg.get_doctype_query("How heavy is Simba?"))
--- a/raggr-frontend/TOKEN_REFRESH_IMPLEMENTATION.md
+++ b/raggr-frontend/TOKEN_REFRESH_IMPLEMENTATION.md
@@ -0,0 +1,63 @@
+# Token Refresh Implementation
+
+## Overview
+
+The API services now automatically handle token refresh when access tokens expire. This provides a seamless user experience without requiring manual re-authentication.
+
+## How It Works
+
+### 1. **userService.ts**
+
+The `userService` now includes:
+
+- **`refreshToken()`**: Automatically gets the refresh token from localStorage, calls the `/api/user/refresh` endpoint, and updates the access token
+- **`fetchWithAuth()`**: A wrapper around `fetch()` that:
+  1. Automatically adds the Authorization header with the access token
+  2. Detects 401 (Unauthorized) responses
+  3. Automatically refreshes the token using the refresh token
+  4. Retries the original request with the new access token
+  5. Throws an error if refresh fails (e.g., refresh token expired)
+
+### 2. **conversationService.ts**
+
+Now uses `userService.fetchWithAuth()` for all API calls:
+- `sendQuery()` - No longer needs token parameter
+- `getMessages()` - No longer needs token parameter
+
+### 3. **Components Updated**
+
+**ChatScreen.tsx**:
+- Removed manual token handling
+- Now simply calls `conversationService.sendQuery(query)` and `conversationService.getMessages()`
+
+## Benefits
+
+✅ **Automatic token refresh** - Users stay logged in longer
+✅ **Transparent retry logic** - Failed requests due to expired tokens are automatically retried
+✅ **Cleaner code** - Components don't need to manage tokens
+✅ **Better UX** - No interruptions when access token expires
+✅ **Centralized auth logic** - All auth handling in one place
+
+## Error Handling
+
+- If refresh token is missing or invalid, the error is thrown
+- Components can catch these errors and redirect to login
+- LocalStorage is automatically cleared when refresh fails
+
+## Usage Example
+
+```typescript
+// Old way (manual token management)
+const token = localStorage.getItem("access_token");
+const result = await conversationService.sendQuery(query, token);
+
+// New way (automatic token refresh)
+const result = await conversationService.sendQuery(query);
+```
+
+## Token Storage
+
+- **Access Token**: `localStorage.getItem("access_token")`
+- **Refresh Token**: `localStorage.getItem("refresh_token")`
+
+Both are automatically managed by the services.
--- a/raggr-frontend/src/.App.tsx.swp
+++ b/raggr-frontend/src/.App.tsx.swp
--- a/raggr-frontend/src/App.tsx
+++ b/raggr-frontend/src/App.tsx
@@ -1,90 +1,71 @@
-import { useState } from "react";
-import axios from "axios";
-import ReactMarkdown from "react-markdown";
+import { useState, useEffect } from "react";

 import "./App.css";
+import { AuthProvider } from "./contexts/AuthContext";
+import { ChatScreen } from "./components/ChatScreen";
+import { LoginScreen } from "./components/LoginScreen";
+import { conversationService } from "./api/conversationService";

-const App = () => {
-	const [query, setQuery] = useState<string>("");
-	const [answer, setAnswer] = useState<string>("");
-	const [loading, setLoading] = useState<boolean>(false);
-	const [simbaMode, setSimbaMode] = useState<boolean>(false);
+const AppContainer = () => {
+  const [isAuthenticated, setAuthenticated] = useState<boolean>(false);
+  const [isChecking, setIsChecking] = useState<boolean>(true);

-	const simbaAnswers = ["meow.", "hiss...", "purrrrrr", "yowOWROWWowowr"];
+  useEffect(() => {
+    const checkAuth = async () => {
+      const accessToken = localStorage.getItem("access_token");
+      const refreshToken = localStorage.getItem("refresh_token");

-	const handleQuestionSubmit = () => {
-		if (simbaMode) {
-			console.log("simba mode activated");
-			setLoading(true);
-			const randomIndex = Math.floor(Math.random() * simbaAnswers.length);
-			const randomElement = simbaAnswers[randomIndex];
-			setAnswer(randomElement);
-			setTimeout(() => setLoading(false), 3500);
+      // No tokens at all, not authenticated
+      if (!accessToken && !refreshToken) {
+        setIsChecking(false);
+        setAuthenticated(false);
        return;
      }
-		const payload = { query: query };
-		setLoading(true);
-		axios
-			.post("/api/query", payload)
-			.then((result) => setAnswer(result.data.response))
-			.finally(() => setLoading(false));
-	};
-	const handleQueryChange = (event) => {
-		setQuery(event.target.value);
-	};
-	return (
-		<div className="bg-[url('./simba_cute.jpeg')] bg-cover bg-center bg-no-repeat h-screen bg-opacity-20">
-			<div className="bg-white/85 h-screen">
-				<div className="flex flex-row justify-center py-4">
-					<div className="flex flex-col gap-4 min-w-xl max-w-xl">
-						<div className="flex flex-row justify-center gap-2 grow">
-							<h1 className="text-3xl">ask simba!</h1>
-						</div>
-						<div className="flex flex-row justify-between gap-2 grow">
-							<textarea
-								type="text"
-								className="p-4 border border-blue-200 rounded-md grow bg-white"
-								onChange={handleQueryChange}
-							/>
-						</div>
-						<div className="flex flex-row justify-between gap-2 grow">
-							<button
-								className="p-4 border border-blue-400 bg-blue-200 hover:bg-blue-400 cursor-pointer rounded-md flex-grow"
-								onClick={() => handleQuestionSubmit()}
-								type="submit"
-							>
-								Submit
-							</button>
-						</div>
-						<div className="flex flex-row justify-center gap-2 grow">
-							<input
-								type="checkbox"
-								onChange={(event) =>
-									setSimbaMode(event.target.checked)
+
+      // Try to verify token by making a request
+      try {
+        await conversationService.getMessages();
+        // If successful, user is authenticated
+        setAuthenticated(true);
+      } catch (error) {
+        // Token is invalid or expired
+        console.error("Authentication check failed:", error);
+        localStorage.removeItem("access_token");
+        localStorage.removeItem("refresh_token");
+        setAuthenticated(false);
+      } finally {
+        setIsChecking(false);
      }
-							/>
-							<p>simba mode?</p>
-						</div>
-						{loading ? (
-							<div className="flex flex-col w-full animate-pulse gap-2">
-								<div className="flex flex-row gap-2 w-full">
-									<div className="bg-gray-400 w-1/2 p-3 rounded-lg" />
-									<div className="bg-gray-400 w-1/2 p-3 rounded-lg" />
-								</div>
-								<div className="flex flex-row gap-2 w-full">
-									<div className="bg-gray-400 w-1/3 p-3 rounded-lg" />
-									<div className="bg-gray-400 w-2/3 p-3 rounded-lg" />
-								</div>
+    };
+
+    checkAuth();
+  }, []);
+
+  // Show loading state while checking authentication
+  if (isChecking) {
+    return (
+      <div className="h-screen flex items-center justify-center bg-white/85">
+        <div className="text-xl">Loading...</div>
      </div>
+    );
+  }
+
+  return (
+    <>
+      {isAuthenticated ? (
+        <ChatScreen setAuthenticated={setAuthenticated} />
      ) : (
-							<div className="flex flex-col">
-								<ReactMarkdown>{answer}</ReactMarkdown>
-							</div>
+        <LoginScreen setAuthenticated={setAuthenticated} />
      )}
-					</div>
-				</div>
-			</div>
-		</div>
+    </>
+  );
+};
+
+const App = () => {
+  return (
+    <AuthProvider>
+      <AppContainer />
+    </AuthProvider>
  );
 };

--- a/raggr-frontend/src/api/conversationService.ts
+++ b/raggr-frontend/src/api/conversationService.ts
@@ -0,0 +1,61 @@
+import { userService } from "./userService";
+
+interface Message {
+  id: string;
+  text: string;
+  speaker: "user" | "simba";
+  created_at: string;
+}
+
+interface Conversation {
+  id: string;
+  name: string;
+  messages: Message[];
+  created_at: string;
+  updated_at: string;
+}
+
+interface QueryRequest {
+  query: string;
+}
+
+interface QueryResponse {
+  response: string;
+}
+
+class ConversationService {
+  private baseUrl = "/api";
+
+  async sendQuery(query: string): Promise<QueryResponse> {
+    const response = await userService.fetchWithRefreshToken(
+      `${this.baseUrl}/query`,
+      {
+        method: "POST",
+        body: JSON.stringify({ query }),
+      },
+    );
+
+    if (!response.ok) {
+      throw new Error("Failed to send query");
+    }
+
+    return await response.json();
+  }
+
+  async getMessages(): Promise<Conversation> {
+    const response = await userService.fetchWithRefreshToken(
+      `${this.baseUrl}/messages`,
+      {
+        method: "GET",
+      },
+    );
+
+    if (!response.ok) {
+      throw new Error("Failed to fetch messages");
+    }
+
+    return await response.json();
+  }
+}
+
+export const conversationService = new ConversationService();
--- a/raggr-frontend/src/api/userService.ts
+++ b/raggr-frontend/src/api/userService.ts
@@ -0,0 +1,123 @@
+interface LoginResponse {
+  access_token: string;
+  refresh_token: string;
+  user: {
+    id: string;
+    username: string;
+  };
+}
+
+interface RefreshResponse {
+  access_token: string;
+}
+
+class UserService {
+  private baseUrl = "/api/user";
+
+  async login(username: string, password: string): Promise<LoginResponse> {
+    const response = await fetch(`${this.baseUrl}/login`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ username, password }),
+    });
+
+    if (!response.ok) {
+      throw new Error("Invalid credentials");
+    }
+
+    return await response.json();
+  }
+
+  async refreshToken(): Promise<string> {
+    const refreshToken = localStorage.getItem("refresh_token");
+
+    if (!refreshToken) {
+      throw new Error("No refresh token available");
+    }
+
+    const response = await fetch(`${this.baseUrl}/refresh`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${refreshToken}`,
+      },
+    });
+
+    if (!response.ok) {
+      // Refresh token is invalid or expired, clear storage
+      localStorage.removeItem("access_token");
+      localStorage.removeItem("refresh_token");
+      throw new Error("Failed to refresh token");
+    }
+
+    const data: RefreshResponse = await response.json();
+    localStorage.setItem("access_token", data.access_token);
+    return data.access_token;
+  }
+
+  async fetchWithAuth(
+    url: string,
+    options: RequestInit = {},
+  ): Promise<Response> {
+    const accessToken = localStorage.getItem("access_token");
+
+    // Add authorization header
+    const headers = {
+      "Content-Type": "application/json",
+      ...(options.headers || {}),
+      ...(accessToken && { Authorization: `Bearer ${accessToken}` }),
+    };
+
+    let response = await fetch(url, { ...options, headers });
+
+    // If unauthorized, try refreshing the token
+    if (response.status === 401) {
+      try {
+        const newAccessToken = await this.refreshToken();
+
+        // Retry the request with new token
+        headers.Authorization = `Bearer ${newAccessToken}`;
+        response = await fetch(url, { ...options, headers });
+      } catch (error) {
+        // Refresh failed, redirect to login or throw error
+        throw new Error("Session expired. Please log in again.");
+      }
+    }
+
+    return response;
+  }
+
+  async fetchWithRefreshToken(
+    url: string,
+    options: RequestInit = {},
+  ): Promise<Response> {
+    const refreshToken = localStorage.getItem("refresh_token");
+
+    // Add authorization header
+    const headers = {
+      "Content-Type": "application/json",
+      ...(options.headers || {}),
+      ...(refreshToken && { Authorization: `Bearer ${refreshToken}` }),
+    };
+
+    let response = await fetch(url, { ...options, headers });
+
+    // If unauthorized, try refreshing the token
+    if (response.status === 401) {
+      try {
+        const newAccessToken = await this.refreshToken();
+
+        // Retry the request with new token
+        headers.Authorization = `Bearer ${newAccessToken}`;
+        response = await fetch(url, { ...options, headers });
+      } catch (error) {
+        // Refresh failed, redirect to login or throw error
+        throw new Error("Session expired. Please log in again.");
+      }
+    }
+
+    return response;
+  }
+}
+
+export const userService = new UserService();
--- a/raggr-frontend/src/components/AnswerBubble.tsx
+++ b/raggr-frontend/src/components/AnswerBubble.tsx
@@ -0,0 +1,29 @@
+import ReactMarkdown from "react-markdown";
+
+type AnswerBubbleProps = {
+  text: string;
+  loading?: boolean;
+};
+
+export const AnswerBubble = ({ text, loading }: AnswerBubbleProps) => {
+  return (
+    <div className="rounded-md bg-orange-100 p-3">
+      {loading ? (
+        <div className="flex flex-col w-full animate-pulse gap-2">
+          <div className="flex flex-row gap-2 w-full">
+            <div className="bg-gray-400 w-1/2 p-3 rounded-lg" />
+            <div className="bg-gray-400 w-1/2 p-3 rounded-lg" />
+          </div>
+          <div className="flex flex-row gap-2 w-full">
+            <div className="bg-gray-400 w-1/3 p-3 rounded-lg" />
+            <div className="bg-gray-400 w-2/3 p-3 rounded-lg" />
+          </div>
+        </div>
+      ) : (
+        <div className="flex flex-col">
+          <ReactMarkdown>{"🐈: " + text}</ReactMarkdown>
+        </div>
+      )}
+    </div>
+  );
+};
--- a/raggr-frontend/src/components/ChatScreen.tsx
+++ b/raggr-frontend/src/components/ChatScreen.tsx
@@ -0,0 +1,150 @@
+import { useEffect, useState } from "react";
+import { conversationService } from "../api/conversationService";
+import { QuestionBubble } from "./QuestionBubble";
+import { AnswerBubble } from "./AnswerBubble";
+
+type Message = {
+  text: string;
+  speaker: "simba" | "user";
+};
+
+type QuestionAnswer = {
+  question: string;
+  answer: string;
+};
+
+type Conversation = {
+  title: string;
+  id: string;
+};
+
+type ChatScreenProps = {
+  setAuthenticated: (isAuth: boolean) => void;
+};
+
+export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
+  const [query, setQuery] = useState<string>("");
+  const [answer, setAnswer] = useState<string>("");
+  const [simbaMode, setSimbaMode] = useState<boolean>(false);
+  const [questionsAnswers, setQuestionsAnswers] = useState<QuestionAnswer[]>(
+    [],
+  );
+  const [messages, setMessages] = useState<Message[]>([]);
+  const [conversations, setConversations] = useState<Conversation[]>([
+    { title: "simba meow meow", id: "uuid" },
+  ]);
+
+  const simbaAnswers = ["meow.", "hiss...", "purrrrrr", "yowOWROWWowowr"];
+
+  useEffect(() => {
+    const loadMessages = async () => {
+      try {
+        const conversation = await conversationService.getMessages();
+        setMessages(
+          conversation.messages.map((message) => ({
+            text: message.text,
+            speaker: message.speaker,
+          })),
+        );
+      } catch (error) {
+        console.error("Failed to load messages:", error);
+      }
+    };
+    loadMessages();
+  }, []);
+
+  const handleQuestionSubmit = async () => {
+    const currMessages = messages.concat([{ text: query, speaker: "user" }]);
+    setMessages(currMessages);
+
+    if (simbaMode) {
+      console.log("simba mode activated");
+      const randomIndex = Math.floor(Math.random() * simbaAnswers.length);
+      const randomElement = simbaAnswers[randomIndex];
+      setAnswer(randomElement);
+      setQuestionsAnswers(
+        questionsAnswers.concat([
+          {
+            question: query,
+            answer: randomElement,
+          },
+        ]),
+      );
+      return;
+    }
+
+    try {
+      const result = await conversationService.sendQuery(query);
+      setQuestionsAnswers(
+        questionsAnswers.concat([{ question: query, answer: result.response }]),
+      );
+      setMessages(
+        currMessages.concat([{ text: result.response, speaker: "simba" }]),
+      );
+      setQuery(""); // Clear input after successful send
+    } catch (error) {
+      console.error("Failed to send query:", error);
+      // If session expired, redirect to login
+      if (error instanceof Error && error.message.includes("Session expired")) {
+        setAuthenticated(false);
+      }
+    }
+  };
+
+  const handleQueryChange = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
+    setQuery(event.target.value);
+  };
+
+  return (
+    <div className="h-screen bg-opacity-20">
+      <div className="bg-white/85 h-screen">
+        <div className="flex flex-row justify-center py-4">
+          <div className="flex flex-col gap-4 min-w-xl max-w-xl">
+            <div className="flex flex-row justify-between">
+              <header className="flex flex-row justify-center gap-2 grow sticky top-0 z-10 bg-white">
+                <h1 className="text-3xl">ask simba!</h1>
+              </header>
+              <button
+                className="p-4 border border-red-400 bg-red-200 hover:bg-red-400 cursor-pointer rounded-md"
+                onClick={() => setAuthenticated(false)}
+              >
+                logout
+              </button>
+            </div>
+            {messages.map((msg, index) => {
+              if (msg.speaker === "simba") {
+                return <AnswerBubble key={index} text={msg.text} />;
+              }
+              return <QuestionBubble key={index} text={msg.text} />;
+            })}
+            <footer className="flex flex-col gap-2 sticky bottom-0">
+              <div className="flex flex-row justify-between gap-2 grow">
+                <textarea
+                  className="p-4 border border-blue-200 rounded-md grow bg-white"
+                  onChange={handleQueryChange}
+                  value={query}
+                />
+              </div>
+              <div className="flex flex-row justify-between gap-2 grow">
+                <button
+                  className="p-4 border border-blue-400 bg-blue-200 hover:bg-blue-400 cursor-pointer rounded-md flex-grow"
+                  onClick={() => handleQuestionSubmit()}
+                  type="submit"
+                >
+                  Submit
+                </button>
+              </div>
+              <div className="flex flex-row justify-center gap-2 grow">
+                <input
+                  type="checkbox"
+                  onChange={(event) => setSimbaMode(event.target.checked)}
+                />
+                <p>simba mode?</p>
+              </div>
+            </footer>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
--- a/raggr-frontend/src/components/ConversationMenu.tsx
+++ b/raggr-frontend/src/components/ConversationMenu.tsx
@@ -0,0 +1,24 @@
+type Conversation = {
+  title: string;
+  id: string;
+};
+
+type ConversationMenuProps = {
+  conversations: Conversation[];
+};
+
+export const ConversationMenu = ({ conversations }: ConversationMenuProps) => {
+  return (
+    <div className="absolute bg-white w-md rounded-md shadow-xl m-4 p-4">
+      <p className="py-2 px-4 rounded-md w-full text-xl font-bold">askSimba!</p>
+      {conversations.map((conversation) => (
+        <p
+          key={conversation.id}
+          className="py-2 px-4 rounded-md hover:bg-stone-200 w-full text-xl font-bold cursor-pointer"
+        >
+          {conversation.title}
+        </p>
+      ))}
+    </div>
+  );
+};
--- a/raggr-frontend/src/components/LoginScreen.tsx
+++ b/raggr-frontend/src/components/LoginScreen.tsx
@@ -0,0 +1,80 @@
+import { useState } from "react";
+import { userService } from "../api/userService";
+
+type LoginScreenProps = {
+  setAuthenticated: (isAuth: boolean) => void;
+};
+
+export const LoginScreen = ({ setAuthenticated }: LoginScreenProps) => {
+  const [username, setUsername] = useState<string>("");
+  const [password, setPassword] = useState<string>("");
+  const [error, setError] = useState<string>("");
+
+  const handleLogin = async () => {
+    if (!username || !password) {
+      setError("Please enter username and password");
+      return;
+    }
+
+    try {
+      const result = await userService.login(username, password);
+      localStorage.setItem("access_token", result.access_token);
+      localStorage.setItem("refresh_token", result.refresh_token);
+      setAuthenticated(true);
+      setError("");
+    } catch (err) {
+      setError("Login failed. Please check your credentials.");
+      console.error("Login error:", err);
+    }
+  };
+
+  return (
+    <div className="h-screen bg-opacity-20">
+      <div className="bg-white/85 h-screen">
+        <div className="flex flex-row justify-center py-4">
+          <div className="flex flex-col gap-4 min-w-xl max-w-xl">
+            <div className="flex flex-col gap-1">
+              <div className="flex flex-grow justify-center w-full bg-amber-400">
+                <h1 className="text-xl font-bold">
+                  I AM LOOKING FOR A DESIGNER. THIS APP WILL REMAIN UGLY UNTIL A
+                  DESIGNER COMES.
+                </h1>
+              </div>
+              <header className="flex flex-row justify-center gap-2 grow sticky top-0 z-10 bg-white">
+                <h1 className="text-3xl">ask simba!</h1>
+              </header>
+              <label htmlFor="username">username</label>
+              <input
+                type="text"
+                id="username"
+                name="username"
+                value={username}
+                onChange={(e) => setUsername(e.target.value)}
+                className="border border-s-slate-950 p-3 rounded-md"
+              />
+              <label htmlFor="password">password</label>
+              <input
+                type="password"
+                id="password"
+                name="password"
+                value={password}
+                onChange={(e) => setPassword(e.target.value)}
+                className="border border-s-slate-950 p-3 rounded-md"
+              />
+              {error && (
+                <div className="text-red-600 font-semibold">{error}</div>
+              )}
+            </div>
+
+            <button
+              className="p-4 border border-blue-400 bg-blue-200 hover:bg-blue-400 cursor-pointer rounded-md flex-grow"
+              onClick={handleLogin}
+            >
+              login
+            </button>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
--- a/raggr-frontend/src/components/QuestionBubble.tsx
+++ b/raggr-frontend/src/components/QuestionBubble.tsx
@@ -0,0 +1,7 @@
+type QuestionBubbleProps = {
+  text: string;
+};
+
+export const QuestionBubble = ({ text }: QuestionBubbleProps) => {
+  return <div className="rounded-md bg-stone-200 p-3">🤦: {text}</div>;
+};
--- a/raggr-frontend/src/contexts/AuthContext.tsx
+++ b/raggr-frontend/src/contexts/AuthContext.tsx
@@ -0,0 +1,56 @@
+import { createContext, useContext, useState, ReactNode } from "react";
+import { userService } from "../api/userService";
+
+interface AuthContextType {
+  token: string | null;
+  login: (username: string, password: string) => Promise<any>;
+  logout: () => void;
+  isAuthenticated: () => boolean;
+}
+
+const AuthContext = createContext<AuthContextType | undefined>(undefined);
+
+interface AuthProviderProps {
+  children: ReactNode;
+}
+
+export const AuthProvider = ({ children }: AuthProviderProps) => {
+  const [token, setToken] = useState(localStorage.getItem("access_token"));
+
+  const login = async (username: string, password: string) => {
+    try {
+      const data = await userService.login(username, password);
+      setToken(data.access_token);
+      localStorage.setItem("access_token", data.access_token);
+      localStorage.setItem("refresh_token", data.refresh_token);
+      return data;
+    } catch (error) {
+      console.error("Login failed:", error);
+      throw error;
+    }
+  };
+
+  const logout = () => {
+    setToken(null);
+    localStorage.removeItem("access_token");
+    localStorage.removeItem("refresh_token");
+  };
+
+  const isAuthenticated = () => {
+    return token !== null && token !== undefined && token !== "";
+  };
+
+  return (
+    <AuthContext.Provider value={{ token, login, logout, isAuthenticated }}>
+      {children}
+    </AuthContext.Provider>
+  );
+};
+
+export const useAuth = () => {
+  const context = useContext(AuthContext);
+  if (context === undefined) {
+    throw new Error("useAuth must be used within an AuthProvider");
+  }
+  return context;
+};
--- a/request.py
+++ b/request.py
@@ -1,23 +1,35 @@
 import os
 import tempfile
 import httpx
+import logging

 from dotenv import load_dotenv

 load_dotenv()

+logging.basicConfig(level=logging.INFO)
+

 class PaperlessNGXService:
    def __init__(self):
        self.base_url = os.getenv("BASE_URL")
        self.token = os.getenv("PAPERLESS_TOKEN")
-        self.url = f"http://{os.getenv('BASE_URL')}/api/documents/?query=simba"
+        self.url = f"http://{os.getenv('BASE_URL')}/api/documents/?tags__id=8"
        self.headers = {"Authorization": f"Token {os.getenv('PAPERLESS_TOKEN')}"}

    def get_data(self):
        print(f"Getting data from: {self.url}")
        r = httpx.get(self.url, headers=self.headers)
-        return r.json()["results"]
+        results = r.json()["results"]
+
+        nextLink = r.json().get("next")
+
+        while nextLink:
+            r = httpx.get(nextLink, headers=self.headers)
+            results += r.json()["results"]
+            nextLink = r.json().get("next")
+
+        return results

    def get_doc_by_id(self, doc_id: int):
        url = f"http://{os.getenv('BASE_URL')}/api/documents/{doc_id}/"
@@ -45,17 +57,29 @@ class PaperlessNGXService:

    def upload_description(self, description_filepath, file, title, exif_date: str):
        POST_URL = f"http://{os.getenv('BASE_URL')}/api/documents/post_document/"
-        files = {'document': ('description_filepath', file, 'application/txt')}
+        files = {"document": ("description_filepath", file, "application/txt")}
        data = {
            "title": title,
            "create": exif_date,
-                "document_type": 3
-                "tags": [7]
+            "document_type": 3,
+            "tags": [7],
        }

        r = httpx.post(POST_URL, headers=self.headers, data=data, files=files)
        r.raise_for_status()

+    def get_tags(self):
+        GET_URL = f"http://{os.getenv('BASE_URL')}/api/tags/"
+        r = httpx.get(GET_URL, headers=self.headers)
+        data = r.json()
+        return {tag["id"]: tag["name"] for tag in data["results"]}
+
+    def get_doctypes(self):
+        GET_URL = f"http://{os.getenv('BASE_URL')}/api/document_types/"
+        r = httpx.get(GET_URL, headers=self.headers)
+        data = r.json()
+        return {doctype["id"]: doctype["name"] for doctype in data["results"]}
+

 if __name__ == "__main__":
    pp = PaperlessNGXService()
--- a/startup.sh
+++ b/startup.sh
@@ -1,5 +1,8 @@
 #!/bin/bash

+echo "Running database migrations..."
+aerich upgrade
+
 echo "Starting reindex process..."
 python main.py "" --reindex

--- a/uv.lock
+++ b/uv.lock
@@ -2,6 +2,42 @@ version = 1
 revision = 2
 requires-python = ">=3.13"

+[[package]]
+name = "aerich"
+version = "0.9.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "asyncclick" },
+    { name = "dictdiffer" },
+    { name = "tortoise-orm" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c4/60/5d3885f531fab2cecec67510e7b821efc403940ed9eefd034b2c21350f3c/aerich-0.9.2.tar.gz", hash = "sha256:02d58658714eebe396fe7bd9f9401db3a60a44dc885910ad3990920d0357317d", size = 74231, upload-time = "2025-10-10T05:53:49.632Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/87/1a/956c6b1e35881bb9835a33c8db1565edcd133f8e45321010489092a0df40/aerich-0.9.2-py3-none-any.whl", hash = "sha256:d0f007acb21f6559f1eccd4e404fb039cf48af2689e0669afa62989389c0582d", size = 46451, upload-time = "2025-10-10T05:53:48.71Z" },
+]
+
+[[package]]
+name = "aiofiles"
+version = "25.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/41/c3/534eac40372d8ee36ef40df62ec129bee4fdb5ad9706e58a29be53b2c970/aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2", size = 46354, upload-time = "2025-10-09T20:51:04.358Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" },
+]
+
+[[package]]
+name = "aiosqlite"
+version = "0.21.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/13/7d/8bca2bf9a247c2c5dfeec1d7a5f40db6518f88d314b8bca9da29670d2671/aiosqlite-0.21.0.tar.gz", hash = "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3", size = 13454, upload-time = "2025-02-03T07:30:16.235Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792, upload-time = "2025-02-03T07:30:13.6Z" },
+]
+
 [[package]]
 name = "annotated-types"
 version = "0.7.0"
@@ -24,6 +60,18 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" },
 ]

+[[package]]
+name = "asyncclick"
+version = "8.3.0.7"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f9/ca/25e426d16bd0e91c1c9259112cecd17b2c2c239bdd8e5dba430f3bd5e3ef/asyncclick-8.3.0.7.tar.gz", hash = "sha256:8a80d8ac613098ee6a9a8f0248f60c66c273e22402cf3f115ed7f071acfc71d3", size = 277634, upload-time = "2025-10-11T08:35:44.841Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/01/d9/782ffcb4c97b889bc12d8276637d2739b99520390ee8fec77c07416c5d12/asyncclick-8.3.0.7-py3-none-any.whl", hash = "sha256:7607046de39a3f315867cad818849f973e29d350c10d92f251db3ff7600c6c7d", size = 109925, upload-time = "2025-10-11T08:35:43.378Z" },
+]
+
 [[package]]
 name = "attrs"
 version = "25.3.0"
@@ -170,6 +218,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" },
 ]

+[[package]]
+name = "cfgv"
+version = "3.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/11/74/539e56497d9bd1d484fd863dd69cbbfa653cd2aa27abfe35653494d85e94/cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560", size = 7114, upload-time = "2023-08-12T20:38:17.776Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", size = 7249, upload-time = "2023-08-12T20:38:16.269Z" },
+]
+
 [[package]]
 name = "charset-normalizer"
 version = "3.4.3"
@@ -276,6 +333,24 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934", size = 46018, upload-time = "2021-06-11T10:22:42.561Z" },
 ]

+[[package]]
+name = "dictdiffer"
+version = "0.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/61/7b/35cbccb7effc5d7e40f4c55e2b79399e1853041997fcda15c9ff160abba0/dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578", size = 31513, upload-time = "2021-07-22T13:24:29.276Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/47/ef/4cb333825d10317a36a1154341ba37e6e9c087bac99c1990ef07ffdb376f/dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595", size = 16754, upload-time = "2021-07-22T13:24:26.783Z" },
+]
+
+[[package]]
+name = "distlib"
+version = "0.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" },
+]
+
 [[package]]
 name = "distro"
 version = "1.9.0"
@@ -320,6 +395,33 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/ec/f9/7f9263c5695f4bd0023734af91bedb2ff8209e8de6ead162f35d8dc762fd/flask-3.1.2-py3-none-any.whl", hash = "sha256:ca1d8112ec8a6158cc29ea4858963350011b5c846a414cdb7a954aa9e967d03c", size = 103308, upload-time = "2025-08-19T21:03:19.499Z" },
 ]

+[[package]]
+name = "flask-jwt-extended"
+version = "4.7.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "flask" },
+    { name = "pyjwt" },
+    { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/51/16/96b101f18cba17ecce3225ab07bc4c8f23e6befd8552dbbed87482e7c7fb/flask_jwt_extended-4.7.1.tar.gz", hash = "sha256:8085d6757505b6f3291a2638c84d207e8f0ad0de662d1f46aa2f77e658a0c976", size = 34411, upload-time = "2024-11-20T23:44:41.044Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/67/34/9a91da47b1565811ab4aa5fb134632c8d1757960bfa7d457f486947c4d75/Flask_JWT_Extended-4.7.1-py2.py3-none-any.whl", hash = "sha256:52f35bf0985354d7fb7b876e2eb0e0b141aaff865a22ff6cc33d9a18aa987978", size = 22588, upload-time = "2024-11-20T23:44:39.435Z" },
+]
+
+[[package]]
+name = "flask-login"
+version = "0.6.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "flask" },
+    { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c3/6e/2f4e13e373bb49e68c02c51ceadd22d172715a06716f9299d9df01b6ddb2/Flask-Login-0.6.3.tar.gz", hash = "sha256:5e23d14a607ef12806c699590b89d0f0e0d67baeec599d75947bf9c147330333", size = 48834, upload-time = "2023-10-30T14:53:21.151Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/59/f5/67e9cc5c2036f58115f9fe0f00d203cf6780c3ff8ae0e705e7a9d9e8ff9e/Flask_Login-0.6.3-py3-none-any.whl", hash = "sha256:849b25b82a436bf830a054e74214074af59097171562ab10bfa999e6b78aae5d", size = 17303, upload-time = "2023-10-30T14:53:19.636Z" },
+]
+
 [[package]]
 name = "flatbuffers"
 version = "25.9.23"
@@ -404,6 +506,19 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
 ]

+[[package]]
+name = "h2"
+version = "4.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "hpack" },
+    { name = "hyperframe" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" },
+]
+
 [[package]]
 name = "hf-xet"
 version = "1.1.10"
@@ -419,6 +534,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/ee/0e/471f0a21db36e71a2f1752767ad77e92d8cde24e974e03d662931b1305ec/hf_xet-1.1.10-cp37-abi3-win_amd64.whl", hash = "sha256:5f54b19cc347c13235ae7ee98b330c26dd65ef1df47e5316ffb1e87713ca7045", size = 2804691, upload-time = "2025-09-12T20:10:28.433Z" },
 ]

+[[package]]
+name = "hpack"
+version = "4.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" },
+]
+
 [[package]]
 name = "httpcore"
 version = "1.0.9"
@@ -493,6 +617,39 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" },
 ]

+[[package]]
+name = "hypercorn"
+version = "0.17.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "h11" },
+    { name = "h2" },
+    { name = "priority" },
+    { name = "wsproto" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/7e/3a/df6c27642e0dcb7aff688ca4be982f0fb5d89f2afd3096dc75347c16140f/hypercorn-0.17.3.tar.gz", hash = "sha256:1b37802ee3ac52d2d85270700d565787ab16cf19e1462ccfa9f089ca17574165", size = 44409, upload-time = "2024-05-28T20:55:53.06Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0e/3b/dfa13a8d96aa24e40ea74a975a9906cfdc2ab2f4e3b498862a57052f04eb/hypercorn-0.17.3-py3-none-any.whl", hash = "sha256:059215dec34537f9d40a69258d323f56344805efb462959e727152b0aa504547", size = 61742, upload-time = "2024-05-28T20:55:48.829Z" },
+]
+
+[[package]]
+name = "hyperframe"
+version = "6.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" },
+]
+
+[[package]]
+name = "identify"
+version = "2.6.15"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ff/e7/685de97986c916a6d93b3876139e00eef26ad5bbbd61925d670ae8013449/identify-2.6.15.tar.gz", hash = "sha256:e4f4864b96c6557ef2a1e1c951771838f4edc9df3a72ec7118b338801b11c7bf", size = 99311, upload-time = "2025-10-02T17:43:40.631Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0f/1c/e5fd8f973d4f375adb21565739498e2e9a1e54c858a97b9a8ccfdc81da9b/identify-2.6.15-py2.py3-none-any.whl", hash = "sha256:1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757", size = 99183, upload-time = "2025-10-02T17:43:39.137Z" },
+]
+
 [[package]]
 name = "idna"
 version = "3.10"
@@ -523,6 +680,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/a4/ed/1f1afb2e9e7f38a545d628f864d562a5ae64fe6f7a10e28ffb9b185b4e89/importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec", size = 37461, upload-time = "2025-01-03T18:51:54.306Z" },
 ]

+[[package]]
+name = "iso8601"
+version = "2.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b9/f3/ef59cee614d5e0accf6fd0cbba025b93b272e626ca89fb70a3e9187c5d15/iso8601-2.1.0.tar.gz", hash = "sha256:6b1d3829ee8921c4301998c909f7829fa9ed3cbdac0d3b16af2d743aed1ba8df", size = 6522, upload-time = "2023-10-03T00:25:39.317Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6c/0c/f37b6a241f0759b7653ffa7213889d89ad49a2b76eb2ddf3b57b2738c347/iso8601-2.1.0-py3-none-any.whl", hash = "sha256:aac4145c4dcb66ad8b648a02830f5e2ff6c24af20f4f482689be402db2429242", size = 7545, upload-time = "2023-10-03T00:25:32.304Z" },
+]
+
 [[package]]
 name = "itsdangerous"
 version = "2.2.0"
@@ -783,6 +949,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" },
 ]

+[[package]]
+name = "nodeenv"
+version = "1.9.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437, upload-time = "2024-06-04T18:44:11.171Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" },
+]
+
 [[package]]
 name = "numpy"
 version = "2.3.3"
@@ -1130,6 +1305,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" },
 ]

+[[package]]
+name = "pony"
+version = "0.7.19"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/60/59/ab6542afa95de0d5a16545ce6ce683960352cfd9a2318722593b81a98123/pony-0.7.19.tar.gz", hash = "sha256:f7f83b2981893e49f7f18e8def52ad8fa8f8e6c5f9583b9aaed62d4d85036a0f", size = 258589, upload-time = "2024-08-27T12:29:29.963Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/16/cb/0ef8429024309fe6f5edf1debc7cf63adeaeb34b2242490cc18710658abd/pony-0.7.19-py3-none-any.whl", hash = "sha256:5112b4cf40d3f24e93ae66dc5ab7dc6813388efa870e750928d60dc699873cf5", size = 317259, upload-time = "2024-08-27T12:29:28.247Z" },
+]
+
 [[package]]
 name = "posthog"
 version = "5.4.0"
@@ -1146,6 +1330,31 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/4f/98/e480cab9a08d1c09b1c59a93dade92c1bb7544826684ff2acbfd10fcfbd4/posthog-5.4.0-py3-none-any.whl", hash = "sha256:284dfa302f64353484420b52d4ad81ff5c2c2d1d607c4e2db602ac72761831bd", size = 105364, upload-time = "2025-06-20T23:19:22.001Z" },
 ]

+[[package]]
+name = "pre-commit"
+version = "4.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cfgv" },
+    { name = "identify" },
+    { name = "nodeenv" },
+    { name = "pyyaml" },
+    { name = "virtualenv" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ff/29/7cf5bbc236333876e4b41f56e06857a87937ce4bf91e117a6991a2dbb02a/pre_commit-4.3.0.tar.gz", hash = "sha256:499fe450cc9d42e9d58e606262795ecb64dd05438943c62b66f6a8673da30b16", size = 193792, upload-time = "2025-08-09T18:56:14.651Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl", hash = "sha256:2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8", size = 220965, upload-time = "2025-08-09T18:56:13.192Z" },
+]
+
+[[package]]
+name = "priority"
+version = "2.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f5/3c/eb7c35f4dcede96fca1842dac5f4f5d15511aa4b52f3a961219e68ae9204/priority-2.0.0.tar.gz", hash = "sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0", size = 24792, upload-time = "2021-06-27T10:15:05.487Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5e/5f/82c8074f7e84978129347c2c6ec8b6c59f3584ff1a20bc3c940a3e061790/priority-2.0.0-py3-none-any.whl", hash = "sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa", size = 8946, upload-time = "2021-06-27T10:15:03.856Z" },
+]
+
 [[package]]
 name = "protobuf"
 version = "6.32.1"
@@ -1325,6 +1534,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
 ]

+[[package]]
+name = "pyjwt"
+version = "2.10.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e7/46/bd74733ff231675599650d3e47f361794b22ef3e3770998dda30d3b63726/pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", size = 87785, upload-time = "2024-11-28T03:43:29.933Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997, upload-time = "2024-11-28T03:43:27.893Z" },
+]
+
 [[package]]
 name = "pymupdf"
 version = "1.26.4"
@@ -1346,6 +1564,15 @@ version = "0.48.9"
 source = { registry = "https://pypi.org/simple" }
 sdist = { url = "https://files.pythonhosted.org/packages/c7/2c/94ed7b91db81d61d7096ac8f2d325ec562fc75e35f3baea8749c85b28784/PyPika-0.48.9.tar.gz", hash = "sha256:838836a61747e7c8380cd1b7ff638694b7a7335345d0f559b04b2cd832ad5378", size = 67259, upload-time = "2022-03-15T11:22:57.066Z" }

+[[package]]
+name = "pypika-tortoise"
+version = "0.6.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/1a/7b/0a31165e22e599ba149ba35d4323d343205a70d91a4f6e8c6565f5b4fa08/pypika_tortoise-0.6.2.tar.gz", hash = "sha256:f95ab59d9b6454db2e8daa0934728458350a1f3d56e81d9d1debc8eebeff26b3", size = 80522, upload-time = "2025-09-02T03:56:33.986Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/67/cf/2d47236c80d6deea85e76c86b959f0ec24369c16db691c6266f7a20ff4bd/pypika_tortoise-0.6.2-py3-none-any.whl", hash = "sha256:425462b02ede0a5ed7b812ec12427419927ed6b19282c55667d1cbc9a440d3cb", size = 46919, upload-time = "2025-09-02T03:56:32.771Z" },
+]
+
 [[package]]
 name = "pyproject-hooks"
 version = "1.2.0"
@@ -1394,6 +1621,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/60/e5/63bed382f6a7a5ba70e7e132b8b7b8abbcf4888ffa6be4877698dcfbed7d/pytokens-0.1.10-py3-none-any.whl", hash = "sha256:db7b72284e480e69fb085d9f251f66b3d2df8b7166059261258ff35f50fb711b", size = 12046, upload-time = "2025-02-19T14:51:18.694Z" },
 ]

+[[package]]
+name = "pytz"
+version = "2025.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" },
+]
+
 [[package]]
 name = "pyyaml"
 version = "6.0.3"
@@ -1430,37 +1666,93 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
 ]

+[[package]]
+name = "quart"
+version = "0.20.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "aiofiles" },
+    { name = "blinker" },
+    { name = "click" },
+    { name = "flask" },
+    { name = "hypercorn" },
+    { name = "itsdangerous" },
+    { name = "jinja2" },
+    { name = "markupsafe" },
+    { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/1d/9d/12e1143a5bd2ccc05c293a6f5ae1df8fd94a8fc1440ecc6c344b2b30ce13/quart-0.20.0.tar.gz", hash = "sha256:08793c206ff832483586f5ae47018c7e40bdd75d886fee3fabbdaa70c2cf505d", size = 63874, upload-time = "2024-12-23T13:53:05.664Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7e/e9/cc28f21f52913adf333f653b9e0a3bf9cb223f5083a26422968ba73edd8d/quart-0.20.0-py3-none-any.whl", hash = "sha256:003c08f551746710acb757de49d9b768986fd431517d0eb127380b656b98b8f1", size = 77960, upload-time = "2024-12-23T13:53:02.842Z" },
+]
+
+[[package]]
+name = "quart-jwt-extended"
+version = "0.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pyjwt" },
+    { name = "quart" },
+    { name = "six" },
+    { name = "werkzeug" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/60/12/e7b37b6dad958470248041f3626ea039337e258cec33ce25e3f316329791/Quart_JWT_Extended-0.1.0-py3-none-any.whl", hash = "sha256:422f04f317a76dc614a55ce01c945534e28a30c4e6e09e746f11f160a618a9f7", size = 22652, upload-time = "2022-10-03T12:53:05.451Z" },
+]
+
 [[package]]
 name = "raggr"
 version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
+    { name = "aerich" },
+    { name = "bcrypt" },
    { name = "black" },
    { name = "chromadb" },
    { name = "flask" },
+    { name = "flask-jwt-extended" },
+    { name = "flask-login" },
    { name = "httpx" },
    { name = "ollama" },
    { name = "openai" },
    { name = "pillow" },
    { name = "pillow-heif" },
+    { name = "pony" },
+    { name = "pre-commit" },
    { name = "pydantic" },
    { name = "pymupdf" },
    { name = "python-dotenv" },
+    { name = "quart" },
+    { name = "quart-jwt-extended" },
+    { name = "tomlkit" },
+    { name = "tortoise-orm" },
+    { name = "tortoise-orm-stubs" },
 ]

 [package.metadata]
 requires-dist = [
+    { name = "aerich", specifier = ">=0.8.0" },
+    { name = "bcrypt", specifier = ">=5.0.0" },
    { name = "black", specifier = ">=25.9.0" },
    { name = "chromadb", specifier = ">=1.1.0" },
    { name = "flask", specifier = ">=3.1.2" },
+    { name = "flask-jwt-extended", specifier = ">=4.7.1" },
+    { name = "flask-login", specifier = ">=0.6.3" },
    { name = "httpx", specifier = ">=0.28.1" },
    { name = "ollama", specifier = ">=0.6.0" },
    { name = "openai", specifier = ">=2.0.1" },
    { name = "pillow", specifier = ">=10.0.0" },
    { name = "pillow-heif", specifier = ">=1.1.1" },
+    { name = "pony", specifier = ">=0.7.19" },
+    { name = "pre-commit", specifier = ">=4.3.0" },
    { name = "pydantic", specifier = ">=2.11.9" },
    { name = "pymupdf", specifier = ">=1.24.0" },
    { name = "python-dotenv", specifier = ">=1.0.0" },
+    { name = "quart", specifier = ">=0.20.0" },
+    { name = "quart-jwt-extended", specifier = ">=0.1.0" },
+    { name = "tomlkit", specifier = ">=0.13.3" },
+    { name = "tortoise-orm", specifier = ">=0.25.1" },
+    { name = "tortoise-orm-stubs", specifier = ">=1.0.2" },
 ]

 [[package]]
@@ -1668,6 +1960,42 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/b3/46/e33a8c93907b631a99377ef4c5f817ab453d0b34f93529421f42ff559671/tokenizers-0.22.1-cp39-abi3-win_amd64.whl", hash = "sha256:65fd6e3fb11ca1e78a6a93602490f134d1fdeb13bcef99389d5102ea318ed138", size = 2674684, upload-time = "2025-09-19T09:49:24.953Z" },
 ]

+[[package]]
+name = "tomlkit"
+version = "0.13.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/cc/18/0bbf3884e9eaa38819ebe46a7bd25dcd56b67434402b66a58c4b8e552575/tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1", size = 185207, upload-time = "2025-06-05T07:13:44.947Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/bd/75/8539d011f6be8e29f339c42e633aae3cb73bffa95dd0f9adec09b9c58e85/tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0", size = 38901, upload-time = "2025-06-05T07:13:43.546Z" },
+]
+
+[[package]]
+name = "tortoise-orm"
+version = "0.25.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "aiosqlite" },
+    { name = "iso8601", marker = "python_full_version < '4.0'" },
+    { name = "pypika-tortoise", marker = "python_full_version < '4.0'" },
+    { name = "pytz" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d7/9b/de966810021fa773fead258efd8deea2bb73bb12479e27f288bd8ceb8763/tortoise_orm-0.25.1.tar.gz", hash = "sha256:4d5bfd13d5750935ffe636a6b25597c5c8f51c47e5b72d7509d712eda1a239fe", size = 128341, upload-time = "2025-06-05T10:43:31.058Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/70/55/2bda7f4445f4c07b734385b46d1647a388d05160cf5b8714a713e8709378/tortoise_orm-0.25.1-py3-none-any.whl", hash = "sha256:df0ef7e06eb0650a7e5074399a51ee6e532043308c612db2cac3882486a3fd9f", size = 167723, upload-time = "2025-06-05T10:43:29.309Z" },
+]
+
+[[package]]
+name = "tortoise-orm-stubs"
+version = "1.0.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "tortoise-orm" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ba/49/45b06cda907e55226b8ed4ddc71d13ff61505bfe366d72276462eeee9d2b/tortoise_orm_stubs-1.0.2.tar.gz", hash = "sha256:f4d6a810f295bebd83aa71b05ebd2decd883517f3c9530bd2376b9209b0777c6", size = 4559, upload-time = "2023-11-20T14:48:26.806Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/33/b1/f0b111dcf9381987f8acb143dd95b77934a3e9120a6c63b2cf4255c2934c/tortoise_orm_stubs-1.0.2-py3-none-any.whl", hash = "sha256:5ae3c2b0eb0286669563634b98202bbdf46349966b1c85659f3160de4fb655d6", size = 4681, upload-time = "2023-11-20T14:48:22.536Z" },
+]
+
 [[package]]
 name = "tqdm"
 version = "4.67.1"
@@ -1763,6 +2091,20 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/63/9a/0962b05b308494e3202d3f794a6e85abe471fe3cafdbcf95c2e8c713aabd/uvloop-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553", size = 4660018, upload-time = "2024-10-14T23:38:10.888Z" },
 ]

+[[package]]
+name = "virtualenv"
+version = "20.35.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "distlib" },
+    { name = "filelock" },
+    { name = "platformdirs" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a4/d5/b0ccd381d55c8f45d46f77df6ae59fbc23d19e901e2d523395598e5f4c93/virtualenv-20.35.3.tar.gz", hash = "sha256:4f1a845d131133bdff10590489610c98c168ff99dc75d6c96853801f7f67af44", size = 6002907, upload-time = "2025-10-10T21:23:33.178Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/27/73/d9a94da0e9d470a543c1b9d3ccbceb0f59455983088e727b8a1824ed90fb/virtualenv-20.35.3-py3-none-any.whl", hash = "sha256:63d106565078d8c8d0b206d48080f938a8b25361e19432d2c9db40d2899c810a", size = 5981061, upload-time = "2025-10-10T21:23:30.433Z" },
+]
+
 [[package]]
 name = "watchfiles"
 version = "1.1.0"
@@ -1858,6 +2200,18 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" },
 ]

+[[package]]
+name = "wsproto"
+version = "1.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c9/4a/44d3c295350d776427904d73c189e10aeae66d7f555bb2feee16d1e4ba5a/wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", size = 53425, upload-time = "2022-08-23T19:58:21.447Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/78/58/e860788190eba3bcce367f74d29c4675466ce8dddfba85f7827588416f01/wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736", size = 24226, upload-time = "2022-08-23T19:58:19.96Z" },
+]
+
 [[package]]
 name = "zipp"
 version = "3.23.0"
Author	SHA1	Message	Date
Ryan Chen	29ac724d50	enabling login btw users	2025-10-25 09:30:54 -04:00
Ryan Chen	6b616137d3	oops	2025-10-23 22:45:14 -04:00
Ryan Chen	841b6ebd4f	i commit	2025-10-23 22:39:26 -04:00
Ryan Chen	45a5e92aee	Added conversation history (#4 ) Reviewed-on: #4 Co-authored-by: Ryan Chen <ryan@torrtle.co> Co-committed-by: Ryan Chen <ryan@torrtle.co>	2025-10-23 22:29:12 -04:00
Ryan Chen	8479898cc4	Logging	2025-10-16 22:43:14 -04:00
Ryan Chen	acaf681927	Metadata filtering	2025-10-16 22:36:21 -04:00
Ryan Chen	2bbe33fedc	Starting attempt #2 at metadata filtering	2025-10-14 22:13:01 -04:00
Ryan Chen	b872750444	Only use OpenAI for embedding	2025-10-14 20:06:32 -04:00
Ryan Chen	376baccadb	message-style frontend	2025-10-10 23:28:41 -04:00
Ryan Chen	c978b1a255	Reducing startup time/cost	2025-10-08 23:21:22 -04:00
Ryan Chen	51b9932389	fixing loal llm	2025-10-08 22:52:49 -04:00
Ryan Chen	ebf39480b6	urf	2025-10-08 22:46:16 -04:00
Ryan Chen	e4a04331cb	add some more debugging	2025-10-08 21:17:45 -04:00
Ryan Chen	166ffb4c09	i only ship bugs	2025-10-08 21:13:15 -04:00
Ryan Chen	64e286e623	oops	2025-10-08 21:07:33 -04:00
Ryan Chen	c6c14729dd	interseting	2025-10-08 21:03:42 -04:00
Ryan Chen	910097d13b	data	2025-10-05 20:31:46 -04:00