Merge pull request 'Validate production config, add CI, and build from locked dependencies' (#78) from feat/config-ci into main
This commit was merged in pull request #78.
This commit is contained in:
+58
-38
@@ -1,14 +1,15 @@
|
||||
# Database Configuration
|
||||
# PostgreSQL is recommended (required for OIDC features)
|
||||
DATABASE_URL=postgres://raggr:changeme@postgres:5432/raggr
|
||||
# PostgreSQL with pgvector is required; production rejects default passwords.
|
||||
DATABASE_URL=postgres://raggr:replace-with-random-password@postgres:5432/raggr
|
||||
|
||||
# PostgreSQL credentials (if using docker-compose postgres service)
|
||||
POSTGRES_USER=raggr
|
||||
POSTGRES_PASSWORD=changeme
|
||||
POSTGRES_PASSWORD=replace-with-random-password
|
||||
POSTGRES_DB=raggr
|
||||
|
||||
# JWT Configuration
|
||||
JWT_SECRET_KEY=your-secret-key-here
|
||||
# Required: generate a random secret with openssl rand -hex 32
|
||||
JWT_SECRET_KEY=
|
||||
|
||||
# Metrics Configuration
|
||||
# Bearer token for the /metrics endpoint (Prometheus scrape + app token gate).
|
||||
@@ -17,13 +18,15 @@ JWT_SECRET_KEY=your-secret-key-here
|
||||
METRICS_TOKEN=
|
||||
|
||||
# Paperless Configuration
|
||||
PAPERLESS_TOKEN=your-paperless-token
|
||||
BASE_URL=192.168.1.5:8000
|
||||
PAPERLESS_TOKEN=
|
||||
# Current Paperless clients prepend http://; use host[:port] without a scheme.
|
||||
BASE_URL=
|
||||
|
||||
# llama-server Configuration (OpenAI-compatible API)
|
||||
# If set, uses llama-server as the primary LLM backend with OpenAI as fallback
|
||||
LLAMA_SERVER_URL=http://192.168.1.213:8080/v1
|
||||
LLAMA_SERVER_URL=
|
||||
LLAMA_MODEL_NAME=llama-3.1-8b-instruct
|
||||
LLAMA_TIMEOUT_SECONDS=120
|
||||
# Abandon a turn when an in-flight model call produces no output (prefill included)
|
||||
# for this long. Thinking tokens count as output, so long reasoning is fine.
|
||||
TTFT_TIMEOUT_SECONDS=30
|
||||
@@ -42,26 +45,30 @@ TOOL_ROUTING_MODE=static
|
||||
# Embedding Server Configuration
|
||||
# If set, uses a custom OpenAI-compatible embedding server (e.g. llama-server)
|
||||
# Falls back to OpenAI embeddings if not set
|
||||
EMBEDDING_SERVER_URL=http://192.168.1.7:8086/v1
|
||||
EMBEDDING_MODEL_NAME=all-minilm
|
||||
EMBEDDING_SERVER_URL=
|
||||
EMBEDDING_MODEL_NAME=text-embedding-3-small
|
||||
|
||||
# OpenAI Configuration
|
||||
OPENAI_API_KEY=your-openai-api-key
|
||||
# Required even with llama-server: the fallback client is always initialized.
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# Tavily Configuration (for web search)
|
||||
TAVILY_API_KEY=your-tavily-api-key
|
||||
# Legacy TAVILIY_API_KEY is accepted by Compose/startup; TAVILY_API_KEY wins.
|
||||
# Current agent initialization needs this key even when web search is unused.
|
||||
TAVILY_API_KEY=
|
||||
|
||||
# Immich Configuration
|
||||
IMMICH_URL=http://192.168.1.5:2283
|
||||
IMMICH_API_KEY=your-immich-api-key
|
||||
SEARCH_QUERY=simba cat
|
||||
IMMICH_API_KEY=
|
||||
SEARCH_QUERY="simba cat"
|
||||
DOWNLOAD_DIR=./simba_photos
|
||||
|
||||
# OIDC Configuration (Authelia)
|
||||
OIDC_ISSUER=https://auth.example.com
|
||||
OIDC_CLIENT_ID=simbarag
|
||||
OIDC_CLIENT_SECRET=your-client-secret-here
|
||||
OIDC_REDIRECT_URI=http://localhost:8080/
|
||||
# Optional OIDC (Authelia): set all four fields together. Production requires HTTPS.
|
||||
# Register the public callback URI: https://your-host/api/user/oidc/callback
|
||||
OIDC_ISSUER=
|
||||
OIDC_CLIENT_ID=
|
||||
OIDC_CLIENT_SECRET=
|
||||
OIDC_REDIRECT_URI=
|
||||
OIDC_USE_DISCOVERY=true
|
||||
|
||||
# Optional: Manual OIDC endpoints (if discovery is disabled)
|
||||
@@ -72,7 +79,7 @@ OIDC_USE_DISCOVERY=true
|
||||
|
||||
# YNAB Configuration
|
||||
# Get your Personal Access Token from https://app.ynab.com/settings/developer
|
||||
YNAB_ACCESS_TOKEN=your-ynab-personal-access-token
|
||||
YNAB_ACCESS_TOKEN=
|
||||
# Optional: Specify a budget ID, or leave empty to use the default/first budget
|
||||
YNAB_BUDGET_ID=
|
||||
# Seconds between background YNAB delta syncs. Keep under the agent tools' 120s
|
||||
@@ -81,21 +88,22 @@ YNAB_SYNC_INTERVAL=90
|
||||
|
||||
# Mealie Configuration
|
||||
# Base URL for your Mealie instance (e.g., http://192.168.1.5:9000 or https://mealie.example.com)
|
||||
MEALIE_BASE_URL=http://192.168.1.5:9000
|
||||
MEALIE_BASE_URL=
|
||||
# Get your API token from Mealie's user settings page
|
||||
MEALIE_API_TOKEN=your-mealie-api-token
|
||||
MEALIE_API_TOKEN=
|
||||
# Group slug used in recipe page links (/g/<slug>/r/<recipe>); defaults to "home"
|
||||
MEALIE_GROUP_SLUG=home
|
||||
|
||||
# Email Integration
|
||||
# Required even without email: encrypted model fields initialize on startup.
|
||||
# Email Encryption Key (32-byte URL-safe base64)
|
||||
# Generate with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
|
||||
FERNET_KEY=your-fernet-key-here
|
||||
FERNET_KEY=
|
||||
|
||||
# Twilio Configuration (WhatsApp)
|
||||
TWILIO_ACCOUNT_SID=your-twilio-account-sid
|
||||
TWILIO_AUTH_TOKEN=your-twilio-auth-token
|
||||
TWILIO_WHATSAPP_NUMBER=whatsapp:+14155238886
|
||||
TWILIO_ACCOUNT_SID=
|
||||
TWILIO_AUTH_TOKEN=
|
||||
TWILIO_WHATSAPP_NUMBER=
|
||||
# Comma-separated list of WhatsApp numbers allowed to use the service (e.g., whatsapp:+1234567890)
|
||||
# Use * to allow any number
|
||||
ALLOWED_WHATSAPP_NUMBERS=
|
||||
@@ -119,10 +127,10 @@ EMAIL_HMAC_SECRET=
|
||||
MAILGUN_SIGNATURE_VALIDATION=true
|
||||
|
||||
# SendBlue Configuration (iMessage)
|
||||
SENDBLUE_API_KEY=your-sendblue-api-key
|
||||
SENDBLUE_API_SECRET=your-sendblue-api-secret
|
||||
SENDBLUE_FROM_NUMBER=+1XXXXXXXXXX
|
||||
SENDBLUE_WEBHOOK_SECRET=your-sendblue-webhook-secret
|
||||
SENDBLUE_API_KEY=
|
||||
SENDBLUE_API_SECRET=
|
||||
SENDBLUE_FROM_NUMBER=
|
||||
SENDBLUE_WEBHOOK_SECRET=
|
||||
# Set to false to disable SendBlue signature validation in development
|
||||
SENDBLUE_SIGNATURE_VALIDATION=true
|
||||
# Comma-separated list of iMessage numbers allowed to use the service (E.164 format)
|
||||
@@ -145,42 +153,43 @@ GOOGLE_TOKEN_FILE=/app/config/google-token.json
|
||||
CRON_SECRET=
|
||||
# Cronicle server URL and API key (from Cronicle Admin → API Keys)
|
||||
# Used to auto-register scheduled jobs on startup.
|
||||
CRONICLE_URL=http://cronicle:3012
|
||||
CRONICLE_URL=
|
||||
CRONICLE_API_KEY=
|
||||
|
||||
# SimbaKit / PetKit Configuration
|
||||
# PetKit account credentials (used by SimbaKit to poll device data)
|
||||
PETKIT_USERNAME=your-petkit-email
|
||||
PETKIT_PASSWORD=your-petkit-password
|
||||
PETKIT_USERNAME=
|
||||
PETKIT_PASSWORD=
|
||||
PETKIT_REGION=US
|
||||
PETKIT_TIMEZONE=America/New_York
|
||||
# Internal URL for SimbaKit service (Docker network)
|
||||
SIMBAKIT_URL=http://simbakit:8585
|
||||
SIMBAKIT_URL=
|
||||
|
||||
# Homebox inventory (admin-only agent tools via homebox-mcp, needs Homebox >= 0.26)
|
||||
# API key from Homebox Profile -> API Keys (starts with hb_). Leave empty to disable.
|
||||
HOMEBOX_URL=https://homebox.example.com
|
||||
HOMEBOX_URL=
|
||||
HOMEBOX_TOKEN=
|
||||
|
||||
# BookStack wiki (admin-only agent tools via bookstack-mcp: read, create, update; no deletes)
|
||||
# API token from BookStack Profile -> API Tokens (user needs the "Access System API" permission).
|
||||
# Leave empty to disable.
|
||||
BOOKSTACK_BASE_URL=https://bookstack.example.com
|
||||
BOOKSTACK_BASE_URL=
|
||||
BOOKSTACK_TOKEN_ID=
|
||||
BOOKSTACK_TOKEN_SECRET=
|
||||
|
||||
# Obsidian Configuration (headless sync)
|
||||
# Obsidian account credentials (used for `ob login` on container startup)
|
||||
OBSIDIAN_EMAIL=your-obsidian-email
|
||||
OBSIDIAN_PASSWORD=your-obsidian-password
|
||||
OBSIDIAN_EMAIL=
|
||||
OBSIDIAN_PASSWORD=
|
||||
# Vault ID to sync (found in Obsidian sync settings)
|
||||
OBSIDIAN_VAULT_ID=your-vault-id
|
||||
OBSIDIAN_VAULT_ID=
|
||||
# End-to-end encryption password (if vault uses E2E encryption)
|
||||
OBSIDIAN_E2E_PASSWORD=
|
||||
# Device name shown in Obsidian sync activity
|
||||
OBSIDIAN_DEVICE_NAME=simbarag
|
||||
# Set to true to run continuous sync in the background
|
||||
OBSIDIAN_CONTINUOUS_SYNC=false
|
||||
OBSIDIAN_SYNC_INTERVAL=60
|
||||
# Local path to Obsidian vault (where files are synced)
|
||||
OBSIDIAN_VAULT_PATH=/app/data/obsidian
|
||||
# Vault name for obsidian://open links in the artifact panel (optional; no link if unset)
|
||||
@@ -193,3 +202,14 @@ CHECKPOINT_DB_PATH=/app/checkpoints/checkpoints.db
|
||||
# Beli restaurant recommendations
|
||||
# Beli account credentials for each profile are stored encrypted in the database.
|
||||
# No additional env vars needed — auth is done via the Beli API at login time.
|
||||
|
||||
# Optional S3 image storage (leave credentials empty to disable)
|
||||
S3_ENDPOINT_URL=
|
||||
S3_ACCESS_KEY_ID=
|
||||
S3_SECRET_ACCESS_KEY=
|
||||
S3_BUCKET_NAME=asksimba-images
|
||||
S3_REGION=garage
|
||||
|
||||
# Ollama is used by image/PDF helper workflows, not the main chat agent.
|
||||
OLLAMA_URL=http://localhost:11434
|
||||
OLLAMA_HOST=http://localhost:11434
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
name: CI
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
backend:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.2.2
|
||||
- uses: actions/setup-python@v5.6.0
|
||||
with:
|
||||
python-version: '3.13.2'
|
||||
- name: Install pinned tooling
|
||||
run: python -m pip install uv==0.7.2 ruff==0.8.2
|
||||
- name: Install locked backend and test dependencies
|
||||
run: |
|
||||
uv lock --check
|
||||
uv sync --frozen --extra test
|
||||
- name: Ruff
|
||||
run: ruff check .
|
||||
- name: Unit tests (no service credentials)
|
||||
env:
|
||||
OPENAI_API_KEY: synthetic-unit-test-key
|
||||
TAVILY_API_KEY: synthetic-unit-test-key
|
||||
run: uv run --frozen --extra test pytest tests/unit -v
|
||||
- name: Deterministic context evaluation
|
||||
run: uv run --frozen --extra test python -m scripts.evaluate_context
|
||||
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: raggr-frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4.2.2
|
||||
- uses: actions/setup-node@v4.4.0
|
||||
with:
|
||||
node-version: '22.14.0'
|
||||
cache: yarn
|
||||
cache-dependency-path: raggr-frontend/yarn.lock
|
||||
- name: Install locked frontend dependencies
|
||||
run: yarn install --frozen-lockfile
|
||||
- name: Renderer tests
|
||||
run: node --test tests/*.test.cjs
|
||||
- name: Typecheck
|
||||
run: yarn tsc --noEmit
|
||||
- name: Build
|
||||
run: yarn build
|
||||
+30
-15
@@ -1,16 +1,30 @@
|
||||
FROM python:3.13-slim
|
||||
FROM node:22.14.0-bookworm-slim AS frontend
|
||||
WORKDIR /frontend
|
||||
COPY raggr-frontend/package.json raggr-frontend/yarn.lock ./
|
||||
RUN yarn install --frozen-lockfile
|
||||
# Copy build inputs explicitly: there is no root .dockerignore, and copying the
|
||||
# whole directory would overwrite locked Linux deps with host node_modules.
|
||||
COPY raggr-frontend/src ./src
|
||||
COPY raggr-frontend/public ./public
|
||||
COPY raggr-frontend/rsbuild.config.ts raggr-frontend/tsconfig.json raggr-frontend/postcss.config.mjs ./
|
||||
RUN yarn build
|
||||
|
||||
FROM ghcr.io/astral-sh/uv:0.7.2 AS uv
|
||||
|
||||
FROM python:3.13.2-slim-bookworm
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install system dependencies, Node.js, Yarn, and uv
|
||||
# Node is required by the optional stdio MCP and Obsidian tools at runtime.
|
||||
COPY --from=frontend /usr/local/bin/node /usr/local/bin/node
|
||||
COPY --from=frontend /usr/local/lib/node_modules /usr/local/lib/node_modules
|
||||
RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm
|
||||
COPY --from=uv /uv /uvx /usr/local/bin/
|
||||
RUN apt-get update && apt-get install -y \
|
||||
build-essential \
|
||||
curl \
|
||||
&& curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
|
||||
&& apt-get install -y nodejs \
|
||||
&& npm install -g yarn obsidian-headless @googleworkspace/cli \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
&& npm install -g obsidian-headless@0.0.8 @googleworkspace/cli@0.8.0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Add uv to PATH
|
||||
ENV PATH="/root/.local/bin:$PATH"
|
||||
@@ -22,10 +36,15 @@ RUN uv tool install homebox-mcp==1.1.0
|
||||
RUN npm install -g bookstack-mcp@5.6.0
|
||||
|
||||
# Copy dependency files
|
||||
COPY pyproject.toml ./
|
||||
COPY pyproject.toml uv.lock ./
|
||||
|
||||
# Install Python dependencies using uv
|
||||
RUN uv pip install --system -e .
|
||||
# Export the frozen graph and install it system-wide: python and aerich remain
|
||||
# available to startup.sh and docker compose exec without activating a venv.
|
||||
RUN uv lock --check \
|
||||
&& uv export --frozen --no-dev --no-emit-project --quiet -o /tmp/requirements.txt \
|
||||
&& uv pip install --system --require-hashes -r /tmp/requirements.txt \
|
||||
&& rm /tmp/requirements.txt
|
||||
|
||||
# Copy application code
|
||||
COPY *.py ./
|
||||
@@ -37,14 +56,10 @@ COPY scripts ./scripts
|
||||
COPY startup.sh ./
|
||||
RUN chmod +x startup.sh
|
||||
|
||||
# Copy frontend code and build
|
||||
COPY raggr-frontend ./raggr-frontend
|
||||
WORKDIR /app/raggr-frontend
|
||||
RUN yarn install && yarn build
|
||||
WORKDIR /app
|
||||
COPY --from=frontend /frontend/dist ./raggr-frontend/dist
|
||||
|
||||
# Create database directory
|
||||
RUN mkdir -p /app/database
|
||||
RUN mkdir -p /app/database /app/checkpoints
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8080
|
||||
|
||||
@@ -69,14 +69,14 @@ migrate-history:
|
||||
|
||||
# Tests
|
||||
test:
|
||||
pytest tests/ -v
|
||||
OPENAI_API_KEY=synthetic-unit-test-key TAVILY_API_KEY=synthetic-unit-test-key uv run --frozen --extra test pytest tests/unit -v
|
||||
|
||||
test-cov:
|
||||
pytest tests/ -v --cov
|
||||
OPENAI_API_KEY=synthetic-unit-test-key TAVILY_API_KEY=synthetic-unit-test-key uv run --frozen --extra test pytest tests/unit -v --cov
|
||||
|
||||
# Frontend
|
||||
frontend:
|
||||
cd raggr-frontend && yarn install && yarn build
|
||||
cd raggr-frontend && npm ci && npm run build
|
||||
|
||||
# Desktop app (Tauri). Point it at the server with SIMBA_URL, e.g.
|
||||
# make desktop-dev SIMBA_URL=https://simba.example.com
|
||||
@@ -88,3 +88,16 @@ desktop:
|
||||
cd desktop && npm install && SIMBA_URL=$(SIMBA_URL) npx tauri build --bundles app
|
||||
rm -rf /Applications/Simba.app
|
||||
cp -R desktop/src-tauri/target/release/bundle/macos/Simba.app /Applications/
|
||||
|
||||
.PHONY: lint evaluate validate-config frontend-check
|
||||
lint:
|
||||
uvx ruff==0.8.2 check .
|
||||
|
||||
evaluate:
|
||||
uv run --frozen --extra test python -m scripts.evaluate_context
|
||||
|
||||
validate-config:
|
||||
uv run --frozen python scripts/validate_config.py
|
||||
|
||||
frontend-check:
|
||||
cd raggr-frontend && npm ci && node --test tests/*.test.cjs && npx --no-install tsc --noEmit && npm run build
|
||||
|
||||
@@ -1,370 +1,60 @@
|
||||
# SimbaRAG 🐱
|
||||
# SimbaRAG
|
||||
|
||||
A Retrieval-Augmented Generation (RAG) conversational AI system for querying information about Simba the cat. Built with LangChain, ChromaDB, and modern web technologies.
|
||||
SimbaRAG is a personal assistant with a Quart backend and React 19 frontend, document retrieval, persistent conversations, and configurable household integrations.
|
||||
|
||||
## Features
|
||||
PostgreSQL stores application data and **pgvector** stores document embeddings (`simba_docs`). LangGraph checkpoints and conversation summaries use separate SQLite files on a persistent volume. Chat uses an OpenAI-compatible **llama-server** when configured, with **OpenAI `gpt-5-mini` fallback**; without llama-server it uses OpenAI directly. Embeddings use a configured OpenAI-compatible server or OpenAI `text-embedding-3-small`. Ollama is used by image/PDF helper workflows.
|
||||
|
||||
- 🤖 **Intelligent Conversations** - LangChain-powered agent with tool use and memory
|
||||
- 📚 **Document Retrieval** - RAG system using ChromaDB vector store
|
||||
- 🔍 **Web Search** - Integrated Tavily API for real-time web searches
|
||||
- 🔐 **OIDC Authentication** - Secure auth via Authelia with LDAP group support
|
||||
- 💬 **Multi-Conversation** - Manage multiple conversation threads per user
|
||||
- 🎨 **Modern UI** - React 19 frontend with Tailwind CSS
|
||||
- 🐳 **Docker Ready** - Containerized deployment with Docker Compose
|
||||
## Setup
|
||||
|
||||
## System Architecture
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph "Client Layer"
|
||||
Browser[Web Browser]
|
||||
end
|
||||
|
||||
subgraph "Frontend - React"
|
||||
UI[React UI<br/>Tailwind CSS]
|
||||
Auth[Auth Service]
|
||||
API[API Client]
|
||||
end
|
||||
|
||||
subgraph "Backend - Quart/Python"
|
||||
App[Quart App<br/>app.py]
|
||||
|
||||
subgraph "Blueprints"
|
||||
Users[Users Blueprint<br/>OIDC + JWT]
|
||||
Conv[Conversation Blueprint<br/>Chat Management]
|
||||
RAG[RAG Blueprint<br/>Document Indexing]
|
||||
end
|
||||
|
||||
Agent[LangChain Agent<br/>main.py]
|
||||
LLM[LLM Client<br/>llm.py]
|
||||
end
|
||||
|
||||
subgraph "Tools & Utilities"
|
||||
Search[Simba Search Tool]
|
||||
Web[Web Search Tool<br/>Tavily]
|
||||
end
|
||||
|
||||
subgraph "Data Layer"
|
||||
Postgres[(PostgreSQL<br/>Users & Conversations)]
|
||||
Chroma[(ChromaDB<br/>Vector Store)]
|
||||
end
|
||||
|
||||
subgraph "External Services"
|
||||
Authelia[Authelia<br/>OIDC Provider]
|
||||
LLDAP[LLDAP<br/>User Directory]
|
||||
Ollama[Ollama<br/>Local LLM]
|
||||
OpenAI[OpenAI<br/>Fallback LLM]
|
||||
Paperless[Paperless-NGX<br/>Documents]
|
||||
TavilyAPI[Tavily API<br/>Web Search]
|
||||
end
|
||||
|
||||
Browser --> UI
|
||||
UI --> Auth
|
||||
UI --> API
|
||||
API --> App
|
||||
|
||||
App --> Users
|
||||
App --> Conv
|
||||
App --> RAG
|
||||
|
||||
Conv --> Agent
|
||||
Agent --> Search
|
||||
Agent --> Web
|
||||
Agent --> LLM
|
||||
|
||||
Search --> Chroma
|
||||
Web --> TavilyAPI
|
||||
RAG --> Chroma
|
||||
RAG --> Paperless
|
||||
|
||||
Users --> Postgres
|
||||
Conv --> Postgres
|
||||
|
||||
Users --> Authelia
|
||||
Authelia --> LLDAP
|
||||
|
||||
LLM --> Ollama
|
||||
LLM -.Fallback.-> OpenAI
|
||||
|
||||
style Browser fill:#e1f5ff
|
||||
style UI fill:#fff3cd
|
||||
style App fill:#d4edda
|
||||
style Agent fill:#d4edda
|
||||
style Postgres fill:#f8d7da
|
||||
style Chroma fill:#f8d7da
|
||||
style Ollama fill:#e2e3e5
|
||||
style OpenAI fill:#e2e3e5
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Docker & Docker Compose
|
||||
- PostgreSQL (or use Docker)
|
||||
- Ollama (optional, for local LLM)
|
||||
- Paperless-NGX instance (for document source)
|
||||
|
||||
### Installation
|
||||
|
||||
1. **Clone the repository**
|
||||
|
||||
```bash
|
||||
git clone https://github.com/yourusername/simbarag.git
|
||||
cd simbarag
|
||||
```
|
||||
|
||||
2. **Configure environment variables**
|
||||
Install Python 3.13, uv 0.7.2, Node 22.14.0, and Docker Compose v2. From the repository root:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# Edit .env with your configuration
|
||||
# Edit .env before starting services; it is deliberately not deployable as-is.
|
||||
uv sync --frozen --extra test
|
||||
cd raggr-frontend
|
||||
npm ci
|
||||
npm run build
|
||||
cd ..
|
||||
```
|
||||
|
||||
3. **Start the services**
|
||||
Set `DATABASE_URL`, `JWT_SECRET_KEY`, `FERNET_KEY`, and `OPENAI_API_KEY`. Use a random database password and match it to `POSTGRES_PASSWORD` if using the included PostgreSQL service. OpenAI fallback and encrypted model fields initialize even when their integrations are otherwise unused. Optional integration credentials can remain empty in validation. The current Tavily client initialization still needs `TAVILY_API_KEY` at app import time; making it optional requires an application change. Configure all OIDC fields together to enable browser login.
|
||||
|
||||
For local backend development, start only PostgreSQL with `docker compose up -d postgres`, set `DATABASE_URL` to the host connection (`localhost:5432`), and use local writable checkpoint paths. Then:
|
||||
|
||||
```bash
|
||||
# Development (local PostgreSQL only)
|
||||
docker compose -f docker-compose.dev.yml up -d
|
||||
|
||||
# Or full Docker deployment (app runs as raggr-blue; see docs/deployment.md)
|
||||
docker compose --profile blue up -d
|
||||
uv run --frozen aerich upgrade
|
||||
uv run --frozen python app.py
|
||||
```
|
||||
|
||||
4. **Access the application**
|
||||
See the [development guide](docs/development.md) for new database initialization, frontend development, and CI checks. For production configuration and blue/green services, see the [deployment guide](docs/deployment.md).
|
||||
|
||||
Open `http://localhost:8080` in your browser.
|
||||
## Agent tools and integrations
|
||||
|
||||
## Development
|
||||
Tools are selected per request by the tool router. The current tool families include:
|
||||
|
||||
### Local Development Setup
|
||||
- Document retrieval (`simba_search`) and Tavily web search (`web_search`).
|
||||
- Obsidian note search/read/create, journal tasks, and saved user memory.
|
||||
- Mealie recipes, meal plans, and shopping lists.
|
||||
- Google Calendar and Gmail, using the OAuth token file; access is gated by the router.
|
||||
- SimbaKit/PetKit device status, weight, litter, and drinking history.
|
||||
- Beli restaurant lists; credentials are stored encrypted per profile.
|
||||
- Admin integrations: YNAB mirror SQL, Homebox inventory through `homebox-mcp`, and BookStack wiki through `bookstack-mcp`.
|
||||
|
||||
The application also supports WhatsApp (Twilio), email (Mailgun), iMessage (SendBlue), scheduled messages, and Cronicle triggers. Integration availability and access depend on credentials and user role. Tool schemas being present does not imply an integration is configured.
|
||||
|
||||
## Checks and operations
|
||||
|
||||
```bash
|
||||
# 1. Start PostgreSQL
|
||||
docker compose -f docker-compose.dev.yml up -d
|
||||
|
||||
# 2. Set environment variables
|
||||
export DATABASE_URL="postgres://raggr:raggr_dev_password@localhost:5432/raggr"
|
||||
export CHROMADB_PATH="./chromadb"
|
||||
export $(grep -v '^#' .env | xargs)
|
||||
|
||||
# 3. Install dependencies
|
||||
pip install -r requirements.txt
|
||||
cd raggr-frontend && yarn install && yarn build && cd ..
|
||||
|
||||
# 4. Run migrations
|
||||
aerich upgrade
|
||||
|
||||
# 5. Start the server
|
||||
python app.py
|
||||
make test # backend unit tests; no deployment secrets required
|
||||
make evaluate # deterministic context fixture evaluation
|
||||
make lint # Ruff 0.8.2, matching pre-commit/CI
|
||||
make frontend-check # locked install, renderer tests, TypeScript, build
|
||||
make validate-config # production policy; prints setting names, not secrets
|
||||
```
|
||||
|
||||
See [docs/development.md](docs/development.md) for detailed development guide.
|
||||
Gitea Actions runs backend Ruff/unit/evaluation and frontend renderer/typecheck/build checks independently. Python dependencies come from `uv.lock`; frontend dependencies come from `raggr-frontend/package-lock.json` with `npm ci`.
|
||||
|
||||
## Project Structure
|
||||
The production Makefile detects the active blue/green color for `make logs`, `make migrate`, and `make migrate-history`. `make redeploy` pulls the current branch and invokes `scripts/deploy.sh`; run it only when an actual deployment is intended. `make reindex` clears and rebuilds documents; `make reindex-obsidian` rebuilds only Obsidian documents. `make ynab-sync` performs a delta mirror sync and `make ynab-resync` a full download.
|
||||
|
||||
```
|
||||
simbarag/
|
||||
├── app.py # Quart application entry point
|
||||
├── main.py # RAG logic & LangChain agent
|
||||
├── llm.py # LLM client with Ollama/OpenAI
|
||||
├── aerich_config.py # Database migration configuration
|
||||
│
|
||||
├── blueprints/ # API route blueprints
|
||||
│ ├── users/ # Authentication & authorization
|
||||
│ ├── conversation/ # Chat conversations
|
||||
│ └── rag/ # Document indexing
|
||||
│
|
||||
├── config/ # Configuration modules
|
||||
│ └── oidc_config.py # OIDC authentication settings
|
||||
│
|
||||
├── utils/ # Reusable utilities
|
||||
│ ├── chunker.py # Document chunking for embeddings
|
||||
│ ├── cleaner.py # PDF cleaning and summarization
|
||||
│ ├── image_process.py # Image description with LLM
|
||||
│ └── request.py # Paperless-NGX API client
|
||||
│
|
||||
├── scripts/ # Administrative scripts
|
||||
│ ├── add_user.py
|
||||
│ ├── user_message_stats.py
|
||||
│ ├── manage_vectorstore.py
|
||||
│ └── inspect_vector_store.py
|
||||
│
|
||||
├── raggr-frontend/ # React frontend
|
||||
│ └── src/
|
||||
│
|
||||
├── migrations/ # Database migrations
|
||||
│
|
||||
├── docs/ # Documentation
|
||||
│ ├── index.md # Documentation hub
|
||||
│ ├── development.md # Development guide
|
||||
│ ├── deployment.md # Deployment & migrations
|
||||
│ ├── VECTORSTORE.md # Vector store management
|
||||
│ ├── MIGRATIONS.md # Migration reference
|
||||
│ └── authentication.md # Authentication setup
|
||||
│
|
||||
├── docker-compose.yml # Production compose
|
||||
├── docker-compose.dev.yml # Development compose
|
||||
├── Dockerfile # Production Dockerfile
|
||||
├── Dockerfile.dev # Development Dockerfile
|
||||
├── CLAUDE.md # AI assistant instructions
|
||||
└── README.md # This file
|
||||
```
|
||||
|
||||
## Key Technologies
|
||||
|
||||
### Backend
|
||||
- **Quart** - Async Python web framework
|
||||
- **LangChain** - Agent framework with tool use
|
||||
- **Tortoise ORM** - Async ORM for PostgreSQL
|
||||
- **Aerich** - Database migration tool
|
||||
- **ChromaDB** - Vector database for embeddings
|
||||
- **OpenAI** - Embeddings & LLM (fallback)
|
||||
- **Ollama** - Local LLM (primary)
|
||||
|
||||
### Frontend
|
||||
- **React 19** - UI framework
|
||||
- **Rsbuild** - Fast bundler
|
||||
- **Tailwind CSS** - Utility-first styling
|
||||
- **Axios** - HTTP client
|
||||
|
||||
### Authentication
|
||||
- **Authelia** - OIDC provider
|
||||
- **LLDAP** - Lightweight LDAP server
|
||||
- **JWT** - Token-based auth
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Authentication
|
||||
- `GET /api/user/oidc/login` - Initiate OIDC login
|
||||
- `GET /api/user/oidc/callback` - OIDC callback handler
|
||||
- `POST /api/user/refresh` - Refresh JWT token
|
||||
|
||||
### Conversations
|
||||
- `POST /api/conversation/` - Create conversation
|
||||
- `GET /api/conversation/` - List conversations
|
||||
- `GET /api/conversation/<id>` - Get conversation with messages
|
||||
- `POST /api/conversation/query` - Send message and get response
|
||||
|
||||
### RAG (Admin Only)
|
||||
- `GET /api/rag/stats` - Vector store statistics
|
||||
- `POST /api/rag/index` - Index new documents
|
||||
- `POST /api/rag/reindex` - Clear and reindex all
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `DATABASE_URL` | PostgreSQL connection string | `postgres://...` |
|
||||
| `CHROMADB_PATH` | ChromaDB storage path | `./chromadb` |
|
||||
| `OLLAMA_URL` | Ollama server URL | `http://localhost:11434` |
|
||||
| `OPENAI_API_KEY` | OpenAI API key | - |
|
||||
| `PAPERLESS_TOKEN` | Paperless-NGX API token | - |
|
||||
| `BASE_URL` | Paperless-NGX base URL | - |
|
||||
| `OIDC_ISSUER` | OIDC provider URL | - |
|
||||
| `OIDC_CLIENT_ID` | OIDC client ID | - |
|
||||
| `OIDC_CLIENT_SECRET` | OIDC client secret | - |
|
||||
| `JWT_SECRET_KEY` | JWT signing key | - |
|
||||
| `TAVILY_KEY` | Tavily web search API key | - |
|
||||
|
||||
See `.env.example` for full list.
|
||||
|
||||
## Scripts
|
||||
|
||||
### User Management
|
||||
```bash
|
||||
# Add a new user
|
||||
python scripts/add_user.py
|
||||
|
||||
# View message statistics
|
||||
python scripts/user_message_stats.py
|
||||
```
|
||||
|
||||
### Vector Store Management
|
||||
```bash
|
||||
# Show vector store statistics
|
||||
python scripts/manage_vectorstore.py stats
|
||||
|
||||
# Index new documents from Paperless
|
||||
python scripts/manage_vectorstore.py index
|
||||
|
||||
# Clear and reindex everything
|
||||
python scripts/manage_vectorstore.py reindex
|
||||
|
||||
# Inspect vector store contents
|
||||
python scripts/inspect_vector_store.py
|
||||
```
|
||||
|
||||
See [docs/vectorstore.md](docs/vectorstore.md) for details.
|
||||
|
||||
## Database Migrations
|
||||
|
||||
```bash
|
||||
# Generate a new migration
|
||||
aerich migrate --name "describe_your_changes"
|
||||
|
||||
# Apply pending migrations
|
||||
aerich upgrade
|
||||
|
||||
# View migration history
|
||||
aerich history
|
||||
|
||||
# Rollback last migration
|
||||
aerich downgrade
|
||||
```
|
||||
|
||||
See [docs/deployment.md](docs/deployment.md) for detailed migration workflows.
|
||||
|
||||
## LangChain Agent
|
||||
|
||||
The conversational agent has access to two tools:
|
||||
|
||||
1. **simba_search** - Query the vector store for Simba's documents
|
||||
- Used for: Medical records, veterinary history, factual information
|
||||
|
||||
2. **web_search** - Search the web via Tavily API
|
||||
- Used for: Recent events, external knowledge, general questions
|
||||
|
||||
The agent automatically selects the appropriate tool based on the user's query.
|
||||
|
||||
## Authentication Flow
|
||||
|
||||
```
|
||||
User → Authelia (OIDC) → Backend (JWT) → Frontend (localStorage)
|
||||
↓
|
||||
LLDAP
|
||||
```
|
||||
|
||||
1. User clicks "Login"
|
||||
2. Frontend redirects to Authelia
|
||||
3. User authenticates via Authelia (backed by LLDAP)
|
||||
4. Authelia redirects back with authorization code
|
||||
5. Backend exchanges code for OIDC tokens
|
||||
6. Backend issues JWT tokens
|
||||
7. Frontend stores tokens in localStorage
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Run tests and linting
|
||||
5. Submit a pull request
|
||||
|
||||
## Documentation
|
||||
|
||||
- [Development Guide](docs/development.md) - Setup and development workflow
|
||||
- [Deployment Guide](docs/deployment.md) - Deployment and migrations
|
||||
- [Vector Store Guide](docs/vectorstore.md) - Managing the vector database
|
||||
- [Authentication Guide](docs/authentication.md) - OIDC and LDAP setup
|
||||
|
||||
## License
|
||||
|
||||
[Your License Here]
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
- Built for Simba, the most important cat in the world 🐱
|
||||
- Powered by LangChain, ChromaDB, and the open-source community
|
||||
Further documentation: [pgvector management](docs/VECTORSTORE.md), [authentication](docs/authentication.md), [context management](docs/context-management.md), and [desktop client](docs/desktop-client.md).
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Explicit startup validation; importing this module never requires secrets.
|
||||
|
||||
Errors name settings, never their values. This is intentionally independent of
|
||||
application imports, database access, and optional integration clients.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import math
|
||||
import posixpath
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
|
||||
class ConfigurationError(ValueError):
|
||||
"""One or more configuration settings are unsafe or malformed."""
|
||||
|
||||
|
||||
def validate_configuration(env: Mapping[str, str], *, production: bool = True) -> None:
|
||||
errors: list[str] = []
|
||||
|
||||
def value(name: str) -> str:
|
||||
return env.get(name, "").strip()
|
||||
|
||||
def problem(name: str, reason: str) -> None:
|
||||
errors.append(f"{name}: {reason}")
|
||||
|
||||
def placeholder(text: str) -> bool:
|
||||
text = text.lower()
|
||||
return text in {"secret_key", "changeme", "password", "raggr_dev_password"} or (
|
||||
text.startswith(("your-", "replace-", "replace_", "example-"))
|
||||
)
|
||||
|
||||
def secret(name: str, *, required: bool = False, minimum: int = 1) -> None:
|
||||
text = value(name)
|
||||
if not text:
|
||||
if required:
|
||||
problem(name, "required for production startup")
|
||||
elif production and (placeholder(text) or len(text) < minimum):
|
||||
problem(
|
||||
name, f"use a non-placeholder secret of at least {minimum} characters"
|
||||
)
|
||||
|
||||
def url(name: str, *, https: bool = False) -> None:
|
||||
text = value(name)
|
||||
if not text:
|
||||
return
|
||||
try:
|
||||
parsed = urlsplit(text)
|
||||
valid = (
|
||||
parsed.scheme in ({"https"} if https else {"http", "https"})
|
||||
and parsed.hostname
|
||||
and not parsed.username
|
||||
and not parsed.password
|
||||
and not parsed.fragment
|
||||
and not any(c.isspace() for c in text)
|
||||
)
|
||||
# Accessing port validates malformed/out-of-range ports too.
|
||||
if parsed.port == 0:
|
||||
valid = False
|
||||
except ValueError:
|
||||
valid = False
|
||||
if not valid:
|
||||
problem(
|
||||
name,
|
||||
"must be a valid HTTPS URL" if https else "must be a valid HTTP(S) URL",
|
||||
)
|
||||
|
||||
secret("JWT_SECRET_KEY", required=production, minimum=32)
|
||||
secret("OPENAI_API_KEY", required=production)
|
||||
secret("FERNET_KEY", required=production)
|
||||
if value("FERNET_KEY"):
|
||||
try:
|
||||
if not re.fullmatch(r"[A-Za-z0-9_-]{43}=", value("FERNET_KEY")):
|
||||
raise ValueError
|
||||
decoded = base64.b64decode(
|
||||
value("FERNET_KEY"), altchars=b"-_", validate=True
|
||||
)
|
||||
if len(decoded) != 32:
|
||||
raise ValueError
|
||||
except (ValueError, binascii.Error):
|
||||
problem("FERNET_KEY", "must encode exactly 32 bytes in URL-safe base64")
|
||||
|
||||
database = value("DATABASE_URL")
|
||||
if production or database:
|
||||
try:
|
||||
parsed = urlsplit(database)
|
||||
valid = (
|
||||
parsed.scheme == "postgres"
|
||||
and parsed.hostname
|
||||
and parsed.username
|
||||
and parsed.password
|
||||
and parsed.path not in {"", "/"}
|
||||
and not parsed.fragment
|
||||
and not any(c.isspace() for c in database)
|
||||
)
|
||||
if parsed.port == 0:
|
||||
valid = False
|
||||
if production and placeholder(unquote(parsed.password or "")):
|
||||
valid = False
|
||||
except ValueError:
|
||||
valid = False
|
||||
if not valid:
|
||||
problem(
|
||||
"DATABASE_URL",
|
||||
"use postgres:// with host, user, non-placeholder password and database",
|
||||
)
|
||||
|
||||
integers = {
|
||||
"CONTEXT_WINDOW_TOKENS": 32768,
|
||||
"PROMPT_BUDGET_TOKENS": 16000,
|
||||
"RESPONSE_RESERVE_TOKENS": 4096,
|
||||
"YNAB_SYNC_INTERVAL": 900,
|
||||
"OBSIDIAN_SYNC_INTERVAL": 60,
|
||||
"EMAIL_RATE_LIMIT_MAX": 5,
|
||||
"EMAIL_RATE_LIMIT_WINDOW": 300,
|
||||
"WHATSAPP_RATE_LIMIT_MAX": 10,
|
||||
"WHATSAPP_RATE_LIMIT_WINDOW": 60,
|
||||
"IMESSAGE_RATE_LIMIT_MAX": 10,
|
||||
"IMESSAGE_RATE_LIMIT_WINDOW": 60,
|
||||
}
|
||||
numbers: dict[str, int] = {}
|
||||
for name, default in integers.items():
|
||||
try:
|
||||
number = int(env.get(name, str(default)))
|
||||
if number <= 0:
|
||||
raise ValueError
|
||||
numbers[name] = number
|
||||
except ValueError:
|
||||
problem(name, "must be a positive integer")
|
||||
if all(
|
||||
name in numbers
|
||||
for name in (
|
||||
"CONTEXT_WINDOW_TOKENS",
|
||||
"PROMPT_BUDGET_TOKENS",
|
||||
"RESPONSE_RESERVE_TOKENS",
|
||||
)
|
||||
):
|
||||
if (
|
||||
numbers["PROMPT_BUDGET_TOKENS"] + numbers["RESPONSE_RESERVE_TOKENS"]
|
||||
> numbers["CONTEXT_WINDOW_TOKENS"]
|
||||
):
|
||||
problem(
|
||||
"PROMPT_BUDGET_TOKENS",
|
||||
"prompt budget plus response reserve exceeds context window",
|
||||
)
|
||||
for name, default in {
|
||||
"TTFT_TIMEOUT_SECONDS": "30",
|
||||
"LLAMA_TIMEOUT_SECONDS": "120",
|
||||
}.items():
|
||||
try:
|
||||
number = float(env.get(name, default))
|
||||
if not math.isfinite(number) or number <= 0:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
problem(name, "must be a finite positive number")
|
||||
|
||||
for name in (
|
||||
"OIDC_USE_DISCOVERY",
|
||||
"OBSIDIAN_CONTINUOUS_SYNC",
|
||||
"TWILIO_SIGNATURE_VALIDATION",
|
||||
"MAILGUN_SIGNATURE_VALIDATION",
|
||||
"SENDBLUE_SIGNATURE_VALIDATION",
|
||||
):
|
||||
if name in env and env[name] not in {"true", "false"}:
|
||||
problem(name, "must be true or false")
|
||||
for name in (
|
||||
"LLAMA_SERVER_URL",
|
||||
"EMBEDDING_SERVER_URL",
|
||||
"MEALIE_BASE_URL",
|
||||
"HOMEBOX_URL",
|
||||
"BOOKSTACK_BASE_URL",
|
||||
"S3_ENDPOINT_URL",
|
||||
"SIMBAKIT_URL",
|
||||
"CRONICLE_URL",
|
||||
"OLLAMA_URL",
|
||||
"OLLAMA_HOST",
|
||||
"TWILIO_WEBHOOK_URL",
|
||||
):
|
||||
url(name)
|
||||
for name in ("LLAMA_MODEL_NAME", "EMBEDDING_MODEL_NAME"):
|
||||
if name in env and not value(name):
|
||||
problem(name, "must not be empty when supplied")
|
||||
|
||||
# OIDC is optional; a partially configured login must not pass validation.
|
||||
oidc = ("OIDC_ISSUER", "OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "OIDC_REDIRECT_URI")
|
||||
if any(value(name) for name in oidc):
|
||||
for name in oidc:
|
||||
if not value(name):
|
||||
problem(name, "required when OIDC is configured")
|
||||
secret("OIDC_CLIENT_SECRET")
|
||||
url("OIDC_ISSUER", https=production)
|
||||
url("OIDC_REDIRECT_URI", https=production)
|
||||
if env.get("OIDC_USE_DISCOVERY", "true") == "false":
|
||||
for name in (
|
||||
"OIDC_AUTHORIZATION_ENDPOINT",
|
||||
"OIDC_TOKEN_ENDPOINT",
|
||||
"OIDC_USERINFO_ENDPOINT",
|
||||
"OIDC_JWKS_URI",
|
||||
):
|
||||
if not value(name):
|
||||
problem(name, "required when OIDC discovery is disabled")
|
||||
url(name, https=production)
|
||||
|
||||
for trigger, signing_key, flag in (
|
||||
("TWILIO_ACCOUNT_SID", "TWILIO_AUTH_TOKEN", "TWILIO_SIGNATURE_VALIDATION"),
|
||||
(
|
||||
"MAILGUN_DOMAIN",
|
||||
"MAILGUN_WEBHOOK_SIGNING_KEY",
|
||||
"MAILGUN_SIGNATURE_VALIDATION",
|
||||
),
|
||||
(
|
||||
"SENDBLUE_API_KEY",
|
||||
"SENDBLUE_WEBHOOK_SECRET",
|
||||
"SENDBLUE_SIGNATURE_VALIDATION",
|
||||
),
|
||||
):
|
||||
if production:
|
||||
if value(trigger) or value(signing_key):
|
||||
secret(signing_key, required=True)
|
||||
# Routes are registered even when outbound integrations are unused.
|
||||
# Missing keys fail closed; disabling verification would bypass that.
|
||||
if env.get(flag, "true") != "true":
|
||||
problem(flag, "signature validation must remain enabled in production")
|
||||
for name in ("METRICS_TOKEN", "CRON_SECRET", "EMAIL_HMAC_SECRET"):
|
||||
secret(name, minimum=32)
|
||||
if env.get("OBSIDIAN_CONTINUOUS_SYNC") == "true":
|
||||
for name in ("OBSIDIAN_EMAIL", "OBSIDIAN_PASSWORD", "OBSIDIAN_VAULT_ID"):
|
||||
if not value(name):
|
||||
problem(name, "required when continuous sync is enabled")
|
||||
|
||||
paths = {
|
||||
"CHECKPOINT_DB_PATH": env.get(
|
||||
"CHECKPOINT_DB_PATH", "/app/checkpoints/checkpoints.db"
|
||||
),
|
||||
"CONVERSATION_SUMMARY_DB_PATH": env.get(
|
||||
"CONVERSATION_SUMMARY_DB_PATH",
|
||||
env.get("CHECKPOINT_DB_PATH", "/app/checkpoints/checkpoints.db")
|
||||
+ ".summaries.db",
|
||||
),
|
||||
}
|
||||
for name, path in paths.items():
|
||||
if (
|
||||
not path
|
||||
or path == ":memory:"
|
||||
or "\x00" in path
|
||||
or (production and not path.startswith("/"))
|
||||
or path.endswith("/")
|
||||
):
|
||||
problem(name, "must be a persistent file path (absolute in production)")
|
||||
if posixpath.normpath(paths["CHECKPOINT_DB_PATH"]) == posixpath.normpath(
|
||||
paths["CONVERSATION_SUMMARY_DB_PATH"]
|
||||
):
|
||||
problem(
|
||||
"CONVERSATION_SUMMARY_DB_PATH", "must differ from checkpoint database path"
|
||||
)
|
||||
if errors:
|
||||
raise ConfigurationError(
|
||||
"Invalid configuration:\n" + "\n".join(f"- {error}" for error in errors)
|
||||
)
|
||||
+21
-5
@@ -1,20 +1,36 @@
|
||||
version: "3.8"
|
||||
|
||||
x-raggr-common: &raggr-common
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: torrtle/simbarag:latest
|
||||
environment:
|
||||
- METRICS_TOKEN=${METRICS_TOKEN:-}
|
||||
- TTFT_TIMEOUT_SECONDS=${TTFT_TIMEOUT_SECONDS:-30}
|
||||
- LLAMA_TIMEOUT_SECONDS=${LLAMA_TIMEOUT_SECONDS:-120}
|
||||
- CONTEXT_WINDOW_TOKENS=${CONTEXT_WINDOW_TOKENS:-32768}
|
||||
- PROMPT_BUDGET_TOKENS=${PROMPT_BUDGET_TOKENS:-16000}
|
||||
- RESPONSE_RESERVE_TOKENS=${RESPONSE_RESERVE_TOKENS:-4096}
|
||||
- CHECKPOINT_DB_PATH=${CHECKPOINT_DB_PATH:-/app/checkpoints/checkpoints.db}
|
||||
- CONVERSATION_SUMMARY_DB_PATH=${CONVERSATION_SUMMARY_DB_PATH:-${CHECKPOINT_DB_PATH:-/app/checkpoints/checkpoints.db}.summaries.db}
|
||||
- OBSIDIAN_SYNC_INTERVAL=${OBSIDIAN_SYNC_INTERVAL:-60}
|
||||
- OIDC_AUTHORIZATION_ENDPOINT=${OIDC_AUTHORIZATION_ENDPOINT:-}
|
||||
- OIDC_TOKEN_ENDPOINT=${OIDC_TOKEN_ENDPOINT:-}
|
||||
- OIDC_USERINFO_ENDPOINT=${OIDC_USERINFO_ENDPOINT:-}
|
||||
- OIDC_JWKS_URI=${OIDC_JWKS_URI:-}
|
||||
- EMAIL_HMAC_SECRET=${EMAIL_HMAC_SECRET:-}
|
||||
- WHATSAPP_RATE_LIMIT_MAX=${WHATSAPP_RATE_LIMIT_MAX:-10}
|
||||
- WHATSAPP_RATE_LIMIT_WINDOW=${WHATSAPP_RATE_LIMIT_WINDOW:-60}
|
||||
- IMESSAGE_RATE_LIMIT_MAX=${IMESSAGE_RATE_LIMIT_MAX:-10}
|
||||
- IMESSAGE_RATE_LIMIT_WINDOW=${IMESSAGE_RATE_LIMIT_WINDOW:-60}
|
||||
- PAPERLESS_TOKEN=${PAPERLESS_TOKEN}
|
||||
- BASE_URL=${BASE_URL}
|
||||
- OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434}
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||
- EMBEDDING_SERVER_URL=${EMBEDDING_SERVER_URL}
|
||||
- EMBEDDING_MODEL_NAME=${EMBEDDING_MODEL_NAME}
|
||||
- EMBEDDING_MODEL_NAME=${EMBEDDING_MODEL_NAME:-text-embedding-3-small}
|
||||
- JWT_SECRET_KEY=${JWT_SECRET_KEY}
|
||||
- LLAMA_SERVER_URL=${LLAMA_SERVER_URL}
|
||||
- LLAMA_MODEL_NAME=${LLAMA_MODEL_NAME}
|
||||
- LLAMA_MODEL_NAME=${LLAMA_MODEL_NAME:-llama-3.1-8b-instruct}
|
||||
- TOOL_ROUTING_MODE=${TOOL_ROUTING_MODE:-static}
|
||||
- OIDC_ISSUER=${OIDC_ISSUER}
|
||||
- OIDC_CLIENT_ID=${OIDC_CLIENT_ID}
|
||||
@@ -22,7 +38,7 @@ x-raggr-common: &raggr-common
|
||||
- OIDC_REDIRECT_URI=${OIDC_REDIRECT_URI}
|
||||
- OIDC_USE_DISCOVERY=${OIDC_USE_DISCOVERY:-true}
|
||||
- DATABASE_URL=${DATABASE_URL:-postgres://raggr:changeme@postgres:5432/raggr}
|
||||
- TAVILY_API_KEY=${TAVILIY_API_KEY}
|
||||
- TAVILY_API_KEY=${TAVILY_API_KEY:-${TAVILIY_API_KEY:-}}
|
||||
- YNAB_ACCESS_TOKEN=${YNAB_ACCESS_TOKEN}
|
||||
- YNAB_BUDGET_ID=${YNAB_BUDGET_ID}
|
||||
- YNAB_SYNC_INTERVAL=${YNAB_SYNC_INTERVAL:-90}
|
||||
|
||||
+45
-218
@@ -1,147 +1,68 @@
|
||||
# Deployment & Migrations Guide
|
||||
# Deployment
|
||||
|
||||
This document covers database migrations and deployment workflows for SimbaRAG.
|
||||
Production Compose defines `postgres` (pgvector on PostgreSQL 16), optional `simbakit`, and two application services: `raggr-blue` on port 8080 and `raggr-green` on port 8081. Enable one application profile for a normal startup. The image builds the React frontend with `npm ci`, exports the frozen `uv.lock` dependency graph, and installs hashed Python packages system-wide so `python` and `aerich` work in startup and Compose exec commands. Runtime Node/MCP tools and uv are version pinned.
|
||||
|
||||
## Migration Workflow
|
||||
## Configuration before deployment
|
||||
|
||||
Migrations are managed by [Aerich](https://github.com/tortoise/aerich), the migration tool for Tortoise ORM.
|
||||
Copy `.env.example` to `.env`, then replace the database placeholders and configure:
|
||||
|
||||
### Key Principles
|
||||
- `DATABASE_URL`: `postgres://` with host, database, user and a non-placeholder password. With the included database, its credentials must match `POSTGRES_USER`, `POSTGRES_PASSWORD`, and `POSTGRES_DB`.
|
||||
- `JWT_SECRET_KEY`: random signing secret of at least 32 characters.
|
||||
- `FERNET_KEY`: URL-safe base64 encoding of 32 random bytes. Required by encrypted model fields even without email. Preserve this key across deployments and restore it with encrypted database backups.
|
||||
- `OPENAI_API_KEY`: required because the OpenAI fallback client always initializes, including when `LLAMA_SERVER_URL` is set.
|
||||
|
||||
1. **Generate migrations in Docker** - Aerich needs database access to detect schema changes
|
||||
2. **Migrations auto-apply on startup** - Both `startup.sh` and `startup-dev.sh` run `aerich upgrade`
|
||||
3. **Commit migrations to git** - Migration files must be in the repo for production deploys
|
||||
|
||||
### Generating a New Migration
|
||||
|
||||
#### Development (Recommended)
|
||||
|
||||
With `docker-compose.dev.yml`, your local `services/raggr` directory is synced to the container. Migrations generated inside the container appear on your host automatically.
|
||||
Generate keys locally with `openssl rand -hex 32` for JWT and `uv run --frozen python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` for Fernet. Store their output securely; never include configuration values in CI logs.
|
||||
|
||||
```bash
|
||||
# 1. Start the dev environment
|
||||
docker compose -f docker-compose.dev.yml up -d
|
||||
|
||||
# 2. Generate migration (runs inside container, syncs to host)
|
||||
docker compose -f docker-compose.dev.yml exec raggr aerich migrate --name describe_your_change
|
||||
|
||||
# 3. Verify migration was created
|
||||
ls services/raggr/migrations/models/
|
||||
|
||||
# 4. Commit the migration
|
||||
git add services/raggr/migrations/
|
||||
git commit -m "Add migration: describe_your_change"
|
||||
make validate-config
|
||||
```
|
||||
|
||||
#### Production Container
|
||||
Validation runs again in `startup.sh` **before migrations**. Failure exits nonzero and names the invalid settings without printing values. It rejects unsafe defaults, malformed database/service URLs, invalid Fernet keys, empty model names, nonpositive/nonfinite timeouts, invalid numeric/boolean settings, context budgets exceeding the window, and invalid checkpoint paths. Successful validation checks syntax/policy, not network connectivity or credential validity.
|
||||
|
||||
For production, migration files are baked into the image. You must generate migrations in dev first.
|
||||
OIDC is optional, but if enabled configure issuer, client ID, client secret and public callback URI together. Production requires HTTPS issuer/callback URLs; register `https://your-host/api/user/oidc/callback` with the provider. With discovery disabled, also set the four manual endpoint variables. Unused optional integration secrets are not required by validation. The current agent module nevertheless constructs Tavily at import time and its SDK rejects an empty key; until that initialization is made optional, application startup also needs a valid Tavily key for web search. Configured Twilio, Mailgun and SendBlue webhook integrations require their signature secret. Signature validation must remain enabled in production even for unused channels, because their routes are still registered. Metrics remain disabled (403) with an empty `METRICS_TOKEN`; when enabled it must be at least 32 characters. `CRON_SECRET` and `EMAIL_HMAC_SECRET` follow the same length policy if supplied.
|
||||
|
||||
`TAVILY_API_KEY` is the correct spelling. Compose and startup accept the legacy environment spelling `TAVILIY_API_KEY` when the canonical value is empty; migrate existing configuration to the canonical name. For local Python started directly, use the canonical name in `.env`.
|
||||
|
||||
Compose forwards context window, prompt budget, response reserve, llama read timeout, stream stall timeout, metrics token, manual OIDC endpoints, rate limits, and checkpoint/summary paths. Defaults are 32768 context tokens, 16000 prompt tokens, 4096 reserved response tokens, 120 seconds llama read timeout, and 30 seconds stream stall timeout. Align context settings with the smallest backend context window, including fallback.
|
||||
|
||||
`raggr-checkpoints` persists `/app/checkpoints` for both colors. The default checkpoint file is `checkpoints.db`; the default summary file is the checkpoint path plus `.summaries.db`. If overriding either path, keep it inside a persistent mounted directory and ensure its parent exists. Use separate files. Back up PostgreSQL, this volume, and encryption keys together.
|
||||
|
||||
The Compose bind mounts include `./obvault` and `./token.json` (Google OAuth, read-only). Provision a token file when enabling Google integrations; otherwise an empty regular file can occupy the mount path. Do not allow Docker to silently create a directory at `token.json`. If SimbaKit is enabled, supply PetKit credentials; app validation does not validate that separate service.
|
||||
|
||||
## Startup and migrations
|
||||
|
||||
```bash
|
||||
# If you need to generate a migration from production (not recommended):
|
||||
docker compose exec raggr aerich migrate --name describe_your_change
|
||||
|
||||
# Copy the file out of the container
|
||||
docker cp $(docker compose ps -q raggr):/app/migrations/models/ ./services/raggr/migrations/
|
||||
```
|
||||
|
||||
### Applying Migrations
|
||||
|
||||
Migrations apply automatically on container start via the startup scripts.
|
||||
|
||||
**Manual application (if needed):**
|
||||
|
||||
```bash
|
||||
# Dev
|
||||
docker compose -f docker-compose.dev.yml exec raggr aerich upgrade
|
||||
|
||||
# Production
|
||||
docker compose exec raggr aerich upgrade
|
||||
```
|
||||
|
||||
### Checking Migration Status
|
||||
|
||||
```bash
|
||||
# View applied migrations
|
||||
docker compose exec raggr aerich history
|
||||
|
||||
# View pending migrations
|
||||
docker compose exec raggr aerich heads
|
||||
```
|
||||
|
||||
### Rolling Back
|
||||
|
||||
```bash
|
||||
# Downgrade one migration
|
||||
docker compose exec raggr aerich downgrade
|
||||
|
||||
# Downgrade to specific version
|
||||
docker compose exec raggr aerich downgrade -v 1
|
||||
```
|
||||
|
||||
## Deployment Workflows
|
||||
|
||||
### Development
|
||||
|
||||
```bash
|
||||
# Start with watch mode (auto-restarts on file changes)
|
||||
docker compose -f docker-compose.dev.yml up
|
||||
|
||||
# Or with docker compose watch (requires Docker Compose v2.22+)
|
||||
docker compose -f docker-compose.dev.yml watch
|
||||
```
|
||||
|
||||
The dev environment:
|
||||
- Syncs `services/raggr/` to `/app` in the container
|
||||
- Rebuilds frontend on changes
|
||||
- Auto-applies migrations on startup
|
||||
|
||||
### Production (zero-downtime blue-green)
|
||||
|
||||
The app runs as one of two identical compose services: `raggr-blue`
|
||||
(host port 8080) or `raggr-green` (host port 8081), gated behind compose
|
||||
profiles so only one color runs at a time. Caddy health-checks both ports and
|
||||
routes traffic to whichever is healthy.
|
||||
|
||||
```bash
|
||||
# Pull latest and swap colors with zero downtime
|
||||
make redeploy
|
||||
|
||||
# Or run the swap directly
|
||||
./scripts/deploy.sh
|
||||
|
||||
# View logs / exec into the active color (Makefile resolves the color)
|
||||
# Build and start a selected color only, after configuration is ready:
|
||||
docker compose --profile blue build raggr-blue
|
||||
docker compose --profile blue up -d --wait raggr-blue
|
||||
make logs
|
||||
make migrate-history
|
||||
```
|
||||
|
||||
`scripts/deploy.sh` does the following:
|
||||
Startup validates configuration, runs `aerich upgrade`, optionally sets up Obsidian pull-only sync, and replaces the shell with the Python process. **A migration failure stops startup; the application is never launched.** Obsidian login/setup failures still log a warning and allow the application to continue.
|
||||
|
||||
1. Detects the running color via `docker compose ps`.
|
||||
2. Builds the inactive color.
|
||||
3. Starts it with `docker compose up -d --wait`, which blocks until the
|
||||
container healthcheck (`GET /api/health`) passes.
|
||||
4. If the new color never becomes healthy, it is stopped and the old color
|
||||
keeps serving (the script exits non-zero).
|
||||
5. Otherwise the old color is stopped with a 30s grace period so in-flight
|
||||
SSE streams can finish, then removed.
|
||||
|
||||
To target a color manually, enable its profile:
|
||||
An empty database without Aerich metadata may need explicit initialization. Inspect its state first. Run the migration tool in a one-off container because the app cannot start until migrations pass:
|
||||
|
||||
```bash
|
||||
docker compose --profile blue up -d raggr-blue
|
||||
docker compose --profile blue --profile green ps
|
||||
docker compose up -d postgres
|
||||
docker compose --profile blue run --rm --no-deps raggr-blue aerich init-db
|
||||
```
|
||||
|
||||
#### Caddy configuration
|
||||
Do not run `init-db` against an existing schema or fake migrations without a reviewed reconciliation plan. The pgvector extension initialization only runs on a fresh PostgreSQL volume; for an existing database, have the database owner ensure the `vector` extension exists before indexing. `aerich init-db` initializes ORM tables, not pgvector document indexing.
|
||||
|
||||
Caddy needs both upstreams with active health checks:
|
||||
To generate and review changes, use `make migrate-new` and inspect `migrations/models/`. Manual application/history use `make migrate` and `make migrate-history`. The Makefile chooses the running color. Production migrations must be backward-compatible while both colors overlap; column removals and renames require staged changes. Make database backups before schema changes.
|
||||
|
||||
## Blue/green rollout
|
||||
|
||||
`./scripts/deploy.sh` builds the inactive color, starts it with `--wait`, and stops the old color only when the new container is healthy. A failed new color is stopped while the old color remains serving. `make redeploy` first performs `git pull`, then calls that script. These commands perform a real deployment and should be run only by the deployment operator.
|
||||
|
||||
An external reverse proxy must route to both ports and check `/api/health`. For example:
|
||||
|
||||
```caddyfile
|
||||
ask.torrtle.co {
|
||||
simba.example.com {
|
||||
reverse_proxy host.docker.internal:8080 host.docker.internal:8081 {
|
||||
lb_policy first # prefer blue when both are healthy
|
||||
lb_try_duration 5s # retry on the other upstream during the swap
|
||||
lb_policy first
|
||||
lb_try_duration 5s
|
||||
lb_try_interval 250ms
|
||||
health_uri /api/health
|
||||
health_interval 2s
|
||||
@@ -151,106 +72,12 @@ ask.torrtle.co {
|
||||
}
|
||||
```
|
||||
|
||||
#### One-time cutover from the old `raggr` service
|
||||
The proxy configuration is outside Compose. During rollouts both colors briefly share checkpoints and Obsidian files; avoid concurrent reindex operations and consider sync ownership when continuous Obsidian sync is enabled.
|
||||
|
||||
The first deploy after this change lands needs manual steps, because the old
|
||||
`raggr` container still holds port 8080 and the deploy script doesn't know
|
||||
about it:
|
||||
The current `app.py` entry point runs Quart in debug mode. Production process serving/debug hardening and broader readiness semantics require separate application changes; configuration validation does not resolve those concerns. Docker's existing health check remains `/api/health`.
|
||||
|
||||
```bash
|
||||
# 1. Update the Caddy config to the two-upstream block below and reload Caddy
|
||||
# 2. On the server:
|
||||
git pull
|
||||
docker compose --profile green up -d --build --wait raggr-green # port 8081, no conflict
|
||||
docker stop -t 30 $(docker ps -q --filter name=raggr-1) # old raggr container
|
||||
docker rm $(docker ps -aq --filter name=raggr-1)
|
||||
# From here on, `make redeploy` handles swaps (green is now the active color)
|
||||
```
|
||||
## CI and release checks
|
||||
|
||||
#### Blue-green caveats
|
||||
Gitea Actions runs backend Ruff, unit tests and deterministic context evaluation, plus frontend renderer tests, TypeScript checks and build. CI receives no production secrets and performs no deployment. Python resolves from `uv.lock` with frozen commands; npm uses `package-lock.json`. Lockfiles must match manifests; do not regenerate them during an image build or CI run.
|
||||
|
||||
- **Migrations must be backward-compatible.** The new color runs
|
||||
`aerich upgrade` on startup while the old color is still serving the
|
||||
previous code. Additive changes (new tables/columns) are safe; destructive
|
||||
changes (drop/rename a column the old code reads) need a two-step deploy:
|
||||
ship code that stops using the column first, then drop it in a later deploy.
|
||||
- **Obsidian sync overlap.** During the ~30s swap window both containers run
|
||||
pull-only `ob sync` against the shared `./obvault` mount. Both are
|
||||
read-only pulls, so this is low risk, but avoid deploying mid-reindex.
|
||||
|
||||
### Fresh Deploy (New Database)
|
||||
|
||||
On first deploy with an empty database, `startup-dev.sh` runs `aerich init-db` instead of `aerich upgrade`. This creates all tables from the current models.
|
||||
|
||||
For production (`startup.sh`), ensure the database exists and run:
|
||||
|
||||
```bash
|
||||
# If aerich table doesn't exist yet
|
||||
docker compose exec raggr aerich init-db
|
||||
|
||||
# Or if migrating from existing schema
|
||||
docker compose exec raggr aerich upgrade
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "No migrations found" on startup
|
||||
|
||||
The `migrations/models/` directory is empty or not copied into the image.
|
||||
|
||||
**Fix:** Ensure migrations are committed and the Dockerfile copies them:
|
||||
```dockerfile
|
||||
COPY migrations ./migrations
|
||||
```
|
||||
|
||||
### Migration fails with "relation already exists"
|
||||
|
||||
The database has tables but aerich doesn't know about them (fresh aerich setup on existing DB).
|
||||
|
||||
**Fix:** Fake the initial migration:
|
||||
```bash
|
||||
# Mark initial migration as applied without running it
|
||||
docker compose exec raggr aerich upgrade --fake
|
||||
```
|
||||
|
||||
### Model changes not detected
|
||||
|
||||
Aerich compares models against the last migration's state. If state is out of sync:
|
||||
|
||||
```bash
|
||||
# Regenerate migration state (dangerous - review carefully)
|
||||
docker compose exec raggr aerich migrate --name fix_state
|
||||
```
|
||||
|
||||
### Database connection errors
|
||||
|
||||
Ensure PostgreSQL is healthy before running migrations:
|
||||
|
||||
```bash
|
||||
# Check postgres status
|
||||
docker compose ps postgres
|
||||
|
||||
# Wait for postgres then run migrations
|
||||
docker compose exec raggr bash -c "sleep 5 && aerich upgrade"
|
||||
```
|
||||
|
||||
## File Reference
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `pyproject.toml` | Aerich config (`[tool.aerich]` section) |
|
||||
| `migrations/models/` | Migration files |
|
||||
| `startup.sh` | Production startup (runs `aerich upgrade`) |
|
||||
| `startup-dev.sh` | Dev startup (runs `aerich upgrade` or `init-db`) |
|
||||
| `app.py` | Contains `TORTOISE_CONFIG` |
|
||||
| `aerich_config.py` | Aerich initialization configuration |
|
||||
|
||||
## Quick Reference
|
||||
|
||||
| Task | Command |
|
||||
|------|---------|
|
||||
| Generate migration | `docker compose -f docker-compose.dev.yml exec raggr aerich migrate --name name` |
|
||||
| Apply migrations | `docker compose exec raggr aerich upgrade` |
|
||||
| View history | `docker compose exec raggr aerich history` |
|
||||
| Rollback | `docker compose exec raggr aerich downgrade` |
|
||||
| Fresh init | `docker compose exec raggr aerich init-db` |
|
||||
Before rollout, require these checks to pass, validate the deployment environment, verify persistent mounts, and ensure migrations support both versions. Frozen application dependencies do not pin OS apt packages or transitive dependencies of globally installed integration CLIs; stricter image reproducibility can additionally use image digests and separately locked tool bundles.
|
||||
|
||||
+44
-226
@@ -1,258 +1,76 @@
|
||||
# Development Guide
|
||||
# Development
|
||||
|
||||
This guide explains how to run SimbaRAG in development mode.
|
||||
The repository uses Python 3.13 with `uv.lock`, React 19 with Rsbuild, and PostgreSQL with pgvector. There is no `docker-compose.dev.yml` or `requirements.txt` workflow. Run the backend locally against the Compose PostgreSQL service.
|
||||
|
||||
## Quick Start
|
||||
## Local setup
|
||||
|
||||
### Option 1: Local Development (Recommended)
|
||||
|
||||
Run PostgreSQL in Docker and the application locally for faster iteration:
|
||||
Install uv 0.7.2, Python 3.13, Node 22.14.0, and Docker Compose v2.
|
||||
|
||||
```bash
|
||||
# 1. Start PostgreSQL
|
||||
docker compose -f docker-compose.dev.yml up -d
|
||||
|
||||
# 2. Set environment variables
|
||||
export DATABASE_URL="postgres://raggr:raggr_dev_password@localhost:5432/raggr"
|
||||
export CHROMADB_PATH="./chromadb"
|
||||
export $(grep -v '^#' .env | xargs) # Load other vars from .env
|
||||
|
||||
# 3. Install dependencies (first time)
|
||||
pip install -r requirements.txt
|
||||
cd raggr-frontend && yarn install && yarn build && cd ..
|
||||
|
||||
# 4. Run migrations
|
||||
aerich upgrade
|
||||
|
||||
# 5. Start the server
|
||||
python app.py
|
||||
cp .env.example .env
|
||||
uv sync --frozen --extra test
|
||||
cd raggr-frontend
|
||||
npm ci
|
||||
npm run build
|
||||
cd ..
|
||||
docker compose up -d postgres
|
||||
```
|
||||
|
||||
The application will be available at `http://localhost:8080`.
|
||||
Edit `.env` before starting the database. Set `POSTGRES_USER`, `POSTGRES_PASSWORD`, and `POSTGRES_DB`; use the same credentials in `DATABASE_URL`, but point its host at `localhost` for local Python. Use `postgres://` so both Tortoise and the pgvector URL conversion work. The PostgreSQL image creates the `vector` extension on an empty data volume using `docker/init-pgvector.sql`.
|
||||
|
||||
### Option 2: Full Docker Development
|
||||
The backend needs `OPENAI_API_KEY` even with llama-server configured, a `JWT_SECRET_KEY`, and `FERNET_KEY` for encrypted model fields. Configure local writable paths, for example `CHECKPOINT_DB_PATH=/tmp/raggr/checkpoints.db` and `CONVERSATION_SUMMARY_DB_PATH=/tmp/raggr/summaries.db`, and create the parent directory. Keep optional integration credentials empty until needed.
|
||||
|
||||
Run everything in Docker with hot reload (slower, but matches production):
|
||||
The backend loads `.env` with python-dotenv; exported process variables take precedence. Do not use `export $(cat .env | xargs)` to load secrets. Validate production policy with `make validate-config`, or check provided settings with relaxed development requirements:
|
||||
|
||||
```bash
|
||||
# Uncomment the raggr service in docker-compose.dev.yml first!
|
||||
|
||||
# Start all services
|
||||
docker compose -f docker-compose.dev.yml up --build
|
||||
|
||||
# View logs
|
||||
docker compose -f docker-compose.dev.yml logs -f raggr
|
||||
uv run --frozen python scripts/validate_config.py --development
|
||||
uv run --frozen aerich upgrade
|
||||
uv run --frozen python app.py
|
||||
```
|
||||
|
||||
## Project Structure
|
||||
The local entry point serves on `http://localhost:8080` in debug mode. A truly new database without Aerich metadata needs an explicit, reviewed `uv run --frozen aerich init-db` first. Do not use `init-db` or fake migrations as a routine repair for an existing database.
|
||||
|
||||
```
|
||||
raggr/
|
||||
├── app.py # Quart application entry point
|
||||
├── main.py # RAG logic and LangChain agent
|
||||
├── llm.py # LLM client (Ollama + OpenAI fallback)
|
||||
├── aerich_config.py # Database migration configuration
|
||||
│
|
||||
├── blueprints/ # API route blueprints
|
||||
│ ├── users/ # Authentication (OIDC, JWT, RBAC)
|
||||
│ ├── conversation/ # Chat conversations and messages
|
||||
│ └── rag/ # Document indexing (admin only)
|
||||
│
|
||||
├── config/ # Configuration modules
|
||||
│ └── oidc_config.py # OIDC authentication settings
|
||||
│
|
||||
├── utils/ # Reusable utilities
|
||||
│ ├── chunker.py # Document chunking for embeddings
|
||||
│ ├── cleaner.py # PDF cleaning and summarization
|
||||
│ ├── image_process.py # Image description with LLM
|
||||
│ └── request.py # Paperless-NGX API client
|
||||
│
|
||||
├── scripts/ # Administrative scripts
|
||||
│ ├── add_user.py # Create users manually
|
||||
│ ├── user_message_stats.py # User message statistics
|
||||
│ ├── manage_vectorstore.py # Vector store management
|
||||
│ ├── inspect_vector_store.py # Inspect ChromaDB contents
|
||||
│ └── query.py # Query generation utilities
|
||||
│
|
||||
├── raggr-frontend/ # React frontend
|
||||
│ └── src/ # Frontend source code
|
||||
│
|
||||
├── migrations/ # Database migrations
|
||||
└── docs/ # Documentation
|
||||
```
|
||||
|
||||
## Making Changes
|
||||
|
||||
### Backend Changes
|
||||
|
||||
**Local development:**
|
||||
1. Edit Python files
|
||||
2. Save
|
||||
3. Restart `python app.py` (or use a tool like `watchdog` for auto-reload)
|
||||
|
||||
**Docker development:**
|
||||
1. Edit Python files
|
||||
2. Files are synced via Docker watch mode
|
||||
3. Container automatically restarts
|
||||
|
||||
### Frontend Changes
|
||||
## Frontend
|
||||
|
||||
```bash
|
||||
cd raggr-frontend
|
||||
|
||||
# Development mode with hot reload
|
||||
yarn dev
|
||||
|
||||
# Production build (for testing)
|
||||
yarn build
|
||||
npm ci
|
||||
npm run dev
|
||||
```
|
||||
|
||||
The backend serves built files from `raggr-frontend/dist/`.
|
||||
|
||||
### Database Model Changes
|
||||
|
||||
When you modify Tortoise ORM models:
|
||||
Rsbuild serves the development UI. The current Rsbuild configuration does not define an API proxy; use the combined build served by Quart for same-origin backend integration, or configure a development proxy before using API-backed flows. `npm run build` writes `dist`, which Quart serves in the combined application. TypeScript and renderer tests run independently of the bundler:
|
||||
|
||||
```bash
|
||||
# Generate migration
|
||||
aerich migrate --name "describe_your_change"
|
||||
|
||||
# Apply migration
|
||||
aerich upgrade
|
||||
|
||||
# View history
|
||||
aerich history
|
||||
node --test tests/*.test.cjs
|
||||
npx --no-install tsc --noEmit
|
||||
npm run build
|
||||
```
|
||||
|
||||
See [deployment.md](deployment.md) for detailed migration workflows.
|
||||
|
||||
### Adding Dependencies
|
||||
|
||||
**Backend:**
|
||||
```bash
|
||||
# Add to requirements.txt or use uv
|
||||
pip install package-name
|
||||
pip freeze > requirements.txt
|
||||
```
|
||||
|
||||
**Frontend:**
|
||||
```bash
|
||||
cd raggr-frontend
|
||||
yarn add package-name
|
||||
```
|
||||
|
||||
## Useful Commands
|
||||
|
||||
### Database
|
||||
## Backend checks
|
||||
|
||||
```bash
|
||||
# Connect to PostgreSQL
|
||||
docker compose -f docker-compose.dev.yml exec postgres psql -U raggr -d raggr
|
||||
|
||||
# Reset database
|
||||
docker compose -f docker-compose.dev.yml down -v
|
||||
docker compose -f docker-compose.dev.yml up -d
|
||||
aerich init-db
|
||||
make test
|
||||
make evaluate
|
||||
make lint
|
||||
make frontend-check
|
||||
```
|
||||
|
||||
### Vector Store
|
||||
Test commands supply synthetic OpenAI/Tavily keys to satisfy existing client constructors; no live API calls are needed. Unit tests use the existing test fixtures and generate an ephemeral Fernet key when needed. They do not invoke startup validation globally or require every integration secret. Configuration tests validate explicit synthetic environments, malformed settings, context budgets, signature policy, Compose forwarding, and startup failure ordering. Compose interpolation tests run when the Docker Compose CLI is installed; they neither start services nor contact a database.
|
||||
|
||||
`make evaluate` runs local context-compaction fixtures without LLM calls. Gitea CI pins uv/Ruff/Python/Node and uses frozen Python dependencies and `npm ci`; TypeScript comes from the frontend lockfile. A lock/manifest mismatch must be fixed by the dependency owner rather than resolved during CI.
|
||||
|
||||
## Changes and migrations
|
||||
|
||||
Backend routes and models live in `blueprints/`; chat/tool routing and context management live in `blueprints/conversation/`. `config/db.py` holds the Tortoise config used by Aerich. Retrieval is in `blueprints/rag/`, and integration clients are in `utils/`.
|
||||
|
||||
```bash
|
||||
# Show statistics
|
||||
python scripts/manage_vectorstore.py stats
|
||||
|
||||
# Index new documents from Paperless
|
||||
python scripts/manage_vectorstore.py index
|
||||
|
||||
# Clear and reindex everything
|
||||
python scripts/manage_vectorstore.py reindex
|
||||
make migrate-new # interactive script: container if running, local DB otherwise
|
||||
# Or, against your explicitly configured local database:
|
||||
uv run --frozen aerich migrate --name describe_your_change
|
||||
uv run --frozen aerich upgrade
|
||||
uv run --frozen aerich history
|
||||
```
|
||||
|
||||
See [vectorstore.md](vectorstore.md) for details.
|
||||
Review generated files in `migrations/models/`. Production blue/green rollouts require migrations compatible with both old and new code. See [deployment.md](deployment.md).
|
||||
|
||||
### Scripts
|
||||
|
||||
```bash
|
||||
# Add a new user
|
||||
python scripts/add_user.py
|
||||
|
||||
# View message statistics
|
||||
python scripts/user_message_stats.py
|
||||
|
||||
# Inspect vector store contents
|
||||
python scripts/inspect_vector_store.py
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Copy `.env.example` to `.env` and configure:
|
||||
|
||||
| Variable | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `DATABASE_URL` | PostgreSQL connection | `postgres://user:pass@localhost:5432/db` |
|
||||
| `CHROMADB_PATH` | ChromaDB storage path | `./chromadb` |
|
||||
| `OLLAMA_URL` | Ollama server URL | `http://localhost:11434` |
|
||||
| `OPENAI_API_KEY` | OpenAI API key (fallback LLM) | `sk-...` |
|
||||
| `PAPERLESS_TOKEN` | Paperless-NGX API token | `...` |
|
||||
| `BASE_URL` | Paperless-NGX URL | `https://paperless.example.com` |
|
||||
| `OIDC_ISSUER` | OIDC provider URL | `https://auth.example.com` |
|
||||
| `OIDC_CLIENT_ID` | OIDC client ID | `simbarag` |
|
||||
| `OIDC_CLIENT_SECRET` | OIDC client secret | `...` |
|
||||
| `JWT_SECRET_KEY` | JWT signing key | `random-secret` |
|
||||
| `TAVILY_KEY` | Tavily web search API key | `tvly-...` |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Port Already in Use
|
||||
|
||||
```bash
|
||||
# Find and kill process on port 8080
|
||||
lsof -ti:8080 | xargs kill -9
|
||||
|
||||
# Or change the port in app.py
|
||||
```
|
||||
|
||||
### Database Connection Errors
|
||||
|
||||
```bash
|
||||
# Check if PostgreSQL is running
|
||||
docker compose -f docker-compose.dev.yml ps postgres
|
||||
|
||||
# View PostgreSQL logs
|
||||
docker compose -f docker-compose.dev.yml logs postgres
|
||||
```
|
||||
|
||||
### Frontend Not Building
|
||||
|
||||
```bash
|
||||
cd raggr-frontend
|
||||
rm -rf node_modules dist
|
||||
yarn install
|
||||
yarn build
|
||||
```
|
||||
|
||||
### ChromaDB Errors
|
||||
|
||||
```bash
|
||||
# Clear and recreate ChromaDB
|
||||
rm -rf chromadb/
|
||||
python scripts/manage_vectorstore.py reindex
|
||||
```
|
||||
|
||||
### Import Errors After Reorganization
|
||||
|
||||
Ensure you're in the project root directory when running scripts, or use:
|
||||
|
||||
```bash
|
||||
# Add project root to Python path
|
||||
export PYTHONPATH="${PYTHONPATH}:$(pwd)"
|
||||
python scripts/your_script.py
|
||||
```
|
||||
|
||||
## Hot Tips
|
||||
|
||||
- Use `python -m pdb app.py` for debugging
|
||||
- Enable Quart debug mode in `app.py`: `app.run(debug=True)`
|
||||
- Check API logs: They appear in the terminal running `python app.py`
|
||||
- Frontend logs: Open browser DevTools console
|
||||
- Use `docker compose -f docker-compose.dev.yml down -v` for a clean slate
|
||||
For dependency changes, use `uv add package-name` for backend dependencies and `npm install package-name` in `raggr-frontend` for frontend dependencies. Review and include the corresponding lockfile changes. Use npm consistently for the frontend CI/build path; a `yarn.lock` also exists, but is not used by production or CI.
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate local .env and process settings without importing the application."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from config.validation import ConfigurationError, validate_configuration
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--development",
|
||||
action="store_true",
|
||||
help="allow missing secrets and local URLs",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
load_dotenv(Path(__file__).resolve().parents[1] / ".env", override=False)
|
||||
try:
|
||||
validate_configuration(os.environ, production=not args.development)
|
||||
except ConfigurationError as error:
|
||||
print(error, file=sys.stderr)
|
||||
return 1
|
||||
print("Configuration valid.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+12
-4
@@ -1,4 +1,12 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Support the historical misspelling for externally supplied environments.
|
||||
if [ -z "${TAVILY_API_KEY:-}" ] && [ -n "${TAVILIY_API_KEY:-}" ]; then
|
||||
export TAVILY_API_KEY="${TAVILIY_API_KEY}"
|
||||
fi
|
||||
|
||||
python scripts/validate_config.py
|
||||
|
||||
echo "Running database migrations..."
|
||||
aerich upgrade
|
||||
@@ -7,8 +15,8 @@ aerich upgrade
|
||||
mkdir -p /app/data/obsidian
|
||||
|
||||
# Start continuous Obsidian sync if enabled
|
||||
if [ "${OBSIDIAN_CONTINUOUS_SYNC}" = "true" ]; then
|
||||
if [ -z "${OBSIDIAN_EMAIL}" ] || [ -z "${OBSIDIAN_PASSWORD}" ] || [ -z "${OBSIDIAN_VAULT_ID}" ]; then
|
||||
if [ "${OBSIDIAN_CONTINUOUS_SYNC:-false}" = "true" ]; then
|
||||
if [ -z "${OBSIDIAN_EMAIL:-}" ] || [ -z "${OBSIDIAN_PASSWORD:-}" ] || [ -z "${OBSIDIAN_VAULT_ID:-}" ]; then
|
||||
echo "WARNING: OBSIDIAN_EMAIL, OBSIDIAN_PASSWORD, or OBSIDIAN_VAULT_ID not set. Skipping sync."
|
||||
else
|
||||
echo "Setting up Obsidian sync..."
|
||||
@@ -20,7 +28,7 @@ if [ "${OBSIDIAN_CONTINUOUS_SYNC}" = "true" ]; then
|
||||
ob sync-setup \
|
||||
--vault "${OBSIDIAN_VAULT_ID}" \
|
||||
--path "${VAULT_PATH}" \
|
||||
--password "${OBSIDIAN_E2E_PASSWORD}" \
|
||||
--password "${OBSIDIAN_E2E_PASSWORD:-}" \
|
||||
--device-name "${OBSIDIAN_DEVICE_NAME:-simbarag}"; then
|
||||
# Remove stale lock from previous container run
|
||||
rm -rf "${VAULT_PATH}/.obsidian/.sync.lock"
|
||||
@@ -36,4 +44,4 @@ if [ "${OBSIDIAN_CONTINUOUS_SYNC}" = "true" ]; then
|
||||
fi
|
||||
|
||||
echo "Starting application..."
|
||||
python app.py
|
||||
exec python app.py
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
"""Startup policy and the environment contract shared by blue/green services."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from config.validation import ConfigurationError, validate_configuration
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def production_env():
|
||||
return {
|
||||
"DATABASE_URL": "postgres://raggr:a-random-test-password@postgres:5432/raggr",
|
||||
"JWT_SECRET_KEY": "a-long-test-signing-secret-with-32-characters",
|
||||
"FERNET_KEY": base64.urlsafe_b64encode(bytes(range(32))).decode(),
|
||||
"OPENAI_API_KEY": "synthetic-test-api-key",
|
||||
}
|
||||
|
||||
|
||||
def test_optional_integrations_are_not_required(production_env):
|
||||
validate_configuration(production_env)
|
||||
validate_configuration({}, production=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name", ["DATABASE_URL", "JWT_SECRET_KEY", "FERNET_KEY", "OPENAI_API_KEY"]
|
||||
)
|
||||
def test_missing_required_settings(production_env, name):
|
||||
del production_env[name]
|
||||
with pytest.raises(ConfigurationError, match=name):
|
||||
validate_configuration(production_env)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name,value",
|
||||
[
|
||||
("JWT_SECRET_KEY", "SECRET_KEY"),
|
||||
("JWT_SECRET_KEY", "your-" + "x" * 40),
|
||||
("JWT_SECRET_KEY", "short"),
|
||||
("OPENAI_API_KEY", "your-openai-api-key"),
|
||||
("FERNET_KEY", "not-base64"),
|
||||
("FERNET_KEY", base64.urlsafe_b64encode(b"short").decode()),
|
||||
("FERNET_KEY", base64.b64encode(b"\xff" * 32).decode()),
|
||||
("DATABASE_URL", "sqlite://test.db"),
|
||||
("DATABASE_URL", "postgres://raggr:change%6de@postgres/raggr"),
|
||||
("DATABASE_URL", "postgres://raggr:password@postgres/raggr"),
|
||||
("DATABASE_URL", "postgres://raggr:strong@postgres:70000/raggr"),
|
||||
("DATABASE_URL", "postgres://raggr@postgres/raggr"),
|
||||
("LLAMA_SERVER_URL", "localhost:8080/v1"),
|
||||
("EMBEDDING_SERVER_URL", "http://server:invalid/v1"),
|
||||
("LLAMA_SERVER_URL", "http://user:private@server/v1"),
|
||||
("LLAMA_MODEL_NAME", ""),
|
||||
("METRICS_TOKEN", "short"),
|
||||
("CHECKPOINT_DB_PATH", ":memory:"),
|
||||
("CHECKPOINT_DB_PATH", "/data/invalid\x00.db"),
|
||||
("CONVERSATION_SUMMARY_DB_PATH", ""),
|
||||
("OBSIDIAN_CONTINUOUS_SYNC", "yes"),
|
||||
],
|
||||
)
|
||||
def test_malformed_and_unsafe_settings(production_env, name, value):
|
||||
production_env[name] = value
|
||||
with pytest.raises(ConfigurationError, match=name) as error:
|
||||
validate_configuration(production_env)
|
||||
if value and value not in name:
|
||||
assert value not in str(error.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["0", "-1", "abc", "", "nan", "inf", "-inf"])
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"TTFT_TIMEOUT_SECONDS",
|
||||
"LLAMA_TIMEOUT_SECONDS",
|
||||
"PROMPT_BUDGET_TOKENS",
|
||||
"WHATSAPP_RATE_LIMIT_MAX",
|
||||
],
|
||||
)
|
||||
def test_invalid_numbers(production_env, name, value):
|
||||
production_env[name] = value
|
||||
with pytest.raises(ConfigurationError, match=name):
|
||||
validate_configuration(production_env)
|
||||
|
||||
|
||||
def test_context_budget_boundary(production_env):
|
||||
production_env.update(
|
||||
CONTEXT_WINDOW_TOKENS="100",
|
||||
PROMPT_BUDGET_TOKENS="80",
|
||||
RESPONSE_RESERVE_TOKENS="20",
|
||||
)
|
||||
validate_configuration(production_env)
|
||||
production_env["RESPONSE_RESERVE_TOKENS"] = "21"
|
||||
with pytest.raises(ConfigurationError, match="exceeds context window"):
|
||||
validate_configuration(production_env)
|
||||
|
||||
|
||||
def test_oidc_partial_insecure_and_manual_endpoints(production_env):
|
||||
production_env["OIDC_ISSUER"] = "https://auth.example.test"
|
||||
with pytest.raises(ConfigurationError, match="OIDC_CLIENT_ID"):
|
||||
validate_configuration(production_env)
|
||||
production_env.update(
|
||||
OIDC_CLIENT_ID="raggr",
|
||||
OIDC_CLIENT_SECRET="synthetic-client-secret",
|
||||
OIDC_REDIRECT_URI="http://localhost:8080/api/user/oidc/callback",
|
||||
)
|
||||
with pytest.raises(ConfigurationError, match="OIDC_REDIRECT_URI"):
|
||||
validate_configuration(production_env)
|
||||
production_env["OIDC_REDIRECT_URI"] = (
|
||||
"https://app.example.test/api/user/oidc/callback"
|
||||
)
|
||||
validate_configuration(production_env)
|
||||
production_env["OIDC_USE_DISCOVERY"] = "false"
|
||||
with pytest.raises(ConfigurationError, match="OIDC_TOKEN_ENDPOINT"):
|
||||
validate_configuration(production_env)
|
||||
for name in (
|
||||
"OIDC_AUTHORIZATION_ENDPOINT",
|
||||
"OIDC_TOKEN_ENDPOINT",
|
||||
"OIDC_USERINFO_ENDPOINT",
|
||||
"OIDC_JWKS_URI",
|
||||
):
|
||||
production_env[name] = "https://auth.example.test/endpoint"
|
||||
validate_configuration(production_env)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"trigger,key,flag",
|
||||
[
|
||||
("TWILIO_ACCOUNT_SID", "TWILIO_AUTH_TOKEN", "TWILIO_SIGNATURE_VALIDATION"),
|
||||
(
|
||||
"MAILGUN_DOMAIN",
|
||||
"MAILGUN_WEBHOOK_SIGNING_KEY",
|
||||
"MAILGUN_SIGNATURE_VALIDATION",
|
||||
),
|
||||
(
|
||||
"SENDBLUE_API_KEY",
|
||||
"SENDBLUE_WEBHOOK_SECRET",
|
||||
"SENDBLUE_SIGNATURE_VALIDATION",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_enabled_webhooks_require_signatures(production_env, trigger, key, flag):
|
||||
production_env[trigger] = "synthetic-test-value"
|
||||
with pytest.raises(ConfigurationError, match=key):
|
||||
validate_configuration(production_env)
|
||||
production_env[key] = "synthetic-signing-key"
|
||||
validate_configuration(production_env)
|
||||
production_env[flag] = "false"
|
||||
with pytest.raises(ConfigurationError, match=flag):
|
||||
validate_configuration(production_env)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"flag",
|
||||
[
|
||||
"TWILIO_SIGNATURE_VALIDATION",
|
||||
"MAILGUN_SIGNATURE_VALIDATION",
|
||||
"SENDBLUE_SIGNATURE_VALIDATION",
|
||||
],
|
||||
)
|
||||
def test_production_never_disables_webhook_verification(production_env, flag):
|
||||
production_env[flag] = "false"
|
||||
with pytest.raises(ConfigurationError, match=flag):
|
||||
validate_configuration(production_env)
|
||||
validate_configuration(production_env, production=False)
|
||||
|
||||
|
||||
def test_sync_and_summary_requirements(production_env):
|
||||
production_env["OBSIDIAN_CONTINUOUS_SYNC"] = "true"
|
||||
with pytest.raises(ConfigurationError, match="OBSIDIAN_VAULT_ID"):
|
||||
validate_configuration(production_env)
|
||||
production_env.update(
|
||||
OBSIDIAN_EMAIL="test@example.test",
|
||||
OBSIDIAN_PASSWORD="test-password",
|
||||
OBSIDIAN_VAULT_ID="test-vault",
|
||||
)
|
||||
validate_configuration(production_env)
|
||||
production_env.update(
|
||||
CHECKPOINT_DB_PATH="/data/store.db",
|
||||
CONVERSATION_SUMMARY_DB_PATH="/data/../data/store.db",
|
||||
)
|
||||
with pytest.raises(ConfigurationError, match="must differ"):
|
||||
validate_configuration(production_env)
|
||||
|
||||
|
||||
def test_compose_forwards_runtime_settings():
|
||||
text = (ROOT / "docker-compose.yml").read_text()
|
||||
common = text.split("services:")[0]
|
||||
for name in (
|
||||
"METRICS_TOKEN",
|
||||
"TTFT_TIMEOUT_SECONDS",
|
||||
"LLAMA_TIMEOUT_SECONDS",
|
||||
"CONTEXT_WINDOW_TOKENS",
|
||||
"PROMPT_BUDGET_TOKENS",
|
||||
"RESPONSE_RESERVE_TOKENS",
|
||||
"CHECKPOINT_DB_PATH",
|
||||
"CONVERSATION_SUMMARY_DB_PATH",
|
||||
"OBSIDIAN_SYNC_INTERVAL",
|
||||
"OIDC_AUTHORIZATION_ENDPOINT",
|
||||
"OIDC_TOKEN_ENDPOINT",
|
||||
"OIDC_USERINFO_ENDPOINT",
|
||||
"OIDC_JWKS_URI",
|
||||
"EMAIL_HMAC_SECRET",
|
||||
"WHATSAPP_RATE_LIMIT_MAX",
|
||||
"WHATSAPP_RATE_LIMIT_WINDOW",
|
||||
"IMESSAGE_RATE_LIMIT_MAX",
|
||||
"IMESSAGE_RATE_LIMIT_WINDOW",
|
||||
):
|
||||
assert f"- {name}=${{{name}" in common
|
||||
assert "TAVILY_API_KEY=${TAVILY_API_KEY:-${TAVILIY_API_KEY:-}}" in common
|
||||
assert text.count("<<: *raggr-common") == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"canonical,legacy,expected",
|
||||
[("new-key", "old-key", "new-key"), ("", "old-key", "old-key")],
|
||||
)
|
||||
def test_compose_interpolation_without_secrets(canonical, legacy, expected):
|
||||
if not shutil.which("docker"):
|
||||
pytest.skip("Docker Compose is unavailable")
|
||||
env = {
|
||||
"PATH": os.environ["PATH"],
|
||||
"HOME": os.environ.get("HOME", "/tmp"),
|
||||
"TAVILY_API_KEY": canonical,
|
||||
"TAVILIY_API_KEY": legacy,
|
||||
"METRICS_TOKEN": "test-metrics",
|
||||
"TTFT_TIMEOUT_SECONDS": "42",
|
||||
"CHECKPOINT_DB_PATH": "/data/custom.db",
|
||||
}
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
"/dev/null",
|
||||
"--profile",
|
||||
"blue",
|
||||
"--profile",
|
||||
"green",
|
||||
"config",
|
||||
"--format",
|
||||
"json",
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode and "not a docker command" in result.stderr:
|
||||
pytest.skip("Docker Compose is unavailable")
|
||||
assert result.returncode == 0, result.stderr
|
||||
services = json.loads(result.stdout)["services"]
|
||||
for color in ("blue", "green"):
|
||||
actual = services[f"raggr-{color}"]["environment"]
|
||||
assert actual["TAVILY_API_KEY"] == expected
|
||||
assert actual["METRICS_TOKEN"] == "test-metrics"
|
||||
assert actual["TTFT_TIMEOUT_SECONDS"] == "42"
|
||||
assert actual["CONVERSATION_SUMMARY_DB_PATH"] == "/data/custom.db.summaries.db"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"validation_status,migration_status,expected", [(0, 19, 19), (23, 0, 23)]
|
||||
)
|
||||
def test_startup_stops_before_app_on_failure(
|
||||
tmp_path, validation_status, migration_status, expected
|
||||
):
|
||||
log = tmp_path / "commands"
|
||||
for name, body in {
|
||||
"python": f'echo "python $*" >> "$COMMAND_LOG"\nexit {validation_status}\n',
|
||||
"aerich": f'echo "aerich $*" >> "$COMMAND_LOG"\nexit {migration_status}\n',
|
||||
}.items():
|
||||
executable = tmp_path / name
|
||||
executable.write_text("#!/bin/sh\n" + body)
|
||||
executable.chmod(0o755)
|
||||
result = subprocess.run(
|
||||
["/bin/bash", str(ROOT / "startup.sh")],
|
||||
cwd=ROOT,
|
||||
env={"PATH": str(tmp_path), "COMMAND_LOG": str(log)},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == expected
|
||||
commands = log.read_text()
|
||||
assert "python scripts/validate_config.py" in commands
|
||||
assert "python app.py" not in commands
|
||||
assert ("aerich upgrade" in commands) == (validation_status == 0)
|
||||
|
||||
|
||||
def test_validation_cli_reports_errors_without_values(tmp_path, production_env):
|
||||
# A separate project tree avoids loading any real .env in the shared checkout.
|
||||
(tmp_path / "scripts").mkdir()
|
||||
(tmp_path / "config").mkdir()
|
||||
(tmp_path / "config" / "__init__.py").touch()
|
||||
shutil.copy(ROOT / "scripts" / "validate_config.py", tmp_path / "scripts")
|
||||
shutil.copy(ROOT / "config" / "validation.py", tmp_path / "config")
|
||||
command = [sys.executable, str(tmp_path / "scripts" / "validate_config.py")]
|
||||
env = {"PATH": os.environ["PATH"], **production_env}
|
||||
result = subprocess.run(command, env=env, capture_output=True, text=True)
|
||||
assert result.returncode == 0
|
||||
assert result.stdout == "Configuration valid.\n"
|
||||
env["JWT_SECRET_KEY"] = "private-invalid-value"
|
||||
result = subprocess.run(command, env=env, capture_output=True, text=True)
|
||||
assert result.returncode == 1
|
||||
assert "JWT_SECRET_KEY" in result.stderr
|
||||
assert env["JWT_SECRET_KEY"] not in result.stderr
|
||||
assert env["OPENAI_API_KEY"] not in result.stderr
|
||||
# dotenv values are validated too, but do not override exported settings.
|
||||
(tmp_path / ".env").write_text("JWT_SECRET_KEY=invalid-dotenv-value\n")
|
||||
env["JWT_SECRET_KEY"] = production_env["JWT_SECRET_KEY"]
|
||||
result = subprocess.run(command, env=env, capture_output=True, text=True)
|
||||
assert result.returncode == 0
|
||||
Reference in New Issue
Block a user