Add durable chat runs, shared agent runtime, reliable jobs, history tools and citations
Shared runtime - One agent entry point (runtime.invoke_agent_for_user / prepare_agent_turn) for web, email, iMessage, WhatsApp, scheduled and cron, so identity, permissions, memories, summaries and metrics match across channels. - Serialize turns per checkpoint thread; /query validates input and returns 409 while a durable run owns the conversation. Reconnectable chat (durable runs) - POST /runs creates an idempotent, owner-scoped run; GET .../events replays an ordered SSE log by cursor, so generation survives refresh and device switches. Stop cancels; disconnect only detaches. /stream-query is a compatibility adapter. - Coalesce streamed deltas (150 ms) and wake same-process subscribers instead of polling; keep buffered text when a run is stopped. - Drain runs on shutdown; serve via Hypercorn with a shutdown timeout covering the drain (and no longer in debug mode). CHAT_RUN_* settings are forwarded. Scheduled messages and cron - Atomic delivery claims, attempt history and explicit retries; ambiguous sends become "uncertain" and are never resent automatically. - Cron triggers need a run_id/Idempotency-Key; legacy jobs need an explicit owner. - Cancelling never rewrites a started delivery; series stop via tombstones. Legacy sent rows are not forked into new series. Malformed bodies return 400. History, integrations and retrieval - Message-content search, pinned conversations, JSON/Markdown export and a memory editor. - Integration status panel and YNAB data-freshness provenance. - Date-filtered document/note search with stable citations and a source viewer. Migration 20 is generated by aerich. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,137 @@
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage
|
||||
from quart import Quart
|
||||
|
||||
from blueprints.conversation import runtime
|
||||
from metrics import AGENT_REQUESTS_TOTAL
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"channel", ["web", "email", "imessage", "whatsapp", "scheduled", "cron"]
|
||||
)
|
||||
@pytest.mark.parametrize("admin", [False, True])
|
||||
async def test_channels_share_identity_memories_permissions_and_summary(
|
||||
monkeypatch, channel, admin
|
||||
):
|
||||
app = Quart(__name__)
|
||||
user = SimpleNamespace(id="owner", is_admin=lambda: admin)
|
||||
app.main_agent = SimpleNamespace(
|
||||
ainvoke=AsyncMock(return_value={"messages": [AIMessage(content="answer")]}),
|
||||
aget_state=AsyncMock(return_value=SimpleNamespace(values={"messages": []})),
|
||||
)
|
||||
app.admin_agent = SimpleNamespace(
|
||||
ainvoke=AsyncMock(
|
||||
return_value={"messages": [AIMessage(content="admin answer")]}
|
||||
),
|
||||
aget_state=AsyncMock(return_value=SimpleNamespace(values={"messages": []})),
|
||||
)
|
||||
summary = AIMessage(content="Saved constraints")
|
||||
app.conversation_summary_store = SimpleNamespace(
|
||||
update=AsyncMock(return_value=SimpleNamespace(as_message=lambda: summary))
|
||||
)
|
||||
prompt = AsyncMock(return_value="Remember the budget")
|
||||
monkeypatch.setattr(runtime, "build_system_prompt_with_memories", prompt)
|
||||
before = AGENT_REQUESTS_TOTAL.labels(channel=channel, result="success")._value.get()
|
||||
async with app.app_context():
|
||||
response = await runtime.invoke_agent_for_user(
|
||||
user, "What is my budget?", "thread", channel=channel
|
||||
)
|
||||
agent = app.admin_agent if admin else app.main_agent
|
||||
assert response["messages"][-1].content == ("admin answer" if admin else "answer")
|
||||
config = agent.ainvoke.call_args.kwargs["config"]["configurable"]
|
||||
assert config["user_id"] == "owner"
|
||||
assert config["is_admin"] is admin
|
||||
assert config["thread_id"] == "thread"
|
||||
assert config["channel"] == channel
|
||||
assert config["system_prompt"] == "Remember the budget"
|
||||
assert config["conversation_summary"] is summary
|
||||
assert config["request_id"]
|
||||
prompt.assert_awaited_once_with("owner", query="What is my budget?")
|
||||
assert (
|
||||
AGENT_REQUESTS_TOTAL.labels(channel=channel, result="success")._value.get()
|
||||
== before + 1
|
||||
)
|
||||
|
||||
|
||||
async def test_anonymous_invocation_is_rejected():
|
||||
with pytest.raises(ValueError, match="authenticated user"):
|
||||
await runtime.invoke_agent_for_user(None, "hello", "thread")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error, outcome",
|
||||
[(RuntimeError("failed"), "error"), (asyncio.CancelledError(), "cancelled")],
|
||||
)
|
||||
async def test_shared_invocation_measures_failed_and_cancelled_generation(
|
||||
error, outcome
|
||||
):
|
||||
agent = SimpleNamespace(ainvoke=AsyncMock(side_effect=error))
|
||||
turn = runtime.PreparedTurn(
|
||||
agent,
|
||||
{"messages": []},
|
||||
{
|
||||
"configurable": {
|
||||
"channel": "web",
|
||||
"request_id": "test",
|
||||
"thread_id": "t",
|
||||
"user_id": "u",
|
||||
}
|
||||
},
|
||||
)
|
||||
before = AGENT_REQUESTS_TOTAL.labels(channel="web", result=outcome)._value.get()
|
||||
with pytest.raises(type(error)):
|
||||
await runtime.invoke_prepared(turn)
|
||||
assert (
|
||||
AGENT_REQUESTS_TOTAL.labels(channel="web", result=outcome)._value.get()
|
||||
== before + 1
|
||||
)
|
||||
|
||||
|
||||
async def test_concurrent_turns_on_one_thread_keep_every_exchange(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
import aiosqlite
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
from langgraph.graph import END, START, MessagesState, StateGraph
|
||||
|
||||
monkeypatch.setattr(
|
||||
runtime, "build_system_prompt_with_memories", AsyncMock(return_value="sys")
|
||||
)
|
||||
async with aiosqlite.connect(tmp_path / "checkpoints.db") as conn:
|
||||
saver = AsyncSqliteSaver(conn)
|
||||
await saver.setup()
|
||||
|
||||
async def answer(state):
|
||||
# Yield so an unserialized second turn would read stale history.
|
||||
await asyncio.sleep(0.05)
|
||||
return {
|
||||
"messages": [
|
||||
AIMessage(content="reply " + state["messages"][-1].content)
|
||||
]
|
||||
}
|
||||
|
||||
graph = StateGraph(MessagesState)
|
||||
graph.add_node("answer", answer)
|
||||
graph.add_edge(START, "answer")
|
||||
graph.add_edge("answer", END)
|
||||
agent = graph.compile(checkpointer=saver)
|
||||
|
||||
app = Quart(__name__)
|
||||
app.main_agent = app.admin_agent = agent
|
||||
user = SimpleNamespace(id="owner", is_admin=lambda: False)
|
||||
async with app.app_context():
|
||||
await asyncio.gather(
|
||||
*[
|
||||
runtime.invoke_agent_for_user(user, q, "same-thread")
|
||||
for q in ("first", "second")
|
||||
]
|
||||
)
|
||||
state = await agent.aget_state(
|
||||
{"configurable": {"thread_id": "same-thread"}}
|
||||
)
|
||||
contents = [m.content for m in state.values["messages"]]
|
||||
assert sorted(contents) == ["first", "reply first", "reply second", "second"]
|
||||
@@ -23,6 +23,7 @@ async def db():
|
||||
"models": [
|
||||
"blueprints.users.models",
|
||||
"blueprints.conversation.models",
|
||||
"blueprints.conversation.run_models",
|
||||
"blueprints.artifacts.models",
|
||||
]
|
||||
},
|
||||
@@ -343,12 +344,12 @@ class TestStreamQuery:
|
||||
"data": {"output": {"messages": [AIMessage(content="Done")]}},
|
||||
},
|
||||
]
|
||||
monkeypatch.setattr(conv, "get_agent_for", lambda u: _FakeAgent(events))
|
||||
monkeypatch.setattr(conv.runtime, "get_agent_for", lambda u: _FakeAgent(events))
|
||||
|
||||
async def _prompt(user_id, query=""):
|
||||
return "system"
|
||||
|
||||
monkeypatch.setattr(conv, "build_system_prompt_with_memories", _prompt)
|
||||
monkeypatch.setattr(conv.runtime, "build_system_prompt_with_memories", _prompt)
|
||||
|
||||
app = Quart(__name__)
|
||||
app.config["JWT_SECRET_KEY"] = "test-secret"
|
||||
|
||||
@@ -0,0 +1,482 @@
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from quart import Quart
|
||||
from tortoise import Tortoise
|
||||
|
||||
from blueprints.conversation import runs
|
||||
from blueprints.conversation.models import Conversation, ConversationMessage
|
||||
from blueprints.conversation.run_models import ChatRun, ChatRunEvent
|
||||
from blueprints.users.models import User
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db(tmp_path):
|
||||
await Tortoise.init(
|
||||
db_url=f"sqlite://{tmp_path / 'runs.sqlite'}",
|
||||
modules={
|
||||
"models": [
|
||||
"blueprints.conversation.models",
|
||||
"blueprints.conversation.run_models",
|
||||
"blueprints.users.models",
|
||||
]
|
||||
},
|
||||
)
|
||||
await Tortoise.generate_schemas()
|
||||
app = Quart(__name__)
|
||||
app.config["RUN_TEST_DB_URL"] = f"sqlite://{tmp_path / 'runs.sqlite'}"
|
||||
user = await User.create(username="owner", email="owner@test")
|
||||
other = await User.create(username="other", email="other@test")
|
||||
conv = await Conversation.create(name="Chat", user=user)
|
||||
yield app, user, other, conv
|
||||
await runs.drain_runs(app, 0)
|
||||
runs._draining_apps.discard(id(app))
|
||||
await Tortoise.close_connections()
|
||||
|
||||
|
||||
async def reserved(user, conv, **kwargs):
|
||||
return await ChatRun.create(
|
||||
owner=user,
|
||||
conversation=conv,
|
||||
request_key=str(uuid.uuid4()),
|
||||
payload={"query": "hello"},
|
||||
active_key=str(conv.id),
|
||||
worker_token=uuid.uuid4(),
|
||||
lease_until=runs.now() + timedelta(seconds=30),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def wait_terminal(run):
|
||||
for _ in range(200):
|
||||
run = await ChatRun.get(id=run.id)
|
||||
if run.status != "running":
|
||||
return run
|
||||
await asyncio.sleep(0.01)
|
||||
pytest.fail("run did not terminate")
|
||||
|
||||
|
||||
async def test_order_cursor_isolation_and_partial_persisted_once(db):
|
||||
app, user, other, conv = db
|
||||
run = await reserved(user, conv)
|
||||
await runs.append(run.id, run.worker_token, {"type": "content", "delta": "one"})
|
||||
await runs.append(run.id, run.worker_token, {"type": "content", "delta": "two"})
|
||||
await runs.finish(run.id, "completed")
|
||||
await runs.finish(run.id, "cancelled")
|
||||
events = await ChatRunEvent.filter(run=run).order_by("sequence")
|
||||
assert [e.sequence for e in events] == [1, 2, 3]
|
||||
assert (
|
||||
await ConversationMessage.filter(conversation=conv, speaker="simba").count()
|
||||
== 1
|
||||
)
|
||||
assert (await ConversationMessage.get(conversation=conv)).text == "onetwo"
|
||||
assert await runs.owned_run(other.id, conv.id, run.id) is None
|
||||
foreign = await Conversation.create(name="other", user=user)
|
||||
assert await runs.owned_run(user.id, foreign.id, run.id) is None
|
||||
async with app.app_context():
|
||||
response = await runs.replay_response(run, 1)
|
||||
text = (await response.get_data()).decode()
|
||||
assert "id: 2" in text and "id: 1" not in text
|
||||
assert text.index("id: 2") < text.index("[DONE]")
|
||||
|
||||
|
||||
async def test_atomic_idempotency_and_concurrent_slot(db, monkeypatch):
|
||||
app, user, other, conv = db
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def fake(*args):
|
||||
await gate.wait()
|
||||
|
||||
monkeypatch.setattr(runs, "_work", fake)
|
||||
data = {"conversation_id": str(conv.id), "query": "hi", "request_id": "same"}
|
||||
results = await asyncio.gather(
|
||||
*(runs.create_run(app, user.id, data) for _ in range(4))
|
||||
)
|
||||
assert len({str(run.id) for run, _ in results}) == 1
|
||||
assert sum(created for _, created in results) == 1
|
||||
assert await ChatRun.all().count() == 1
|
||||
with pytest.raises(runs.RunConflict, match="different input"):
|
||||
await runs.create_run(app, user.id, {**data, "query": "changed"})
|
||||
with pytest.raises(runs.RunConflict, match="already active"):
|
||||
await runs.create_run(app, user.id, {**data, "request_id": "another"})
|
||||
with pytest.raises(runs.RunConflict) as err:
|
||||
await runs.create_run(app, other.id, data)
|
||||
assert err.value.status == 404
|
||||
gate.set()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
async def test_expired_restart_never_regenerates_and_releases_slot(db):
|
||||
app, user, _, conv = db
|
||||
run = await reserved(user, conv)
|
||||
await runs.append(run.id, run.worker_token, {"type": "content", "delta": "partial"})
|
||||
await ChatRun.filter(id=run.id).update(
|
||||
lease_until=runs.now() - timedelta(seconds=1)
|
||||
)
|
||||
# Close/reopen the real file database, discarding process-local ORM state.
|
||||
await Tortoise.close_connections()
|
||||
await Tortoise.init(
|
||||
db_url=app.config["RUN_TEST_DB_URL"],
|
||||
modules={
|
||||
"models": [
|
||||
"blueprints.conversation.models",
|
||||
"blueprints.conversation.run_models",
|
||||
"blueprints.users.models",
|
||||
]
|
||||
},
|
||||
)
|
||||
await runs.expire_runs(str(conv.id))
|
||||
await runs.expire_runs(str(conv.id))
|
||||
latest = await ChatRun.get(id=run.id)
|
||||
assert latest.status == "interrupted" and latest.active_key is None
|
||||
assert await ConversationMessage.filter(conversation=conv).count() == 1
|
||||
assert (await ChatRunEvent.filter(run=run).order_by("sequence"))[1].payload[
|
||||
"type"
|
||||
] == "error"
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await runs.append(
|
||||
run.id, run.worker_token, {"type": "content", "delta": "zombie"}
|
||||
)
|
||||
|
||||
|
||||
async def test_background_context_detach_and_cross_process_cancel(db, monkeypatch):
|
||||
app, user, _, conv = db
|
||||
gate = asyncio.Event()
|
||||
started = asyncio.Event()
|
||||
from quart import current_app
|
||||
import blueprints.conversation as conversation
|
||||
|
||||
async def fake(data, user, conv, save_user, save_answer):
|
||||
assert current_app._get_current_object() is app
|
||||
await save_user("hi", None)
|
||||
yield 'data: {"type":"content","delta":"partial"}\n\n'
|
||||
started.set()
|
||||
await gate.wait()
|
||||
await save_answer("complete")
|
||||
yield 'data: {"type":"response","message":"complete"}\n\n'
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
monkeypatch.setattr(conversation, "_chat_run_stream", fake)
|
||||
run, _ = await runs.create_run(
|
||||
app,
|
||||
user.id,
|
||||
{"conversation_id": str(conv.id), "query": "hi", "request_id": "turn"},
|
||||
)
|
||||
await started.wait()
|
||||
async with app.app_context():
|
||||
response = await runs.replay_response(run, 0)
|
||||
subscriber = response.response.__aiter__()
|
||||
frame = ""
|
||||
while "partial" not in frame: # skip keepalives until the delta is written
|
||||
frame = await asyncio.wait_for(subscriber.__anext__(), 2)
|
||||
await subscriber.aclose() # browser disconnect
|
||||
assert (await ChatRun.get(id=run.id)).status == "running"
|
||||
# Same write made by another process, no local task cancellation API involved.
|
||||
await ChatRun.filter(id=run.id).update(cancel_requested=True)
|
||||
latest = await wait_terminal(run)
|
||||
assert latest.status == "cancelled"
|
||||
assert (
|
||||
await ConversationMessage.filter(conversation=conv, speaker="user").count() == 1
|
||||
)
|
||||
assert (
|
||||
await ConversationMessage.filter(conversation=conv, speaker="simba").count()
|
||||
== 1
|
||||
)
|
||||
assert (
|
||||
await ConversationMessage.get(conversation=conv, speaker="simba")
|
||||
).text == "partial"
|
||||
|
||||
|
||||
async def test_completed_answer_once_and_drain(db, monkeypatch):
|
||||
app, user, _, conv = db
|
||||
import blueprints.conversation as conversation
|
||||
|
||||
async def fake(data, user, conv, save_user, save_answer):
|
||||
await save_user("hi", None)
|
||||
yield 'data: {"type":"content","delta":"answer"}\n\n'
|
||||
await save_answer("answer")
|
||||
yield 'data: {"type":"response","message":"answer"}\n\n'
|
||||
|
||||
monkeypatch.setattr(conversation, "_chat_run_stream", fake)
|
||||
run, _ = await runs.create_run(
|
||||
app,
|
||||
user.id,
|
||||
{"conversation_id": str(conv.id), "query": "hi", "request_id": "turn"},
|
||||
)
|
||||
await runs.drain_runs(app, 2)
|
||||
latest = await ChatRun.get(id=run.id)
|
||||
assert latest.status == "completed"
|
||||
assert (
|
||||
await ConversationMessage.filter(conversation=conv, speaker="simba").count()
|
||||
== 1
|
||||
)
|
||||
with pytest.raises(runs.RunConflict) as err:
|
||||
await runs.create_run(
|
||||
app, user.id, {"conversation_id": str(conv.id), "query": "new"}
|
||||
)
|
||||
assert err.value.status == 503
|
||||
|
||||
|
||||
async def test_authenticated_routes_validate_owner_conversation_cursor_and_cancel(db):
|
||||
from quart_jwt_extended import JWTManager, create_access_token
|
||||
|
||||
app, owner, foreign, conv = db
|
||||
app.config.update(
|
||||
TESTING=True, JWT_SECRET_KEY="chat-runs-test-secret-at-least-32-characters"
|
||||
)
|
||||
JWTManager(app)
|
||||
app.register_blueprint(runs.runs_blueprint)
|
||||
async with app.app_context():
|
||||
headers = {
|
||||
"Authorization": "Bearer " + create_access_token(identity=str(owner.id))
|
||||
}
|
||||
other_headers = {
|
||||
"Authorization": "Bearer " + create_access_token(identity=str(foreign.id))
|
||||
}
|
||||
client = app.test_client()
|
||||
run = await reserved(owner, conv)
|
||||
base = f"/api/conversation/{conv.id}/runs/{run.id}"
|
||||
assert (await client.get(base + "/events")).status_code == 401
|
||||
assert (
|
||||
await client.get(base + "/events", headers=other_headers)
|
||||
).status_code == 404
|
||||
assert (
|
||||
await client.post(base + "/cancel", headers=other_headers)
|
||||
).status_code == 404
|
||||
assert not (await ChatRun.get(id=run.id)).cancel_requested
|
||||
for cursor in ("-1", "bad", "999"):
|
||||
assert (
|
||||
await client.get(base + f"/events?cursor={cursor}", headers=headers)
|
||||
).status_code == 400
|
||||
assert (await client.post(base + "/cancel", headers=headers)).status_code == 200
|
||||
assert (await ChatRun.get(id=run.id)).cancel_requested
|
||||
assert (
|
||||
await client.get(f"/api/conversation/{conv.id}/runs", headers=other_headers)
|
||||
).status_code == 404
|
||||
discovery = await (
|
||||
await client.get(f"/api/conversation/{conv.id}/runs", headers=headers)
|
||||
).get_json()
|
||||
assert discovery["run"]["id"] == str(run.id)
|
||||
assert (
|
||||
await client.post(
|
||||
"/api/conversation/runs",
|
||||
headers=other_headers,
|
||||
json={"conversation_id": str(conv.id), "query": "hi"},
|
||||
)
|
||||
).status_code == 404
|
||||
assert (
|
||||
await client.post("/api/conversation/runs", headers=headers, json=[])
|
||||
).status_code == 400
|
||||
|
||||
|
||||
async def test_real_shared_runtime_legacy_endpoint_persists_and_replays(
|
||||
db, monkeypatch
|
||||
):
|
||||
from quart_jwt_extended import JWTManager, create_access_token
|
||||
from langchain_core.messages import AIMessageChunk, AIMessage
|
||||
import blueprints.conversation as conversation
|
||||
|
||||
app, owner, _, conv = db
|
||||
|
||||
class Agent:
|
||||
async def astream_events(self, payload, version, config):
|
||||
assert config["configurable"]["user_id"] == str(owner.id)
|
||||
yield {"event": "on_chat_model_start", "run_id": "llm", "name": "model"}
|
||||
yield {
|
||||
"event": "on_chat_model_stream",
|
||||
"run_id": "llm",
|
||||
"data": {"chunk": AIMessageChunk(content="answer")},
|
||||
}
|
||||
yield {
|
||||
"event": "on_chat_model_end",
|
||||
"run_id": "llm",
|
||||
"data": {"output": AIMessage(content="answer")},
|
||||
}
|
||||
yield {
|
||||
"event": "on_chain_end",
|
||||
"data": {"output": {"messages": [AIMessage(content="answer")]}},
|
||||
}
|
||||
|
||||
app.main_agent = Agent()
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
monkeypatch.setattr(
|
||||
conversation.runtime,
|
||||
"build_system_prompt_with_memories",
|
||||
AsyncMock(return_value="system"),
|
||||
)
|
||||
app.config.update(
|
||||
TESTING=True, JWT_SECRET_KEY="chat-runs-test-secret-at-least-32-characters"
|
||||
)
|
||||
JWTManager(app)
|
||||
app.register_blueprint(conversation.conversation_blueprint)
|
||||
app.register_blueprint(runs.runs_blueprint)
|
||||
async with app.app_context():
|
||||
headers = {
|
||||
"Authorization": "Bearer " + create_access_token(identity=str(owner.id))
|
||||
}
|
||||
response = await app.test_client().post(
|
||||
"/api/conversation/stream-query",
|
||||
headers=headers,
|
||||
json={"conversation_id": str(conv.id), "query": "hi", "request_id": "legacy"},
|
||||
)
|
||||
body = (await response.get_data()).decode()
|
||||
assert (
|
||||
response.status_code == 200
|
||||
and '"type": "response"' in body
|
||||
and "[DONE]" in body
|
||||
)
|
||||
run = await ChatRun.get(request_key="legacy")
|
||||
assert run.status == "completed"
|
||||
assert (
|
||||
await ConversationMessage.filter(conversation=conv, speaker="simba").count()
|
||||
== 1
|
||||
)
|
||||
replay = await app.test_client().get(
|
||||
f"/api/conversation/{conv.id}/runs/{run.id}/events?cursor=0", headers=headers
|
||||
)
|
||||
assert (await replay.get_data()).decode() == body.replace(": keepalive\n\n", "")
|
||||
|
||||
|
||||
async def test_nonstream_query_validates_input_and_respects_active_run(db):
|
||||
from quart_jwt_extended import JWTManager, create_access_token
|
||||
|
||||
from blueprints.conversation import conversation_blueprint
|
||||
|
||||
app, owner, _, conv = db
|
||||
app.config.update(
|
||||
TESTING=True, JWT_SECRET_KEY="chat-runs-test-secret-at-least-32-characters"
|
||||
)
|
||||
JWTManager(app)
|
||||
app.register_blueprint(conversation_blueprint)
|
||||
async with app.app_context():
|
||||
headers = {
|
||||
"Authorization": "Bearer " + create_access_token(identity=str(owner.id))
|
||||
}
|
||||
client = app.test_client()
|
||||
for body in (
|
||||
["not", "an", "object"],
|
||||
{"conversation_id": str(conv.id), "query": 123},
|
||||
{"conversation_id": str(conv.id), "query": " "},
|
||||
):
|
||||
resp = await client.post("/api/conversation/query", headers=headers, json=body)
|
||||
assert resp.status_code == 400, body
|
||||
resp = await client.post(
|
||||
"/api/conversation/query",
|
||||
headers=headers,
|
||||
json={"conversation_id": ["x"], "query": "hi"},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
# A durable run owns the checkpoint thread: reject before saving anything.
|
||||
await reserved(owner, conv)
|
||||
resp = await client.post(
|
||||
"/api/conversation/query",
|
||||
headers=headers,
|
||||
json={"conversation_id": str(conv.id), "query": "hi"},
|
||||
)
|
||||
assert resp.status_code == 409
|
||||
assert await ConversationMessage.filter(conversation=conv).count() == 0
|
||||
|
||||
|
||||
async def test_deltas_coalesce_and_local_subscribers_wake_immediately(db, monkeypatch):
|
||||
app, user, _, conv = db
|
||||
import blueprints.conversation as conversation
|
||||
|
||||
release = asyncio.Event()
|
||||
|
||||
async def fake(data, user, conv, save_user, save_answer):
|
||||
await save_user("hi", None)
|
||||
for word in ("one ", "two ", "three"):
|
||||
yield f'data: {{"type":"content","delta":"{word}"}}\n\n'
|
||||
yield 'data: {"type":"tool_start","tool":"t"}\n\n'
|
||||
await release.wait()
|
||||
await save_answer("one two three")
|
||||
yield 'data: {"type":"response","message":"one two three"}\n\n'
|
||||
|
||||
monkeypatch.setattr(conversation, "_chat_run_stream", fake)
|
||||
monkeypatch.setattr(runs, "POLL_SECONDS", 30) # only a wakeup can be fast
|
||||
run, _ = await runs.create_run(
|
||||
app, user.id, {"conversation_id": str(conv.id), "query": "hi"}
|
||||
)
|
||||
async with app.app_context():
|
||||
response = await runs.replay_response(run, 0)
|
||||
frames = response.response.__aiter__()
|
||||
seen = []
|
||||
while not any('"tool_start"' in f for f in seen):
|
||||
seen.append(await asyncio.wait_for(frames.__anext__(), 2))
|
||||
release.set()
|
||||
while "[DONE]" not in seen[-1]:
|
||||
seen.append(await asyncio.wait_for(frames.__anext__(), 2))
|
||||
deltas = [
|
||||
e.payload["delta"]
|
||||
for e in await ChatRunEvent.filter(run=run).order_by("sequence")
|
||||
if e.payload["type"] == "content"
|
||||
]
|
||||
# First delta is written immediately; the burst behind it shares one write.
|
||||
assert "".join(deltas) == "one two three"
|
||||
assert len(deltas) == 2
|
||||
await wait_terminal(run)
|
||||
|
||||
|
||||
async def _content_deltas(run):
|
||||
return [
|
||||
e.payload["delta"]
|
||||
for e in await ChatRunEvent.filter(run=run).order_by("sequence")
|
||||
if e.payload["type"] == "content"
|
||||
]
|
||||
|
||||
|
||||
async def test_buffered_delta_flushes_while_model_is_silent(db, monkeypatch):
|
||||
app, user, _, conv = db
|
||||
import blueprints.conversation as conversation
|
||||
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def fake(data, user, conv, save_user, save_answer):
|
||||
await save_user("hi", None)
|
||||
yield 'data: {"type":"content","delta":"a"}\n\n'
|
||||
yield 'data: {"type":"content","delta":"b"}\n\n'
|
||||
await gate.wait() # e.g. the model is writing tool-call arguments
|
||||
yield 'data: {"type":"response","message":"ab"}\n\n'
|
||||
|
||||
monkeypatch.setattr(conversation, "_chat_run_stream", fake)
|
||||
monkeypatch.setattr(runs, "FLUSH_SECONDS", 0.05)
|
||||
run, _ = await runs.create_run(
|
||||
app, user.id, {"conversation_id": str(conv.id), "query": "hi"}
|
||||
)
|
||||
for _ in range(100):
|
||||
if "".join(await _content_deltas(run)) == "ab":
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
assert "".join(await _content_deltas(run)) == "ab"
|
||||
gate.set()
|
||||
await wait_terminal(run)
|
||||
|
||||
|
||||
async def test_stop_keeps_buffered_text_in_saved_answer(db, monkeypatch):
|
||||
app, user, _, conv = db
|
||||
import blueprints.conversation as conversation
|
||||
|
||||
buffered = asyncio.Event()
|
||||
|
||||
async def fake(data, user, conv, save_user, save_answer):
|
||||
await save_user("hi", None)
|
||||
yield 'data: {"type":"content","delta":"first "}\n\n'
|
||||
yield 'data: {"type":"content","delta":"tail"}\n\n'
|
||||
buffered.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
monkeypatch.setattr(conversation, "_chat_run_stream", fake)
|
||||
monkeypatch.setattr(runs, "FLUSH_SECONDS", 60) # "tail" stays buffered
|
||||
run, _ = await runs.create_run(
|
||||
app, user.id, {"conversation_id": str(conv.id), "query": "hi"}
|
||||
)
|
||||
await buffered.wait()
|
||||
await ChatRun.filter(id=run.id).update(cancel_requested=True)
|
||||
assert (await wait_terminal(run)).status == "cancelled"
|
||||
answer = await ConversationMessage.get(conversation=conv, speaker="simba")
|
||||
assert answer.text == "first tail"
|
||||
@@ -263,10 +263,14 @@ class TestUploadRoute:
|
||||
|
||||
class TestStreamQueryFileKey:
|
||||
async def test_rejects_file_key_from_other_conversation(self, app, headers, s3):
|
||||
from blueprints.conversation import runs
|
||||
|
||||
add = AsyncMock()
|
||||
owned = AsyncMock(return_value=SimpleNamespace(id=CONV_ID))
|
||||
with (
|
||||
patch.object(conv.blueprints.users.models.User, "get", AsyncMock()),
|
||||
patch.object(conv, "add_message_to_conversation", add),
|
||||
patch.object(runs.Conversation, "get_or_none", owned),
|
||||
patch.object(runs.ChatRun, "create", AsyncMock()) as create,
|
||||
patch.object(runs, "add_message_to_conversation", add),
|
||||
):
|
||||
resp = await app.test_client().post(
|
||||
"/api/conversation/stream-query",
|
||||
@@ -278,6 +282,7 @@ class TestStreamQueryFileKey:
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
create.assert_not_awaited()
|
||||
add.assert_not_awaited()
|
||||
|
||||
async def test_load_attached_file_recovers_name_and_text(self):
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Real SQLite tests: ownership, joined pagination, exports and malformed requests."""
|
||||
|
||||
import base64
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import zlib
|
||||
|
||||
import pytest
|
||||
from quart import Quart
|
||||
from quart_jwt_extended import JWTManager, create_access_token
|
||||
from tortoise import Tortoise
|
||||
|
||||
from blueprints.conversation.history import history_blueprint, list_owned_conversations
|
||||
from blueprints.conversation.models import (
|
||||
Conversation,
|
||||
ConversationMessage,
|
||||
Speaker,
|
||||
UserMemory,
|
||||
)
|
||||
from blueprints.users.models import User
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def history_app():
|
||||
await Tortoise.init(
|
||||
db_url="sqlite://:memory:",
|
||||
modules={
|
||||
"models": ["blueprints.users.models", "blueprints.conversation.models"]
|
||||
},
|
||||
)
|
||||
await Tortoise.generate_schemas()
|
||||
app = Quart(__name__)
|
||||
app.config.update(
|
||||
TESTING=True,
|
||||
JWT_SECRET_KEY="history-test-secret-at-least-thirty-two-characters",
|
||||
)
|
||||
JWTManager(app)
|
||||
app.register_blueprint(history_blueprint)
|
||||
owner = await User.create(username="owner", email="owner@example.com")
|
||||
foreign = await User.create(username="foreign", email="foreign@example.com")
|
||||
async with app.app_context():
|
||||
headers = {
|
||||
"Authorization": f"Bearer {create_access_token(identity=str(owner.id))}"
|
||||
}
|
||||
try:
|
||||
yield app.test_client(), headers, owner, foreign
|
||||
finally:
|
||||
await Tortoise.close_connections()
|
||||
|
||||
|
||||
async def message(conversation, text, speaker=Speaker.USER, **kwargs):
|
||||
return await ConversationMessage.create(
|
||||
conversation=conversation, text=text, speaker=speaker, **kwargs
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_unique_pagination_and_owner_isolation(history_app):
|
||||
client, headers, owner, foreign = history_app
|
||||
a = await Conversation.create(user=owner, name="ordinary")
|
||||
b = await Conversation.create(user=owner, name="needle title", pinned=True)
|
||||
secret = await Conversation.create(user=foreign, name="needle foreign", pinned=True)
|
||||
await Conversation.create(name="needle unowned")
|
||||
for _ in range(5):
|
||||
await message(a, "many needle message matches")
|
||||
await message(secret, "needle secret")
|
||||
first = await list_owned_conversations(owner.id, "needle", 1, 0)
|
||||
second = await list_owned_conversations(owner.id, "needle", 1, 1)
|
||||
assert [first[0]["id"], second[0]["id"]] == [str(b.id), str(a.id)]
|
||||
assert await list_owned_conversations(owner.id, "needle", 1, 2) == []
|
||||
assert "needle" in second[0]["snippet"]
|
||||
assert second[0]["match_message_id"]
|
||||
response = await client.get(
|
||||
"/api/history/conversations?search=NEEDLE&limit=10", headers=headers
|
||||
)
|
||||
assert {row["id"] for row in await response.get_json()} == {str(a.id), str(b.id)}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pin_owner_only_boolean_and_recency(history_app):
|
||||
client, headers, owner, foreign = history_app
|
||||
own = await Conversation.create(user=owner, name="own")
|
||||
other = await Conversation.create(user=foreign, name="other")
|
||||
for identifier in (other.id, "malformed"):
|
||||
response = await client.patch(
|
||||
f"/api/history/conversations/{identifier}/pin",
|
||||
headers=headers,
|
||||
json={"pinned": True},
|
||||
)
|
||||
assert response.status_code == 404
|
||||
for body in ({}, {"pinned": "false"}, {"pinned": 1}, [], None):
|
||||
response = await client.patch(
|
||||
f"/api/history/conversations/{own.id}/pin", headers=headers, json=body
|
||||
)
|
||||
assert response.status_code == 400
|
||||
original = own.updated_at
|
||||
for pinned in (True, False):
|
||||
response = await client.patch(
|
||||
f"/api/history/conversations/{own.id}/pin",
|
||||
headers=headers,
|
||||
json={"pinned": pinned},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
await own.refresh_from_db()
|
||||
assert own.pinned is pinned
|
||||
assert own.updated_at == original
|
||||
await other.refresh_from_db()
|
||||
assert not other.pinned
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_export_display_and_attachment_provenance(history_app):
|
||||
client, headers, owner, foreign = history_app
|
||||
own = await Conversation.create(user=owner, name='quoted "title"')
|
||||
other = await Conversation.create(user=foreign, name="FOREIGN SECRET")
|
||||
await message(other, "FOREIGN CONTENT")
|
||||
await message(own, "hello\n\n📎 notes.md", image_key="foreign/private/secret.png")
|
||||
await message(own, "[web_search result]\nHIDDEN RAW TOOL CONTEXT", Speaker.TOOL)
|
||||
await message(own, "answer", Speaker.SIMBA)
|
||||
await message(own, "malformed tool payload\nHIDDEN MORE", Speaker.TOOL)
|
||||
response = await client.get(
|
||||
f"/api/history/conversations/{own.id}/export", headers=headers
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = await response.get_json()
|
||||
text = json.dumps(data)
|
||||
for secret in ("HIDDEN RAW", "HIDDEN MORE", "FOREIGN", "foreign/private"):
|
||||
assert secret not in text
|
||||
assert data["messages"][1]["text"] == "[web_search result]"
|
||||
assert data["messages"][1]["provenance"] == "displayed_tool_header"
|
||||
assert data["messages"][0]["attachments"] == [
|
||||
{"kind": "file", "recorded_name": "notes.md", "contents_included": False},
|
||||
{"kind": "image", "stored_reference_present": True, "contents_included": False},
|
||||
]
|
||||
assert "attachment; filename=" in response.headers["Content-Disposition"]
|
||||
assert response.headers["Cache-Control"] == "no-store"
|
||||
md = await client.get(
|
||||
f"/api/history/conversations/{own.id}/export?format=markdown", headers=headers
|
||||
)
|
||||
markdown = await md.get_data(as_text=True)
|
||||
assert md.status_code == 200 and "text/markdown" in md.content_type
|
||||
assert "📎 notes.md" in markdown and "Raw tool context" in markdown
|
||||
assert "HIDDEN" not in markdown
|
||||
for identifier in (other.id, "bad-uuid"):
|
||||
assert (
|
||||
await client.get(
|
||||
f"/api/history/conversations/{identifier}/export", headers=headers
|
||||
)
|
||||
).status_code == 404
|
||||
assert (
|
||||
await client.get(
|
||||
f"/api/history/conversations/{own.id}/export?format=html", headers=headers
|
||||
)
|
||||
).status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_owner_only_correction_forget_and_validation(history_app):
|
||||
client, headers, owner, foreign = history_app
|
||||
own = await UserMemory.create(user=owner, content="old fact")
|
||||
other = await UserMemory.create(user=foreign, content="FOREIGN MEMORY")
|
||||
rows = await (await client.get("/api/history/memories", headers=headers)).get_json()
|
||||
assert [row["id"] for row in rows] == [str(own.id)]
|
||||
for identifier in (other.id, "invalid"):
|
||||
for method in (client.patch, client.delete):
|
||||
assert (
|
||||
await method(
|
||||
f"/api/history/memories/{identifier}",
|
||||
headers=headers,
|
||||
json={"content": "stolen"},
|
||||
)
|
||||
).status_code == 404
|
||||
for body in (
|
||||
{},
|
||||
[],
|
||||
{"content": None},
|
||||
{"content": 3},
|
||||
{"content": " "},
|
||||
{"content": "x" * 10001},
|
||||
):
|
||||
assert (
|
||||
await client.patch(
|
||||
f"/api/history/memories/{own.id}", headers=headers, json=body
|
||||
)
|
||||
).status_code == 400
|
||||
response = await client.patch(
|
||||
f"/api/history/memories/{own.id}",
|
||||
headers=headers,
|
||||
json={"content": " corrected fact "},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert (await response.get_json())["content"] == "corrected fact"
|
||||
await own.refresh_from_db()
|
||||
assert own.content == "corrected fact"
|
||||
assert (
|
||||
await client.delete(f"/api/history/memories/{own.id}", headers=headers)
|
||||
).status_code == 204
|
||||
assert await UserMemory.get_or_none(id=own.id) is None
|
||||
await other.refresh_from_db()
|
||||
assert other.content == "FOREIGN MEMORY"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_pagination(history_app):
|
||||
client, headers, owner, _ = history_app
|
||||
for i in range(5):
|
||||
await UserMemory.create(user=owner, content=f"fact {i}")
|
||||
first = await (
|
||||
await client.get("/api/history/memories?limit=2&offset=0", headers=headers)
|
||||
).get_json()
|
||||
second = await (
|
||||
await client.get("/api/history/memories?limit=2&offset=2", headers=headers)
|
||||
).get_json()
|
||||
assert len(first) == len(second) == 2
|
||||
assert {r["id"] for r in first}.isdisjoint(r["id"] for r in second)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authentication_and_malformed_pagination(history_app):
|
||||
client, headers, owner, _ = history_app
|
||||
own = await Conversation.create(user=owner, name="own")
|
||||
memory = await UserMemory.create(user=owner, content="fact")
|
||||
for method, path in (
|
||||
(client.get, "/conversations"),
|
||||
(client.get, "/memories"),
|
||||
(client.get, f"/conversations/{own.id}/export"),
|
||||
(client.patch, f"/conversations/{own.id}/pin"),
|
||||
(client.patch, f"/memories/{memory.id}"),
|
||||
(client.delete, f"/memories/{memory.id}"),
|
||||
):
|
||||
assert (await method("/api/history" + path)).status_code == 401
|
||||
for path in ("conversations", "memories"):
|
||||
for query in ("limit=0", "limit=101", "offset=-1", "limit=oops", "offset=1.5"):
|
||||
assert (
|
||||
await client.get(f"/api/history/{path}?{query}", headers=headers)
|
||||
).status_code == 400
|
||||
assert (
|
||||
await client.get(
|
||||
"/api/history/conversations?search=" + "x" * 501, headers=headers
|
||||
)
|
||||
).status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_adds_pinned_and_records_state(history_app):
|
||||
migration_path = (
|
||||
Path(__file__).parents[2]
|
||||
/ "migrations/models/20_20261012231017_application_foundation.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("history_migration", migration_path)
|
||||
migration = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(migration)
|
||||
sql = await migration.upgrade(None)
|
||||
assert 'ALTER TABLE "conversations" ADD "pinned" BOOL NOT NULL DEFAULT False' in sql
|
||||
state = json.loads(zlib.decompress(base64.b64decode(migration.MODELS_STATE)))
|
||||
assert any(
|
||||
field["name"] == "pinned" and field["default"] is False
|
||||
for field in state["models.Conversation"]["data_fields"]
|
||||
)
|
||||
@@ -0,0 +1,111 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from quart import Quart
|
||||
from quart_jwt_extended import JWTManager, create_access_token
|
||||
from tortoise import Tortoise
|
||||
|
||||
from blueprints import integrations
|
||||
from blueprints.users.models import User
|
||||
from blueprints.ynab import tools
|
||||
from blueprints.ynab.sync import tool_data_freshness
|
||||
|
||||
|
||||
async def test_stale_ynab_status_hides_provider_error(monkeypatch):
|
||||
monkeypatch.setattr(integrations, "ynab_configured", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
integrations,
|
||||
"_current_state",
|
||||
AsyncMock(
|
||||
return_value=SimpleNamespace(
|
||||
last_synced_at=datetime.now(timezone.utc) - timedelta(hours=1),
|
||||
last_error="Authorization: secret-provider-key",
|
||||
)
|
||||
),
|
||||
)
|
||||
result = await integrations._ynab_status()
|
||||
assert result["status"] == "stale"
|
||||
assert result["last_success_at"]
|
||||
assert "secret-provider-key" not in str(result)
|
||||
|
||||
|
||||
async def test_fresh_ynab_status_is_ready(monkeypatch):
|
||||
monkeypatch.setattr(integrations, "ynab_configured", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
integrations,
|
||||
"_current_state",
|
||||
AsyncMock(
|
||||
return_value=SimpleNamespace(
|
||||
last_synced_at=datetime.now(timezone.utc), last_error=None
|
||||
)
|
||||
),
|
||||
)
|
||||
assert (await integrations._ynab_status())["status"] == "ready"
|
||||
|
||||
|
||||
async def test_stale_tool_data_warns_model_and_carries_artifact_provenance():
|
||||
@tools.ynab_tool_errors
|
||||
async def result():
|
||||
tool_data_freshness.set(
|
||||
{"system": "ynab", "as_of": "2026-10-01T00:00:00+00:00", "stale": True}
|
||||
)
|
||||
return "Total: $100", {"kind": "ynab.summary", "data": {"total": 100}}
|
||||
|
||||
text, artifact = await result()
|
||||
assert "cached budget data is stale" in text
|
||||
assert "2026-10-01" in text
|
||||
assert artifact["source_ref"]["stale"] is True
|
||||
assert artifact["data"] == {"total": 100}
|
||||
|
||||
|
||||
async def test_tool_freshness_does_not_leak_from_previous_call():
|
||||
tool_data_freshness.set({"system": "ynab", "as_of": "old", "stale": True})
|
||||
|
||||
@tools.ynab_tool_errors
|
||||
async def result():
|
||||
return "Dates must be YYYY-MM-DD."
|
||||
|
||||
assert await result() == "Dates must be YYYY-MM-DD."
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def status_app(monkeypatch):
|
||||
await Tortoise.init(
|
||||
db_url="sqlite://:memory:", modules={"models": ["blueprints.users.models"]}
|
||||
)
|
||||
await Tortoise.generate_schemas()
|
||||
user = await User.create(username="owner", email="owner@example.com")
|
||||
app = Quart(__name__)
|
||||
app.config["JWT_SECRET_KEY"] = "test-secret"
|
||||
JWTManager(app)
|
||||
app.register_blueprint(integrations.integrations_blueprint)
|
||||
monkeypatch.setattr(
|
||||
integrations,
|
||||
"integration_statuses",
|
||||
AsyncMock(
|
||||
return_value=[
|
||||
{
|
||||
"name": "YNAB",
|
||||
"status": "stale",
|
||||
"message": "Cached data",
|
||||
"last_success_at": None,
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
async with app.app_context():
|
||||
token = create_access_token(identity=str(user.id))
|
||||
yield app, token
|
||||
await Tortoise.close_connections()
|
||||
|
||||
|
||||
async def test_status_requires_access_token(status_app):
|
||||
app, token = status_app
|
||||
assert (await app.test_client().get("/api/integrations/")).status_code == 401
|
||||
response = await app.test_client().get(
|
||||
"/api/integrations/", headers={"Authorization": f"Bearer {token}"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert (await response.get_json())["integrations"][0]["status"] == "stale"
|
||||
@@ -0,0 +1,690 @@
|
||||
"""Real SQLite ORM delivery claims; providers/runtime are replaced locally."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from quart import Quart
|
||||
from tortoise import Tortoise
|
||||
|
||||
from blueprints.cron.models import CronJob
|
||||
from blueprints.cron.scheduler import execute_job
|
||||
from blueprints.scheduled_messages import delivery
|
||||
from blueprints.scheduled_messages.models import (
|
||||
JobAttempt,
|
||||
JobDelivery,
|
||||
MessageChannel,
|
||||
MessageStatus,
|
||||
Recurrence,
|
||||
ScheduledMessage,
|
||||
)
|
||||
from blueprints.scheduled_messages.scheduler import (
|
||||
_schedule_next_occurrence,
|
||||
process_due_messages,
|
||||
process_message,
|
||||
run_id,
|
||||
)
|
||||
from blueprints.users.models import User
|
||||
|
||||
MODULES = [
|
||||
"blueprints.users.models",
|
||||
"blueprints.scheduled_messages.models",
|
||||
"blueprints.cron.models",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db(tmp_path):
|
||||
path = tmp_path / "jobs.sqlite3"
|
||||
await Tortoise.init(db_url=f"sqlite://{path}", modules={"models": MODULES})
|
||||
await Tortoise.generate_schemas()
|
||||
yield path
|
||||
await Tortoise.close_connections()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def owner(db):
|
||||
return await User.create(username="owner", email="owner@example.com")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def message(owner):
|
||||
return await ScheduledMessage.create(
|
||||
recipient="recipient",
|
||||
channel=MessageChannel.EMAIL,
|
||||
subject="subject",
|
||||
content="prompt",
|
||||
scheduled_at=delivery.now() - timedelta(minutes=1),
|
||||
created_by=owner,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def job(owner):
|
||||
return await CronJob.create(
|
||||
name="cron",
|
||||
prompt="prompt",
|
||||
recipient="recipient",
|
||||
channel=MessageChannel.EMAIL,
|
||||
created_by=owner,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def providers(monkeypatch):
|
||||
send = AsyncMock()
|
||||
agent = AsyncMock(return_value="response")
|
||||
monkeypatch.setattr(delivery, "send", send)
|
||||
monkeypatch.setattr(delivery, "agent_content", agent)
|
||||
return send, agent
|
||||
|
||||
|
||||
async def test_atomic_concurrent_scheduled_claims(message, providers):
|
||||
results = await asyncio.gather(*[process_message(message) for _ in range(8)])
|
||||
assert results.count("sent") == 1
|
||||
assert providers[0].await_count == 1
|
||||
assert await JobAttempt.all().count() == 1
|
||||
await message.refresh_from_db()
|
||||
assert message.status == MessageStatus.SENT
|
||||
|
||||
|
||||
async def test_claims_across_independent_processes(db):
|
||||
# Both processes use their own SQLite connection and ORM registry.
|
||||
code = """
|
||||
import asyncio, sys
|
||||
from tortoise import Tortoise
|
||||
from blueprints.scheduled_messages.delivery import claim
|
||||
async def main():
|
||||
await Tortoise.init(db_url="sqlite://" + sys.argv[1], modules={"models": ["blueprints.users.models", "blueprints.scheduled_messages.models", "blueprints.cron.models"]})
|
||||
result = await claim("cron", "same-job", "same-run")
|
||||
print("won" if result else "lost")
|
||||
await Tortoise.close_connections()
|
||||
asyncio.run(main())
|
||||
"""
|
||||
|
||||
async def worker():
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-c",
|
||||
code,
|
||||
str(db),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=os.environ.copy(),
|
||||
)
|
||||
out, err = await p.communicate()
|
||||
assert p.returncode == 0, err.decode()
|
||||
return out.decode().strip()
|
||||
|
||||
assert sorted(await asyncio.gather(worker(), worker())) == ["lost", "won"]
|
||||
assert await JobAttempt.all().count() == 1
|
||||
|
||||
|
||||
async def test_safe_send_failure_explicit_retry(message, providers):
|
||||
providers[0].side_effect = delivery.SafeDeliveryError("rejected before send")
|
||||
assert await process_message(message) == "failed"
|
||||
assert (await process_due_messages())["sent"] == 0
|
||||
providers[0].side_effect = None
|
||||
await message.refresh_from_db()
|
||||
assert await process_message(message, retry=True) == "sent"
|
||||
assert [a.status for a in await JobAttempt.all().order_by("started_at")] == [
|
||||
"failed",
|
||||
"sent",
|
||||
]
|
||||
assert providers[0].await_count == 2
|
||||
|
||||
|
||||
async def test_ambiguous_provider_failure_never_resends(message, providers):
|
||||
providers[0].side_effect = TimeoutError("may have delivered")
|
||||
assert await process_message(message) == "uncertain"
|
||||
await message.refresh_from_db()
|
||||
assert await process_message(message, retry=True) == "skipped"
|
||||
await process_due_messages()
|
||||
assert providers[0].await_count == 1
|
||||
assert (await JobAttempt.first()).status == "uncertain"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("phase", ["processing", "sending"])
|
||||
async def test_abandoned_claim_is_uncertain(message, providers, phase):
|
||||
claimed = await delivery.claim("scheduled", message.id, run_id(message))
|
||||
if phase == "sending":
|
||||
await delivery.transition(claimed, "sending")
|
||||
await JobDelivery.filter(id=claimed.id).update(
|
||||
claimed_at=delivery.now() - timedelta(hours=2)
|
||||
)
|
||||
await ScheduledMessage.filter(id=message.id).update(status=MessageStatus.PROCESSING)
|
||||
await process_due_messages()
|
||||
await message.refresh_from_db()
|
||||
assert message.status == MessageStatus.UNCERTAIN
|
||||
assert (await JobAttempt.first()).status == "uncertain"
|
||||
providers[0].assert_not_awaited()
|
||||
|
||||
|
||||
async def test_crash_after_provider_send_never_resends(message, providers):
|
||||
class WorkerCrash(BaseException):
|
||||
pass
|
||||
|
||||
async def delivered_then_crash(*args):
|
||||
raise WorkerCrash()
|
||||
|
||||
providers[0].side_effect = delivered_then_crash
|
||||
with pytest.raises(WorkerCrash):
|
||||
await process_message(message)
|
||||
assert (await JobDelivery.first()).status == "sending"
|
||||
await JobDelivery.all().update(claimed_at=delivery.now() - timedelta(hours=2))
|
||||
await process_due_messages()
|
||||
await message.refresh_from_db()
|
||||
assert message.status == MessageStatus.UNCERTAIN
|
||||
assert providers[0].await_count == 1
|
||||
|
||||
|
||||
async def test_agent_failure_safe_retry_uses_owner(message, providers):
|
||||
message.use_agent = True
|
||||
await message.save()
|
||||
providers[1].side_effect = ValueError("no text")
|
||||
assert await process_message(message) == "failed"
|
||||
providers[0].assert_not_awaited()
|
||||
providers[1].side_effect = None
|
||||
await message.refresh_from_db()
|
||||
assert await process_message(message, retry=True) == "sent"
|
||||
args = providers[1].await_args.args
|
||||
assert args[0].id == message.created_by_id
|
||||
assert args[-1] == "scheduled"
|
||||
|
||||
|
||||
async def test_recurrence_deduplicates_and_skips_missed_occurrences(message, providers):
|
||||
message.recurrence = Recurrence.DAILY
|
||||
message.scheduled_at = delivery.now() - timedelta(days=3)
|
||||
await message.save()
|
||||
assert await process_message(message) == "sent"
|
||||
await asyncio.gather(*[_schedule_next_occurrence(message) for _ in range(5)])
|
||||
assert await ScheduledMessage.all().count() == 2
|
||||
successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
|
||||
assert successor.scheduled_at > delivery.now()
|
||||
assert successor.series_id == message.id
|
||||
await process_due_messages()
|
||||
assert await ScheduledMessage.all().count() == 2
|
||||
|
||||
|
||||
async def test_cancelled_series_does_not_regenerate(message, providers):
|
||||
message.recurrence = Recurrence.DAILY
|
||||
await message.save()
|
||||
await process_message(message)
|
||||
successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
|
||||
await ScheduledMessage.filter(id=successor.id).update(
|
||||
status=MessageStatus.CANCELLED
|
||||
)
|
||||
await _schedule_next_occurrence(message)
|
||||
await process_due_messages()
|
||||
assert await ScheduledMessage.all().count() == 2
|
||||
assert providers[0].await_count == 1
|
||||
|
||||
|
||||
async def test_legacy_sent_rows_are_not_forked_into_new_series(owner, providers):
|
||||
# Rows sent before delivery claims existed have no successor links or
|
||||
# delivery records; the repair pass must leave them alone.
|
||||
common = dict(
|
||||
recipient="recipient",
|
||||
channel=MessageChannel.EMAIL,
|
||||
subject="subject",
|
||||
content="prompt",
|
||||
recurrence=Recurrence.DAILY,
|
||||
created_by=owner,
|
||||
)
|
||||
for days in (2, 1):
|
||||
await ScheduledMessage.create(
|
||||
**common,
|
||||
status=MessageStatus.SENT,
|
||||
scheduled_at=delivery.now() - timedelta(days=days),
|
||||
)
|
||||
await ScheduledMessage.create(
|
||||
**common, scheduled_at=delivery.now() + timedelta(hours=12)
|
||||
)
|
||||
await process_due_messages()
|
||||
assert await ScheduledMessage.filter(status=MessageStatus.PENDING).count() == 1
|
||||
assert await ScheduledMessage.all().count() == 3
|
||||
providers[0].assert_not_awaited()
|
||||
|
||||
|
||||
async def test_claimed_send_without_successor_is_repaired(message, providers):
|
||||
message.recurrence = Recurrence.DAILY
|
||||
await message.save()
|
||||
assert await process_message(message) == "sent"
|
||||
# Simulate a crash after the send committed but before the successor existed.
|
||||
await ScheduledMessage.filter(previous_occurrence_id=message.id).delete()
|
||||
await process_due_messages()
|
||||
assert await ScheduledMessage.filter(previous_occurrence_id=message.id).exists()
|
||||
|
||||
|
||||
async def test_cancellation_during_agent_prevents_delivery(message, providers):
|
||||
message.use_agent = True
|
||||
await message.save()
|
||||
|
||||
async def cancel(*args):
|
||||
await ScheduledMessage.filter(id=message.id).update(
|
||||
status=MessageStatus.CANCELLED
|
||||
)
|
||||
return "response"
|
||||
|
||||
providers[1].side_effect = cancel
|
||||
assert await process_message(message) == "cancelled"
|
||||
providers[0].assert_not_awaited()
|
||||
assert (await JobDelivery.first()).status == "cancelled"
|
||||
|
||||
|
||||
async def test_duplicate_cron_triggers_scoped_to_run(job, providers):
|
||||
results = await asyncio.gather(
|
||||
*[execute_job(job, "occurrence-1") for _ in range(6)]
|
||||
)
|
||||
assert sum(bool(r.get("ok")) for r in results) == 1
|
||||
assert providers[0].await_count == 1
|
||||
assert await execute_job(job, "occurrence-2") == {
|
||||
"ok": True,
|
||||
"status": "sent",
|
||||
"error": None,
|
||||
}
|
||||
assert await JobAttempt.all().count() == 2
|
||||
assert providers[1].await_args.args[0].id == job.created_by_id
|
||||
assert providers[1].await_args.args[-1] == "cron"
|
||||
|
||||
|
||||
async def test_legacy_cron_has_no_implicit_admin(job, providers):
|
||||
job.created_by_id = None
|
||||
await job.save()
|
||||
result = await execute_job(job, "legacy")
|
||||
assert result["status"] == "failed"
|
||||
assert "explicit" in result["error"]
|
||||
providers[1].assert_not_awaited()
|
||||
providers[0].assert_not_awaited()
|
||||
|
||||
|
||||
async def test_cron_failure_retry_and_ambiguous_rejection(job, providers):
|
||||
providers[1].side_effect = ValueError("agent failed")
|
||||
assert (await execute_job(job, "run"))["status"] == "failed"
|
||||
assert (await execute_job(job, "run"))["skipped"]
|
||||
providers[1].side_effect = None
|
||||
providers[0].side_effect = TimeoutError("ambiguous")
|
||||
assert (await execute_job(job, "run", retry=True))["status"] == "uncertain"
|
||||
assert (await execute_job(job, "run", retry=True))["skipped"]
|
||||
assert await JobAttempt.all().count() == 2
|
||||
assert providers[0].await_count == 1
|
||||
|
||||
|
||||
async def test_disabled_cron_prevents_send_after_agent(job, providers):
|
||||
async def disable(*args):
|
||||
await CronJob.filter(id=job.id).update(enabled=False)
|
||||
return "response"
|
||||
|
||||
providers[1].side_effect = disable
|
||||
assert (await execute_job(job, "run"))["status"] == "cancelled"
|
||||
providers[0].assert_not_awaited()
|
||||
|
||||
|
||||
async def test_history_records_attempts(message, providers):
|
||||
await process_message(message)
|
||||
result = await delivery.history("scheduled", message.id)
|
||||
assert result[0]["status"] == "sent"
|
||||
assert not result[0]["retryable"]
|
||||
assert result[0]["attempts"][0]["finished_at"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def api(owner, monkeypatch):
|
||||
from quart_jwt_extended import JWTManager, create_access_token
|
||||
from blueprints.cron import cron_blueprint
|
||||
from blueprints.scheduled_messages import scheduled_messages_blueprint
|
||||
|
||||
app = Quart(__name__)
|
||||
app.config["JWT_SECRET_KEY"] = "test-secret" * 8
|
||||
JWTManager(app)
|
||||
app.register_blueprint(cron_blueprint)
|
||||
app.register_blueprint(scheduled_messages_blueprint)
|
||||
monkeypatch.setenv("CRON_SECRET", "cron-secret")
|
||||
owner.ldap_groups = ["lldap_admin"]
|
||||
await owner.save()
|
||||
async with app.app_context():
|
||||
token = create_access_token(identity=str(owner.id))
|
||||
return app.test_client(), {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
async def test_trigger_auth_and_required_cron_identity(job, api, providers):
|
||||
client, _ = api
|
||||
url = f"/api/cron/{job.id}/trigger"
|
||||
assert (await client.post(url, json={"run_id": "run"})).status_code == 401
|
||||
assert (
|
||||
await client.post(url, headers={"X-Cron-Secret": "cron-secret"})
|
||||
).status_code == 400
|
||||
headers = {"X-Cron-Secret": "cron-secret", "Idempotency-Key": "run"}
|
||||
assert (await client.post(url, headers=headers)).status_code == 200
|
||||
assert (await (await client.post(url, headers=headers)).get_json())["skipped"]
|
||||
assert providers[0].await_count == 1
|
||||
|
||||
|
||||
async def test_admin_history_retry_and_failed_reschedule(message, providers, api):
|
||||
client, headers = api
|
||||
url = f"/api/scheduled-messages/{message.id}"
|
||||
providers[0].side_effect = delivery.SafeDeliveryError("not delivered")
|
||||
await process_message(message)
|
||||
response = await client.put(
|
||||
url,
|
||||
headers=headers,
|
||||
json={"scheduled_at": (delivery.now() + timedelta(hours=1)).isoformat()},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert (await response.get_json())["status"] == "failed"
|
||||
assert (await client.post(url + "/retry")).status_code == 401
|
||||
providers[0].side_effect = None
|
||||
assert (await client.post(url + "/retry", headers=headers)).status_code == 200
|
||||
data = await (await client.get(url + "/deliveries", headers=headers)).get_json()
|
||||
assert len(data[0]["attempts"]) == 2
|
||||
|
||||
|
||||
async def test_api_rejects_ambiguous_retry(message, providers, api):
|
||||
client, headers = api
|
||||
providers[0].side_effect = TimeoutError("unknown")
|
||||
await process_message(message)
|
||||
assert (
|
||||
await client.post(
|
||||
f"/api/scheduled-messages/{message.id}/retry", headers=headers
|
||||
)
|
||||
).status_code == 409
|
||||
assert providers[0].await_count == 1
|
||||
|
||||
|
||||
async def test_api_cancel_recurring_series(message, providers, api):
|
||||
client, headers = api
|
||||
message.recurrence = Recurrence.DAILY
|
||||
await message.save()
|
||||
await process_message(message)
|
||||
successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
|
||||
response = await client.put(
|
||||
f"/api/scheduled-messages/{message.id}",
|
||||
headers=headers,
|
||||
json={"status": "cancelled"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
await successor.refresh_from_db()
|
||||
assert successor.status == MessageStatus.CANCELLED
|
||||
await process_due_messages()
|
||||
assert await ScheduledMessage.all().count() == 2
|
||||
|
||||
|
||||
async def test_cancel_keeps_delivered_status_and_stops_series(message, providers, api):
|
||||
client, headers = api
|
||||
message.recurrence = Recurrence.DAILY
|
||||
await message.save()
|
||||
await process_message(message)
|
||||
response = await client.put(
|
||||
f"/api/scheduled-messages/{message.id}",
|
||||
headers=headers,
|
||||
json={"status": "cancelled"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
await message.refresh_from_db()
|
||||
assert message.status == MessageStatus.SENT # delivered stays delivered
|
||||
successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
|
||||
assert successor.status == MessageStatus.CANCELLED
|
||||
|
||||
|
||||
async def test_cancel_during_send_does_not_claim_cancelled(message, providers, api):
|
||||
client, headers = api
|
||||
message.recurrence = Recurrence.DAILY
|
||||
await message.save()
|
||||
in_flight, release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def slow_send(*args):
|
||||
in_flight.set()
|
||||
await release.wait()
|
||||
|
||||
providers[0].side_effect = slow_send
|
||||
task = asyncio.create_task(process_message(message))
|
||||
await in_flight.wait()
|
||||
response = await client.put(
|
||||
f"/api/scheduled-messages/{message.id}",
|
||||
headers=headers,
|
||||
json={"status": "cancelled"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
release.set()
|
||||
assert await task == "sent"
|
||||
await message.refresh_from_db()
|
||||
assert message.status == MessageStatus.SENT
|
||||
# The series stops even though nothing pending existed when cancelled.
|
||||
await process_due_messages()
|
||||
assert not await ScheduledMessage.filter(
|
||||
status=MessageStatus.PENDING, deleted_at__isnull=True
|
||||
).exists()
|
||||
|
||||
|
||||
async def test_deleting_edited_occurrence_does_not_revive_series(
|
||||
message, providers, api
|
||||
):
|
||||
client, headers = api
|
||||
message.recurrence = Recurrence.DAILY
|
||||
await message.save()
|
||||
await process_message(message)
|
||||
successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
|
||||
url = f"/api/scheduled-messages/{successor.id}"
|
||||
assert (
|
||||
await client.put(url, headers=headers, json={"recurrence": "none"})
|
||||
).status_code == 200
|
||||
assert (await client.delete(url, headers=headers)).status_code == 200
|
||||
await process_due_messages()
|
||||
assert not await ScheduledMessage.filter(
|
||||
status=MessageStatus.PENDING, deleted_at__isnull=True
|
||||
).exists()
|
||||
|
||||
|
||||
async def test_api_assign_legacy_owner(job, api, providers, owner):
|
||||
job.created_by_id = None
|
||||
await job.save()
|
||||
client, headers = api
|
||||
response = await client.put(
|
||||
f"/api/cron/{job.id}", headers=headers, json={"created_by": str(owner.id)}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
await job.refresh_from_db()
|
||||
assert (await execute_job(job, "run"))["ok"]
|
||||
|
||||
|
||||
async def test_due_snapshot_does_not_send_rescheduled_message(message, providers):
|
||||
await ScheduledMessage.filter(id=message.id).update(
|
||||
scheduled_at=delivery.now() + timedelta(hours=1), content="edited"
|
||||
)
|
||||
assert await process_message(message) == "skipped"
|
||||
providers[0].assert_not_awaited()
|
||||
|
||||
|
||||
async def test_claim_loads_current_content(message, providers):
|
||||
await ScheduledMessage.filter(id=message.id).update(content="edited")
|
||||
assert await process_message(message) == "sent"
|
||||
assert providers[0].await_args.args[2] == "edited"
|
||||
|
||||
|
||||
async def test_delete_recurring_hides_tombstone_and_prevents_repair(
|
||||
message, providers, api
|
||||
):
|
||||
message.recurrence = Recurrence.DAILY
|
||||
await message.save()
|
||||
await process_message(message)
|
||||
successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
|
||||
client, headers = api
|
||||
response = await client.delete(
|
||||
f"/api/scheduled-messages/{successor.id}", headers=headers
|
||||
)
|
||||
assert response.status_code == 200
|
||||
rows = await (
|
||||
await client.get("/api/scheduled-messages/", headers=headers)
|
||||
).get_json()
|
||||
assert [r["id"] for r in rows] == [str(message.id)]
|
||||
await process_due_messages()
|
||||
assert await ScheduledMessage.all().count() == 2
|
||||
assert (await ScheduledMessage.get(id=successor.id)).deleted_at is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status, safe", [(200, None), (401, True), (429, True), (500, False)]
|
||||
)
|
||||
async def test_email_provider_acknowledgement(monkeypatch, status, safe):
|
||||
import httpx
|
||||
|
||||
monkeypatch.setenv("MAILGUN_API_KEY", "test")
|
||||
monkeypatch.setenv("MAILGUN_DOMAIN", "test.invalid")
|
||||
transport = httpx.MockTransport(
|
||||
lambda request: httpx.Response(status, json={"id": "test"})
|
||||
)
|
||||
original = httpx.AsyncClient
|
||||
monkeypatch.setattr(httpx, "AsyncClient", lambda: original(transport=transport))
|
||||
if safe is None:
|
||||
await delivery.send(MessageChannel.EMAIL, "recipient", "body", "subject")
|
||||
else:
|
||||
with pytest.raises(
|
||||
delivery.SafeDeliveryError if safe else httpx.HTTPStatusError
|
||||
):
|
||||
await delivery.send(MessageChannel.EMAIL, "recipient", "body", "subject")
|
||||
|
||||
|
||||
async def test_missing_email_configuration_is_safe_failure(monkeypatch):
|
||||
monkeypatch.delenv("MAILGUN_API_KEY", raising=False)
|
||||
monkeypatch.delenv("MAILGUN_DOMAIN", raising=False)
|
||||
with pytest.raises(delivery.SafeDeliveryError):
|
||||
await delivery.send(MessageChannel.EMAIL, "recipient", "body")
|
||||
|
||||
|
||||
async def test_agent_adapter_uses_shared_runtime(owner, monkeypatch):
|
||||
import types
|
||||
|
||||
runtime = types.ModuleType("blueprints.conversation.runtime")
|
||||
invoke = AsyncMock(return_value={"messages": [{"content": "answer"}]})
|
||||
runtime.invoke_agent_for_user = invoke
|
||||
monkeypatch.setitem(sys.modules, "blueprints.conversation.runtime", runtime)
|
||||
assert (
|
||||
await delivery.agent_content(owner, "query", "thread", "scheduled") == "answer"
|
||||
)
|
||||
invoke.assert_awaited_once_with(owner, "query", "thread", channel="scheduled")
|
||||
|
||||
|
||||
async def test_migration_snapshot_preserves_prior_models_and_describes_new_models(db):
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from aerich.utils import compress_dict, decompress_dict, get_models_describe
|
||||
|
||||
def load(path):
|
||||
spec = importlib.util.spec_from_file_location("test_migration", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
migration_dir = Path(__file__).resolve().parents[2] / "migrations" / "models"
|
||||
previous = load(migration_dir / "19_20261012225640_add_fk_indexes.py")
|
||||
migration = load(migration_dir / "20_20261012231017_application_foundation.py")
|
||||
state = decompress_dict(migration.MODELS_STATE)
|
||||
owned = {
|
||||
"models." + name
|
||||
for name in ("ScheduledMessage", "CronJob", "JobDelivery", "JobAttempt")
|
||||
}
|
||||
current = decompress_dict(compress_dict(get_models_describe("models")))
|
||||
for name in owned:
|
||||
assert state[name] == current[name]
|
||||
assert "models.User" in decompress_dict(previous.MODELS_STATE)
|
||||
sql = await migration.upgrade(None)
|
||||
assert 'UNIQUE ("kind", "job_id", "run_id")' in sql
|
||||
assert '"previous_occurrence_id" UUID UNIQUE' in sql
|
||||
assert '"created_by_id" UUID' in sql
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content, expected",
|
||||
[
|
||||
([{"type": "text", "text": "answer"}], "answer"),
|
||||
("<think>private reasoning</think>answer", "answer"),
|
||||
],
|
||||
)
|
||||
async def test_agent_adapter_extracts_supported_content(
|
||||
owner, monkeypatch, content, expected
|
||||
):
|
||||
import types
|
||||
|
||||
runtime = types.ModuleType("blueprints.conversation.runtime")
|
||||
runtime.invoke_agent_for_user = AsyncMock(
|
||||
return_value={"messages": [{"content": content}]}
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "blueprints.conversation.runtime", runtime)
|
||||
assert await delivery.agent_content(owner, "query", "thread", "cron") == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path, body",
|
||||
[
|
||||
("/api/scheduled-messages/", ["array"]),
|
||||
("/api/scheduled-messages/", {"recipient": 5, "channel": "email"}),
|
||||
("/api/cron/", ["array"]),
|
||||
("/api/cron/", {"name": 7, "prompt": "p", "recipient": "r"}),
|
||||
],
|
||||
)
|
||||
async def test_malformed_create_bodies_are_rejected(api, path, body):
|
||||
client, headers = api
|
||||
resp = await client.post(path, headers=headers, json=body)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
async def test_malformed_update_and_trigger_bodies(job, message, api, providers):
|
||||
client, headers = api
|
||||
for path in (f"/api/scheduled-messages/{message.id}", f"/api/cron/{job.id}"):
|
||||
resp = await client.put(path, headers=headers, json={"recipient": ["x"]})
|
||||
assert resp.status_code == 400
|
||||
resp = await client.post(
|
||||
f"/api/cron/{job.id}/trigger",
|
||||
headers={"X-Cron-Secret": "cron-secret"},
|
||||
json=["run"],
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
async def test_update_rejects_null_and_blank_required_fields(job, message, api):
|
||||
client, headers = api
|
||||
for path, field in (
|
||||
(f"/api/scheduled-messages/{message.id}", "recipient"),
|
||||
(f"/api/scheduled-messages/{message.id}", "content"),
|
||||
(f"/api/cron/{job.id}", "name"),
|
||||
(f"/api/cron/{job.id}", "prompt"),
|
||||
):
|
||||
for value in (None, " "):
|
||||
resp = await client.put(path, headers=headers, json={field: value})
|
||||
assert resp.status_code == 400, (path, field, value)
|
||||
# subject may be cleared, except where the channel requires it (email).
|
||||
resp = await client.put(
|
||||
f"/api/scheduled-messages/{message.id}", headers=headers, json={"subject": None}
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
resp = await client.put(
|
||||
f"/api/scheduled-messages/{message.id}",
|
||||
headers=headers,
|
||||
json={"subject": None, "channel": "imessage"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
async def test_cancel_started_one_off_message_is_rejected(message, providers, api):
|
||||
client, headers = api
|
||||
in_flight, release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def slow_send(*args):
|
||||
in_flight.set()
|
||||
await release.wait()
|
||||
|
||||
providers[0].side_effect = slow_send
|
||||
task = asyncio.create_task(process_message(message))
|
||||
await in_flight.wait()
|
||||
response = await client.put(
|
||||
f"/api/scheduled-messages/{message.id}",
|
||||
headers=headers,
|
||||
json={"status": "cancelled"},
|
||||
)
|
||||
assert response.status_code == 409
|
||||
release.set()
|
||||
assert await task == "sent"
|
||||
assert await ScheduledMessage.all().count() == 1 # no tombstone for one-offs
|
||||
@@ -0,0 +1,164 @@
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from langchain_core.documents import Document
|
||||
from langchain_postgres import PGVector
|
||||
from quart import Quart
|
||||
from quart_jwt_extended import JWTManager, create_access_token
|
||||
from sqlalchemy import Column
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
from blueprints.rag.retrieval import build_filter, citation_url, document_date
|
||||
from blueprints.rag import logic, source_api
|
||||
from blueprints.conversation.artifacts_util import search_documents_artifact
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", [None, "paperless", "obsidian"])
|
||||
@pytest.mark.parametrize(
|
||||
"start,end",
|
||||
[("2026-09-01", "2026-09-30"), ("2026-09-01", None), (None, "2026-09-30")],
|
||||
)
|
||||
def test_filters_compile_with_actual_pgvector_operator_rules(source, start, end):
|
||||
store = PGVector.__new__(PGVector)
|
||||
store.EmbeddingStore = SimpleNamespace(cmetadata=Column("cmetadata", JSONB))
|
||||
expression = store._create_filter_clause(build_filter(source, start, end))
|
||||
assert expression is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"start,end",
|
||||
[
|
||||
("20260901", None),
|
||||
("bad", None),
|
||||
("2026-10-01", "2026-09-01"),
|
||||
(None, "9999-12-31"),
|
||||
],
|
||||
)
|
||||
def test_date_filters_validate_before_vector_lookup(start, end):
|
||||
with pytest.raises(ValueError):
|
||||
build_filter("obsidian", start, end)
|
||||
|
||||
|
||||
async def test_filtered_query_preserves_citations_and_excludes_unknown_dates(
|
||||
monkeypatch,
|
||||
):
|
||||
identifier = str(uuid4())
|
||||
docs = [
|
||||
Document(
|
||||
id=identifier,
|
||||
page_content="September groceries plan",
|
||||
metadata={
|
||||
"source": "obsidian",
|
||||
"date": "2026-09-15",
|
||||
"filepath": "notes/budget.md",
|
||||
},
|
||||
),
|
||||
Document(
|
||||
id=str(uuid4()),
|
||||
page_content="Undated plan",
|
||||
metadata={"source": "obsidian"},
|
||||
),
|
||||
]
|
||||
store = SimpleNamespace(amax_marginal_relevance_search=AsyncMock(return_value=docs))
|
||||
monkeypatch.setattr(logic, "_get_vector_store", lambda: store)
|
||||
text, results = await logic.query_vector_store(
|
||||
"budget", source="obsidian", start_date="2026-09-01", end_date="2026-09-30"
|
||||
)
|
||||
assert results == docs[:1]
|
||||
assert f"Citation: /sources/{identifier}" in text
|
||||
assert store.amax_marginal_relevance_search.call_args.kwargs[
|
||||
"filter"
|
||||
] == build_filter("obsidian", "2026-09-01", "2026-09-30")
|
||||
|
||||
|
||||
def test_artifact_citations_identify_exact_retrieved_chunk(monkeypatch):
|
||||
monkeypatch.setenv("OBSIDIAN_VAULT_PATH", "/vault")
|
||||
identifier = str(uuid4())
|
||||
doc = Document(
|
||||
id=identifier,
|
||||
page_content="Savings target",
|
||||
metadata={
|
||||
"filepath": "/vault/notes/budget.md",
|
||||
"h1": "Budget",
|
||||
"h2": "Savings",
|
||||
},
|
||||
)
|
||||
artifact = search_documents_artifact("target", "obsidian", [doc])
|
||||
result = artifact["data"]["results"][0]
|
||||
assert result["url"] == f"/sources/{identifier}"
|
||||
assert result["source_id"] == identifier
|
||||
assert result["path"] == "notes/budget.md"
|
||||
assert result["headings"] == ["Budget", "Savings"]
|
||||
|
||||
|
||||
def test_document_date_and_unknown_citations():
|
||||
assert (
|
||||
document_date(
|
||||
{"created_date": datetime(2026, 9, 1, tzinfo=timezone.utc).timestamp()}
|
||||
)
|
||||
== "2026-09-01"
|
||||
)
|
||||
assert document_date({"date": "bad"}) is None
|
||||
assert citation_url(Document(page_content="no stable ID")) is None
|
||||
|
||||
|
||||
async def test_source_read_requires_auth_and_handles_missing_chunk(monkeypatch):
|
||||
app = Quart(__name__)
|
||||
app.config["JWT_SECRET_KEY"] = "test-secret"
|
||||
JWTManager(app)
|
||||
app.register_blueprint(source_api.sources_blueprint)
|
||||
calls = []
|
||||
|
||||
def load(identifier):
|
||||
calls.append(identifier)
|
||||
return {
|
||||
"id": identifier,
|
||||
"source": "paperless",
|
||||
"title": "Vet report",
|
||||
"content": "Retrieved section",
|
||||
"headings": [],
|
||||
}
|
||||
|
||||
monkeypatch.setattr(source_api, "_load_chunk", load)
|
||||
identifier = str(uuid4())
|
||||
assert (
|
||||
await app.test_client().get(f"/api/sources/{identifier}")
|
||||
).status_code == 401
|
||||
async with app.app_context():
|
||||
token = create_access_token(identity=str(uuid4()))
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
response = await app.test_client().get(
|
||||
f"/api/sources/{identifier}", headers=headers
|
||||
)
|
||||
assert (await response.get_json())["content"] == "Retrieved section"
|
||||
assert response.headers["Cache-Control"] == "no-store"
|
||||
assert (
|
||||
await app.test_client().get("/api/sources/not-an-id", headers=headers)
|
||||
).status_code == 404
|
||||
assert calls == [identifier]
|
||||
|
||||
|
||||
async def test_post_filter_keeps_note_matched_by_any_date_field(monkeypatch):
|
||||
# The database filter matches if any date field is in range; the post-filter
|
||||
# must agree or matching notes silently vanish from results.
|
||||
doc = Document(
|
||||
id=str(uuid4()),
|
||||
page_content="Lease renewal notes",
|
||||
metadata={
|
||||
"source": "obsidian",
|
||||
"date": "2023-05-01",
|
||||
"created_at": "2024-06-12T09:00:00",
|
||||
"filepath": "notes/lease.md",
|
||||
},
|
||||
)
|
||||
store = SimpleNamespace(
|
||||
amax_marginal_relevance_search=AsyncMock(return_value=[doc])
|
||||
)
|
||||
monkeypatch.setattr(logic, "_get_vector_store", lambda: store)
|
||||
_, results = await logic.query_vector_store(
|
||||
"lease", source="obsidian", start_date="2024-06-01", end_date="2024-06-30"
|
||||
)
|
||||
assert results == [doc]
|
||||
@@ -26,6 +26,7 @@ async def owner():
|
||||
"models": [
|
||||
"blueprints.users.models",
|
||||
"blueprints.conversation.models",
|
||||
"blueprints.conversation.run_models",
|
||||
"blueprints.artifacts.models",
|
||||
]
|
||||
},
|
||||
@@ -91,12 +92,12 @@ def _final(text):
|
||||
async def _run(owner, monkeypatch, script, *, response_timeout=60, direct=False):
|
||||
user, conversation = owner
|
||||
monkeypatch.setattr(conv, "MODEL_STALL_TIMEOUT", LIMIT)
|
||||
monkeypatch.setattr(conv, "get_agent_for", lambda u: _TimedAgent(script))
|
||||
monkeypatch.setattr(conv.runtime, "get_agent_for", lambda u: _TimedAgent(script))
|
||||
|
||||
async def _prompt(user_id, query=""):
|
||||
return "system"
|
||||
|
||||
monkeypatch.setattr(conv, "build_system_prompt_with_memories", _prompt)
|
||||
monkeypatch.setattr(conv.runtime, "build_system_prompt_with_memories", _prompt)
|
||||
|
||||
app = Quart(__name__)
|
||||
app.config["RESPONSE_TIMEOUT"] = response_timeout
|
||||
@@ -300,18 +301,22 @@ async def test_no_response_is_counted_as_error(owner, monkeypatch):
|
||||
async def test_final_persistence_exit_has_exactly_one_outcome(
|
||||
owner, monkeypatch, caplog, cancelled
|
||||
):
|
||||
results, durations, writes = [], [], []
|
||||
original = conv.add_message_to_conversation
|
||||
# Answers are persisted by the detached run worker, so the request only
|
||||
# observes the terminal event; cancellation must not escape the worker.
|
||||
from blueprints.conversation import runs
|
||||
|
||||
async def persist(**kwargs):
|
||||
if kwargs["speaker"] == "simba":
|
||||
writes.append(kwargs)
|
||||
results, durations, writes = [], [], []
|
||||
original = runs.add_message_to_conversation
|
||||
|
||||
async def persist(conversation, message, speaker, *args, **kwargs):
|
||||
if speaker == "simba":
|
||||
writes.append(message)
|
||||
if cancelled:
|
||||
raise asyncio.CancelledError()
|
||||
raise RuntimeError("persistence failed")
|
||||
return await original(**kwargs)
|
||||
return await original(conversation, message, speaker, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(conv, "add_message_to_conversation", persist)
|
||||
monkeypatch.setattr(runs, "add_message_to_conversation", persist)
|
||||
monkeypatch.setattr(
|
||||
conv, "count_result", lambda **kwargs: results.append(kwargs["result"])
|
||||
)
|
||||
@@ -325,13 +330,8 @@ async def test_final_persistence_exit_has_exactly_one_outcome(
|
||||
(0, _final("Answer")),
|
||||
]
|
||||
with caplog.at_level(logging.INFO):
|
||||
if cancelled:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await _run(owner, monkeypatch, script, direct=True)
|
||||
else:
|
||||
frames = await _run(owner, monkeypatch, script, direct=True)
|
||||
assert frames[-1]["type"] == "error"
|
||||
assert "persistence failed" in frames[-1]["message"]
|
||||
frames = await _run(owner, monkeypatch, script, direct=True)
|
||||
assert frames[-1]["type"] == "error"
|
||||
assert len(writes) == 1
|
||||
assert results == ["cancelled" if cancelled else "error"]
|
||||
assert len(durations) == 1
|
||||
|
||||
Reference in New Issue
Block a user