Files
simbarag/tests/unit/test_tool_artifacts.py
T
ryan b8ccc88b58
CI / backend (pull_request) Successful in 41s
CI / frontend (pull_request) Failing after 7s
CI / backend (push) Successful in 42s
CI / frontend (push) Failing after 6s
Give the model full retrieved chunks instead of 400-character snippets
Compaction replaces a tool's text with its artifact, and the search artifact
only carries UI snippets, so the full chunk the retriever found never reached
the model. Search tools now attach the full text as model_context, which
compaction prefers and shortens under budget pressure while keeping source
ids; the persisted artifact and UI are unchanged. The per-chunk header drops
the raw metadata dict (about a fifth of search output tokens) for title, date
and source.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 03:40:52 -04:00

673 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Tests for tools returning (text, artifact) via content_and_artifact.
The text half must stay identical to the pre-artifact output (the LLM's view),
and the artifact must match the contract in docs/cowork-ui-plan.md.
"""
import datetime
import json
import os
from unittest.mock import AsyncMock, MagicMock
import pytest
from langchain_core.documents import Document
from langchain_core.messages import ToolMessage
os.environ.setdefault("TAVILY_API_KEY", "dummy")
from blueprints.conversation import agents # noqa: E402
from blueprints.conversation.artifacts_util import ( # noqa: E402
NOTE_MAX_CHARS,
SNIPPET_MAX_CHARS,
format_date_range,
search_documents_artifact,
)
from utils.obsidian_service import ObsidianService # noqa: E402
KIND_DATA_KEYS = {
"mealie.recipe": {
"slug",
"name",
"description",
"servings",
"prep_time",
"cook_time",
"total_time",
"ingredients",
"instructions",
"tags",
},
"mealie.meal_plan": {"start_date", "end_date", "days"},
"mealie.shopping_lists": {"lists"},
"obsidian.note": {"path", "title", "markdown", "tags", "frontmatter"},
"search.documents": {"query", "source", "results"},
}
async def call(tool, args=None) -> ToolMessage:
"""Invoke a tool the way the agent does: with a ToolCall dict."""
msg = await tool.ainvoke(
{"type": "tool_call", "id": "call-1", "name": tool.name, "args": args or {}}
)
assert isinstance(msg, ToolMessage)
return msg
def assert_artifact(artifact, kind):
assert artifact is not None
assert artifact["kind"] == kind
assert isinstance(artifact["title"], str) and artifact["title"]
assert set(artifact["data"]) == KIND_DATA_KEYS[kind]
json.dumps(artifact)
@pytest.fixture
def mealie(monkeypatch):
service = MagicMock()
monkeypatch.setattr(agents, "mealie_enabled", True)
monkeypatch.setattr(agents, "mealie_service", service, raising=False)
monkeypatch.setenv("MEALIE_BASE_URL", "http://mealie.local:9000/")
monkeypatch.delenv("MEALIE_GROUP_SLUG", raising=False)
return service
@pytest.fixture
def obsidian(monkeypatch, tmp_path):
(tmp_path / "seed.md").write_text("# seed")
monkeypatch.setenv("OBSIDIAN_VAULT_PATH", str(tmp_path))
monkeypatch.delenv("OBSIDIAN_VAULT_NAME", raising=False)
service = ObsidianService()
monkeypatch.setattr(agents, "obsidian_enabled", True)
monkeypatch.setattr(agents, "obsidian_service", service, raising=False)
return service
# --- Mealie -----------------------------------------------------------------
RECIPE = {
"name": "Chicken Tikka Masala",
"slug": "chicken-tikka-masala",
"description": "Creamy and spiced.",
"ingredients": [{"display": "1 lb chicken"}, {"display": "1 cup yogurt"}],
"instructions": [
{"title": "Marinate", "text": "Mix chicken and yogurt."},
{"title": "", "text": "Cook it."},
],
"prep_time": "20 minutes",
"cook_time": "30 minutes",
"total_time": "",
"servings": "4",
"tags": ["Indian", "Dinner"],
}
RECIPE_TEXT = """**Chicken Tikka Masala**
Creamy and spiced.
Servings: 4
Prep: 20 minutes | Cook: 30 minutes
Ingredients:
- 1 lb chicken
- 1 cup yogurt
Instructions:
1. Mix chicken and yogurt.
2. Cook it.
Tags: Indian, Dinner"""
async def test_mealie_get_recipe(mealie):
mealie.get_recipe = AsyncMock(return_value=RECIPE)
msg = await call(agents.mealie_get_recipe, {"recipe_slug": "chicken-tikka-masala"})
assert msg.content == RECIPE_TEXT
art = msg.artifact
assert_artifact(art, "mealie.recipe")
assert art["title"] == "Chicken Tikka Masala"
assert art["data"]["ingredients"] == ["1 lb chicken", "1 cup yogurt"]
assert art["data"]["instructions"] == [
{"title": "Marinate", "text": "Mix chicken and yogurt."},
{"text": "Cook it."},
]
assert art["source_ref"] == {
"system": "mealie",
"id": "chicken-tikka-masala",
"url": "http://mealie.local:9000/g/home/r/chicken-tikka-masala",
}
async def test_mealie_recipe_url_uses_group_slug(mealie, monkeypatch):
monkeypatch.setenv("MEALIE_GROUP_SLUG", "family")
mealie.get_recipe = AsyncMock(return_value=RECIPE)
msg = await call(agents.mealie_get_recipe, {"recipe_slug": "chicken-tikka-masala"})
assert msg.artifact["source_ref"]["url"].endswith(
"/g/family/r/chicken-tikka-masala"
)
async def test_mealie_get_recipe_error(mealie):
mealie.get_recipe = AsyncMock(side_effect=Exception("404"))
msg = await call(agents.mealie_get_recipe, {"recipe_slug": "nope"})
assert msg.content == "Error fetching recipe: 404"
assert msg.artifact is None
MEAL_PLANS = {
"start_date": "2026-10-12",
"end_date": "2026-10-19",
"total_plans": 3,
"plans_by_date": {
"2026-10-13": [
{
"entry_type": "dinner",
"title": "",
"recipe_name": "Tacos",
"recipe_slug": "tacos",
"note": "",
}
],
"2026-10-12": [
{
"entry_type": "lunch",
"title": "Leftovers",
"recipe_name": "",
"recipe_slug": "",
"note": "from Sunday",
},
{
"entry_type": "dinner",
"title": "",
"recipe_name": "Chicken Tikka Masala",
"recipe_slug": "chicken-tikka-masala",
"note": "",
},
],
},
}
MEAL_PLANS_TEXT = """Meal plans from 2026-10-12 to 2026-10-19 (3 meals):
2026-10-12:
- Lunch: Leftovers
Note: from Sunday
- Dinner: Chicken Tikka Masala
2026-10-13:
- Dinner: Tacos"""
async def test_mealie_meal_plans(mealie):
mealie.get_meal_plans = AsyncMock(return_value=MEAL_PLANS)
msg = await call(agents.mealie_meal_plans, {})
assert msg.content == MEAL_PLANS_TEXT
art = msg.artifact
assert_artifact(art, "mealie.meal_plan")
assert art["title"] == "Meal plan · Oct 12–19"
assert [d["date"] for d in art["data"]["days"]] == ["2026-10-12", "2026-10-13"]
assert art["data"]["days"][0]["meals"] == [
{"entry_type": "lunch", "name": "Leftovers", "note": "from Sunday"},
{
"entry_type": "dinner",
"name": "Chicken Tikka Masala",
"recipe_slug": "chicken-tikka-masala",
},
]
async def test_mealie_meal_plans_empty(mealie):
mealie.get_meal_plans = AsyncMock(
return_value={
"start_date": "2026-10-12",
"end_date": "2026-10-19",
"total_plans": 0,
"plans_by_date": {},
}
)
msg = await call(agents.mealie_meal_plans, {})
assert msg.content == "No meal plans found from 2026-10-12 to 2026-10-19."
assert msg.artifact is None
async def test_mealie_todays_meals(mealie):
mealie.get_todays_meals = AsyncMock(
return_value={
"today": "2026-10-12",
"total_meals": 1,
"meals": [
{
"entry_type": "dinner",
"title": "",
"recipe_name": "Tacos",
"recipe_slug": "tacos",
"note": "extra salsa",
}
],
}
)
msg = await call(agents.mealie_todays_meals)
assert msg.content == (
"Meals planned for today (2026-10-12):\n- Dinner: Tacos\n Note: extra salsa"
)
art = msg.artifact
assert_artifact(art, "mealie.meal_plan")
assert art["title"] == "Today's meals · Oct 12"
assert art["data"] == {
"start_date": "2026-10-12",
"end_date": "2026-10-12",
"days": [
{
"date": "2026-10-12",
"meals": [
{
"entry_type": "dinner",
"name": "Tacos",
"recipe_slug": "tacos",
"note": "extra salsa",
}
],
}
],
}
async def test_mealie_todays_meals_empty(mealie):
mealie.get_todays_meals = AsyncMock(
return_value={"today": "2026-10-12", "total_meals": 0, "meals": []}
)
msg = await call(agents.mealie_todays_meals)
assert msg.content == "No meals planned for today (2026-10-12)."
assert msg.artifact is None
async def test_mealie_shopping_lists(mealie):
mealie.get_shopping_lists = AsyncMock(
return_value={
"total_lists": 1,
"total_unchecked_items": 2,
"shopping_lists": [
{
"name": "Groceries",
"unchecked_items": [
{"name": "Milk", "quantity": 1, "note": "", "food": None},
{"name": "Eggs", "quantity": 12.0, "note": "", "food": None},
],
"checked_items": [
{"name": "Bread", "quantity": 1, "note": "", "food": None}
],
}
],
}
)
msg = await call(agents.mealie_shopping_lists)
assert msg.content == (
"Shopping lists (2 items needed):\n"
"\n**Groceries** (2 items needed)\n"
" - [ ] Milk\n"
" - [ ] Eggs x12.0\n"
" (1 items already checked off)"
)
art = msg.artifact
assert_artifact(art, "mealie.shopping_lists")
assert art["data"]["lists"] == [
{
"name": "Groceries",
"items": [
{"name": "Milk", "quantity": 1, "checked": False},
{"name": "Eggs", "quantity": 12.0, "checked": False},
{"name": "Bread", "quantity": 1, "checked": True},
],
}
]
async def test_mealie_shopping_lists_empty(mealie):
mealie.get_shopping_lists = AsyncMock(
return_value={
"total_lists": 0,
"total_unchecked_items": 0,
"shopping_lists": [],
}
)
msg = await call(agents.mealie_shopping_lists)
assert msg.content == "No shopping lists found."
assert msg.artifact is None
@pytest.mark.parametrize(
"tool,args",
[
(agents.mealie_get_recipe, {"recipe_slug": "x"}),
(agents.mealie_meal_plans, {}),
(agents.mealie_todays_meals, {}),
(agents.mealie_shopping_lists, {}),
],
)
async def test_mealie_not_configured(monkeypatch, tool, args):
monkeypatch.setattr(agents, "mealie_enabled", False)
msg = await call(tool, args)
assert msg.content == (
"Mealie integration is not configured. Please set MEALIE_BASE_URL and "
"MEALIE_API_TOKEN environment variables."
)
assert msg.artifact is None
# --- Obsidian ---------------------------------------------------------------
NOTE = """---
title: Vet visit
date: 2026-10-01
tags: [simba]
---
Simba saw the [[Dr Smith|vet]] today. #simba #health #simba
"""
async def test_obsidian_read_note(obsidian, monkeypatch):
monkeypatch.setenv("OBSIDIAN_VAULT_NAME", "Home Vault")
os.makedirs(os.path.join(obsidian.vault_path, "notes"))
with open(os.path.join(obsidian.vault_path, "notes", "vet visit.md"), "w") as f:
f.write(NOTE)
msg = await call(agents.obsidian_read_note, {"relative_path": "notes/vet visit.md"})
assert msg.content == (
"File: notes/vet visit.md\n\n"
"Frontmatter:\n{'title': 'Vet visit', 'date': datetime.date(2026, 10, 1), "
"'tags': ['simba']}\n\n"
"Content:\nSimba saw the vet today. #simba #health #simba\n\n"
"Tags: simba, health, simba\n"
"Contains 1 wikilinks and 0 embeds"
)
art = msg.artifact
assert_artifact(art, "obsidian.note")
assert art["title"] == "Vet visit"
assert art["data"]["frontmatter"] == {
"title": "Vet visit",
"date": "2026-10-01",
"tags": ["simba"],
}
assert art["data"]["tags"] == ["simba", "health"]
assert art["data"]["markdown"] == "Simba saw the vet today. #simba #health #simba"
assert art["source_ref"] == {
"system": "obsidian",
"path": "notes/vet visit.md",
"url": "obsidian://open?vault=Home%20Vault&file=notes%2Fvet%20visit.md",
}
async def test_obsidian_read_note_caps_markdown_and_omits_url(obsidian):
with open(os.path.join(obsidian.vault_path, "big.md"), "w") as f:
f.write("x" * (NOTE_MAX_CHARS + 500))
msg = await call(agents.obsidian_read_note, {"relative_path": "big.md"})
art = msg.artifact
assert art["title"] == "big"
assert len(art["data"]["markdown"]) <= NOTE_MAX_CHARS
assert art["source_ref"] == {"system": "obsidian", "path": "big.md"}
async def test_obsidian_read_note_missing(obsidian):
msg = await call(agents.obsidian_read_note, {"relative_path": "nope.md"})
assert msg.content == (
"Note not found at 'nope.md'. Please check the path is correct."
)
assert msg.artifact is None
async def test_journal_get_today(obsidian):
path = obsidian.get_daily_note_path()
full = os.path.join(obsidian.vault_path, path)
os.makedirs(os.path.dirname(full), exist_ok=True)
raw = "## Tasks\n- [ ] Feed Simba\n"
with open(full, "w") as f:
f.write(raw)
today = datetime.datetime.now().strftime("%Y-%m-%d")
msg = await call(agents.journal_get_today)
assert msg.content == f"Daily note for {today}:\n\n{raw}"
art = msg.artifact
assert_artifact(art, "obsidian.note")
assert art["title"].startswith("Daily note · ")
assert art["data"]["path"] == path
assert art["data"]["markdown"] == "## Tasks\n- [ ] Feed Simba"
async def test_journal_get_today_missing(obsidian):
today = datetime.datetime.now().strftime("%Y-%m-%d")
msg = await call(agents.journal_get_today)
assert msg.content == (
f"No daily note found for {today}. Use journal_add_task to create one."
)
assert msg.artifact is None
async def test_obsidian_not_configured(monkeypatch):
monkeypatch.setattr(agents, "obsidian_enabled", False)
for tool, args in [
(agents.obsidian_read_note, {"relative_path": "a.md"}),
(agents.obsidian_search_notes, {"query": "a"}),
]:
msg = await call(tool, args)
assert msg.content == (
"Obsidian integration is not configured. Please set "
"OBSIDIAN_VAULT_PATH environment variable."
)
assert msg.artifact is None
msg = await call(agents.journal_get_today)
assert msg.content == "Obsidian integration is not configured."
assert msg.artifact is None
# --- Search -----------------------------------------------------------------
PAPERLESS_DOCS = [
Document(
page_content="Rabies vaccine\nadministered. " + "y" * 600,
metadata={
"source": "paperless",
"created_date": 1760227200.0,
"filename": "vet_invoice.pdf",
"document_type": "Invoice",
},
)
]
async def test_simba_search(monkeypatch):
query_mock = AsyncMock(return_value=("Source: {...}\nContent: x", PAPERLESS_DOCS))
monkeypatch.setattr(agents, "query_vector_store", query_mock)
msg = await call(agents.simba_search, {"query": "vet records"})
assert msg.content == "Source: {...}\nContent: x"
query_mock.assert_awaited_once_with(query="vet records", source="paperless")
art = msg.artifact
assert_artifact(art, "search.documents")
assert art["title"] == "Search: vet records"
assert art["data"]["source"] == "paperless"
(result,) = art["data"]["results"]
assert result["title"] == "vet_invoice"
assert result["filename"] == "vet_invoice.pdf"
assert result["document_type"] == "Invoice"
assert len(result["date"]) == 10
assert result["snippet"].startswith("Rabies vaccine administered. ")
assert len(result["snippet"]) <= SNIPPET_MAX_CHARS
(model_result,) = art["model_context"]["results"]
assert model_result["content"] == PAPERLESS_DOCS[0].page_content
assert len(model_result["content"]) > SNIPPET_MAX_CHARS
assert model_result["title"] == result["title"]
assert model_result["filename"] == result["filename"]
assert "snippet" not in model_result
async def test_simba_search_empty(monkeypatch):
monkeypatch.setattr(agents, "query_vector_store", AsyncMock(return_value=("", [])))
msg = await call(agents.simba_search, {"query": "nothing"})
assert msg.content == ""
assert msg.artifact is None
async def test_obsidian_search_notes(obsidian, monkeypatch):
docs = [
Document(
page_content="Simba's vet is Dr Smith.",
metadata={
"source": "obsidian",
"filepath": os.path.join(obsidian.vault_path, "pets", "simba.md"),
"tags": ["simba"],
"created_at": "2026-01-02T10:00:00",
},
)
]
monkeypatch.setattr(
agents, "query_vector_store", AsyncMock(return_value=("serialized", docs))
)
msg = await call(agents.obsidian_search_notes, {"query": "vet"})
assert msg.content == "serialized"
art = msg.artifact
assert_artifact(art, "search.documents")
assert art["data"]["results"] == [
{
"snippet": "Simba's vet is Dr Smith.",
"title": "simba",
"path": "pets/simba.md",
"date": "2026-01-02",
}
]
assert art["model_context"]["results"] == [
{
"content": "Simba's vet is Dr Smith.",
"title": "simba",
"path": "pets/simba.md",
"date": "2026-01-02",
}
]
async def test_obsidian_search_notes_error(obsidian, monkeypatch):
monkeypatch.setattr(
agents, "query_vector_store", AsyncMock(side_effect=Exception("db down"))
)
msg = await call(agents.obsidian_search_notes, {"query": "vet"})
assert msg.content == "Error searching Obsidian notes: db down"
assert msg.artifact is None
def test_search_documents_artifact_empty():
assert search_documents_artifact("q", "paperless", []) is None
@pytest.mark.parametrize(
"start,end,expected",
[
("2026-10-12", "2026-10-19", "Oct 12–19"),
("2026-10-28", "2026-11-03", "Oct 28–Nov 3"),
("2026-12-28", "2027-01-03", "Dec 28, 2026–Jan 3, 2027"),
("2026-10-12", "2026-10-12", "Oct 12"),
("bad", "2026-10-12", "bad–2026-10-12"),
],
)
def test_format_date_range(start, end, expected):
assert format_date_range(start, end) == expected
async def test_obsidian_note_line_range_retains_full_artifact(obsidian):
with open(os.path.join(obsidian.vault_path, "range.md"), "w") as f:
f.write("first\nsecond\nthird\nfourth")
msg = await call(
agents.obsidian_read_note,
{"relative_path": "range.md", "start_line": 2, "max_lines": 2},
)
assert "Content lines 2-3 of 4:\nsecond\nthird\n\n" in msg.content
assert "first" not in msg.content
assert msg.artifact["data"]["markdown"] == "first\nsecond\nthird\nfourth"
@pytest.mark.parametrize(
"args", [{"start_line": 0}, {"max_lines": 0}, {"start_line": 99}]
)
async def test_obsidian_note_invalid_line_range(obsidian, args):
with open(os.path.join(obsidian.vault_path, "range.md"), "w") as f:
f.write("first\nsecond")
msg = await call(agents.obsidian_read_note, {"relative_path": "range.md", **args})
assert msg.artifact is None
assert "positive integers" in msg.content or "beyond the end" in msg.content
async def test_focused_note_read_survives_model_compaction(obsidian):
from langchain_core.messages import AIMessage, HumanMessage
from blueprints.conversation.context import trim_history
content = "\n".join(
["background details"] * 1000 + ["The requested savings target is $500."]
)
with open(os.path.join(obsidian.vault_path, "late.md"), "w") as f:
f.write(content)
msg = await call(
agents.obsidian_read_note,
{"relative_path": "late.md", "start_line": 1001, "max_lines": 1},
)
messages = [
HumanMessage(content="Read line 1001"),
AIMessage(
content="",
tool_calls=[
{
"id": "call-1",
"name": "obsidian_read_note",
"args": {
"relative_path": "late.md",
"start_line": 1001,
"max_lines": 1,
},
}
],
),
msg,
]
projected = trim_history(messages, token_budget=2500)[-1]
parsed = json.loads(projected.content)
assert parsed["data"]["markdown"] == "The requested savings target is $500."
assert parsed["data"]["read_range"] == {
"start_line": 1001,
"end_line": 1001,
"total_lines": 1001,
}
assert msg.artifact["data"]["markdown"] == content
assert projected.artifact is msg.artifact