47 lines
1.4 KiB
Python
47 lines
1.4 KiB
Python
from scripts.evaluate_context import evaluate
|
|
|
|
|
|
def test_representative_context_scenarios_preserve_required_facts():
|
|
report = evaluate()
|
|
assert len(report) == 3
|
|
assert all(case["passed"] for case in report), report
|
|
assert all(
|
|
case["output_estimated_tokens"] < case["input_estimated_tokens"]
|
|
for case in report
|
|
)
|
|
|
|
|
|
def test_fact_checks_reject_similar_but_wrong_numbers_and_source_ids():
|
|
from scripts.evaluate_context import check_facts
|
|
|
|
payload = {
|
|
"data": {
|
|
"total": 1710.5,
|
|
"net_outflows": 1610.5,
|
|
"rows_compaction": {"omitted_source_refs": [{"id": "transaction-990"}]},
|
|
}
|
|
}
|
|
expected = {
|
|
"required_fields": {
|
|
"data.total": 710.5,
|
|
"data.net_outflows": 610.5,
|
|
"data.inflows": 100,
|
|
"data.total_basis": "outflows",
|
|
},
|
|
"required_source_ids": ["transaction-99"],
|
|
}
|
|
missing, total = check_facts(payload, expected)
|
|
assert len(missing) == total == 5
|
|
|
|
|
|
def test_fact_checks_reject_incorrect_heading_line_numbers():
|
|
from scripts.evaluate_context import check_facts
|
|
|
|
payload = {
|
|
"data": {
|
|
"markdown_compaction": {"headings": [{"line": 1, "heading": "# Target"}]}
|
|
}
|
|
}
|
|
expected = {"required_headings": [{"line": 1001, "heading": "# Target"}]}
|
|
assert check_facts(payload, expected) == (["heading:# Target"], 1)
|