Regression Suites in CI

# Colab cell — pure Python, no key needed. Stub agent + judge stand in for
# your real ones so the mixed-tier runner executes end to end.
import json
from collections import namedtuple

Usage = namedtuple("Usage", "input_tokens output_tokens")
ToolCall = namedtuple("ToolCall", "name")
Result = namedtuple("Result", "text tool_calls usage")

def run_agent(prompt: str) -> Result:                # stub for your real agent
    if "cart" in prompt.lower():
        return Result("Here is your cart.", [ToolCall("lookup_cart")], Usage(1200, 300))
    return Result("Returns are accepted within 30 days.", [], Usage(1500, 400))

FAITHFULNESS_RUBRIC = "PASS if the answer only states facts from context."
def run_judge(text: str, rubric: str) -> str:        # stub for your real judge
    return "pass" if "30 days" in text else "fail"

# Each case declares HOW it should be scored, so the runner can mix tiers.
# In production these live as cases/*.json; inlined here so the cell runs.
CASES = [
    {"id": "bug_412_empty_cart", "prompt": "Show my cart", "check": "assert",
     "must_call": "lookup_cart", "must_not_call": "issue_refund"},
    {"id": "bug_419_hallucinated_policy", "prompt": "What's the return policy?",
     "check": "judge", "rubric": "faithfulness"},
]

def load_cases():
    return iter(CASES)

# $/1M tokens at list — keep rates in one constant you can update
PRICE = {"input": 3.00, "output": 15.00}          # claude-sonnet-5 list price

def usd(usage) -> float:
    # No SDK returns dollars — you always compute them from token counts.
    return (usage.input_tokens * PRICE["input"]
            + usage.output_tokens * PRICE["output"]) / 1e6

def score_case(case) -> tuple[bool, float]:
    result = run_agent(case["prompt"])
    cost = usd(result.usage)         # Lesson 4 automates this via Langfuse

    if case["check"] == "assert":
        called = {c.name for c in result.tool_calls}
        ok = (case.get("must_call", None) in called or "must_call" not in case) \
             and case.get("must_not_call", "___none___") not in called
        return ok, cost

    if case["check"] == "judge":
        verdict = run_judge(result.text, FAITHFULNESS_RUBRIC)
        return verdict == "pass", cost

    raise ValueError(f"unknown check type: {case['check']}")

def main():
    passed = failed = 0
    total_cost = 0.0
    failures = []
    for case in load_cases():
        ok, cost = score_case(case)
        total_cost += cost
        if ok:
            passed += 1
        else:
            failed += 1
            failures.append(case["id"])
    print(f"PASS {passed}  FAIL {failed}  COST $" + f"{total_cost:.3f}")
    if failures:
        print("failing cases:", ", ".join(failures))
    raise SystemExit(1 if failed else 0)   # non-zero fails the CI job

try:
    main()
except SystemExit as e:
    print(f"(in CI this exit code {e.code} gates the merge)")

name: agent-evals
on:
  pull_request:
    paths:
      - "prompts/**"
      - "src/agent/**"
      - "cases/**"
jobs:
  regression:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install -r requirements.txt
      - name: Run deterministic + judged regression suite
        env:
          ANTHROPIC_API_KEY: {{ secrets.ANTHROPIC_API_KEY }}
        run: python -m evals.run_suite     # exits non-zero on any failure

# Colab cell — pure Python, no key needed; run it as-is.
import math

def wilson_interval(successes: int, n: int, z: float = 1.96) -> tuple[float, float]:
    """Approximate 95% CI for a pass rate. Cheap gut-check before trusting a delta."""
    if n == 0:
        return (0.0, 0.0)
    p = successes / n
    denom = 1 + z**2 / n
    center = p + z**2 / (2 * n)
    margin = z * math.sqrt(p * (1 - p) / n + z**2 / (4 * n**2))
    return ((center - margin) / denom, (center + margin) / denom)

baseline = wilson_interval(41, 50)    # 82%
candidate = wilson_interval(42, 50)   # 84%
print(f"baseline  95% CI: {baseline[0]:.2f}-{baseline[1]:.2f}")
print(f"candidate 95% CI: {candidate[0]:.2f}-{candidate[1]:.2f}")
# These overlap heavily -> the 2-point move is noise, not signal at n=50.

def paired_sign_test(baseline_pass: list[bool], candidate_pass: list[bool]) -> dict:
    """Same cases, both versions -> count flips. Far more powerful than
    comparing two independent aggregate percentages."""
    newly_passing = sum(not b and c for b, c in zip(baseline_pass, candidate_pass))
    newly_failing = sum(b and not c for b, c in zip(baseline_pass, candidate_pass))
    return {"newly_passing": newly_passing, "newly_failing": newly_failing,
            "net": newly_passing - newly_failing}

# same 50 cases scored under both versions -> count the flips directly:
baseline_pass  = [True]*41 + [False]*9
candidate_pass = [True]*40 + [False]*7 + [True]*3   # 3 newly pass, 1 flips the other way
print(paired_sign_test(baseline_pass, candidate_pass))