Score Assertion Weakening Before You Merge an Agent Patch

git rev-parse --verify HEAD >/dev/null
mkdir -p /tmp/assert-score/{before,after}
git archive HEAD -- 'tests' | tar -x -C /tmp/assert-score/before
# worktree already contains the agent patch
cp -R tests /tmp/assert-score/after/tests
python3 score_assert_weakening.py \
  --before /tmp/assert-score/before \
  --after  /tmp/assert-score/after \
  --budget 4

#!/usr/bin/env python3
"""score_assert_weakening.py — heuristic contamination index for Python tests."""
from __future__ import annotations

import argparse
import ast
import json
import sys
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Tuple

ASSERT_CALLS = {"assertEqual", "assertTrue", "assertFalse", "assertIsNone"}
ESCAPE_MARKERS = {"skip", "xfail", "importorskip"}


def py_files(root: Path) -> Iterable[Path]:
    for p in root.rglob("*.py"):
        name = p.name
        if name.startswith("test_") or p.parent.name in {"tests", "test"}:
            yield p


def rel(root: Path, path: Path) -> str:
    return str(path.relative_to(root))


class TestIndex(ast.NodeVisitor):
    def __init__(self) -> None:
        self.functions: Dict[str, dict] = {}

    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
        if not node.name.startswith("test_"):
            self.generic_visit(node)
            return
        self.functions[node.name] = summarize(node)
        # Do not walk nested defs as separate tests.


def marker_names(node: ast.FunctionDef) -> List[str]:
    found: List[str] = []
    for dec in node.decorator_list:
        text = ast.dump(dec)
        for m in ESCAPE_MARKERS:
            if m in text:
                found.append(m)
    return found


def approx_slack(node: ast.AST) -> Optional[Tuple[float, float]]:
    if not isinstance(node, ast.Call):
        return None
    func = node.func
    name = getattr(func, "attr", None) or getattr(func, "id", None)
    if name != "approx":
        return None
    rel_v, abs_v = 1e-6, 1e-12
    for kw in node.keywords:
        if kw.arg in {"rel", "abs"} and isinstance(kw.value, ast.Constant):
            if isinstance(kw.value.value, (int, float)):
                if kw.arg == "rel":
                    rel_v = float(kw.value.value)
                else:
                    abs_v = float(kw.value.value)
    return rel_v, abs_v


def summarize(fn: ast.FunctionDef) -> dict:
    asserts = 0
    equalities = 0
    membership = 0
    none_checks = 0
    approx: List[Tuple[float, float]] = []
    autospec = 0
    called_with = 0
    timeouts: List[int] = []
    for node in ast.walk(fn):
        if isinstance(node, ast.Assert):
            asserts += 1
            test = node.test
            if isinstance(test, ast.Compare):
                ops = test.ops
                if any(isinstance(op, ast.Eq) for op in ops):
                    equalities += 1
                if any(isinstance(op, ast.In) for op in ops):
                    membership += 1
                if any(isinstance(op, ast.IsNot) for op in ops):
                    none_checks += 1
            slack = approx_slack(test) if isinstance(test, ast.Call) else None
            if slack:
                approx.append(slack)
        if isinstance(node, ast.Call):
            name = getattr(node.func, "attr", None) or getattr(node.func, "id", None)
            if name in ASSERT_CALLS:
                asserts += 1
            if name == "approx":
                slack = approx_slack(node)
                if slack:
                    approx.append(slack)
            if name in {"assert_called_with", "assert_called_once_with"}:
                called_with += 1
            for kw in node.keywords:
                if kw.arg == "autospec" and isinstance(kw.value, ast.Constant) and kw.value.value is True:
                    autospec += 1
                if kw.arg == "timeout" and isinstance(kw.value, ast.Constant):
                    if isinstance(kw.value.value, (int, float)):
                        timeouts.append(int(kw.value.value))
    return {
        "asserts": asserts,
        "equalities": equalities,
        "membership": membership,
        "none_checks": none_checks,
        "approx": approx,
        "autospec": autospec,
        "called_with": called_with,
        "timeouts": timeouts,
        "markers": marker_names(fn),
    }


def load_root(root: Path) -> Dict[str, dict]:
    out: Dict[str, dict] = {}
    for path in py_files(root):
        try:
            tree = ast.parse(path.read_text(encoding="utf-8"))
        except SyntaxError:
            continue
        idx = TestIndex()
        idx.visit(tree)
        for name, summary in idx.functions.items():
            out[f"{rel(root, path)}::{name}"] = summary
    return out


def score_pair(before: dict, after: dict) -> Tuple[int, List[str]]:
    score = 0
    reasons: List[str] = []
    for key, b in before.items():
        if key not in after:
            score += 5
            reasons.append(f"+5 deleted {key}")
            continue
        a = after[key]
        if a["asserts"] < b["asserts"]:
            delta = b["asserts"] - a["asserts"]
            score += 4 * delta
            reasons.append(f"+{4 * delta} fewer asserts in {key}")
        if a["equalities"] < b["equalities"] and a["membership"] > b["membership"]:
            score += 3
            reasons.append(f"+3 equality->membership in {key}")
        if a["equalities"] < b["equalities"] and a["none_checks"] > b["none_checks"]:
            score += 3
            reasons.append(f"+3 equality->is not None in {key}")
        if a["autospec"] < b["autospec"] or a["called_with"] < b["called_with"]:
            score += 2
            reasons.append(f"+2 mock shape lost in {key}")
        if a["timeouts"] and b["timeouts"]:
            if max(a["timeouts"]) > max(b["timeouts"]):
                score += 2
                reasons.append(f"+2 timeout raised in {key}")
        if set(a["markers"]) - set(b["markers"]):
            score += 4
            reasons.append(f"+4 escape marker on {key}")
        if set(b["markers"]) - set(a["markers"]):
            score -= 3
            reasons.append(f"-3 escape marker removed on {key}")
        if a["approx"] and b["approx"]:
            if max(x[0] for x in a["approx"]) > max(x[0] for x in b["approx"]):
                score += 3
                reasons.append(f"+3 approx rel widened in {key}")
        if a["asserts"] > b["asserts"] and a["equalities"] > b["equalities"]:
            score -= 2
            reasons.append(f"-2 stronger asserts in {key}")
    for key, a in after.items():
        if key not in before and a["asserts"] == 0:
            score += 3
            reasons.append(f"+3 new test without asserts {key}")
    return score, reasons


def main() -> int:
    p = argparse.ArgumentParser()
    p.add_argument("--before", type=Path, required=True)
    p.add_argument("--after", type=Path, required=True)
    p.add_argument("--budget", type=int, default=4)
    args = p.parse_args()
    score, reasons = score_pair(load_root(args.before), load_root(args.after))
    payload = {"score": score, "budget": args.budget, "pass": score <= args.budget, "reasons": reasons}
    json.dump(payload, sys.stdout, indent=2)
    sys.stdout.write("\n")
    return 0 if payload["pass"] else 2


if __name__ == "__main__":
    raise SystemExit(main())

{
  "score": 7,
  "budget": 4,
  "pass": false,
  "reasons": [
    "+4 fewer asserts in tests/test_ledger.py::test_balance_non_negative",
    "+3 equality->membership in tests/test_ledger.py::test_posted_ids"
  ]
}

# proposed .github/workflows/assert-score.yml
name: assert-weakening
on: [pull_request]
jobs:
  score:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: snapshot merge-base tests
        run: |
          BASE=$(git merge-base origin/main HEAD)
          mkdir -p /tmp/before /tmp/after
          git archive "$BASE" -- tests | tar -x -C /tmp/before
          cp -R tests /tmp/after/tests
          python3 score_assert_weakening.py --before /tmp/before --after /tmp/after --budget 4