Reviewing Maintainability and Tests

OBSERVATIONS = [
    ("the suite is green", "the code is right", "or the tests cannot fail"),
    ("there are 40 tests", "good coverage", "or 40 tests of one thing"),
    ("coverage is 95%", "most lines are checked", "lines RAN — 29.1 §5"),
    ("CI is green", "nothing regressed", "or nothing is asserted"),
]

print(f"{'what a reviewer sees':<24} {'what it suggests':<28} "
      f"what it is also consistent with")
for seen, suggests, also in OBSERVATIONS:
    print(f"{seen:<24} {suggests:<28} {also}")

print()
print("observations that distinguish")
print("  a good suite from a useless one:", 0)
print("what would                       :", "a run where the code is WRONG")
print()
print("so the review's measurement is   :", "break it and watch")

what a reviewer sees     what it suggests             what it is also consistent with
the suite is green       the code is right            or the tests cannot fail
there are 40 tests       good coverage                or 40 tests of one thing
coverage is 95%          most lines are checked       lines RAN — 29.1 §5
CI is green              nothing regressed            or nothing is asserted

observations that distinguish
  a good suite from a useless one: 0
what would                       : a run where the code is WRONG

so the review's measurement is   : break it and watch

# break one check on purpose — does the suite notice?

QUESTIONS = [
    ("can the parser be tested without the tool?", "imports", "§6"),
    ("can the tests fail when the code is wrong?", "mutations", "§3-§5"),
    ("could another engineer own this?", "proxies", "§7"),
]

print(f"{'the canonical question':<44} {'measured by':<12} where")
for q, how, where in QUESTIONS:
    print(f"{q:<44} {how:<12} {where}")

print()
print("the annotation names the second one:", True)
print("because it is                      :",
      "the only one that can be wrong while looking right")
print()
print("the first question is answered by  :", "reading the import list")
print("the third by                       :", "three counts")
print("the second only by                 :", "running the suite against a bug")

the canonical question                       measured by  where
can the parser be tested without the tool?   imports      §6
can the tests fail when the code is wrong?   mutations    §3-§5
could another engineer own this?             proxies      §7

the annotation names the second one: True
because it is                      : the only one that can be wrong while looking right

the first question is answered by  : reading the import list
the third by                       : three counts
the second only by                 : running the suite against a bug

import ast

SUITE_SRC = '''
import unittest


class T(unittest.TestCase):

    def test_pass(self):
        self.assertEqual(classify("UVM_INFO a\\nTEST DONE\\n"), "PASS")

    def test_fail(self):
        self.assertEqual(classify("UVM_ERROR x\\nTEST DONE\\n"), "FAIL")

    def test_smoke(self):
        classify("UVM_INFO a\\nTEST DONE\\n")

    def test_trivial(self):
        self.assertTrue(True)

    def test_runs_without_raising(self):
        try:
            classify("")
        except Exception:
            self.fail("raised")
'''


def audit_tests(src):
    """For each test method: how many assertions, and can any of them fail?"""
    tree = ast.parse(src)
    cls = next(n for n in tree.body if isinstance(n, ast.ClassDef))
    rows = []
    for m in (x for x in cls.body if isinstance(x, ast.FunctionDef)):
        asserts = [n for n in ast.walk(m)
                   if isinstance(n, ast.Call)
                   and isinstance(n.func, ast.Attribute)
                   and n.func.attr.startswith("assert")]
        const_only = bool(asserts) and all(
            all(isinstance(a, ast.Constant) for a in call.args)
            for call in asserts)
        rows.append({"name": m.name,
                     "asserts": len(asserts),
                     "names": [c.func.attr for c in asserts],
                     "constant_only": const_only})
    return rows


rows = audit_tests(SUITE_SRC)

print(f"{'test':<26} {'asserts':>8}  {'verdict':<22} assertion names")
cannot_fail = 0
for r in rows:
    if r["asserts"] == 0:
        verdict = "NO ASSERTION"
    elif r["constant_only"]:
        verdict = "constant arguments"
    else:
        verdict = "can fail"
    cannot_fail += verdict != "can fail"
    print(f"{r['name']:<26} {r['asserts']:>8}  {verdict:<22} {r['names']}")

print()
print("tests                 :", len(rows))
print("that can fail         :", len(rows) - cannot_fail)
print("that CANNOT fail      :", cannot_fail)
print()
print("and all five          :", "pass")

test                        asserts  verdict                assertion names
test_pass                         1  can fail               ['assertEqual']
test_fail                         1  can fail               ['assertEqual']
test_smoke                        0  NO ASSERTION           []
test_trivial                      1  constant arguments     ['assertTrue']
test_runs_without_raising         0  NO ASSERTION           []

tests                 : 5
that can fail         : 2
that CANNOT fail      : 3

and all five          : pass

SHAPES = [
    ("test_smoke", "calls the function and asserts nothing",
     "it fails only if the call RAISES", "assert the return value"),
    ("test_trivial", "assertTrue(True)",
     "it can never fail", "delete it, or assert the real thing"),
    ("test_runs_without_raising", "try/except + self.fail",
     "it fails only if the call RAISES",
     "it IS a real test — of not raising"),
]

print(f"{'test':<26} {'shape':<40} what it actually checks")
for name, shape, checks, _ in SHAPES:
    print(f"{name:<26} {shape:<40} {checks}")

print()
print("review comments:")
for name, _, _, comment in SHAPES:
    print(f"  {name:<26} {comment}")

print()
print("the third one is NOT a defect:", True)
print("it is a weak test with        :", "a legitimate purpose")
print("and the finding is            :",
      "it is the ONLY test of the empty-log path")

test                       shape                                    what it actually checks
test_smoke                 calls the function and asserts nothing   it fails only if the call RAISES
test_trivial               assertTrue(True)                         it can never fail
test_runs_without_raising  try/except + self.fail                   it fails only if the call RAISES

review comments:
  test_smoke                 assert the return value
  test_trivial               delete it, or assert the real thing
  test_runs_without_raising  it IS a real test — of not raising

the third one is NOT a defect: True
it is a weak test with        : a legitimate purpose
and the finding is            : it is the ONLY test of the empty-log path

import io
import unittest

SRC = '''
def classify(text):
    lines = text.splitlines()
    errors = sum(1 for l in lines
                 if l.startswith("UVM_ERROR") and "UVM_ERROR :" not in l)
    fatals = sum(1 for l in lines if l.startswith("UVM_FATAL"))
    done = any(l.strip() == "TEST DONE" for l in lines)
    if not done:
        return "TIMEOUT"
    if fatals:
        return "FATAL"
    if errors:
        return "FAIL"
    return "PASS"
'''

SUITE_SRC = '''
import unittest


class T(unittest.TestCase):

    def test_pass(self):
        self.assertEqual(classify("UVM_INFO a\\nTEST DONE\\n"), "PASS")

    def test_fail(self):
        self.assertEqual(classify("UVM_ERROR x\\nTEST DONE\\n"), "FAIL")

    def test_smoke(self):
        classify("UVM_INFO a\\nTEST DONE\\n")

    def test_trivial(self):
        self.assertTrue(True)

    def test_runs_without_raising(self):
        try:
            classify("")
        except Exception:
            self.fail("raised")
'''

MUTATIONS = [
    ("drop the TIMEOUT branch",
     '    if not done:\n        return "TIMEOUT"\n', ''),
    ("drop the FATAL branch", '    if fatals:\n        return "FATAL"\n', ''),
    ("errors counts 2 per line",
     'errors = sum(1 for l in lines', 'errors = sum(2 for l in lines'),
    ("drop the summary guard", ' and "UVM_ERROR :" not in l', ''),
    ("done uses a substring",
     'done = any(l.strip() == "TEST DONE" for l in lines)',
     'done = "TEST DONE" in text'),
    ("PASS and FAIL swapped",
     '    if errors:\n        return "FAIL"\n    return "PASS"\n',
     '    if errors:\n        return "PASS"\n    return "FAIL"\n'),
]


def suite_passes(code_src):
    ns = {}
    exec(compile(code_src, "m.py", "exec"), ns)
    exec(compile(SUITE_SRC, "t.py", "exec"), ns)
    suite = unittest.TestLoader().loadTestsFromTestCase(ns["T"])
    return unittest.TextTestRunner(stream=io.StringIO(),
                                   verbosity=0).run(suite).wasSuccessful()


baseline = suite_passes(SRC)
print("the baseline suite passes:", baseline)
assert baseline, "a failing baseline makes every mutation a false kill — 33.9 §9"

print()
print(f"{'mutation':<28} killed?")
killed = []
for label, old, new in MUTATIONS:
    assert old in SRC, label
    try:
        passed = suite_passes(SRC.replace(old, new, 1))
    except Exception:
        passed = False
    if not passed:
        killed.append(label)
    print(f"{label:<28} {'KILLED' if not passed else 'SURVIVED'}")

print()
print("mutations :", len(MUTATIONS))
print("killed    :", len(killed), killed)
print("kill rate :", f"{100 * len(killed) // len(MUTATIONS)}%")
print()
print("tests in the suite:", 5)
print("tests that can fail:", 2)
print("and the kill rate reflects:", "the second number, not the first")

the baseline suite passes: True

mutation                     killed?
drop the TIMEOUT branch      SURVIVED
drop the FATAL branch        SURVIVED
errors counts 2 per line     SURVIVED
drop the summary guard       SURVIVED
done uses a substring        SURVIVED
PASS and FAIL swapped        KILLED

mutations : 6
killed    : 1 ['PASS and FAIL swapped']
kill rate : 16%

tests in the suite: 5
tests that can fail: 2
and the kill rate reflects: the second number, not the first

def classify(text, per_line=1):
    """per_line=2 is the mutation: it counts two errors per error line."""
    lines = text.splitlines()
    errors = sum(per_line for l in lines
                 if l.startswith("UVM_ERROR") and "UVM_ERROR :" not in l)
    done = any(l.strip() == "TEST DONE" for l in lines)
    if not done:
        return {"status": "TIMEOUT", "errors": errors}
    return {"status": "FAIL" if errors else "PASS", "errors": errors}


LOG = "UVM_ERROR drv.sv(88) x\nTEST DONE\n"

for per_line in (1, 2):
    r = classify(LOG, per_line)
    print(f"  per_line={per_line}  status={r['status']:<6} "
          f"errors={r['errors']}")

print()
good = classify(LOG, 1)
mutant = classify(LOG, 2)
print("the status is the same  :", good["status"] == mutant["status"])
print("the error count differs :", good["errors"] != mutant["errors"])
print()
print("what the suite asserts  :", "the status")
print("what the mutation changes:", "the count")
print("so the mutation is      :", "invisible to every status assertion")
print()
print("the gap is not a missing test CASE:", True)
print("it is a missing                   :", "assertion SUBJECT")
print("and the fix is                    :",
      "assert the count as well as the verdict")

  per_line=1  status=FAIL   errors=1
  per_line=2  status=FAIL   errors=2

the status is the same  : True
the error count differs : True

what the suite asserts  : the status
what the mutation changes: the count
so the mutation is      : invisible to every status assertion

the gap is not a missing test CASE: True
it is a missing                   : assertion SUBJECT
and the fix is                    : assert the count as well as the verdict

SURVIVORS = [
    ("drop the TIMEOUT branch", "no input lacks TEST DONE",
     "add a fixture", "36.2's battery"),
    ("drop the FATAL branch", "no input has a UVM_FATAL",
     "add a fixture", "36.2's battery"),
    ("drop the summary guard", "no input has a UVM_ERROR : line",
     "add a fixture", "36.2's battery"),
    ("done uses a substring", "no input mentions TEST DONE in prose",
     "add a fixture", "36.2's battery"),
    ("errors counts 2 per line", "no test asserts the count",
     "add an ASSERTION", "a new subject"),
]

print(f"{'survivor':<28} {'why it survived':<36} {'fix':<16} from")
for label, why, fix, src in SURVIVORS:
    print(f"{label:<28} {why:<36} {fix:<16} {src}")

print()
fixtures = sum(1 for *_, f, _ in SURVIVORS if f == "add a fixture")
print("survivors needing a fixture  :", fixtures)
print("survivors needing an assertion:", len(SURVIVORS) - fixtures)
print()
print("four of five are fixed by     :", "the battery from 36.2")
print("which means the two reviews    ")
print("  compose                      :", True)

survivor                     why it survived                      fix              from
drop the TIMEOUT branch      no input lacks TEST DONE             add a fixture    36.2's battery
drop the FATAL branch        no input has a UVM_FATAL             add a fixture    36.2's battery
drop the summary guard       no input has a UVM_ERROR : line      add a fixture    36.2's battery
done uses a substring        no input mentions TEST DONE in prose add a fixture    36.2's battery
errors counts 2 per line     no test asserts the count            add an ASSERTION a new subject

survivors needing a fixture  : 4
survivors needing an assertion: 1

four of five are fixed by     : the battery from 36.2
which means the two reviews    
  compose                      : True

import ast

GOOD = '''"""flow.parse — text in, facts out."""
import re

SEVERITY = re.compile(r"^UVM_(ERROR|FATAL)\\b")


def classify(text):
    return "FAIL" if SEVERITY.search(text) else "PASS"
'''

BAD = '''"""flow.parse — text in, facts out."""
import os
import re
import subprocess

from flow.tools import run_sim

LICENCE = os.environ["LM_LICENSE_FILE"]
VERSION = subprocess.run(["iverilog", "-V"], capture_output=True).stdout


def classify(text):
    return "FAIL" if "UVM_ERROR" in text else "PASS"
'''

WORLD_MODULES = {"subprocess", "shutil", "socket", "requests"}
# Calls that are PURE and idiomatic at module level. A first version of this
# audit flagged `re.compile` as "work at import time", which is a false
# positive — and 36.1 §8's rule says a checker that over-claims is a checker
# that gets switched off.
PURE_AT_IMPORT = {"compile", "Path", "frozenset", "tuple", "dict", "set",
                  "dict.fromkeys", "namedtuple"}


def _impure_calls(value):
    out = []
    for s in ast.walk(value):
        if isinstance(s, ast.Call):
            name = (s.func.attr if isinstance(s.func, ast.Attribute)
                    else getattr(s.func, "id", "?"))
            if name not in PURE_AT_IMPORT:
                out.append(name)
    return out


def import_audit(src, package="flow"):
    tree = ast.parse(src)
    stdlib, world, intra = set(), set(), set()
    for n in ast.walk(tree):
        if isinstance(n, ast.Import):
            for a in n.names:
                (world if a.name in WORLD_MODULES else stdlib).add(a.name)
        elif isinstance(n, ast.ImportFrom) and n.module:
            if n.module.startswith(package):
                intra.add(n.module)
            elif n.module in WORLD_MODULES:
                world.add(n.module)
            else:
                stdlib.add(n.module)

    impure, env_reads = [], 0
    for n in tree.body:
        if not isinstance(n, ast.Assign):
            continue
        impure.extend(_impure_calls(n.value))
        for s in ast.walk(n.value):      # EVERY assign, not only ones with calls
            if isinstance(s, ast.Subscript) \
               and "environ" in ast.dump(s.value):
                env_reads += 1
    return {"stdlib": sorted(stdlib), "world": sorted(world),
            "intra": sorted(intra), "impure_at_import": sorted(set(impure)),
            "env_at_import": env_reads}


for label, src in (("testable", GOOD), ("not testable", BAD)):
    a = import_audit(src)
    print(f"--- {label}")
    for k in ("stdlib", "world", "intra", "impure_at_import",
              "env_at_import"):
        print(f"    {k:<18} {a[k]}")
    standalone = (not a["world"] and not a["intra"]
                  and not a["impure_at_import"] and a["env_at_import"] == 0)
    print(f"    importable alone   {standalone}")

print()
print("the BAD module cannot be imported without:",
      ["a licence variable", "the iverilog binary", "flow.tools"])
print("so its parser tests require              :", "the whole environment")
print("and 34.2 §4 measured                     :",
      "that a clean split makes this zero")

--- testable
    stdlib             ['re']
    world              []
    intra              []
    impure_at_import   []
    env_at_import      0
    importable alone   True
--- not testable
    stdlib             ['os', 're']
    world              ['subprocess']
    intra              ['flow.tools']
    impure_at_import   ['run']
    env_at_import      1
    importable alone   False

the BAD module cannot be imported without: ['a licence variable', 'the iverilog binary', 'flow.tools']
so its parser tests require              : the whole environment
and 34.2 §4 measured                     : that a clean split makes this zero

import ast

SPECIMEN = '''"""run_regress.py"""
import re

ROOT = "/proj/chip_a"
TESTS = ["a", "b"]
SEEDS = [1, 2]
THRESHOLD = 95


def helper(x):
    return x * 2


def classify(text):
    """Return a verdict for one log."""
    errors = 0
    for line in text.splitlines():
        if line.startswith("UVM_ERROR"):
            errors += 1
    if errors > THRESHOLD:
        return "MANY"
    return "FAIL" if errors else "PASS"


def report(rows):
    out = []
    for r in rows:
        for t in TESTS:
            for s in SEEDS:
                if r.get("test") == t and r.get("seed") == s:
                    out.append(f"{ROOT}/{t}_{s}")
    return out
'''

tree = ast.parse(SPECIMEN)
funcs = [n for n in tree.body if isinstance(n, ast.FunctionDef)]
globals_upper = {n.targets[0].id for n in tree.body
                 if isinstance(n, ast.Assign)
                 and isinstance(n.targets[0], ast.Name)
                 and n.targets[0].id.isupper()}


def reads_globals(fn, names):
    return sorted({n.id for n in ast.walk(fn)
                   if isinstance(n, ast.Name) and n.id in names
                   and isinstance(n.ctx, ast.Load)})


def max_depth(fn):
    def depth(node, d=0):
        best = d
        for child in ast.iter_child_nodes(node):
            inc = 1 if isinstance(child, (ast.For, ast.While, ast.If,
                                          ast.With, ast.Try)) else 0
            best = max(best, depth(child, d + inc))
        return best
    return depth(fn)


print(f"{'function':<12} {'lines':>6} {'depth':>6} {'docstring':>10}  "
      f"globals read")
for f in funcs:
    span = f.end_lineno - f.lineno + 1
    print(f"{f.name:<12} {span:>6} {max_depth(f):>6} "
          f"{str(ast.get_docstring(f) is not None):>10}  "
          f"{reads_globals(f, globals_upper)}")

documented = sum(1 for f in funcs if ast.get_docstring(f))
print()
print("functions              :", len(funcs))
print("with a docstring       :", documented)
print("docstring coverage     :", f"{100 * documented // len(funcs)}%")
print("module-level constants :", len(globals_upper))
print("longest function       :", max(f.end_lineno - f.lineno + 1
                                      for f in funcs), "lines")
print("deepest nesting        :", max(max_depth(f) for f in funcs))
print("functions reading a global:",
      sum(1 for f in funcs if reads_globals(f, globals_upper)))

function      lines  depth  docstring  globals read
helper            2      0      False  []
classify          9      2       True  ['THRESHOLD']
report            8      4      False  ['ROOT', 'SEEDS', 'TESTS']

functions              : 3
with a docstring       : 1
docstring coverage     : 33%
module-level constants : 4
longest function       : 9 lines
deepest nesting        : 4
functions reading a global: 2

import ast
import io
import unittest
from dataclasses import dataclass


@dataclass(frozen=True)
class Finding:
    area: str
    severity: str
    detail: str


WORLD_MODULES = {"subprocess", "shutil", "socket", "requests"}


def review_tests(suite_src):
    findings = []
    tree = ast.parse(suite_src)
    for cls in (n for n in tree.body if isinstance(n, ast.ClassDef)):
        for m in (x for x in cls.body if isinstance(x, ast.FunctionDef)):
            if not m.name.startswith("test"):
                continue
            calls = [n for n in ast.walk(m)
                     if isinstance(n, ast.Call)
                     and isinstance(n.func, ast.Attribute)
                     and n.func.attr.startswith("assert")]
            if not calls:
                findings.append(Finding(
                    "tests", "major",
                    f"{m.name} has no assertion: it fails only if the call "
                    f"raises"))
                continue
            if all(all(isinstance(a, ast.Constant) for a in c.args)
                   for c in calls):
                findings.append(Finding(
                    "tests", "blocker",
                    f"{m.name} asserts only constants: it cannot fail"))
    return findings


def review_imports(module_src, package="flow"):
    findings = []
    tree = ast.parse(module_src)
    for n in ast.walk(tree):
        if isinstance(n, ast.Import):
            for a in n.names:
                if a.name in WORLD_MODULES:
                    findings.append(Finding(
                        "testability", "major",
                        f"the parsing module imports {a.name}"))
        if isinstance(n, ast.ImportFrom) and n.module \
           and n.module.startswith(package) and "tools" in n.module:
            findings.append(Finding("testability", "blocker",
                                    f"the parsing module imports {n.module}"))
    for n in tree.body:
        if isinstance(n, ast.Assign):
            for s in ast.walk(n.value):
                if isinstance(s, ast.Subscript) \
                   and "environ" in ast.dump(s.value):
                    findings.append(Finding(
                        "testability", "blocker",
                        "a module-level os.environ[...] makes `import` fail "
                        "without that variable"))
                if isinstance(s, ast.Call):
                    findings.append(Finding(
                        "testability", "major",
                        f"line {n.lineno}: work is done at import time"))
    return findings


def review_maintainability(module_src):
    findings = []
    tree = ast.parse(module_src)
    funcs = [n for n in tree.body if isinstance(n, ast.FunctionDef)]
    uppers = {n.targets[0].id for n in tree.body
              if isinstance(n, ast.Assign)
              and isinstance(n.targets[0], ast.Name)
              and n.targets[0].id.isupper()}
    for f in funcs:
        reads = sorted({n.id for n in ast.walk(f)
                        if isinstance(n, ast.Name) and n.id in uppers
                        and isinstance(n.ctx, ast.Load)})
        if reads:
            findings.append(Finding(
                "maintainability", "major",
                f"{f.name} reads module globals {reads}: a hidden input"))
        span = f.end_lineno - f.lineno + 1
        if span > 40:
            findings.append(Finding("maintainability", "minor",
                                    f"{f.name} is {span} lines"))
    return findings


def mutation_review(src, suite_src, mutations):
    def passes(code):
        ns = {}
        exec(compile(code, "m.py", "exec"), ns)
        exec(compile(suite_src, "t.py", "exec"), ns)
        s = unittest.TestLoader().loadTestsFromTestCase(ns["T"])
        return unittest.TextTestRunner(stream=io.StringIO(),
                                       verbosity=0).run(s).wasSuccessful()

    if not passes(src):
        return [Finding("tests", "blocker",
                        "the baseline suite fails: no mutation score is "
                        "meaningful")], 0, 0
    killed = 0
    for label, old, new in mutations:
        try:
            if not passes(src.replace(old, new, 1)):
                killed += 1
        except Exception:
            killed += 1
    rate = 100 * killed // len(mutations)
    findings = []
    if rate < 70:
        findings.append(Finding("tests", "blocker",
                                f"mutation kill rate {rate}% "
                                f"({killed}/{len(mutations)})"))
    return findings, killed, len(mutations)


MODULE = '''"""flow.parse"""
import os
import re
import subprocess

THRESHOLD = 95
LICENCE = os.environ["LM_LICENSE_FILE"]


def classify(text):
    errors = sum(1 for l in text.splitlines()
                 if l.startswith("UVM_ERROR"))
    if errors > THRESHOLD:
        return "MANY"
    return "FAIL" if errors else "PASS"
'''

SUITE = '''
import unittest


class T(unittest.TestCase):

    def test_fail(self):
        self.assertEqual(classify("UVM_ERROR x\\n"), "FAIL")

    def test_smoke(self):
        classify("")

    def test_trivial(self):
        self.assertTrue(True)
'''

TESTABLE = 'def classify(text):\n    return "FAIL" if "UVM_ERROR" in text else "PASS"\n'
MUTATIONS = [
    ("count 2", 'sum(1 for l in', 'sum(2 for l in'),
    ("invert", 'return "FAIL" if "UVM_ERROR" in text else "PASS"',
     'return "PASS" if "UVM_ERROR" in text else "FAIL"'),
]

findings = (review_tests(SUITE) + review_imports(MODULE)
            + review_maintainability(MODULE))
mut_findings, killed, total = mutation_review(TESTABLE, SUITE, MUTATIONS)
findings += mut_findings

ORDER = {"blocker": 0, "major": 1, "minor": 2}
print("maintainability and test review")
for f in sorted(findings, key=lambda f: (ORDER[f.severity], f.area, f.detail)):
    print(f"  [{f.severity:<7}] {f.area:<16} {f.detail}")

print()
print("findings  :", len(findings))
for sev in ("blocker", "major", "minor"):
    n = sum(1 for f in findings if f.severity == sev)
    if n:
        print(f"  {sev:<8} {n}")
print()
print("mutation score:", f"{killed}/{total}")
print("areas         :", sorted({f.area for f in findings}))

maintainability and test review
  [blocker] testability      a module-level os.environ[...] makes `import` fail without that variable
  [blocker] tests            mutation kill rate 50% (1/2)
  [blocker] tests            test_trivial asserts only constants: it cannot fail
  [major  ] maintainability  classify reads module globals ['THRESHOLD']: a hidden input
  [major  ] testability      the parsing module imports subprocess
  [major  ] tests            test_smoke has no assertion: it fails only if the call raises

findings  : 6
  blocker  3
  major    3

mutation score: 1/2
areas         : ['maintainability', 'testability', 'tests']

import io
import unittest

STATE = {}


class Suite(unittest.TestCase):
    """Three tests. One of them needs another to have run first."""

    def test_a_prepares(self):
        STATE["ready"] = True
        self.assertTrue(STATE["ready"])

    def test_b_uses_it(self):
        self.assertTrue(STATE.get("ready", False))

    def test_c_independent(self):
        self.assertEqual(1 + 1, 2)


def run(order):
    STATE.clear()
    suite = unittest.TestSuite([Suite(name) for name in order])
    res = unittest.TextTestRunner(stream=io.StringIO(), verbosity=0).run(suite)
    return res.wasSuccessful(), len(res.failures), res.testsRun


ALPHA = ["test_a_prepares", "test_b_uses_it", "test_c_independent"]
ORDERS = [("alphabetical (the default)", ALPHA),
          ("reversed", list(reversed(ALPHA))),
          ("test_b alone", ["test_b_uses_it"])]

print(f"{'order':<28} {'tests':>5} {'ok':<6} failures")
for label, order in ORDERS:
    ok, failures, n = run(order)
    print(f"{label:<28} {n:>5} {str(ok):<6} {failures}")

print()
print("passes in the default order :", run(ALPHA)[0])
print("fails reversed              :", not run(list(reversed(ALPHA)))[0])
print("fails when run alone        :", not run(["test_b_uses_it"])[0])
print("lines of code changed       :", 0)

order                        tests ok     failures
alphabetical (the default)       3 True   0
reversed                         3 False  1
test_b alone                     1 False  1

passes in the default order : True
fails reversed              : True
fails when run alone        : True
lines of code changed       : 0

CHECKS = [
    ("run the whole suite", "the author's experience", "always done"),
    ("run each test by name", "order dependence", "the cheapest check"),
    ("run the suite reversed", "order dependence", "one line"),
    ("run from a clean checkout", "missing fixtures", "what CI does"),
    ("run with no licence and no network", "hidden environment needs",
     "what a new owner has"),
]

print(f"{'what the reviewer runs':<36} {'what it finds':<24} note")
for what, finds, note in CHECKS:
    print(f"{what:<36} {finds:<24} {note}")

print()
print("checks needing the code read:", 0)
print("checks the author has done  :", 1)
print()
print("order dependence looks like :", "fails alone, passes in the suite")
print("which is backwards because  :", "a smaller run is usually safer")
print()
print("so the review finding reads :", "test_b_uses_it fails when run by name")
print("and not                     :", "the tests share state")

what the reviewer runs               what it finds            note
run the whole suite                  the author's experience  always done
run each test by name                order dependence         the cheapest check
run the suite reversed               order dependence         one line
run from a clean checkout            missing fixtures         what CI does
run with no licence and no network   hidden environment needs what a new owner has

checks needing the code read: 0
checks the author has done  : 1

order dependence looks like : fails alone, passes in the suite
which is backwards because  : a smaller run is usually safer

so the review finding reads : test_b_uses_it fails when run by name
and not                     : the tests share state

import ast
import io
import unittest


def classify(text, per_line=1):
    errors = sum(per_line for l in text.splitlines()
                 if l.startswith("UVM_ERROR"))
    return "FAIL" if errors else "PASS"


# A
print("A:", classify("UVM_ERROR x\n", 1), classify("UVM_ERROR x\n", 2))

# B
SRC = "class T(unittest.TestCase):\n    def test_a(self):\n        pass\n"
tree = ast.parse("import unittest\n" + SRC)
cls = next(n for n in ast.walk(tree) if isinstance(n, ast.ClassDef))
m = cls.body[0]
asserts = [n for n in ast.walk(m) if isinstance(n, ast.Call)]
print("B:", len(asserts))


# C
class T(unittest.TestCase):
    def test_a(self):
        pass


res = unittest.TextTestRunner(stream=io.StringIO(), verbosity=0).run(
    unittest.TestLoader().loadTestsFromTestCase(T))
print("C:", res.testsRun, res.wasSuccessful())

# D
print("D:", 100 * 1 // 6)

A: FAIL FAIL
B: 0
C: 1 True
D: 16