What a Regression Must Guarantee

GUARANTEES = [
    ("every job accounted for",
     "the summary's parts add up to the number of jobs submitted"),
    ("every failure reproducible",
     "each failing result carries the one command that re-runs it"),
    ("a summary wrong in neither direction",
     "it must not flatter the design, and must not blame it unfairly"),
]

for name, what in GUARANTEES:
    print(f"{name:<38} {what}")

print()
print("none of the three is about speed")
print("all three are about whether the number at the top can be believed")

every job accounted for                the summary's parts add up to the number of jobs submitted
every failure reproducible             each failing result carries the one command that re-runs it
a summary wrong in neither direction   it must not flatter the design, and must not blame it unfairly

none of the three is about speed
all three are about whether the number at the top can be believed

WAYS = [
    ("the machine running the job was rebooted", "no result was ever written"),
    ("the scheduler killed the job for exceeding a limit", "the runner never returned"),
    ("the result file was written to a shared path two jobs used", "one overwrote the other"),
    ("the parent crashed while collecting", "results after that point were lost"),
    ("the job was never submitted", "a filter dropped it silently"),
    ("the result file is corrupt", "json.loads raised, the exception was caught and logged"),
]

print(f"{'what happened':<52} {'consequence'}")
for what, consequence in WAYS:
    print(f"{what:<52} {consequence}")

print()
print("ways a job goes missing          :", len(WAYS))
print("of which the LOOP would notice   :", 0)
print("of which a COUNT would notice    :", len(WAYS))

what happened                                        consequence
the machine running the job was rebooted             no result was ever written
the scheduler killed the job for exceeding a limit   the runner never returned
the result file was written to a shared path two jobs used one overwrote the other
the parent crashed while collecting                  results after that point were lost
the job was never submitted                          a filter dropped it silently
the result file is corrupt                           json.loads raised, the exception was caught and logged

ways a job goes missing          : 6
of which the LOOP would notice   : 0
of which a COUNT would notice    : 6

Summary(total, passed, failed, timeout, infra_error, not_run)

from dataclasses import dataclass


@dataclass(frozen=True)
class Summary:
    total: int
    passed: int
    failed: int
    timeout: int
    infra_error: int
    not_run: int

    @property
    def accounted(self):
        return self.passed + self.failed + self.timeout + self.infra_error + self.not_run

    @property
    def balanced(self):
        return self.total == self.accounted


balanced = Summary(total=400, passed=380, failed=12, timeout=3, infra_error=5, not_run=0)
lost = Summary(total=400, passed=380, failed=12, timeout=3, infra_error=0, not_run=0)

for name, s in (("balanced", balanced), ("five jobs lost", lost)):
    print(f"{name:<16} total={s.total} accounted={s.accounted} balanced={s.balanced}")

balanced         total=400 accounted=400 balanced=True
five jobs lost   total=400 accounted=395 balanced=False

from dataclasses import dataclass


@dataclass(frozen=True)
class NoNotRun:
    passed: int
    failed: int
    timeout: int
    infra_error: int


naive = NoNotRun(passed=380, failed=12, timeout=3, infra_error=0)
parts = naive.passed + naive.failed + naive.timeout + naive.infra_error

print("the parts sum to :", parts)
print("compared against :", "nothing — there is no total")
print()
print("the report reads :", f"{naive.passed}/{parts}", f"= {100 * naive.passed / parts:.0f}%")
print("jobs submitted   :", 400)
print("jobs mentioned   :", parts)
print("silently absent  :", 400 - parts)

the parts sum to : 395
compared against : nothing — there is no total

the report reads : 380/395 = 96%
jobs submitted   : 400
jobs mentioned   : 395
silently absent  : 5

PASSED, TOTAL, INFRA, NOT_RUN = 380, 400, 5, 0
tested = TOTAL - INFRA - NOT_RUN

print("jobs submitted      :", TOTAL)
print("infrastructure errors:", INFRA)
print("jobs that tested the design:", tested)
print()
print("naive  passed/total :", f"{PASSED}/{TOTAL}", f"= {100 * PASSED / TOTAL:.1f}%")
print("honest passed/tested:", f"{PASSED}/{tested}", f"= {100 * PASSED / tested:.1f}%")
print("difference          :", f"{100 * PASSED / tested - 100 * PASSED / TOTAL:.2f} points")

jobs submitted      : 400
infrastructure errors: 5
jobs that tested the design: 395

naive  passed/total : 380/400 = 95.0%
honest passed/tested: 380/395 = 96.2%
difference          : 1.20 points

SCENARIOS = [
    ("count a lost job as nothing", "shrinks the denominator", "too optimistic"),
    ("count an infra error as a failure", "blames the design", "too pessimistic"),
    ("count a timeout as a pass", "hides a hang", "too optimistic"),
    ("count a compile error as 400 failures", "one cause, four hundred rows", "too pessimistic"),
    ("report only what arrived", "silently drops the missing", "too optimistic"),
]

print(f"{'mistake':<40} {'effect':<32} direction")
for mistake, effect, direction in SCENARIOS:
    print(f"{mistake:<40} {effect:<32} {direction}")

opt = sum(1 for *_, d in SCENARIOS if d == "too optimistic")
print()
print("too optimistic :", opt)
print("too pessimistic:", len(SCENARIOS) - opt)
print("which is worse :", "the optimistic ones — nobody investigates a good number")

mistake                                  effect                           direction
count a lost job as nothing              shrinks the denominator          too optimistic
count an infra error as a failure        blames the design                too pessimistic
count a timeout as a pass                hides a hang                     too optimistic
count a compile error as 400 failures    one cause, four hundred rows     too pessimistic
report only what arrived                 silently drops the missing       too optimistic

too optimistic : 3
too pessimistic: 2
which is worse : the optimistic ones — nobody investigates a good number

RESULT_STATUSES = ("PASS", "FAIL", "COMPILE_ERROR", "NO_FINISH", "ERROR")
SUMMARY_FIELDS = ("passed", "failed", "timeout", "infra_error", "not_run")

print("Chapter 25.1 statuses:", RESULT_STATUSES)
print("summary count fields :", SUMMARY_FIELDS)
print()
print("same length:", len(RESULT_STATUSES) == len(SUMMARY_FIELDS))
print("same meanings:", False)
print()
for note in ("timeout promotes a *reason* to a column",
             "infra_error renames ERROR",
             "COMPILE_ERROR has no column of its own",
             "not_run corresponds to no status at all"):
    print("   " + note)

Chapter 25.1 statuses: ('PASS', 'FAIL', 'COMPILE_ERROR', 'NO_FINISH', 'ERROR')
summary count fields : ('passed', 'failed', 'timeout', 'infra_error', 'not_run')

same length: True
same meanings: False

   timeout promotes a *reason* to a column
   infra_error renames ERROR
   COMPILE_ERROR has no column of its own
   not_run corresponds to no status at all

from collections import Counter

STATUS_TO_FIELD = {
    "PASS": "passed",
    "FAIL": "failed",
    "ERROR": "infra_error",
    "COMPILE_ERROR": "infra_error",
}


def to_field(status, reason=None):
    if status == "NO_FINISH":
        return "timeout" if reason == "TIMEOUT" else "failed"
    try:
        return STATUS_TO_FIELD[status]
    except KeyError:
        raise ValueError(f"no summary column for status {status!r}") from None


CASES = [("PASS", None), ("FAIL", "UVM_ERROR"), ("NO_FINISH", "TIMEOUT"),
         ("NO_FINISH", "NO_END_MARKER"), ("ERROR", "no licence"),
         ("COMPILE_ERROR", None)]

for status, reason in CASES:
    print(f"{status:<14} reason={str(reason):<15} -> {to_field(status, reason)}")

print()
print("every status is mapped:", sorted({to_field(s, r) for s, r in CASES}))
try:
    to_field("MAYBE")
except ValueError as exc:
    print("unknown status      :", exc)

PASS           reason=None            -> passed
FAIL           reason=UVM_ERROR       -> failed
NO_FINISH      reason=TIMEOUT         -> timeout
NO_FINISH      reason=NO_END_MARKER   -> failed
ERROR          reason=no licence      -> infra_error
COMPILE_ERROR  reason=None            -> infra_error

every status is mapped: ['failed', 'infra_error', 'passed', 'timeout']
unknown status      : no summary column for status 'MAYBE'

from dataclasses import dataclass
from typing import Optional


@dataclass(frozen=True)
class JobResult:
    test: str
    seed: int
    sim: str
    status: str
    reason: Optional[str] = None
    run_dir: Optional[str] = None

    @property
    def reproduce(self):
        return (f"flow run --test {self.test} --seed {self.seed} "
                f"--sim {self.sim} --waves")


FAILURES = [
    JobResult("axi_burst", 1839274, "vcs", "FAIL", "UVM_ERROR", "runs/axi_burst_1839274"),
    JobResult("axi_hang", 771002, "vcs", "NO_FINISH", "TIMEOUT", "runs/axi_hang_771002"),
]

for f in FAILURES:
    print(f"{f.test:<11} {f.status:<10} {f.reproduce}")

print()
print("every argument comes from the record:", True)
print("nothing needs the regression's own state:", True)
print("usable a month later:", True)

axi_burst   FAIL       flow run --test axi_burst --seed 1839274 --sim vcs --waves
axi_hang    NO_FINISH  flow run --test axi_hang --seed 771002 --sim vcs --waves

every argument comes from the record: True
nothing needs the regression's own state: True
usable a month later: True

OWNS = [
    "expanding the suite into a concrete job list, before anything runs",
    "giving every job its own directory",
    "bounding every job and deciding what may be retried",
    "collecting a result for every job, or recording that none arrived",
    "reconciling the counts against the job list",
    "producing one summary and one bucketed failure report",
]
DELEGATES = [
    ("running one test", "Module 25's run_test"),
    ("deciding a status", "Chapter 25.6's classifier"),
    ("parsing a log", "Module 20"),
    ("the environment", "Module 24"),
    ("running jobs at the same time", "Module 27"),
    ("owning the CPU", "the scheduler — Chapter 26.9"),
]

print("the regression owns:")
for item in OWNS:
    print("   " + item)
print()
print("and delegates:")
for what, to in DELEGATES:
    print(f"   {what:<32} {to}")

the regression owns:
   expanding the suite into a concrete job list, before anything runs
   giving every job its own directory
   bounding every job and deciding what may be retried
   collecting a result for every job, or recording that none arrived
   reconciling the counts against the job list
   producing one summary and one bucketed failure report

and delegates:
   running one test                 Module 25's run_test
   deciding a status                Chapter 25.6's classifier
   parsing a log                    Module 20
   the environment                  Module 24
   running jobs at the same time    Module 27
   owning the CPU                   the scheduler — Chapter 26.9

from dataclasses import dataclass, field
from typing import Optional, Tuple

SUMMARY_FIELDS = ("passed", "failed", "timeout", "infra_error", "not_run")


@dataclass(frozen=True)
class Summary:
    total: int
    passed: int = 0
    failed: int = 0
    timeout: int = 0
    infra_error: int = 0
    not_run: int = 0

    def __post_init__(self):
        if self.total < 0 or any(getattr(self, f) < 0 for f in SUMMARY_FIELDS):
            raise ValueError("counts must not be negative")
        if self.accounted != self.total:
            raise ValueError(
                f"summary does not balance: total={self.total} "
                f"but parts sum to {self.accounted}")

    @property
    def accounted(self):
        return sum(getattr(self, f) for f in SUMMARY_FIELDS)

    @property
    def tested(self):
        return self.total - self.infra_error - self.not_run

    @property
    def pass_rate(self):
        return None if self.tested == 0 else self.passed / self.tested


def run_regression(suite, *, sim, seeds, retries=0) -> Summary:
    """Expand, run, reconcile. Never raises for a failing test."""
    raise NotImplementedError("Chapters 26.2 to 26.9")


good = Summary(total=400, passed=380, failed=12, timeout=3, infra_error=5)
print("balanced   :", good.accounted == good.total)
print("tested     :", good.tested)
print("pass rate  :", f"{good.pass_rate:.3f}")
print("as a headline:", f"{good.passed}/{good.tested}")

try:
    Summary(total=400, passed=380, failed=12, timeout=3)
except ValueError as exc:
    print()
    print("guard      :", exc)

empty = Summary(total=0)
print()
print("empty regression pass rate:", empty.pass_rate, "(not a division by zero)")

balanced   : True
tested     : 395
pass rate  : 0.962
as a headline: 380/395

guard      : summary does not balance: total=400 but parts sum to 395

empty regression pass rate: None (not a division by zero)

from dataclasses import dataclass

FIELDS = ("passed", "failed", "timeout", "infra_error", "not_run")


@dataclass(frozen=True)
class S:
    total: int
    passed: int = 0
    failed: int = 0
    timeout: int = 0
    infra_error: int = 0
    not_run: int = 0

    @property
    def accounted(self):
        return sum(getattr(self, f) for f in FIELDS)

    @property
    def tested(self):
        return self.total - self.infra_error - self.not_run


a = S(10, passed=8, failed=2)
b = S(10, passed=8, failed=1, not_run=1)
c = S(10, passed=8, failed=1, infra_error=1)

# A
print("A:", a.accounted, a.accounted == a.total)

# B
print("B:", S(10, passed=8, failed=1).accounted)

# C
print("C:", b.tested, c.tested)

# D
print("D:", f"{a.passed}/{a.total}", f"{a.passed}/{a.tested}")

# E
print("E:", f"{c.passed / c.total:.2f}", f"{c.passed / c.tested:.2f}")

# F
print("F:", a == S(10, passed=8, failed=2))

# G
print("G:", S(0).accounted, S(0).tested)

# H
print("H:", sorted({a.accounted, b.accounted, c.accounted}))

A: 10 True
B: 9
C: 9 9
D: 8/10 8/10
E: 0.80 0.89
F: True
G: 0 0
H: [10]