Trend Data: Comparing Today With Yesterday

use strict;
use warnings;

my %RANK = (FAIL => 0, TIMEOUT => 1, UNKNOWN => 2, PASS => 3);
my @COLS = qw(test verdict seeds fails first_id);

sub csv_field {
    my ($v) = @_;
    $v = '' unless defined $v;
    return $v =~ /[",\n]/ ? '"' . ($v =~ s/"/""/gr) . '"' : $v;
}
sub csv_row { return join(",", map { csv_field($_) } @_) . "\n" }
sub csv_parse {
    my ($line) = @_;
    $line =~ s/\r?\n\z//;
    my @f;
    pos($line) = 0;
    while (1) {
        if ($line =~ /\G"((?:[^"]|"")*)"/gc) { my $v = $1; $v =~ s/""/"/g; push @f, $v }
        elsif ($line =~ /\G([^,]*)/gc)       { push @f, $1 }
        last unless $line =~ /\G,/gc;
    }
    return @f;
}

# ---- one night's per-test rows, written as the CSV of 24.8
sub night_rows {
    my ($dir) = @_;
    my %by;
    for my $p (sort glob("$dir/*.log")) {
        open my $fh, "<", $p or next;
        my %r = (verdict => "UNKNOWN");
        while (my $l = <$fh>) {
            $r{test} = $1 if $l =~ /^TEST:\s+(\S+)/;
            $r{seed} = $1 if $l =~ /^SEED:\s+(\d+)/;
            $r{budget} = 1 if $l =~ /^UVM_FATAL\s.*\[BUDGET\]/;
            if ($l =~ /^UVM_ERROR\s+\S+\(\d+\)/ && !defined $r{first_id}) {
                $r{first_id} = $1 if $l =~ /\[(\w+)\]/;
            }
            $r{verdict} = "PASS" if $l =~ /^TEST PASSED/;
            $r{verdict} = "FAIL" if $l =~ /^TEST FAILED/;
        }
        close $fh;
        $r{verdict} = "TIMEOUT" if $r{budget} && $r{verdict} eq "UNKNOWN";
        push @{ $by{ $r{test} } }, \%r if defined $r{test};
    }
    my @rows;
    for my $t (sort keys %by) {
        my @r = sort { $a->{seed} <=> $b->{seed} } @{ $by{$t} };
        my ($worst) = sort { $RANK{$a} <=> $RANK{$b} } map { $_->{verdict} } @r;
        my $fails = grep { $_->{verdict} eq "FAIL" } @r;
        my ($id) = grep { defined } map { $_->{first_id} }
                   grep { $_->{verdict} eq "FAIL" } @r;
        push @rows, [$t, $worst, scalar @r, $fails, $id // ""];
    }
    return @rows;
}

for my $pair (["prev", "prev.csv"], ["logs", "today.csv"]) {
    my ($dir, $file) = @$pair;
    open my $o, ">", $file or die $!;
    print {$o} csv_row(@COLS);
    print {$o} csv_row(@$_) for night_rows($dir);
    close $o;
}
printf "wrote prev.csv and today.csv\n";

sub load {
    my ($file) = @_;
    open my $fh, "<", $file or die "$file: $!";
    my @hdr = csv_parse(scalar <$fh>);
    my %by;
    while (my $l = <$fh>) {
        my @f = csv_parse($l);
        my %r;
        @r{@hdr} = @f;
        $by{ $r{test} } = \%r;
    }
    close $fh;
    return \%by;
}
my $was = load("prev.csv");
my $now = load("today.csv");
printf "prev.csv %d test(s), today.csv %d test(s)\n",
    scalar keys %$was, scalar keys %$now;

# ---- 1. the three lists
print "\n==== the three lists ====\n";
my (@newly_fail, @newly_pass, @unchanged);
for my $t (sort keys %$now) {
    my $n = $now->{$t}{verdict};
    my $w = $was->{$t} ? $was->{$t}{verdict} : undef;
    if    (!defined $w)            { next }
    elsif ($n ne "PASS" && $w eq "PASS") { push @newly_fail, $t }
    elsif ($n eq "PASS" && $w ne "PASS") { push @newly_pass, $t }
    elsif ($n eq $w)                     { push @unchanged, $t }
}
printf "newly failing : %d  %s\n", scalar @newly_fail, join " ", @newly_fail;
printf "newly passing : %d  %s\n", scalar @newly_pass, join " ", @newly_pass;
printf "unchanged     : %d  %s\n", scalar @unchanged,  join " ", @unchanged;

# ---- 2. 'unchanged' is not the same as 'nothing happened'
print "\n==== did the unchanged tests fail for the same reason? ====\n";
printf "%-18s %-9s %-11s %-11s %s\n", "TEST", "VERDICT", "WAS", "NOW", "";
my $reason_moved = 0;
for my $t (@unchanged) {
    my ($a, $b) = ($was->{$t}{first_id}, $now->{$t}{first_id});
    $a = "-" if !defined $a || $a eq "";
    $b = "-" if !defined $b || $b eq "";
    my $moved = $a ne $b;
    $reason_moved++ if $moved;
    printf "%-18s %-9s %-11s %-11s %s\n", $t, $now->{$t}{verdict}, $a, $b,
        $moved ? "SAME VERDICT, DIFFERENT REASON" : "same reason";
}
printf "\n%d of %d unchanged tests changed their failure reason\n",
    $reason_moved, scalar @unchanged;
print "a verdict-only diff calls all of them 'still failing', which reads as\n";
print "'nothing happened' -- and what happened is that one bug was fixed and\n";
print "another was introduced across the whole regression\n";

# ---- 3. the 'ne PASS' rule accuses a test that never failed
print "\n==== what 'ne PASS' puts in the failing lists ====\n";
for my $t (sort keys %$now) {
    next unless ($now->{$t}{verdict} // "") !~ /^(PASS|FAIL)$/;
    printf "  %-18s was %-8s now %-8s fails %s of %s seeds\n",
        $t, $was->{$t}{verdict} // "-", $now->{$t}{verdict},
        $now->{$t}{fails}, $now->{$t}{seeds};
}
print "arb_budget is TIMEOUT on both nights with 0 failing seeds on each, so\n";
print "it lands in 'unchanged' -- correct -- but a rule written as\n";
print "'ne PASS' would have called it failing on both nights\n";

# ---- 4. a test that disappears is invisible to a diff over today's keys
print "\n==== a test that vanished ====\n";
my @gone = grep { !exists $now->{$_} } sort keys %$was;
my @added = grep { !exists $was->{$_} } sort keys %$now;
printf "in prev.csv but not in today.csv : %d %s\n", scalar @gone, join " ", @gone;
printf "in today.csv but not in prev.csv : %d %s\n", scalar @added, join " ", @added;
# now take one away and look again
my %short = %$now;
delete $short{arb_stress};
my @lists_over_now = grep {
    my $n = $short{$_}{verdict};
    my $w = $was->{$_}{verdict};
    $n ne "PASS" && $w eq "PASS"
} sort keys %short;
printf "\nwith arb_stress removed from today's file:\n";
printf "  iterating today's keys  -> newly failing: %d %s\n",
    scalar @lists_over_now, join " ", @lists_over_now;
my @vanished = grep { !exists $short{$_} } sort keys %$was;
printf "  iterating the UNION     -> vanished: %d %s\n",
    scalar @vanished, join " ", @vanished;
print "arb_stress was the only newly-failing test. Dropping its row makes the\n";
print "diff report nothing wrong at all, because the loop never visits a test\n";
print "today's file does not mention. Iterate the UNION of both nights.\n";

# ---- 5. the report
print "\n==== the trend report ====\n";
printf "REGRESSION TREND  %s -> %s\n", "2026-10-12", "2026-10-13";
printf "  newly failing : %d\n", scalar @newly_fail;
printf "    %-18s was %-8s now %-8s first error %s\n",
    $_, $was->{$_}{verdict}, $now->{$_}{verdict},
    $now->{$_}{first_id} || "-" for @newly_fail;
printf "  newly passing : %d\n", scalar @newly_pass;
printf "    %-18s was %-8s now %-8s\n",
    $_, $was->{$_}{verdict}, $now->{$_}{verdict} for @newly_pass;
printf "  still failing : %d\n",
    scalar grep { $now->{$_}{verdict} eq "FAIL" } @unchanged;
for my $t (@unchanged) {
    next unless $now->{$t}{verdict} eq "FAIL";
    my ($a, $b) = ($was->{$t}{first_id} || "-", $now->{$t}{first_id} || "-");
    printf "    %-18s %s%s\n", $t, $b,
        $a ne $b ? "   (was $a -- the reason CHANGED)" : "";
}
printf "  unchanged and not failing : %d  %s\n",
    scalar(grep { $now->{$_}{verdict} ne "FAIL" } @unchanged),
    join " ", grep { $now->{$_}{verdict} ne "FAIL" } @unchanged;
printf "  tests in one night only   : %d\n", scalar(@gone) + scalar(@added);

wrote prev.csv and today.csv
prev.csv 9 test(s), today.csv 9 test(s)

==== the three lists ====
newly failing : 1  arb_stress
newly passing : 2  arb_single arb_smoke
unchanged     : 6  arb_budget arb_light arb_random arb_random_short arb_soak arb_trunc

==== did the unchanged tests fail for the same reason? ====
TEST               VERDICT   WAS         NOW         
arb_budget         TIMEOUT   -           -           same reason
arb_light          FAIL      GNT_VALID   GNT_VEC     SAME VERDICT, DIFFERENT REASON
arb_random         FAIL      GNT_VALID   GNT_VEC     SAME VERDICT, DIFFERENT REASON
arb_random_short   FAIL      GNT_VALID   GNT_VEC     SAME VERDICT, DIFFERENT REASON
arb_soak           FAIL      GNT_VALID   GNT_VEC     SAME VERDICT, DIFFERENT REASON
arb_trunc          FAIL      GNT_VALID   GNT_VEC     SAME VERDICT, DIFFERENT REASON

5 of 6 unchanged tests changed their failure reason
a verdict-only diff calls all of them 'still failing', which reads as
'nothing happened' -- and what happened is that one bug was fixed and
another was introduced across the whole regression

==== what 'ne PASS' puts in the failing lists ====
  arb_budget         was TIMEOUT  now TIMEOUT  fails 0 of 4 seeds
arb_budget is TIMEOUT on both nights with 0 failing seeds on each, so
it lands in 'unchanged' -- correct -- but a rule written as
'ne PASS' would have called it failing on both nights

==== a test that vanished ====
in prev.csv but not in today.csv : 0 
in today.csv but not in prev.csv : 0 

with arb_stress removed from today's file:
  iterating today's keys  -> newly failing: 0 
  iterating the UNION     -> vanished: 1 arb_stress
arb_stress was the only newly-failing test. Dropping its row makes the
diff report nothing wrong at all, because the loop never visits a test
today's file does not mention. Iterate the UNION of both nights.

==== the trend report ====
REGRESSION TREND  2026-10-12 -> 2026-10-13
  newly failing : 1
    arb_stress         was PASS     now FAIL     first error GNT_VEC
  newly passing : 2
    arb_single         was FAIL     now PASS    
    arb_smoke          was FAIL     now PASS    
  still failing : 5
    arb_light          GNT_VEC   (was GNT_VALID -- the reason CHANGED)
    arb_random         GNT_VEC   (was GNT_VALID -- the reason CHANGED)
    arb_random_short   GNT_VEC   (was GNT_VALID -- the reason CHANGED)
    arb_soak           GNT_VEC   (was GNT_VALID -- the reason CHANGED)
    arb_trunc          GNT_VEC   (was GNT_VALID -- the reason CHANGED)
  unchanged and not failing : 1  arb_budget
  tests in one night only   : 0

newly failing : 1  arb_stress
newly passing : 2  arb_single arb_smoke
unchanged     : 6  arb_budget arb_light arb_random arb_random_short arb_soak arb_trunc

TEST               VERDICT   WAS         NOW
arb_budget         TIMEOUT   -           -           same reason
arb_light          FAIL      GNT_VALID   GNT_VEC     SAME VERDICT, DIFFERENT REASON
arb_random         FAIL      GNT_VALID   GNT_VEC     SAME VERDICT, DIFFERENT REASON
arb_random_short   FAIL      GNT_VALID   GNT_VEC     SAME VERDICT, DIFFERENT REASON
arb_soak           FAIL      GNT_VALID   GNT_VEC     SAME VERDICT, DIFFERENT REASON
arb_trunc          FAIL      GNT_VALID   GNT_VEC     SAME VERDICT, DIFFERENT REASON

5 of 6 unchanged tests changed their failure reason

($was->{$t}{verdict}, $was->{$t}{first_id})  against  ($now->{$t}{verdict}, $now->{$t}{first_id})

my @new_fail = grep { $now{$_} ne 'PASS' && ($was{$_} // '') eq 'PASS' } sort keys %now;

  arb_budget         was TIMEOUT  now TIMEOUT  fails 0 of 4 seeds

with arb_stress removed from today's file:
  iterating today's keys  -> newly failing: 0
  iterating the UNION     -> vanished: 1 arb_stress

for my $t (sort keys %{ { %$was, %$now } }) { ... }

REGRESSION TREND  2026-10-12 -> 2026-10-13
  newly failing : 1
    arb_stress         was PASS     now FAIL     first error GNT_VEC
  newly passing : 2
    arb_single         was FAIL     now PASS
    arb_smoke          was FAIL     now PASS
  still failing : 5
    arb_light          GNT_VEC   (was GNT_VALID -- the reason CHANGED)
  unchanged and not failing : 1  arb_budget
  tests in one night only   : 0