DMA Integration

   n_full = L / M          full packets
   last   = L % M          then ONE more packet of this length

   total  = n_full + 1     always. For every L, including zero.

   L = 63   ->  one 63-byte packet.            Short: done.
   L = 64   ->  one 64-byte packet + a ZLP.    The ZLP is the terminator.
   L = 65   ->  one 64 + one 1-byte packet.    Short: done.
   L = 0    ->  a single ZLP.                  A real transfer.

   residual = length requested - bytes actually moved

   fewer bytes than asked for   ->  a residual. Normal.
   more bytes than a packet     ->  BABBLE. A buffer overrun.

// usb_dma_xfer -- the arithmetic that decides where a USB transfer ends, and
// the two ways a DMA engine gets it wrong.
//
// A DESCRIPTOR HAS A LENGTH. THE WIRE HAS PACKETS.
//
// Software hands the controller a buffer and a byte count. The controller
// has to turn that into a sequence of packets, none longer than the
// endpoint's maximum packet size, and the host has to be able to tell when
// the sequence has ended.
//
// The rule is not "send ceil(L/M) packets". It is:
//
//     n_full = L / M          full packets
//     last   = L % M          then ONE more packet of this length
//
// and `last` may be ZERO, in which case that final packet is a
// ZERO-LENGTH PACKET and it is not optional.
//
// WHY THE ZLP IS NOT OPTIONAL
//
// The receiver knows a transfer has ended when it sees a packet SHORTER
// than the maximum. That is the only in-band terminator there is. So if the
// byte count happens to be an exact multiple of the packet size, every
// packet is full, nothing is short, and the receiver has no way to know the
// transfer finished -- it waits for more.
//
//     L = 63   ->  one 63-byte packet.           Short: done.
//     L = 64   ->  one 64-byte packet + a ZLP.   The ZLP is the terminator.
//     L = 65   ->  one 64 + one 1-byte packet.   Short: done.
//     L = 0    ->  a single ZLP.                 A real transfer.
//
// The total is n_full + 1 packets, always, for every L including zero. An
// engine that emits ceil(L/M) is correct for every length except the exact
// multiples -- which is to say it works in testing and hangs on the one
// buffer size everybody actually uses, because everybody rounds buffers up
// to a power of two.
//
// AND IN THE OTHER DIRECTION, A SHORT PACKET ENDS IT EARLY
//
// On a receive, the host may simply send less than the descriptor asked
// for. The first short packet terminates the transfer and the difference is
// the RESIDUAL:
//
//     residual = length requested - bytes actually moved
//
// A residual is NOT an error. It is the normal way a variable-length
// response arrives, and an engine that reports it as an under-run produces
// one false error per short read -- which on a bulk pipe is most of them.
//
// THE COMPLETION CODE SAYS WHICH OF THE TWO HAPPENED
//
// Both endings are successes and they mean different things to the layer
// above, so they are reported separately: D_LENGTH means the full count
// moved, D_SHORT means the peer ended it early and there is a residual.
// Collapsing them into one "done" bit throws away the only information the
// driver needs to decide whether to issue another transfer.
module usb_dma_xfer #(
  parameter integer MAXP = 64      // endpoint maximum packet size
) (
  input  wire        clk,
  input  wire        rst_n,

  input  wire        start,        // a descriptor is being handed over
  input  wire [15:0] len,          // its byte count
  input  wire        dir,          // 0 = TX (device sends), 1 = RX (receives)

  input  wire        pkt_ack,      // the packet on the wire completed
  input  wire [7:0]  rx_len,       // RX only: how many bytes actually arrived
  input  wire        eot,

  output wire        busy,
  output wire        pkt_valid,    // a packet is offered this cycle
  output wire [7:0]  pkt_len,      // its length; 0 means a ZLP
  output wire        pkt_is_zlp,
  output wire [15:0] pkt_index,    // 0-based position in the sequence
  output wire [15:0] pkt_total,    // how many there will be: n_full + 1
  output wire [15:0] moved,
  output wire [15:0] residual,
  output wire        done_pulse,
  output wire [1:0]  done_code,
  output wire        err_pulse,
  output wire [1:0]  err_code,

  output reg [31:0] n_start,
  output reg [31:0] n_pkt,
  output reg [31:0] n_zlp,
  output reg [31:0] n_done_len,
  output reg [31:0] n_done_short,
  output reg [31:0] n_restart,
  output reg [31:0] n_ack_idle,
  output reg [31:0] n_babble
);

  // Named, not bare bits. `dir_r` appears in three conditions and a bare
  // `if (dir_r)` reads as "if direction", which is not a question. The two
  // values behave completely differently -- one of them can end a transfer
  // early -- so they get names.
  localparam DIR_TX = 1'b0,   // the device sends
             DIR_RX = 1'b1;   // the device receives

  localparam [1:0] D_NONE   = 2'd0,
                   D_LENGTH = 2'd1,   // the full count moved
                   D_SHORT  = 2'd2;   // the peer ended it early; see residual

  localparam [1:0] E_NONE    = 2'd0,
                   E_RESTART = 2'd1,  // a new descriptor while one is running
                   E_ACKIDLE = 2'd2,  // a packet completed with none offered
                   E_BABBLE  = 2'd3;  // the peer sent more than was asked for

  // Widths derived from the parameter's own type. A packet length needs
  // eight bits because MAXP is 64 and the count runs 0..MAXP inclusive --
  // not because 64 happens to fit in seven.
  localparam [15:0] MAXP16 = MAXP;
  localparam [7:0]  MAXP8  = MAXP;

  reg [15:0] len_r;       // the descriptor's byte count
  reg [15:0] idx_r;       // which packet we are on
  reg [15:0] tot_r;       // how many there will be
  reg [15:0] moved_r;
  reg        busy_r, dir_r;
  reg        dn_r, er_r;
  reg [1:0]  dc_r, ec_r;

  // ---- The sequence, computed once when the descriptor arrives. ----
  //
  // n_full + 1, for every length including zero. Writing it as ceil(L/M) is
  // correct for every length that is NOT an exact multiple of the packet
  // size, which is why it survives testing.
  function [15:0] total_pkts;
    input [15:0] l;
    begin
      total_pkts = (l / MAXP16) + 16'd1;
    end
  endfunction

  // The length of packet `i` of a transfer of `l` bytes: full until the
  // last, and the last is the remainder -- which is zero exactly when the
  // ZLP is required.
  function [7:0] pkt_len_of;
    input [15:0] l;
    input [15:0] i;
    begin
      if (i < (l / MAXP16)) pkt_len_of = MAXP8;
      else                  pkt_len_of = (l % MAXP16) & 16'h00FF;
    end
  endfunction

  assign busy       = busy_r;
  assign pkt_valid  = busy_r;
  assign pkt_len    = pkt_len_of(len_r, idx_r);
  assign pkt_is_zlp = busy_r && (pkt_len_of(len_r, idx_r) == 8'd0);
  assign pkt_index  = idx_r;
  assign pkt_total  = tot_r;
  assign moved      = moved_r;
  // A residual is what was asked for and did not arrive. It is meaningful
  // only once the transfer has ended, and it is not an error.
  assign residual   = len_r - moved_r;
  assign done_pulse = dn_r;
  assign done_code  = dc_r;
  assign err_pulse  = er_r;
  assign err_code   = ec_r;

  reg [15:0] len_n, idx_n, tot_n, moved_n;
  reg        busy_n, dir_n, dn_n, er_n;
  reg [1:0]  dc_n, ec_n;
  reg        st_n, pk_n, zl_n, dl_n, ds_n, rs_n, ai_n, bb_n;
  reg [7:0]  this_len;
  reg [15:0] take;

  always @* begin
    len_n   = len_r;
    idx_n   = idx_r;
    tot_n   = tot_r;
    moved_n = moved_r;
    busy_n  = busy_r;
    dir_n   = dir_r;
    dn_n    = 1'b0; er_n = 1'b0;
    dc_n    = D_NONE; ec_n = E_NONE;
    st_n = 1'b0; pk_n = 1'b0; zl_n = 1'b0; dl_n = 1'b0;
    ds_n = 1'b0; rs_n = 1'b0; ai_n = 1'b0; bb_n = 1'b0;
    this_len = pkt_len_of(len_r, idx_r);

    if (eot) begin
      // Nothing: the counters are the report.
    end else if (start) begin
      if (busy_r) begin
        // A second descriptor while one is still running. The hardware can
        // only track one, so the new one would silently replace the old and
        // the driver would wait for a completion that never comes.
        er_n = 1'b1; ec_n = E_RESTART;
        rs_n = 1'b1;
      end else begin
        len_n   = len;
        idx_n   = 16'd0;
        tot_n   = total_pkts(len);
        moved_n = 16'd0;
        busy_n  = 1'b1;
        dir_n   = dir;
        st_n    = 1'b1;
      end
    end else if (pkt_ack) begin
      if (!busy_r) begin
        // A packet completed with nothing offered. On real hardware this is
        // a controller and a driver that disagree about whose turn it is.
        er_n = 1'b1; ec_n = E_ACKIDLE;
        ai_n = 1'b1;
      end else begin
        pk_n = 1'b1;
        if (this_len == 8'd0) zl_n = 1'b1;

        // ---- How many bytes actually moved. ----
        //
        // On TX it is what we offered. On RX it is what arrived, which is
        // the peer's choice and may be less.
        take = (dir_r == DIR_RX) ? {8'd0, rx_len} : {8'd0, this_len};

        if ((dir_r == DIR_RX) && ({8'd0, rx_len} > {8'd0, this_len})) begin
          // The peer sent more than the packet could hold. This is babble,
          // and it is a real error rather than a short read: the extra bytes
          // went somewhere.
          er_n = 1'b1; ec_n = E_BABBLE;
          bb_n = 1'b1;
          take = {8'd0, this_len};
        end

        moved_n = moved_r + take;

        if ((dir_r == DIR_RX) && (take < {8'd0, this_len})) begin
          // ---- A SHORT packet on receive ends the transfer. ----
          //
          // Not an error. The peer had less to send than we asked for, which
          // is how every variable-length response arrives. The difference is
          // the residual and the driver needs it.
          busy_n = 1'b0;
          dn_n   = 1'b1; dc_n = D_SHORT;
          ds_n   = 1'b1;
        end else if (idx_r + 16'd1 >= tot_r) begin
          // ---- The sequence is complete. ----
          //
          // Including the case where the last packet was a ZLP, which is a
          // packet like any other and has to be acknowledged before the
          // transfer is done.
          busy_n = 1'b0;
          dn_n   = 1'b1; dc_n = D_LENGTH;
          dl_n   = 1'b1;
        end else begin
          idx_n = idx_r + 16'd1;
        end
      end
    end
  end

  always @(posedge clk or negedge rst_n) begin
    if (!rst_n) begin
      len_r   <= 16'd0;
      idx_r   <= 16'd0;
      tot_r   <= 16'd0;
      moved_r <= 16'd0;
      busy_r  <= 1'b0;
      dir_r   <= DIR_TX;
      dn_r    <= 1'b0; er_r <= 1'b0;
      dc_r    <= D_NONE; ec_r <= E_NONE;
      n_start      <= 32'd0;
      n_pkt        <= 32'd0;
      n_zlp        <= 32'd0;
      n_done_len   <= 32'd0;
      n_done_short <= 32'd0;
      n_restart    <= 32'd0;
      n_ack_idle   <= 32'd0;
      n_babble     <= 32'd0;
    end else begin
      len_r   <= len_n;
      idx_r   <= idx_n;
      tot_r   <= tot_n;
      moved_r <= moved_n;
      busy_r  <= busy_n;
      dir_r   <= dir_n;
      dn_r    <= dn_n; er_r <= er_n;
      dc_r    <= dc_n; ec_r <= ec_n;

      if (st_n) n_start      <= n_start      + 32'd1;
      if (pk_n) n_pkt        <= n_pkt        + 32'd1;
      if (zl_n) n_zlp        <= n_zlp        + 32'd1;
      if (dl_n) n_done_len   <= n_done_len   + 32'd1;
      if (ds_n) n_done_short <= n_done_short + 32'd1;
      if (rs_n) n_restart    <= n_restart    + 32'd1;
      if (ai_n) n_ack_idle   <= n_ack_idle   + 32'd1;
      if (bb_n) n_babble     <= n_babble     + 32'd1;
    end
  end
endmodule

// usb_dma_xfer -- the arithmetic that decides where a USB transfer ends, and
// the two ways a DMA engine gets it wrong.
//
// A DESCRIPTOR HAS A LENGTH. THE WIRE HAS PACKETS.
//
// Software hands the controller a buffer and a byte count. The controller
// has to turn that into a sequence of packets, none longer than the
// endpoint's maximum packet size, and the host has to be able to tell when
// the sequence has ended.
//
// The rule is not "send ceil(L/M) packets". It is:
//
//     n_full = L / M          full packets
//     last   = L % M          then ONE more packet of this length
//
// and `last` may be ZERO, in which case that final packet is a
// ZERO-LENGTH PACKET and it is not optional.
//
// WHY THE ZLP IS NOT OPTIONAL
//
// The receiver knows a transfer has ended when it sees a packet SHORTER
// than the maximum. That is the only in-band terminator there is. So if the
// byte count happens to be an exact multiple of the packet size, every
// packet is full, nothing is short, and the receiver has no way to know the
// transfer finished -- it waits for more.
//
//     L = 63   ->  one 63-byte packet.           Short: done.
//     L = 64   ->  one 64-byte packet + a ZLP.   The ZLP is the terminator.
//     L = 65   ->  one 64 + one 1-byte packet.   Short: done.
//     L = 0    ->  a single ZLP.                 A real transfer.
//
// The total is n_full + 1 packets, always, for every L including zero. An
// engine that emits ceil(L/M) is correct for every length except the exact
// multiples -- which is to say it works in testing and hangs on the one
// buffer size everybody actually uses, because everybody rounds buffers up
// to a power of two.
//
// AND IN THE OTHER DIRECTION, A SHORT PACKET ENDS IT EARLY
//
// On a receive, the host may simply send less than the descriptor asked
// for. The first short packet terminates the transfer and the difference is
// the RESIDUAL:
//
//     residual = length requested - bytes actually moved
//
// A residual is NOT an error. It is the normal way a variable-length
// response arrives, and an engine that reports it as an under-run produces
// one false error per short read -- which on a bulk pipe is most of them.
//
// THE COMPLETION CODE SAYS WHICH OF THE TWO HAPPENED
//
// Both endings are successes and they mean different things to the layer
// above, so they are reported separately: D_LENGTH means the full count
// moved, D_SHORT means the peer ended it early and there is a residual.
// Collapsing them into one "done" bit throws away the only information the
// driver needs to decide whether to issue another transfer.
package usb_dma_pkg;
  typedef enum logic {
    DIR_TX = 1'b0,    // the device sends
    DIR_RX = 1'b1     // the device receives
  } xfer_dir_e;

  // Both endings are SUCCESSES and they mean different things to the layer
  // above, so they are separate values rather than one "done" bit.
  typedef enum logic [1:0] {
    D_NONE   = 2'd0,
    D_LENGTH = 2'd1,  // the full count moved
    D_SHORT  = 2'd2   // the peer ended it early; see residual
  } done_code_e;

  typedef enum logic [1:0] {
    E_NONE    = 2'd0,
    E_RESTART = 2'd1, // a new descriptor while one is running
    E_ACKIDLE = 2'd2, // a packet completed with none offered
    E_BABBLE  = 2'd3  // the peer sent more than was asked for
  } dma_err_e;
endpackage

module usb_dma_xfer
  import usb_dma_pkg::*;
 #(
  parameter int MAXP = 64          // endpoint maximum packet size
) (
  input  logic        clk,
  input  logic        rst_n,

  input  logic        start,       // a descriptor is being handed over
  input  logic [15:0] len,         // its byte count
  input  xfer_dir_e   dir,

  input  logic        pkt_ack,     // the packet on the wire completed
  input  logic [7:0]  rx_len,      // RX only: how many bytes actually arrived
  input  logic        eot,

  output logic        busy,
  output logic        pkt_valid,   // a packet is offered this cycle
  output logic [7:0]  pkt_len,     // its length; 0 means a ZLP
  output logic        pkt_is_zlp,
  output logic [15:0] pkt_index,   // 0-based position in the sequence
  output logic [15:0] pkt_total,   // how many there will be: n_full + 1
  output logic [15:0] moved,
  output logic [15:0] residual,
  output logic        done_pulse,
  output done_code_e  done_code,
  output logic        err_pulse,
  output dma_err_e    err_code,

  output logic [31:0] n_start,
  output logic [31:0] n_pkt,
  output logic [31:0] n_zlp,
  output logic [31:0] n_done_len,
  output logic [31:0] n_done_short,
  output logic [31:0] n_restart,
  output logic [31:0] n_ack_idle,
  output logic [31:0] n_babble
);

  // Widths derived from the parameter's own type. A packet length needs
  // eight bits because MAXP is 64 and the count runs 0..MAXP inclusive --
  // not because 64 happens to fit in seven.
  localparam logic [15:0] MAXP16 = 16'(MAXP);
  localparam logic [7:0]  MAXP8  = 8'(MAXP);

  logic [15:0] len_r;     // the descriptor's byte count
  logic [15:0] idx_r;     // which packet we are on
  logic [15:0] tot_r;     // how many there will be
  logic [15:0] moved_r;
  logic        busy_r;
  xfer_dir_e   dir_r;
  logic        dn_r, er_r;
  done_code_e  dc_r;
  dma_err_e    ec_r;

  // ---- The sequence, computed once when the descriptor arrives. ----
  //
  // n_full + 1, for every length including zero. Writing it as ceil(L/M) is
  // correct for every length that is NOT an exact multiple of the packet
  // size, which is why it survives testing.
  function automatic logic [15:0] total_pkts(input logic [15:0] l);
    begin
      total_pkts = (l / MAXP16) + 16'd1;
    end
  endfunction

  // The length of packet `i` of a transfer of `l` bytes: full until the
  // last, and the last is the remainder -- which is zero exactly when the
  // ZLP is required.
  function automatic logic [7:0] pkt_len_of(input logic [15:0] l,
                                            input logic [15:0] i);
    begin
      if (i < (l / MAXP16)) pkt_len_of = MAXP8;
      else                  pkt_len_of = (l % MAXP16) & 16'h00FF;
    end
  endfunction

  assign busy       = busy_r;
  assign pkt_valid  = busy_r;
  assign pkt_len    = pkt_len_of(len_r, idx_r);
  assign pkt_is_zlp = busy_r && (pkt_len_of(len_r, idx_r) == 8'd0);
  assign pkt_index  = idx_r;
  assign pkt_total  = tot_r;
  assign moved      = moved_r;
  // A residual is what was asked for and did not arrive. It is meaningful
  // only once the transfer has ended, and it is not an error.
  assign residual   = len_r - moved_r;
  assign done_pulse = dn_r;
  assign done_code  = dc_r;
  assign err_pulse  = er_r;
  assign err_code   = ec_r;

  logic [15:0] len_n, idx_n, tot_n, moved_n;
  logic        busy_n, dn_n, er_n;
  xfer_dir_e   dir_n;
  done_code_e  dc_n;
  dma_err_e    ec_n;
  logic        st_n, pk_n, zl_n, dl_n, ds_n, rs_n, ai_n, bb_n;
  logic [7:0]  this_len;
  logic [15:0] take;

  always_comb begin
    len_n   = len_r;
    idx_n   = idx_r;
    tot_n   = tot_r;
    moved_n = moved_r;
    busy_n  = busy_r;
    dir_n   = dir_r;
    dn_n    = 1'b0; er_n = 1'b0;
    dc_n    = D_NONE; ec_n = E_NONE;
    st_n = 1'b0; pk_n = 1'b0; zl_n = 1'b0; dl_n = 1'b0;
    ds_n = 1'b0; rs_n = 1'b0; ai_n = 1'b0; bb_n = 1'b0;
    this_len = pkt_len_of(len_r, idx_r);

    if (eot) begin
      // Nothing: the counters are the report.
    end else if (start) begin
      if (busy_r) begin
        // A second descriptor while one is still running. The hardware can
        // only track one, so the new one would silently replace the old and
        // the driver would wait for a completion that never comes.
        er_n = 1'b1; ec_n = E_RESTART;
        rs_n = 1'b1;
      end else begin
        len_n   = len;
        idx_n   = 16'd0;
        tot_n   = total_pkts(len);
        moved_n = 16'd0;
        busy_n  = 1'b1;
        dir_n   = dir;
        st_n    = 1'b1;
      end
    end else if (pkt_ack) begin
      if (!busy_r) begin
        // A packet completed with nothing offered. On real hardware this is
        // a controller and a driver that disagree about whose turn it is.
        er_n = 1'b1; ec_n = E_ACKIDLE;
        ai_n = 1'b1;
      end else begin
        pk_n = 1'b1;
        if (this_len == 8'd0) zl_n = 1'b1;

        // ---- How many bytes actually moved. ----
        //
        // On TX it is what we offered. On RX it is what arrived, which is
        // the peer's choice and may be less.
        take = (dir_r == DIR_RX) ? {8'd0, rx_len} : {8'd0, this_len};

        if ((dir_r == DIR_RX) && ({8'd0, rx_len} > {8'd0, this_len})) begin
          // The peer sent more than the packet could hold. This is babble,
          // and it is a real error rather than a short read: the extra bytes
          // went somewhere.
          er_n = 1'b1; ec_n = E_BABBLE;
          bb_n = 1'b1;
          take = {8'd0, this_len};
        end

        moved_n = moved_r + take;

        if ((dir_r == DIR_RX) && (take < {8'd0, this_len})) begin
          // ---- A SHORT packet on receive ends the transfer. ----
          //
          // Not an error. The peer had less to send than we asked for, which
          // is how every variable-length response arrives. The difference is
          // the residual and the driver needs it.
          busy_n = 1'b0;
          dn_n   = 1'b1; dc_n = D_SHORT;
          ds_n   = 1'b1;
        end else if (idx_r + 16'd1 >= tot_r) begin
          // ---- The sequence is complete. ----
          //
          // Including the case where the last packet was a ZLP, which is a
          // packet like any other and has to be acknowledged before the
          // transfer is done.
          busy_n = 1'b0;
          dn_n   = 1'b1; dc_n = D_LENGTH;
          dl_n   = 1'b1;
        end else begin
          idx_n = idx_r + 16'd1;
        end
      end
    end
  end

  always_ff @(posedge clk or negedge rst_n) begin
    if (!rst_n) begin
      len_r   <= 16'd0;
      idx_r   <= 16'd0;
      tot_r   <= 16'd0;
      moved_r <= 16'd0;
      busy_r  <= 1'b0;
      dir_r   <= DIR_TX;
      dn_r    <= 1'b0; er_r <= 1'b0;
      dc_r    <= D_NONE; ec_r <= E_NONE;
      n_start      <= 32'd0;
      n_pkt        <= 32'd0;
      n_zlp        <= 32'd0;
      n_done_len   <= 32'd0;
      n_done_short <= 32'd0;
      n_restart    <= 32'd0;
      n_ack_idle   <= 32'd0;
      n_babble     <= 32'd0;
    end else begin
      len_r   <= len_n;
      idx_r   <= idx_n;
      tot_r   <= tot_n;
      moved_r <= moved_n;
      busy_r  <= busy_n;
      dir_r   <= dir_n;
      dn_r    <= dn_n; er_r <= er_n;
      dc_r    <= dc_n; ec_r <= ec_n;

      if (st_n) n_start      <= n_start      + 32'd1;
      if (pk_n) n_pkt        <= n_pkt        + 32'd1;
      if (zl_n) n_zlp        <= n_zlp        + 32'd1;
      if (dl_n) n_done_len   <= n_done_len   + 32'd1;
      if (ds_n) n_done_short <= n_done_short + 32'd1;
      if (rs_n) n_restart    <= n_restart    + 32'd1;
      if (ai_n) n_ack_idle   <= n_ack_idle   + 32'd1;
      if (bb_n) n_babble     <= n_babble     + 32'd1;
    end
  end
endmodule

-- usb_dma_xfer -- the arithmetic that decides where a USB transfer ends, and
-- the two ways a DMA engine gets it wrong.
--
-- A DESCRIPTOR HAS A LENGTH. THE WIRE HAS PACKETS.
--
-- Software hands the controller a buffer and a byte count. The controller
-- has to turn that into a sequence of packets, none longer than the
-- endpoint's maximum packet size, and the host has to be able to tell when
-- the sequence has ended.
--
-- The rule is not "send ceil(L/M) packets". It is:
--
--     n_full = L / M          full packets
--     last   = L mod M        then ONE more packet of this length
--
-- and `last` may be ZERO, in which case that final packet is a ZERO-LENGTH
-- PACKET and it is not optional.
--
-- WHY THE ZLP IS NOT OPTIONAL
--
-- The receiver knows a transfer has ended when it sees a packet SHORTER than
-- the maximum. That is the only in-band terminator there is. So if the byte
-- count happens to be an exact multiple of the packet size, every packet is
-- full, nothing is short, and the receiver has no way to know the transfer
-- finished -- it waits for more.
--
--     L = 63   ->  one 63-byte packet.           Short: done.
--     L = 64   ->  one 64-byte packet + a ZLP.   The ZLP is the terminator.
--     L = 65   ->  one 64 + one 1-byte packet.   Short: done.
--     L = 0    ->  a single ZLP.                 A real transfer.
--
-- The total is n_full + 1 packets, always, for every L including zero. An
-- engine that emits ceil(L/M) is correct for every length except the exact
-- multiples -- which is to say it works in testing and hangs on the one
-- buffer size everybody actually uses, because everybody rounds buffers up
-- to a power of two.
--
-- AND IN THE OTHER DIRECTION, A SHORT PACKET ENDS IT EARLY
--
-- On a receive, the host may simply send less than the descriptor asked for.
-- The first short packet terminates the transfer and the difference is the
-- RESIDUAL:
--
--     residual = length requested - bytes actually moved
--
-- A residual is NOT an error. It is the normal way a variable-length
-- response arrives, and an engine that reports it as an under-run produces
-- one false error per short read -- which on a bulk pipe is most of them.
--
-- THE COMPLETION CODE SAYS WHICH OF THE TWO HAPPENED
--
-- Both endings are successes and they mean different things to the layer
-- above, so they are reported separately: D_LENGTH means the full count
-- moved, D_SHORT means the peer ended it early and there is a residual.
library ieee;
use ieee.std_logic_1164.all;
use ieee.numeric_std.all;

package usb_dma_pkg is
  constant DIR_TX : std_logic := '0';   -- the device sends
  constant DIR_RX : std_logic := '1';   -- the device receives

  constant D_NONE   : std_logic_vector(1 downto 0) := "00";
  constant D_LENGTH : std_logic_vector(1 downto 0) := "01";
  constant D_SHORT  : std_logic_vector(1 downto 0) := "10";

  constant E_NONE    : std_logic_vector(1 downto 0) := "00";
  constant E_RESTART : std_logic_vector(1 downto 0) := "01";
  constant E_ACKIDLE : std_logic_vector(1 downto 0) := "10";
  constant E_BABBLE  : std_logic_vector(1 downto 0) := "11";
end package;

library ieee;
use ieee.std_logic_1164.all;
use ieee.numeric_std.all;
use work.usb_dma_pkg.all;

entity usb_dma_xfer is
  generic (
    MAXP : integer := 64          -- endpoint maximum packet size
  );
  port (
    clk     : in std_logic;
    rst_n   : in std_logic;

    start   : in std_logic;       -- a descriptor is being handed over
    len     : in unsigned(15 downto 0);
    dir     : in std_logic;       -- DIR_TX or DIR_RX

    pkt_ack : in std_logic;       -- the packet on the wire completed
    rx_len  : in unsigned(7 downto 0);
    eot     : in std_logic;

    busy       : out std_logic;
    pkt_valid  : out std_logic;
    pkt_len    : out unsigned(7 downto 0);   -- 0 means a ZLP
    pkt_is_zlp : out std_logic;
    pkt_index  : out unsigned(15 downto 0);
    pkt_total  : out unsigned(15 downto 0);  -- n_full + 1
    moved      : out unsigned(15 downto 0);
    residual   : out unsigned(15 downto 0);
    done_pulse : out std_logic;
    done_code  : out std_logic_vector(1 downto 0);
    err_pulse  : out std_logic;
    err_code   : out std_logic_vector(1 downto 0);

    n_start      : out unsigned(31 downto 0);
    n_pkt        : out unsigned(31 downto 0);
    n_zlp        : out unsigned(31 downto 0);
    n_done_len   : out unsigned(31 downto 0);
    n_done_short : out unsigned(31 downto 0);
    n_restart    : out unsigned(31 downto 0);
    n_ack_idle   : out unsigned(31 downto 0);
    n_babble     : out unsigned(31 downto 0)
  );
end entity;

architecture rtl of usb_dma_xfer is
  signal len_r, idx_r, tot_r, moved_r : unsigned(15 downto 0)
    := (others => '0');
  signal busy_r, dir_r, dn_r, er_r : std_logic := '0';
  signal dc_r : std_logic_vector(1 downto 0) := D_NONE;
  signal ec_r : std_logic_vector(1 downto 0) := E_NONE;

  signal st_c, pk_c, zl_c, dl_c : unsigned(31 downto 0) := (others => '0');
  signal ds_c, rs_c, ai_c, bb_c : unsigned(31 downto 0) := (others => '0');

  -- ---- The sequence, computed once when the descriptor arrives. ----
  --
  -- n_full + 1, for every length including zero. Writing it as ceil(L/M) is
  -- correct for every length that is NOT an exact multiple of the packet
  -- size, which is why it survives testing.
  function total_pkts (l : unsigned(15 downto 0)) return unsigned is
  begin
    return (l / to_unsigned(MAXP, 16)) + 1;
  end function;

  -- The length of packet `i` of a transfer of `l` bytes: full until the
  -- last, and the last is the remainder -- which is zero exactly when the
  -- ZLP is required.
  function pkt_len_of (l, i : unsigned(15 downto 0)) return unsigned is
    variable rem16 : unsigned(15 downto 0);
  begin
    if i < (l / to_unsigned(MAXP, 16)) then
      return to_unsigned(MAXP, 8);
    else
      rem16 := l mod to_unsigned(MAXP, 16);
      return rem16(7 downto 0);
    end if;
  end function;
begin
  busy       <= busy_r;
  pkt_valid  <= busy_r;
  pkt_len    <= pkt_len_of(len_r, idx_r);
  pkt_is_zlp <= '1' when (busy_r = '1') and (pkt_len_of(len_r, idx_r) = 0)
                else '0';
  pkt_index  <= idx_r;
  pkt_total  <= tot_r;
  moved      <= moved_r;
  -- A residual is what was asked for and did not arrive. It is meaningful
  -- only once the transfer has ended, and it is not an error.
  residual   <= len_r - moved_r;
  done_pulse <= dn_r;
  done_code  <= dc_r;
  err_pulse  <= er_r;
  err_code   <= ec_r;

  n_start      <= st_c;
  n_pkt        <= pk_c;
  n_zlp        <= zl_c;
  n_done_len   <= dl_c;
  n_done_short <= ds_c;
  n_restart    <= rs_c;
  n_ack_idle   <= ai_c;
  n_babble     <= bb_c;

  process (clk, rst_n)
    variable len_v, idx_v, tot_v, moved_v : unsigned(15 downto 0);
    variable busy_v, dir_v, dn_v, er_v    : std_logic;
    variable dc_v, ec_v                   : std_logic_vector(1 downto 0);
    variable this_len : unsigned(7 downto 0);
    variable take     : unsigned(15 downto 0);
  begin
    if rst_n = '0' then
      len_r   <= (others => '0');
      idx_r   <= (others => '0');
      tot_r   <= (others => '0');
      moved_r <= (others => '0');
      busy_r  <= '0';
      dir_r   <= DIR_TX;
      dn_r    <= '0'; er_r <= '0';
      dc_r    <= D_NONE; ec_r <= E_NONE;
      st_c <= (others => '0'); pk_c <= (others => '0');
      zl_c <= (others => '0'); dl_c <= (others => '0');
      ds_c <= (others => '0'); rs_c <= (others => '0');
      ai_c <= (others => '0'); bb_c <= (others => '0');
    elsif rising_edge(clk) then
      len_v   := len_r;
      idx_v   := idx_r;
      tot_v   := tot_r;
      moved_v := moved_r;
      busy_v  := busy_r;
      dir_v   := dir_r;
      dn_v := '0'; er_v := '0';
      dc_v := D_NONE; ec_v := E_NONE;
      this_len := pkt_len_of(len_r, idx_r);

      if eot = '1' then
        -- Nothing: the counters are the report.
        null;
      elsif start = '1' then
        if busy_r = '1' then
          -- A second descriptor while one is still running. The hardware can
          -- only track one, so the new one would silently replace the old
          -- and the driver would wait for a completion that never comes.
          er_v := '1'; ec_v := E_RESTART;
          rs_c <= rs_c + 1;
        else
          len_v   := len;
          idx_v   := (others => '0');
          tot_v   := total_pkts(len);
          moved_v := (others => '0');
          busy_v  := '1';
          dir_v   := dir;
          st_c    <= st_c + 1;
        end if;
      elsif pkt_ack = '1' then
        if busy_r = '0' then
          -- A packet completed with nothing offered. On real hardware this
          -- is a controller and a driver that disagree about whose turn it
          -- is.
          er_v := '1'; ec_v := E_ACKIDLE;
          ai_c <= ai_c + 1;
        else
          pk_c <= pk_c + 1;
          if this_len = 0 then zl_c <= zl_c + 1; end if;

          -- ---- How many bytes actually moved. ----
          --
          -- On TX it is what we offered. On RX it is what arrived, which is
          -- the peer's choice and may be less.
          if dir_r = DIR_RX then
            take := resize(rx_len, 16);
          else
            take := resize(this_len, 16);
          end if;

          if dir_r = DIR_RX and rx_len > this_len then
            -- The peer sent more than the packet could hold. This is babble,
            -- and it is a real error rather than a short read: the extra
            -- bytes went somewhere.
            er_v := '1'; ec_v := E_BABBLE;
            bb_c <= bb_c + 1;
            take := resize(this_len, 16);
          end if;

          moved_v := moved_v + take;

          if dir_r = DIR_RX and take < resize(this_len, 16) then
            -- ---- A SHORT packet on receive ends the transfer. ----
            --
            -- Not an error. The peer had less to send than we asked for,
            -- which is how every variable-length response arrives.
            busy_v := '0';
            dn_v   := '1'; dc_v := D_SHORT;
            ds_c   <= ds_c + 1;
          elsif (idx_v + 1) >= tot_v then
            -- ---- The sequence is complete. ----
            --
            -- Including the case where the last packet was a ZLP, which is a
            -- packet like any other and has to be acknowledged before the
            -- transfer is done.
            busy_v := '0';
            dn_v   := '1'; dc_v := D_LENGTH;
            dl_c   <= dl_c + 1;
          else
            idx_v := idx_v + 1;
          end if;
        end if;
      end if;

      len_r   <= len_v;
      idx_r   <= idx_v;
      tot_r   <= tot_v;
      moved_r <= moved_v;
      busy_r  <= busy_v;
      dir_r   <= dir_v;
      dn_r    <= dn_v; er_r <= er_v;
      dc_r    <= dc_v; ec_r <= ec_v;
    end if;
  end process;
end architecture;

   every L in 0 .. 1024, in BOTH directions
       -> 2050 complete transfers, every packet of every one
          compared against an expectation computed from the rule

   and for every L in 0 .. 256, a short packet injected at
   EVERY position in the sequence
       -> 640 residual cases, because the arithmetic depends on
          where the peer stopped

`timescale 1ns/1ps
// Testbench for usb_dma_xfer.
//
// The oracle is a shadow model written from the chapter's rules rather than
// from the RTL, re-derived every cycle and compared against every output.
//
// THE EXHAUSTIVE CLAIM IS OVER EVERY DESCRIPTOR LENGTH
//
// The packet sequence is a function of one number, and the interesting
// values of that number are the ones where the arithmetic changes: the exact
// multiples of the maximum packet size, where the zero-length terminator
// becomes necessary. Testing "a short one, a long one and one in the middle"
// is exactly the test that misses them.
//
//     every L in 0 .. 1024, in both directions
//     -> 2050 complete transfers, every packet of every one checked
//
// and separately, for every L in 0 .. 256, a short packet injected at EVERY
// position in the sequence, because the residual arithmetic depends on where
// the peer stopped.
module tb_dx_v;

  localparam integer MAXP  = 64;
  localparam integer L_MAX = 1024;
  localparam integer S_MAX = 256;

  localparam [1:0] D_NONE = 2'd0, D_LENGTH = 2'd1, D_SHORT = 2'd2;
  localparam [1:0] E_NONE = 2'd0, E_RESTART = 2'd1, E_ACKIDLE = 2'd2,
                   E_BABBLE = 2'd3;

  reg         clk = 1'b0, rst_n = 1'b0;
  reg         start = 1'b0, dir = 1'b0, pkt_ack = 1'b0, eot = 1'b0;
  reg  [15:0] len = 16'd0;
  reg  [7:0]  rx_len = 8'd0;

  wire        busy, pkt_valid, pkt_is_zlp, done_pulse, err_pulse;
  wire [7:0]  pkt_len;
  wire [15:0] pkt_index, pkt_total, moved, residual;
  wire [1:0]  done_code, err_code;
  wire [31:0] n_start, n_pkt, n_zlp, n_done_len, n_done_short,
              n_restart, n_ack_idle, n_babble;

  usb_dma_xfer #(.MAXP(MAXP)) dut (
    .clk(clk), .rst_n(rst_n),
    .start(start), .len(len), .dir(dir),
    .pkt_ack(pkt_ack), .rx_len(rx_len), .eot(eot),
    .busy(busy), .pkt_valid(pkt_valid), .pkt_len(pkt_len),
    .pkt_is_zlp(pkt_is_zlp), .pkt_index(pkt_index), .pkt_total(pkt_total),
    .moved(moved), .residual(residual),
    .done_pulse(done_pulse), .done_code(done_code),
    .err_pulse(err_pulse), .err_code(err_code),
    .n_start(n_start), .n_pkt(n_pkt), .n_zlp(n_zlp),
    .n_done_len(n_done_len), .n_done_short(n_done_short),
    .n_restart(n_restart), .n_ack_idle(n_ack_idle), .n_babble(n_babble)
  );

  always #5 clk = ~clk;

  // ---------------- the shadow model ----------------
  reg [15:0] m_len, m_idx, m_tot, m_moved;
  reg        m_busy, m_dir, m_dn, m_er;
  reg [1:0]  m_dc, m_ec;
  reg [31:0] c_start, c_pkt, c_zlp, c_dlen, c_dshort, c_rs, c_ai, c_bb;

  integer errors = 0, checks = 0, steps = 0;
  integer k;

  // The independent expectation, written from the chapter rather than from
  // the design: n_full + 1 packets, for every length including zero.
  function [15:0] exp_total; input [15:0] l;
    begin exp_total = (l / MAXP) + 16'd1; end
  endfunction

  function [7:0] exp_plen; input [15:0] l; input [15:0] i;
    begin
      if (i < (l / MAXP)) exp_plen = MAXP;
      else                exp_plen = (l % MAXP);
    end
  endfunction

  task ck;
    input [255:0] nm;
    input [31:0]  got, exp;
    begin
      checks = checks + 1;
      if (got !== exp) begin
        errors = errors + 1;
        if (errors < 25)
          $display("FAIL t=%0t step=%0d %0s got=%0d exp=%0d",
                   $time, steps, nm, got, exp);
      end
    end
  endtask

  task model_step;
    reg [7:0]  this_len;
    reg [15:0] take;
    begin
      m_dn = 1'b0; m_er = 1'b0; m_dc = D_NONE; m_ec = E_NONE;
      this_len = exp_plen(m_len, m_idx);

      if (eot) begin
        // nothing
      end else if (start) begin
        if (m_busy) begin
          m_er = 1'b1; m_ec = E_RESTART; c_rs = c_rs + 1;
        end else begin
          m_len = len; m_idx = 16'd0; m_tot = exp_total(len);
          m_moved = 16'd0; m_busy = 1'b1; m_dir = dir;
          c_start = c_start + 1;
        end
      end else if (pkt_ack) begin
        if (!m_busy) begin
          m_er = 1'b1; m_ec = E_ACKIDLE; c_ai = c_ai + 1;
        end else begin
          c_pkt = c_pkt + 1;
          if (this_len == 8'd0) c_zlp = c_zlp + 1;
          take = m_dir ? {8'd0, rx_len} : {8'd0, this_len};
          if (m_dir && ({8'd0, rx_len} > {8'd0, this_len})) begin
            m_er = 1'b1; m_ec = E_BABBLE; c_bb = c_bb + 1;
            take = {8'd0, this_len};
          end
          m_moved = m_moved + take;
          if (m_dir && (take < {8'd0, this_len})) begin
            // A SHORT packet on receive ends the transfer. Not an error.
            m_busy = 1'b0; m_dn = 1'b1; m_dc = D_SHORT;
            c_dshort = c_dshort + 1;
          end else if (m_idx + 16'd1 >= m_tot) begin
            m_busy = 1'b0; m_dn = 1'b1; m_dc = D_LENGTH;
            c_dlen = c_dlen + 1;
          end else begin
            m_idx = m_idx + 16'd1;
          end
        end
      end
    end
  endtask

  task check_out;
    begin
      ck("busy",       {31'd0, busy},      {31'd0, m_busy});
      ck("pkt_valid",  {31'd0, pkt_valid}, {31'd0, m_busy});
      ck("pkt_len",    {24'd0, pkt_len},   {24'd0, exp_plen(m_len, m_idx)});
      ck("pkt_is_zlp", {31'd0, pkt_is_zlp},
         {31'd0, m_busy && (exp_plen(m_len, m_idx) == 8'd0)});
      ck("pkt_index",  {16'd0, pkt_index}, {16'd0, m_idx});
      ck("pkt_total",  {16'd0, pkt_total}, {16'd0, m_tot});
      ck("moved",      {16'd0, moved},     {16'd0, m_moved});
      ck("residual",   {16'd0, residual},  {16'd0, m_len - m_moved});
      ck("done_pulse", {31'd0, done_pulse},{31'd0, m_dn});
      ck("done_code",  {30'd0, done_code}, {30'd0, m_dc});
      ck("err_pulse",  {31'd0, err_pulse}, {31'd0, m_er});
      ck("err_code",   {30'd0, err_code},  {30'd0, m_ec});
      ck("n_start",      n_start,      c_start);
      ck("n_pkt",        n_pkt,        c_pkt);
      ck("n_zlp",        n_zlp,        c_zlp);
      ck("n_done_len",   n_done_len,   c_dlen);
      ck("n_done_short", n_done_short, c_dshort);
      ck("n_restart",    n_restart,    c_rs);
      ck("n_ack_idle",   n_ack_idle,   c_ai);
      ck("n_babble",     n_babble,     c_bb);
      // ---- the structural invariant ----
      //
      // Every transfer ends exactly once, by length or by a short packet.
      // If these ever drift apart, a completion has been produced with no
      // transfer behind it -- which is a driver waiting on an event that
      // already fired, or worse, one that fired twice.
      ck("completions", n_done_len + n_done_short, c_dlen + c_dshort);
    end
  endtask

  task step;
    begin
      model_step;
      @(posedge clk);
      #1;
      steps = steps + 1;
      check_out;
    end
  endtask

  task do_start; input [15:0] l; input d;
    begin
      start = 1'b1; pkt_ack = 1'b0; eot = 1'b0; len = l; dir = d;
      step;
      start = 1'b0;
    end
  endtask

  task do_ack; input [7:0] rl;
    begin
      start = 1'b0; pkt_ack = 1'b1; eot = 1'b0; rx_len = rl;
      step;
      pkt_ack = 1'b0;
    end
  endtask

  task idle;
    begin
      start = 1'b0; pkt_ack = 1'b0; eot = 1'b0;
      step;
    end
  endtask

  task reset_all;
    begin
      start = 1'b0; pkt_ack = 1'b0; eot = 1'b0;
      rst_n = 1'b0;
      @(posedge clk); #1;
      rst_n = 1'b1;
      m_len = 16'd0; m_idx = 16'd0; m_tot = 16'd0; m_moved = 16'd0;
      m_busy = 1'b0; m_dir = 1'b0; m_dn = 1'b0; m_er = 1'b0;
      m_dc = D_NONE; m_ec = E_NONE;
      c_start=0; c_pkt=0; c_zlp=0; c_dlen=0; c_dshort=0;
      c_rs=0; c_ai=0; c_bb=0;
      @(negedge clk);
    end
  endtask

  integer L, d, i, sp, w;
  integer n_zlp_cases, n_short_cases, base_zlp, base_dlen, base_dshort;
  integer n_bab_cases, n_rst_cases;
  integer seen_total, seen_pkts;

  initial begin
    n_zlp_cases = 0; n_short_cases = 0;
    n_bab_cases = 0; n_rst_cases = 0;

    repeat (3) @(posedge clk);
    rst_n = 1'b1;
    @(negedge clk);
    reset_all;

    // ================= PHASE 1 -- EVERY length, both directions ==========
    //
    // The whole packet sequence is driven and every packet's length, index
    // and total is compared against an expectation computed from the
    // chapter's rule rather than from the design.
    for (d = 0; d < 2; d = d + 1)
      for (L = 0; L <= L_MAX; L = L + 1) begin
        do_start(L[15:0], d[0]);
        if (pkt_total !== exp_total(L[15:0])) begin
          errors = errors + 1;
          if (errors < 25)
            $display("FAIL L=%0d total %0d expected %0d", L, pkt_total,
                     exp_total(L[15:0]));
        end
        base_dlen = c_dlen;
        seen_pkts = 0;
        // acknowledge every packet the engine offers, at its full length so
        // nothing terminates early
        // BOUNDED. An unbounded `while (busy)` in a testbench is an
        // infinite loop the moment a mutation stops the design advancing,
        // and the run never reaches the summary that would have told you
        // which mutation. A correct sequence is n_full + 1 packets, so twice
        // that plus a margin is a bound no correct design can reach.
        while (m_busy && (seen_pkts < 2 * exp_total(L[15:0]) + 8)) begin
          if (exp_plen(m_len, m_idx) == 8'd0) n_zlp_cases = n_zlp_cases + 1;
          seen_pkts = seen_pkts + 1;
          do_ack(exp_plen(m_len, m_idx));
        end
        if (m_busy) begin
          errors = errors + 1;
          if (errors < 25)
            $display("FAIL L=%0d never completed after %0d packets", L, seen_pkts);
          reset_all;
        end
        if (seen_pkts != exp_total(L[15:0])) begin
          errors = errors + 1;
          if (errors < 25)
            $display("FAIL L=%0d emitted %0d packets, expected %0d", L,
                     seen_pkts, exp_total(L[15:0]));
        end
        if (c_dlen != base_dlen + 1) begin
          errors = errors + 1;
          $display("FAIL L=%0d did not complete by length", L);
        end
        if (m_moved != L[15:0]) begin
          errors = errors + 1;
          $display("FAIL L=%0d moved %0d bytes", L, m_moved);
        end
      end
    // Every exact multiple of MAXP needs a ZLP, and there are L_MAX/MAXP + 1
    // of them in each direction. If that count is wrong the sweep did not
    // reach the cases it exists for.
    if (n_zlp_cases != 2 * ((L_MAX / MAXP) + 1)) begin
      errors = errors + 1;
      $display("FAIL saw %0d zero-length packets, expected %0d",
               n_zlp_cases, 2 * ((L_MAX / MAXP) + 1));
    end

    // ================= PHASE 2 -- the ZLP boundary, named ================
    //
    // The three lengths either side of one multiple, spelled out, because
    // this is the case the whole chapter is about.
    reset_all;
    do_start(16'd63, 1'b0);
    if (pkt_total !== 16'd1 || pkt_len !== 8'd63) begin
      errors = errors + 1;
      $display("FAIL L=63: total=%0d len=%0d, expected 1 and 63",
               pkt_total, pkt_len);
    end
    do_ack(8'd63);
    do_start(16'd64, 1'b0);
    if (pkt_total !== 16'd2) begin
      errors = errors + 1;
      $display("FAIL L=64: total=%0d, expected 2 (a full packet and a ZLP)",
               pkt_total);
    end
    do_ack(8'd64);
    if (!pkt_is_zlp || (pkt_len !== 8'd0)) begin
      errors = errors + 1;
      $display("FAIL L=64: the second packet is not a ZLP (len=%0d)", pkt_len);
    end
    do_ack(8'd0);
    do_start(16'd65, 1'b0);
    if (pkt_total !== 16'd2) begin
      errors = errors + 1;
      $display("FAIL L=65: total=%0d, expected 2", pkt_total);
    end
    do_ack(8'd64);
    if (pkt_len !== 8'd1) begin
      errors = errors + 1;
      $display("FAIL L=65: the second packet is %0d bytes, expected 1",
               pkt_len);
    end
    do_ack(8'd1);
    // ...and a zero-length transfer is one ZLP, not nothing at all.
    do_start(16'd0, 1'b0);
    if (pkt_total !== 16'd1 || !pkt_is_zlp) begin
      errors = errors + 1;
      $display("FAIL L=0: total=%0d zlp=%0b, expected 1 and a ZLP",
               pkt_total, pkt_is_zlp);
    end
    do_ack(8'd0);

    // ================= PHASE 3 -- a short packet at EVERY position =======
    //
    // On receive, the peer may stop early at any packet. The residual
    // arithmetic depends on where, so every position is driven for every
    // length up to S_MAX.
    for (L = 0; L <= S_MAX; L = L + 1)
      for (sp = 0; sp < exp_total(L[15:0]); sp = sp + 1) begin
        reset_all;
        do_start(L[15:0], 1'b1);
        base_dshort = c_dshort;
        seen_total = 0;
        for (i = 0; i < sp; i = i + 1) begin
          seen_total = seen_total + exp_plen(L[15:0], i[15:0]);
          do_ack(exp_plen(L[15:0], i[15:0]));
        end
        // the short one: one byte fewer than the engine offered, or skip
        // this case when the offered packet is already zero-length
        if (exp_plen(L[15:0], sp[15:0]) != 8'd0) begin
          seen_total = seen_total + exp_plen(L[15:0], sp[15:0]) - 1;
          do_ack(exp_plen(L[15:0], sp[15:0]) - 8'd1);
          n_short_cases = n_short_cases + 1;
          if (c_dshort != base_dshort + 1) begin
            errors = errors + 1;
            $display("FAIL L=%0d short at %0d did not complete SHORT", L, sp);
          end
          if (done_code !== D_SHORT) begin
            errors = errors + 1;
            $display("FAIL L=%0d short at %0d code=%0d", L, sp, done_code);
          end
          // ---- THE arithmetic. ----
          if (residual !== (L[15:0] - seen_total[15:0])) begin
            errors = errors + 1;
            if (errors < 25)
              $display("FAIL L=%0d short at %0d: residual %0d expected %0d",
                       L, sp, residual, L - seen_total);
          end
          if (err_pulse !== 1'b0) begin
            errors = errors + 1;
            $display("FAIL a short packet raised an error: it is not one");
          end
        end
      end

    // ================= PHASE 4 -- babble, at EVERY position ==============
    //
    // A short packet is the peer choosing to send less. More than the packet
    // can hold is the peer sending bytes that went somewhere, and the two
    // must not be reported the same way.
    //
    // The first version of this phase drove one babble case and the mutation
    // that accepts babble silently died on FIVE checks. A property with one
    // instance is a property that is nearly untested, so every packet
    // position of a multi-packet transfer is driven, at four different
    // over-lengths.
    n_bab_cases = 0;
    for (L = 0; L < 4; L = L + 1)            // over-length index
      for (sp = 0; sp < 4; sp = sp + 1) begin
        reset_all;
        do_start(16'd256, 1'b1);
        for (i = 0; i < sp; i = i + 1) do_ack(8'd64);
        base_dshort = c_dshort;
        k = c_bb;
        // 65, 100, 200, 255 -- all longer than the 64-byte packet
        do_ack((L == 0) ? 8'd65 : (L == 1) ? 8'd100 :
               (L == 2) ? 8'd200 : 8'd255);
        if (c_bb != k + 1) begin
          errors = errors + 1;
          $display("FAIL babble at position %0d, len case %0d not reported",
                   sp, L);
        end
        // ...and only MAXP bytes are counted as moved, whatever arrived.
        if (m_moved != (sp + 1) * MAXP) begin
          errors = errors + 1;
          $display("FAIL babble at %0d moved %0d, expected %0d",
                   sp, m_moved, (sp + 1) * MAXP);
        end
        // ...and it is NOT reported as a short read, because it is not one.
        if (c_dshort != base_dshort) begin
          errors = errors + 1;
          $display("FAIL babble at %0d was reported as a short packet", sp);
        end
        n_bab_cases = n_bab_cases + 1;
      end
    if (n_bab_cases != 16) begin
      errors = errors + 1;
      $display("FAIL babble cases %0d, expected 16", n_bab_cases);
    end

    // ================= PHASE 5 -- a restart, at EVERY position ============
    //
    // A second descriptor arriving while one is running. The hardware can
    // only track one, so the new one would silently replace the old and the
    // driver would wait for a completion that never comes.
    //
    // Driven at every position in the sequence and with three different
    // replacement lengths, for the same reason as phase 4: the mutation that
    // accepts the replacement died on 39 checks when this was one case.
    n_rst_cases = 0;
    for (L = 0; L < 3; L = L + 1)
      for (sp = 0; sp < 4; sp = sp + 1) begin
        reset_all;
        do_start(16'd256, 1'b0);
        for (i = 0; i < sp; i = i + 1) do_ack(8'd64);
        k = c_rs;
        do_start((L == 0) ? 16'd64 : (L == 1) ? 16'd512 : 16'd1, 1'b1);
        if (c_rs != k + 1) begin
          errors = errors + 1;
          $display("FAIL restart at position %0d was not reported", sp);
        end
        // THE check: the running descriptor is untouched. Its length, its
        // direction, its position and its total all survive.
        if (m_len != 16'd256) begin
          errors = errors + 1;
          $display("FAIL restart at %0d replaced the length: %0d", sp, m_len);
        end
        if (m_idx != sp[15:0]) begin
          errors = errors + 1;
          $display("FAIL restart at %0d moved the position to %0d", sp, m_idx);
        end
        if (m_tot != 16'd5) begin
          errors = errors + 1;
          $display("FAIL restart at %0d changed the total to %0d", sp, m_tot);
        end
        if (m_dir != 1'b0) begin
          errors = errors + 1;
          $display("FAIL restart at %0d flipped the direction", sp);
        end
        // ...and the original transfer still completes normally.
        w = 0;
        while (m_busy && (w < 64)) begin
          do_ack(exp_plen(m_len, m_idx));
          w = w + 1;
        end
        if (m_busy) begin
          errors = errors + 1;
          if (errors < 25)
            $display("FAIL the transfer after a restart at %0d never finished", sp);
          reset_all;
        end
        if (m_busy == 1'b0 && m_moved != 16'd256) begin
          errors = errors + 1;
          $display("FAIL after a restart at %0d the transfer moved %0d",
                   sp, m_moved);
        end
        n_rst_cases = n_rst_cases + 1;
      end
    if (n_rst_cases != 12) begin
      errors = errors + 1;
      $display("FAIL restart cases %0d, expected 12", n_rst_cases);
    end

    // ...and an acknowledgement with nothing outstanding, which is the other
    // half of the same disagreement between controller and driver.
    reset_all;
    k = c_ai;
    do_ack(8'd64);
    if (c_ai != k + 1) begin
      errors = errors + 1;
      $display("FAIL an idle ack was not reported");
    end

    // The random phase is switchable, because a mutation score is only
    // interesting once it is DECOMPOSED. Phases 1 to 5 already sweep every
    // length and every short-packet position, so nothing in the exhaustive
    // claim depends on it.
`ifndef DIRECTED_ONLY
    // ================= PHASE 6 -- random =================================
    reset_all;
    for (i = 0; i < 20000; i = i + 1) begin
      w = $unsigned($random) % 100;
      if (!m_busy) begin
        if (w < 90) do_start($unsigned($random) % 600, $unsigned($random) % 2);
        else if (w < 95) do_ack($unsigned($random) % 80);
        else idle;
      end else begin
        if (w < 88) begin
          // Mostly full-length acks, with a realistic fraction of short ones
          // and a small fraction of babble. Leaving babble out of the mix
          // means the one error that is NOT a short read is exercised by a
          // single directed case, and the check that separates them is under
          // no load at all.
          sp = $unsigned($random) % 100;
          if (sp < 12)
            do_ack((exp_plen(m_len, m_idx) == 8'd0) ? 8'd0
                   : (exp_plen(m_len, m_idx) - 8'd1));
          else if (sp < 15 && m_dir)
            do_ack(8'd200);                 // more than any packet can hold
          else
            do_ack(exp_plen(m_len, m_idx));
        end else if (w < 94) begin
          do_start($unsigned($random) % 600, $unsigned($random) % 2);
        end else idle;
      end
    end

`endif

    $display("steps=%0d checks=%0d errors=%0d", steps, checks, errors);
    $display("lengths swept=%0d (x2 directions)  ZLP cases=%0d  short cases=%0d",
             L_MAX + 1, n_zlp_cases, n_short_cases);
    $display("start=%0d pkt=%0d zlp=%0d done_len=%0d done_short=%0d",
             n_start, n_pkt, n_zlp, n_done_len, n_done_short);
    $display("restart=%0d ack_idle=%0d babble=%0d",
             n_restart, n_ack_idle, n_babble);
    $display("%0s: %0d errors in %0d checks",
             (errors == 0) ? "PASS" : "FAIL", errors, checks);
    $finish;
  end
endmodule

`timescale 1ns/1ps
// Testbench for usb_dma_xfer.
//
// The oracle is a shadow model written from the chapter's rules rather than
// from the RTL, re-derived every cycle and compared against every output.
//
// THE EXHAUSTIVE CLAIM IS OVER EVERY DESCRIPTOR LENGTH
//
// The packet sequence is a function of one number, and the interesting
// values of that number are the ones where the arithmetic changes: the exact
// multiples of the maximum packet size, where the zero-length terminator
// becomes necessary. Testing "a short one, a long one and one in the middle"
// is exactly the test that misses them.
//
//     every L in 0 .. 1024, in both directions
//     -> 2050 complete transfers, every packet of every one checked
//
// and separately, for every L in 0 .. 256, a short packet injected at EVERY
// position in the sequence, because the residual arithmetic depends on where
// the peer stopped.
module tb_dx_sv;
  import usb_dma_pkg::*;

  localparam int MAXP  = 64;
  localparam int L_MAX = 1024;
  localparam int S_MAX = 256;

  logic        clk = 1'b0, rst_n = 1'b0;
  logic        start = 1'b0, pkt_ack = 1'b0, eot = 1'b0;
  xfer_dir_e   dir = DIR_TX;
  logic [15:0] len = 16'd0;
  logic [7:0]  rx_len = 8'd0;

  logic        busy, pkt_valid, pkt_is_zlp, done_pulse, err_pulse;
  logic [7:0]  pkt_len;
  logic [15:0] pkt_index, pkt_total, moved, residual;
  done_code_e  done_code;
  dma_err_e    err_code;
  logic [31:0] n_start, n_pkt, n_zlp, n_done_len, n_done_short,
               n_restart, n_ack_idle, n_babble;

  usb_dma_xfer #(.MAXP(MAXP)) dut (
    .clk(clk), .rst_n(rst_n),
    .start(start), .len(len), .dir(dir),
    .pkt_ack(pkt_ack), .rx_len(rx_len), .eot(eot),
    .busy(busy), .pkt_valid(pkt_valid), .pkt_len(pkt_len),
    .pkt_is_zlp(pkt_is_zlp), .pkt_index(pkt_index), .pkt_total(pkt_total),
    .moved(moved), .residual(residual),
    .done_pulse(done_pulse), .done_code(done_code),
    .err_pulse(err_pulse), .err_code(err_code),
    .n_start(n_start), .n_pkt(n_pkt), .n_zlp(n_zlp),
    .n_done_len(n_done_len), .n_done_short(n_done_short),
    .n_restart(n_restart), .n_ack_idle(n_ack_idle), .n_babble(n_babble)
  );

  always #5 clk = ~clk;

  // ---------------- the shadow model ----------------
  logic [15:0] m_len, m_idx, m_tot, m_moved;
  logic        m_busy, m_dn, m_er;
  xfer_dir_e   m_dir;
  done_code_e  m_dc;
  dma_err_e    m_ec;
  int unsigned c_start, c_pkt, c_zlp, c_dlen, c_dshort, c_rs, c_ai, c_bb;

  int errors = 0, checks = 0, steps = 0;
  int k;

  // The independent expectation, written from the chapter rather than from
  // the design: n_full + 1 packets, for every length including zero.
  function automatic logic [15:0] exp_total(input logic [15:0] l);
    begin exp_total = (l / 16'(MAXP)) + 16'd1; end
  endfunction

  function automatic logic [7:0] exp_plen(input logic [15:0] l,
                                          input logic [15:0] i);
    begin
      if (i < (l / 16'(MAXP))) exp_plen = 8'(MAXP);
      else                     exp_plen = 8'(l % 16'(MAXP));
    end
  endfunction

  task automatic ck(string nm, int unsigned got, int unsigned exp);
    checks++;
    if (got !== exp) begin
      errors++;
      if (errors < 25)
        $display("FAIL t=%0t step=%0d %0s got=%0d exp=%0d",
                 $time, steps, nm, got, exp);
    end
  endtask

  task automatic model_step;
    logic [7:0]  this_len;
    logic [15:0] take;
    begin
      m_dn = 1'b0; m_er = 1'b0; m_dc = D_NONE; m_ec = E_NONE;
      this_len = exp_plen(m_len, m_idx);

      if (eot) begin
        // nothing
      end else if (start) begin
        if (m_busy) begin
          m_er = 1'b1; m_ec = E_RESTART; c_rs = c_rs + 1;
        end else begin
          m_len = len; m_idx = 16'd0; m_tot = exp_total(len);
          m_moved = 16'd0; m_busy = 1'b1; m_dir = dir;
          c_start = c_start + 1;
        end
      end else if (pkt_ack) begin
        if (!m_busy) begin
          m_er = 1'b1; m_ec = E_ACKIDLE; c_ai = c_ai + 1;
        end else begin
          c_pkt = c_pkt + 1;
          if (this_len == 8'd0) c_zlp = c_zlp + 1;
          take = (m_dir == DIR_RX) ? {8'd0, rx_len} : {8'd0, this_len};
          if ((m_dir == DIR_RX) && ({8'd0, rx_len} > {8'd0, this_len})) begin
            m_er = 1'b1; m_ec = E_BABBLE; c_bb = c_bb + 1;
            take = {8'd0, this_len};
          end
          m_moved = m_moved + take;
          if ((m_dir == DIR_RX) && (take < {8'd0, this_len})) begin
            // A SHORT packet on receive ends the transfer. Not an error.
            m_busy = 1'b0; m_dn = 1'b1; m_dc = D_SHORT;
            c_dshort = c_dshort + 1;
          end else if (m_idx + 16'd1 >= m_tot) begin
            m_busy = 1'b0; m_dn = 1'b1; m_dc = D_LENGTH;
            c_dlen = c_dlen + 1;
          end else begin
            m_idx = m_idx + 16'd1;
          end
        end
      end
    end
  endtask

  task automatic check_out;
    begin
      ck("busy",       {31'd0, busy},      {31'd0, m_busy});
      ck("pkt_valid",  {31'd0, pkt_valid}, {31'd0, m_busy});
      ck("pkt_len",    {24'd0, pkt_len},   {24'd0, exp_plen(m_len, m_idx)});
      ck("pkt_is_zlp", {31'd0, pkt_is_zlp},
         {31'd0, m_busy && (exp_plen(m_len, m_idx) == 8'd0)});
      ck("pkt_index",  {16'd0, pkt_index}, {16'd0, m_idx});
      ck("pkt_total",  {16'd0, pkt_total}, {16'd0, m_tot});
      ck("moved",      {16'd0, moved},     {16'd0, m_moved});
      ck("residual",   {16'd0, residual},  {16'd0, m_len - m_moved});
      ck("done_pulse", {31'd0, done_pulse},{31'd0, m_dn});
      ck("done_code",  done_code, m_dc);
      ck("err_pulse",  {31'd0, err_pulse}, {31'd0, m_er});
      ck("err_code",   err_code, m_ec);
      ck("n_start",      n_start,      c_start);
      ck("n_pkt",        n_pkt,        c_pkt);
      ck("n_zlp",        n_zlp,        c_zlp);
      ck("n_done_len",   n_done_len,   c_dlen);
      ck("n_done_short", n_done_short, c_dshort);
      ck("n_restart",    n_restart,    c_rs);
      ck("n_ack_idle",   n_ack_idle,   c_ai);
      ck("n_babble",     n_babble,     c_bb);
      // ---- the structural invariant ----
      //
      // Every transfer ends exactly once, by length or by a short packet.
      // If these ever drift apart, a completion has been produced with no
      // transfer behind it -- which is a driver waiting on an event that
      // already fired, or worse, one that fired twice.
      ck("completions", n_done_len + n_done_short, c_dlen + c_dshort);
    end
  endtask

  task automatic step;
    begin
      model_step;
      @(posedge clk);
      #1;
      steps = steps + 1;
      check_out;
    end
  endtask

  task automatic do_start(logic [15:0] l, xfer_dir_e d);
    begin
      start = 1'b1; pkt_ack = 1'b0; eot = 1'b0; len = l; dir = d;
      step;
      start = 1'b0;
    end
  endtask

  task automatic do_ack(logic [7:0] rl);
    begin
      start = 1'b0; pkt_ack = 1'b1; eot = 1'b0; rx_len = rl;
      step;
      pkt_ack = 1'b0;
    end
  endtask

  task automatic idle;
    begin
      start = 1'b0; pkt_ack = 1'b0; eot = 1'b0;
      step;
    end
  endtask

  task automatic reset_all;
    begin
      start = 1'b0; pkt_ack = 1'b0; eot = 1'b0;
      rst_n = 1'b0;
      @(posedge clk); #1;
      rst_n = 1'b1;
      m_len = 16'd0; m_idx = 16'd0; m_tot = 16'd0; m_moved = 16'd0;
      m_busy = 1'b0; m_dir = DIR_TX; m_dn = 1'b0; m_er = 1'b0;
      m_dc = D_NONE; m_ec = E_NONE;
      c_start=0; c_pkt=0; c_zlp=0; c_dlen=0; c_dshort=0;
      c_rs=0; c_ai=0; c_bb=0;
      @(negedge clk);
    end
  endtask

  integer L, d, i, sp, w;
  integer n_zlp_cases, n_short_cases, base_zlp, base_dlen, base_dshort;
  integer n_bab_cases, n_rst_cases;
  integer seen_total, seen_pkts;

  initial begin
    n_zlp_cases = 0; n_short_cases = 0;
    n_bab_cases = 0; n_rst_cases = 0;

    repeat (3) @(posedge clk);
    rst_n = 1'b1;
    @(negedge clk);
    reset_all;

    // ================= PHASE 1 -- EVERY length, both directions ==========
    //
    // The whole packet sequence is driven and every packet's length, index
    // and total is compared against an expectation computed from the
    // chapter's rule rather than from the design.
    for (d = 0; d < 2; d = d + 1)
      for (L = 0; L <= L_MAX; L = L + 1) begin
        do_start(16'(L), xfer_dir_e'(d[0]));
        if (pkt_total !== exp_total(16'(L))) begin
          errors = errors + 1;
          if (errors < 25)
            $display("FAIL L=%0d total %0d expected %0d", L, pkt_total,
                     exp_total(16'(L)));
        end
        base_dlen = c_dlen;
        seen_pkts = 0;
        // acknowledge every packet the engine offers, at its full length so
        // nothing terminates early
        // BOUNDED. An unbounded `while (busy)` in a testbench is an
        // infinite loop the moment a mutation stops the design advancing,
        // and the run never reaches the summary that would have told you
        // which mutation. A correct sequence is n_full + 1 packets, so twice
        // that plus a margin is a bound no correct design can reach.
        while (m_busy && (seen_pkts < 2 * int'(exp_total(16'(L))) + 8)) begin
          if (exp_plen(m_len, m_idx) == 8'd0) n_zlp_cases = n_zlp_cases + 1;
          seen_pkts = seen_pkts + 1;
          do_ack(exp_plen(m_len, m_idx));
        end
        if (m_busy) begin
          errors = errors + 1;
          if (errors < 25)
            $display("FAIL L=%0d never completed after %0d packets", L, seen_pkts);
          reset_all;
        end
        if (seen_pkts != exp_total(16'(L))) begin
          errors = errors + 1;
          if (errors < 25)
            $display("FAIL L=%0d emitted %0d packets, expected %0d", L,
                     seen_pkts, exp_total(16'(L)));
        end
        if (c_dlen != base_dlen + 1) begin
          errors = errors + 1;
          $display("FAIL L=%0d did not complete by length", L);
        end
        if (m_moved != 16'(L)) begin
          errors = errors + 1;
          $display("FAIL L=%0d moved %0d bytes", L, m_moved);
        end
      end
    // Every exact multiple of MAXP needs a ZLP, and there are L_MAX/MAXP + 1
    // of them in each direction. If that count is wrong the sweep did not
    // reach the cases it exists for.
    if (n_zlp_cases != 2 * ((L_MAX / MAXP) + 1)) begin
      errors = errors + 1;
      $display("FAIL saw %0d zero-length packets, expected %0d",
               n_zlp_cases, 2 * ((L_MAX / MAXP) + 1));
    end

    // ================= PHASE 2 -- the ZLP boundary, named ================
    //
    // The three lengths either side of one multiple, spelled out, because
    // this is the case the whole chapter is about.
    reset_all;
    do_start(16'd63, DIR_TX);
    if (pkt_total !== 16'd1 || pkt_len !== 8'd63) begin
      errors = errors + 1;
      $display("FAIL L=63: total=%0d len=%0d, expected 1 and 63",
               pkt_total, pkt_len);
    end
    do_ack(8'd63);
    do_start(16'd64, DIR_TX);
    if (pkt_total !== 16'd2) begin
      errors = errors + 1;
      $display("FAIL L=64: total=%0d, expected 2 (a full packet and a ZLP)",
               pkt_total);
    end
    do_ack(8'd64);
    if (!pkt_is_zlp || (pkt_len !== 8'd0)) begin
      errors = errors + 1;
      $display("FAIL L=64: the second packet is not a ZLP (len=%0d)", pkt_len);
    end
    do_ack(8'd0);
    do_start(16'd65, DIR_TX);
    if (pkt_total !== 16'd2) begin
      errors = errors + 1;
      $display("FAIL L=65: total=%0d, expected 2", pkt_total);
    end
    do_ack(8'd64);
    if (pkt_len !== 8'd1) begin
      errors = errors + 1;
      $display("FAIL L=65: the second packet is %0d bytes, expected 1",
               pkt_len);
    end
    do_ack(8'd1);
    // ...and a zero-length transfer is one ZLP, not nothing at all.
    do_start(16'd0, DIR_TX);
    if (pkt_total !== 16'd1 || !pkt_is_zlp) begin
      errors = errors + 1;
      $display("FAIL L=0: total=%0d zlp=%0b, expected 1 and a ZLP",
               pkt_total, pkt_is_zlp);
    end
    do_ack(8'd0);

    // ================= PHASE 3 -- a short packet at EVERY position =======
    //
    // On receive, the peer may stop early at any packet. The residual
    // arithmetic depends on where, so every position is driven for every
    // length up to S_MAX.
    for (L = 0; L <= S_MAX; L = L + 1)
      for (sp = 0; sp < exp_total(16'(L)); sp = sp + 1) begin
        reset_all;
        do_start(16'(L), DIR_RX);
        base_dshort = int'(c_dshort);
        seen_total = 0;
        for (i = 0; i < sp; i = i + 1) begin
          seen_total = seen_total + int'(exp_plen(16'(L), 16'(i)));
          do_ack(exp_plen(16'(L), 16'(i)));
        end
        // the short one: one byte fewer than the engine offered, or skip
        // this case when the offered packet is already zero-length
        if (exp_plen(16'(L), 16'(sp)) != 8'd0) begin
          seen_total = seen_total + int'(exp_plen(16'(L), 16'(sp))) - 1;
          do_ack(exp_plen(16'(L), 16'(sp)) - 8'd1);
          n_short_cases = n_short_cases + 1;
          if (c_dshort != base_dshort + 1) begin
            errors = errors + 1;
            $display("FAIL L=%0d short at %0d did not complete SHORT", L, sp);
          end
          if (done_code !== D_SHORT) begin
            errors = errors + 1;
            $display("FAIL L=%0d short at %0d code=%0d", L, sp, done_code);
          end
          // ---- THE arithmetic. ----
          if (residual !== (16'(L) - 16'(seen_total))) begin
            errors = errors + 1;
            if (errors < 25)
              $display("FAIL L=%0d short at %0d: residual %0d expected %0d",
                       L, sp, residual, L - seen_total);
          end
          if (err_pulse !== 1'b0) begin
            errors = errors + 1;
            $display("FAIL a short packet raised an error: it is not one");
          end
        end
      end

    // ================= PHASE 4 -- babble, at EVERY position ==============
    //
    // A short packet is the peer choosing to send less. More than the packet
    // can hold is the peer sending bytes that went somewhere, and the two
    // must not be reported the same way.
    //
    // The first version of this phase drove one babble case and the mutation
    // that accepts babble silently died on FIVE checks. A property with one
    // instance is a property that is nearly untested, so every packet
    // position of a multi-packet transfer is driven, at four different
    // over-lengths.
    n_bab_cases = 0;
    for (L = 0; L < 4; L = L + 1)            // over-length index
      for (sp = 0; sp < 4; sp = sp + 1) begin
        reset_all;
        do_start(16'd256, DIR_RX);
        for (i = 0; i < sp; i = i + 1) do_ack(8'd64);
        base_dshort = int'(c_dshort);
        k = int'(c_bb);
        // 65, 100, 200, 255 -- all longer than the 64-byte packet
        do_ack((L == 0) ? 8'd65 : (L == 1) ? 8'd100 :
               (L == 2) ? 8'd200 : 8'd255);
        if (int'(c_bb) != k + 1) begin
          errors = errors + 1;
          $display("FAIL babble at position %0d, len case %0d not reported",
                   sp, L);
        end
        // ...and only MAXP bytes are counted as moved, whatever arrived.
        if (m_moved != 16'((sp + 1) * MAXP)) begin
          errors = errors + 1;
          $display("FAIL babble at %0d moved %0d, expected %0d",
                   sp, m_moved, (sp + 1) * MAXP);
        end
        // ...and it is NOT reported as a short read, because it is not one.
        if (int'(c_dshort) != base_dshort) begin
          errors = errors + 1;
          $display("FAIL babble at %0d was reported as a short packet", sp);
        end
        n_bab_cases = n_bab_cases + 1;
      end
    if (n_bab_cases != 16) begin
      errors = errors + 1;
      $display("FAIL babble cases %0d, expected 16", n_bab_cases);
    end

    // ================= PHASE 5 -- a restart, at EVERY position ============
    //
    // A second descriptor arriving while one is running. The hardware can
    // only track one, so the new one would silently replace the old and the
    // driver would wait for a completion that never comes.
    //
    // Driven at every position in the sequence and with three different
    // replacement lengths, for the same reason as phase 4: the mutation that
    // accepts the replacement died on 39 checks when this was one case.
    n_rst_cases = 0;
    for (L = 0; L < 3; L = L + 1)
      for (sp = 0; sp < 4; sp = sp + 1) begin
        reset_all;
        do_start(16'd256, DIR_TX);
        for (i = 0; i < sp; i = i + 1) do_ack(8'd64);
        k = int'(c_rs);
        do_start((L == 0) ? 16'd64 : (L == 1) ? 16'd512 : 16'd1, DIR_RX);
        if (int'(c_rs) != k + 1) begin
          errors = errors + 1;
          $display("FAIL restart at position %0d was not reported", sp);
        end
        // THE check: the running descriptor is untouched. Its length, its
        // direction, its position and its total all survive.
        if (m_len != 16'd256) begin
          errors = errors + 1;
          $display("FAIL restart at %0d replaced the length: %0d", sp, m_len);
        end
        if (m_idx != 16'(sp)) begin
          errors = errors + 1;
          $display("FAIL restart at %0d moved the position to %0d", sp, m_idx);
        end
        if (m_tot != 16'd5) begin
          errors = errors + 1;
          $display("FAIL restart at %0d changed the total to %0d", sp, m_tot);
        end
        if (m_dir != DIR_TX) begin
          errors = errors + 1;
          $display("FAIL restart at %0d flipped the direction", sp);
        end
        // ...and the original transfer still completes normally.
        w = 0;
        while (m_busy && (w < 64)) begin
          do_ack(exp_plen(m_len, m_idx));
          w = w + 1;
        end
        if (m_busy) begin
          errors = errors + 1;
          if (errors < 25)
            $display("FAIL the transfer after a restart at %0d never finished", sp);
          reset_all;
        end
        if (m_busy == 1'b0 && m_moved != 16'd256) begin
          errors = errors + 1;
          $display("FAIL after a restart at %0d the transfer moved %0d",
                   sp, m_moved);
        end
        n_rst_cases = n_rst_cases + 1;
      end
    if (n_rst_cases != 12) begin
      errors = errors + 1;
      $display("FAIL restart cases %0d, expected 12", n_rst_cases);
    end

    // ...and an acknowledgement with nothing outstanding, which is the other
    // half of the same disagreement between controller and driver.
    reset_all;
    k = int'(c_ai);
    do_ack(8'd64);
    if (int'(c_ai) != k + 1) begin
      errors = errors + 1;
      $display("FAIL an idle ack was not reported");
    end

    // The random phase is switchable, because a mutation score is only
    // interesting once it is DECOMPOSED. Phases 1 to 5 already sweep every
    // length and every short-packet position, so nothing in the exhaustive
    // claim depends on it.
`ifndef DIRECTED_ONLY
    // ================= PHASE 6 -- random =================================
    reset_all;
    for (i = 0; i < 20000; i = i + 1) begin
      w = $unsigned($random) % 100;
      if (!m_busy) begin
        if (w < 90) do_start(16'($unsigned($random) % 600), xfer_dir_e'($unsigned($random) % 2));
        else if (w < 95) do_ack(8'($unsigned($random) % 80));
        else idle;
      end else begin
        if (w < 88) begin
          // Mostly full-length acks, with a realistic fraction of short ones
          // and a small fraction of babble. Leaving babble out of the mix
          // means the one error that is NOT a short read is exercised by a
          // single directed case, and the check that separates them is under
          // no load at all.
          sp = $unsigned($random) % 100;
          if (sp < 12)
            do_ack((exp_plen(m_len, m_idx) == 8'd0) ? 8'd0
                   : (exp_plen(m_len, m_idx) - 8'd1));
          else if (sp < 15 && (m_dir == DIR_RX))
            do_ack(8'd200);                 // more than any packet can hold
          else
            do_ack(exp_plen(m_len, m_idx));
        end else if (w < 94) begin
          do_start(16'($unsigned($random) % 600), xfer_dir_e'($unsigned($random) % 2));
        end else idle;
      end
    end

`endif

    $display("steps=%0d checks=%0d errors=%0d", steps, checks, errors);
    $display("lengths swept=%0d (x2 directions)  ZLP cases=%0d  short cases=%0d",
             L_MAX + 1, n_zlp_cases, n_short_cases);
    $display("start=%0d pkt=%0d zlp=%0d done_len=%0d done_short=%0d",
             n_start, n_pkt, n_zlp, n_done_len, n_done_short);
    $display("restart=%0d ack_idle=%0d babble=%0d",
             n_restart, n_ack_idle, n_babble);
    $display("%0s: %0d errors in %0d checks",
             (errors == 0) ? "PASS" : "FAIL", errors, checks);
    $finish;
  end
endmodule