PCIe Debugging — One Method, Applied to Any Failure

1. Link state    is the Link usable at all?
2. Data Link     is packet delivery over this Link healthy?
3. Transaction   are requests, completions and configuration correct?
4. System        is the operation even being issued or recognized?

// SYNTHESIZABLE. Diagnostic event types.
// INTERNAL DEBUG FORMAT -- PCIe defines no such record (§1).
package pcie_dbg_pkg;
 
  parameter int EVENT_W   = 8;
  parameter int STATE_W   = 4;
  parameter int PORT_W    = 3;
  parameter int CTX_W     = 32;
  parameter int HIST_DEPTH = 32;
  parameter int HIST_W    = (HIST_DEPTH <= 1) ? 1 : $clog2(HIST_DEPTH);
 
  typedef enum logic [EVENT_W-1:0] {
    EV_LINK_STATE_CHANGE = 8'd0,
    EV_CFG_REQ_ACCEPTED  = 8'd1,
    EV_CFG_CPL_RETURNED  = 8'd2,
    EV_DESC_ACCEPTED     = 8'd3,
    EV_REQ_TRANSMITTED   = 8'd4,
    EV_CPL_RECEIVED      = 8'd5,
    EV_TXN_RETIRED       = 8'd6,
    EV_STATUS_PUBLISHED  = 8'd7,
    EV_REPLAY            = 8'd8,
    EV_ERROR             = 8'd9
  } dbg_event_e;
 
  typedef struct packed {
    dbg_event_e        ev;
    logic [STATE_W-1:0] link_state;
    logic [PORT_W-1:0]  port;
    logic [CTX_W-1:0]   context;      // identity, offset, or cause code
    logic [CTX_W-1:0]   cycle;
  } dbg_record_t;
 
endpackage

import pcie_dbg_pkg::*;
 
// SYNTHESIZABLE. FIRST-FAILURE CAPTURE (§6). The single highest-value
// block in this chapter. 21.4 §11 measured last-write-wins naming a
// non-root cause 96.8% of the time; this latches once and locks.
module first_failure_capture (
  input  logic clk,
  input  logic rst_n,
  input  logic              err_fire,
  input  logic [EVENT_W-1:0] err_code,
  input  logic [STATE_W-1:0] link_state,
  input  logic [CTX_W-1:0]   context,
  input  logic [CTX_W-1:0]   cycle,
  input  logic              dbg_clear,        // EXPLICIT clear only
 
  output logic              captured,
  output logic [EVENT_W-1:0] first_code,
  output logic [STATE_W-1:0] first_link_state,
  output logic [CTX_W-1:0]   first_context,
  output logic [CTX_W-1:0]   first_cycle,
  output logic [31:0]        later_count
);
  logic cap_q; logic [EVENT_W-1:0] code_q; logic [STATE_W-1:0] ls_q;
  logic [CTX_W-1:0] ctx_q, cyc_q; logic [31:0] later_q;
 
  assign captured=cap_q; assign first_code=code_q; assign first_link_state=ls_q;
  assign first_context=ctx_q; assign first_cycle=cyc_q; assign later_count=later_q;
 
  always_ff @(posedge clk or negedge rst_n) begin
    if (!rst_n || dbg_clear) begin
      cap_q<=1'b0; code_q<='0; ls_q<='0; ctx_q<='0; cyc_q<='0; later_q<='0;
    end else if (err_fire) begin
      if (!cap_q) begin
        // Capture the LINK STATE too: an error during Recovery means
        // something very different from the same error in L0 (§4).
        cap_q<=1'b1; code_q<=err_code; ls_q<=link_state;
        ctx_q<=context; cyc_q<=cycle;
      end else if (!(&later_q)) later_q <= later_q + 32'd1;
    end
  end
endmodule

import pcie_dbg_pkg::*;
 
// SYNTHESIZABLE. Bounded event history (§5). Enough to answer "what is
// the last thing that provably happened", not a protocol analyzer.
// Freezes on failure so the window around the fault survives.
module dbg_event_history (
  input  logic clk,
  input  logic rst_n,
  input  logic        push,
  input  dbg_record_t rec,
  input  logic        freeze,          // level: stop writing, keep contents
  input  logic        dbg_clear,
 
  output dbg_record_t history [HIST_DEPTH],
  output logic [HIST_W-1:0] wr_ptr,
  output logic              wrapped,
  output logic              frozen,
  output logic [31:0]       dropped_while_frozen
);
  dbg_record_t h_q [HIST_DEPTH];
  logic [HIST_W-1:0] wr_q;
  logic wrap_q, froz_q;
  logic [31:0] drop_q;
 
  always_comb for (int i=0;i<HIST_DEPTH;i++) history[i]=h_q[i];
  assign wr_ptr=wr_q; assign wrapped=wrap_q; assign frozen=froz_q;
  assign dropped_while_frozen=drop_q;
 
  always_ff @(posedge clk or negedge rst_n) begin
    if (!rst_n || dbg_clear) begin
      wr_q<='0; wrap_q<=1'b0; froz_q<=1'b0; drop_q<='0;
      for (int i=0;i<HIST_DEPTH;i++) h_q[i]<='0;
    end else begin
      if (freeze) froz_q <= 1'b1;
      if (push) begin
        if (froz_q) begin
          // Count what we could not record. A frozen buffer that silently
          // drops makes the window look quieter than it was.
          if (!(&drop_q)) drop_q <= drop_q + 32'd1;
        end else begin
          h_q[wr_q] <= rec;
          // Guarded so HIST_DEPTH = 1 is legal: the pointer is always 0.
          if (HIST_DEPTH == 1) wrap_q <= 1'b1;
          else if (wr_q == HIST_W'(HIST_DEPTH-1)) begin wr_q<='0; wrap_q<=1'b1; end
          else wr_q <= wr_q + HIST_W'(1);
        end
      end
    end
  end
endmodule

import pcie_dbg_pkg::*;
 
// SYNTHESIZABLE. PROGRESS HEARTBEAT (§4's fourth row).
// "Meaningful progress" is a DECLARED list of events -- a clock that keeps
// toggling is not progress, and neither is a counter that keeps counting.
module progress_heartbeat #(parameter int unsigned STALL_BUDGET = 65536) (
  input  logic clk,
  input  logic rst_n,
  input  logic progress_event,        // OR of the declared meaningful events
  input  logic dbg_clear,
 
  output logic [31:0] progress_epoch,
  output logic [31:0] cycles_since_progress,
  output logic        no_progress,     // sticky: exceeded the DEBUG budget
  output logic [31:0] worst_gap
);
  logic [31:0] ep_q, since_q, worst_q;
  logic        np_q;
 
  assign progress_epoch=ep_q; assign cycles_since_progress=since_q;
  assign no_progress=np_q;    assign worst_gap=worst_q;
 
  always_ff @(posedge clk or negedge rst_n) begin
    if (!rst_n || dbg_clear) begin ep_q<='0; since_q<='0; np_q<=1'b0; worst_q<='0; end
    else if (progress_event) begin
      ep_q <= ep_q + 32'd1;
      if (since_q > worst_q) worst_q <= since_q;
      since_q <= '0;
    end else begin
      since_q <= since_q + 32'd1;
      // A DEBUG HEURISTIC, not a PCIe timeout (§1). It says "nothing
      // meaningful has advanced for this long", which is a fact about the
      // design, not a protocol violation.
      if (since_q >= 32'(STALL_BUDGET)) np_q <= 1'b1;
    end
  end
endmodule

import pcie_dbg_pkg::*;
 
// SYNTHESIZABLE. State residency counters (§4).
// Distinguishes "sat in one state forever" from "cycled rapidly" -- two
// completely different faults with the same external symptom.
module state_residency #(parameter int N_STATE = 16) (
  input  logic clk,
  input  logic rst_n,
  input  logic [STATE_W-1:0] state,
  input  logic dbg_clear,
 
  output logic [31:0] residency [N_STATE],
  output logic [31:0] entries   [N_STATE],
  output logic        state_enter,
  output logic        saturated
);
  logic [31:0] res_q [N_STATE], ent_q [N_STATE];
  logic [STATE_W-1:0] prev_q;
  logic sat_q;
 
  // ONE definition of state entry, shared by every consumer -- computing
  // it separately in each debug block is how they disagree (25.4 §5).
  assign state_enter = (state != prev_q);
  assign saturated   = sat_q;
  always_comb for (int i=0;i<N_STATE;i++) begin residency[i]=res_q[i]; entries[i]=ent_q[i]; end
 
  always_ff @(posedge clk or negedge rst_n) begin
    if (!rst_n || dbg_clear) begin
      prev_q<='0; sat_q<=1'b0;
      for (int i=0;i<N_STATE;i++) begin res_q[i]<='0; ent_q[i]<='0; end
    end else begin
      prev_q <= state;
      if (state < STATE_W'(N_STATE)) begin           // range-safe (23.6 pattern 11)
        if (state_enter) begin
          if (!(&ent_q[state])) ent_q[state] <= ent_q[state] + 32'd1;
        end
        if (&res_q[state]) sat_q <= 1'b1;            // saturate, report
        else res_q[state] <= res_q[state] + 32'd1;
      end
    end
  end
endmodule

import pcie_dbg_pkg::*;
 
// SYNTHESIZABLE. Outstanding-work snapshot -- the "last known good vs
// expected next" instrument for the transaction layer (§5).
module outstanding_snapshot (
  input  logic clk,
  input  logic rst_n,
  input  logic req_transmitted,
  input  logic cpl_received,
  input  logic txn_retired,
  input  logic freeze,
  input  logic dbg_clear,
 
  output logic [15:0] outstanding,
  output logic [31:0] req_count,
  output logic [31:0] cpl_count,
  output logic [31:0] retired_count,
  output logic [15:0] frozen_outstanding,
  output logic        err_negative_outstanding    // sticky
);
  logic [15:0] out_q, froz_q;
  logic [31:0] rq_q, cp_q, rt_q;
  logic e_q;
 
  assign outstanding=out_q; assign req_count=rq_q; assign cpl_count=cp_q;
  assign retired_count=rt_q; assign frozen_outstanding=froz_q;
  assign err_negative_outstanding=e_q;
 
  always_ff @(posedge clk or negedge rst_n) begin
    if (!rst_n || dbg_clear) begin
      out_q<='0; rq_q<='0; cp_q<='0; rt_q<='0; froz_q<='0; e_q<=1'b0;
    end else begin
      // NEXT-STATE arithmetic so a same-cycle transmit and retire is exact
      // (23.6 pattern 9).
      automatic logic [16:0] nxt = {1'b0,out_q} + 17'(req_transmitted) - 17'(txn_retired);
      if (nxt[16]) e_q <= 1'b1;                     // went negative: a lost retire
      else out_q <= nxt[15:0];
      if (req_transmitted && !(&rq_q)) rq_q <= rq_q + 32'd1;
      if (cpl_received    && !(&cp_q)) cp_q <= cp_q + 32'd1;
      if (txn_retired     && !(&rt_q)) rt_q <= rt_q + 32'd1;
      if (freeze) froz_q <= out_q;                  // the value AT the failure
    end
  end
endmodule

import pcie_dbg_pkg::*;
 
// SYNTHESIZABLE. The debug top level -- one place that decides WHEN to
// freeze, so every instrument freezes on the same event (§10's audit).
module pcie_debug_top #(parameter int unsigned STALL_BUDGET = 65536) (
  input  logic clk,
  input  logic rst_n,
  input  logic        err_fire,
  input  logic        no_progress_in,
  input  logic        host_freeze_req,
  input  logic        dbg_clear,
 
  output logic        freeze,
  output logic [1:0]  freeze_reason      // 0 none, 1 error, 2 no-progress, 3 host
);
  logic froz_q; logic [1:0] rsn_q;
  assign freeze = froz_q; assign freeze_reason = rsn_q;
 
  always_ff @(posedge clk or negedge rst_n) begin
    if (!rst_n || dbg_clear) begin froz_q<=1'b0; rsn_q<=2'd0; end
    else if (!froz_q) begin
      // PRIORITY, declared: an error is more informative than a timeout,
      // and both beat a host request (§10).
      if (err_fire)            begin froz_q<=1'b1; rsn_q<=2'd1; end
      else if (no_progress_in) begin froz_q<=1'b1; rsn_q<=2'd2; end
      else if (host_freeze_req)begin froz_q<=1'b1; rsn_q<=2'd3; end
    end
  end
endmodule

// ---- FIRST FAILURE (§6) -----------------------------------------
// P1: the first error is latched exactly once.
property p_first_capture_once;
  @(posedge clk) disable iff (!rst_n)
    ($rose(captured)) |-> $past(err_fire && !captured);
endproperty
 
// P2: the record is immutable until an EXPLICIT clear.
property p_first_record_sticky;
  @(posedge clk) disable iff (!rst_n)
    (captured && !dbg_clear) |=> (captured && $stable(first_code)
                                            && $stable(first_context));
endproperty
 
// P3: later errors are counted, never stored over the first. 21.4 §11:
// last-write-wins named a non-root cause 96.8% of the time.
property p_later_counted_not_stored;
  @(posedge clk) disable iff (!rst_n)
    (captured && err_fire && !dbg_clear) |=> $stable(first_cycle);
endproperty
 
// P4: the captured link state is the one at the failure, not afterwards.
property p_link_state_captured_at_failure;
  @(posedge clk) disable iff (!rst_n)
    ($rose(captured)) |-> (first_link_state == $past(link_state));
endproperty
 
// ---- EVENT HISTORY (§5) -----------------------------------------
// P5: an event is recorded once per push.
property p_push_once;
  @(posedge clk) disable iff (!rst_n)
    (push && !frozen) |=> (wr_ptr != $past(wr_ptr)) || wrapped;
endproperty
 
// P6: the write pointer is always in range -- HIST_DEPTH = 1 included.
property p_ptr_in_range;
  @(posedge clk) disable iff (!rst_n) (wr_ptr < HIST_W'(HIST_DEPTH));
endproperty
 
// P7: history records TRANSFERS, not offers -- the same rule as every
// counter in Modules 22-24.
property p_history_on_transfer;
  @(posedge clk) disable iff (!rst_n)
    push |-> event_transferred;
endproperty
 
// P8: a frozen history is IMMUTABLE.
property p_frozen_immutable;
  @(posedge clk) disable iff (!rst_n)
    frozen |=> $stable(history[0]);
endproperty
 
// P9: events arriving while frozen are COUNTED, not silently discarded.
property p_frozen_drops_counted;
  @(posedge clk) disable iff (!rst_n)
    (frozen && push) |=> (dropped_while_frozen == $past(dropped_while_frozen) + 1);
endproperty
 
// ---- PROGRESS AND RESIDENCY (§4) --------------------------------
// P10: the progress epoch advances only on a DECLARED progress event --
// not on clock activity, not on a counter ticking.
property p_progress_on_declared_event;
  @(posedge clk) disable iff (!rst_n)
    (progress_epoch != $past(progress_epoch)) |-> $past(progress_event);
endproperty
 
// P11: the state-entry pulse is exactly one cycle.
property p_state_enter_one_cycle;
  @(posedge clk) disable iff (!rst_n)
    state_enter |=> !state_enter || (state != $past(state));
endproperty
 
// P12: residency accrues to the state actually occupied.
property p_residency_correct_state;
  @(posedge clk) disable iff (!rst_n)
    (residency[state] != $past(residency[state])) |-> $past(state == state);
endproperty
 
// P13: entry counters increment once per entry, not per resident cycle.
property p_entries_once_per_entry;
  @(posedge clk) disable iff (!rst_n)
    (entries[0] != $past(entries[0])) |-> $past(state_enter);
endproperty
 
// P14: progress resets the stall counter and records the gap.
property p_progress_resets_gap;
  @(posedge clk) disable iff (!rst_n)
    progress_event |=> (cycles_since_progress == '0);
endproperty
 
// P15: the no-progress flag is a DEBUG HEURISTIC and is sticky.
property p_no_progress_sticky;
  @(posedge clk) disable iff (!rst_n)
    no_progress |=> (no_progress || dbg_clear);
endproperty
 
// P16: counters saturate rather than wrap -- a wrapped diagnostic makes a
// busy system look idle (22.1 §11 measured 232 after saturating at 255).
property p_counters_saturate;
  @(posedge clk) disable iff (!rst_n)
    (req_count == 32'hFFFF_FFFF) |=> (req_count == 32'hFFFF_FFFF);
endproperty
 
// ---- FREEZE AND COHERENCE ---------------------------------------
// P17: freeze reason follows the declared priority (§10's audit).
property p_freeze_priority;
  @(posedge clk) disable iff (!rst_n)
    (err_fire && no_progress_in && !freeze) |=> (freeze_reason == 2'd1);
endproperty
 
// P18: every instrument freezes on the SAME event -- one freeze signal.
property p_single_freeze_source;
  @(posedge clk) disable iff (!rst_n)
    freeze |-> (history_frozen && outstanding_frozen);
endproperty
 
// P19: no diagnostic output drives functional logic.
property p_diagnostics_non_functional;
  @(posedge clk) disable iff (!rst_n)
    $stable({req_transmitted, cpl_received}) or !$stable({req_count, cpl_count});
endproperty
 
// P20: outstanding never goes negative -- a retire with no transmit is a
// lost-ownership bug, reported rather than wrapped.
property p_outstanding_non_negative;
  @(posedge clk) disable iff (!rst_n) !err_negative_outstanding;
endproperty
 
// P21: the frozen outstanding value is the one at the failure instant.
property p_frozen_outstanding_at_failure;
  @(posedge clk) disable iff (!rst_n)
    $rose(freeze) |=> (frozen_outstanding == $past(outstanding, 2));
endproperty
 
// P22: EVIDENCE -- the instruments were actually exercised.
property c_error_captured;
  @(posedge clk) disable iff (!rst_n) captured;
endproperty