[UVM] Fixes for error prediction edge cases and some scoreboard race …

…conditions (#586)

* Disable soc_req_lock interrupts during sanitize operation

* Wait for sanitize operation to start before kicking off multi_agent seq, since 'exp_error' calc is difficult

* Try harder to acquire lock using force-unlock (up to attempt_count)

* Use a legal constraint for edge cases where dlen == 0

* Disable 'error' message for unexpected dlen value when an unlock occurred

* Calculate correct expected sts txn if reset occurs simultaneous with a soc_ifc_status signal change

* Refactor the single-cycle pulse of 'fall' signals, for maintainability

* Only start tracking unlock events after lock is acquired

* MICROSOFT AUTOMATED PIPELINE: Stamp 'cwhitehead-msft-uvm-exp-error-multi-agent-fix' with updated timestamp and hash after successful run

.github/workflow_metadata/pr_hash

@@ -1 +1 @@
-f3cb8ff4aa4c7b5a753ebf231de215e8ce1934d851bf919332b53f0042fe2e151aa82a4ec67a58e448f1a4ebf2d14d3c
+fab6c87e214d418f246590abc80ce5acbff6ab3211181509da704c57546ef46fccefa05b529bb463d06fbf9fefdeee36

.github/workflow_metadata/pr_timestamp

@@ -1 +1 @@
-1725662210
+1725922621

src/integration/test_suites/libs/soc_ifc/soc_ifc.c

@@ -236,22 +236,33 @@ void soc_ifc_fw_update(mbox_op_s op) {
 }
 
 uint8_t soc_ifc_sanitize_mbox_n_bytes(uint32_t byte_count, uint32_t attempt_count) {
+    uint32_t notif_intr_en;
     if (byte_count > MBOX_DIR_SPAN) {
         VPRINTF(FATAL, "SOC_IFC: Illegal byte_count 0x%x\n", byte_count);
         SEND_STDOUT_CTRL(0x1);
     }
-    if (soc_ifc_mbox_acquire_lock(attempt_count) != 0) {
+    if(soc_ifc_mbox_acquire_lock(attempt_count) != 0) {
         VPRINTF(WARNING, "Failed to acquire lock - mbox sanitize\n");
-        lsu_write_32(CLP_MBOX_CSR_MBOX_UNLOCK, MBOX_CSR_MBOX_UNLOCK_UNLOCK_MASK);
-        if (soc_ifc_mbox_acquire_lock(1) != 0) {
-            VPRINTF(FATAL, "FATAL: Failed to acquire lock after force unlock\n");
-            return 1;
+        for(uint32_t ii=1; ii<attempt_count; ii++) {
+            lsu_write_32(CLP_MBOX_CSR_MBOX_UNLOCK, MBOX_CSR_MBOX_UNLOCK_UNLOCK_MASK);
+            if((lsu_read_32(CLP_MBOX_CSR_MBOX_LOCK) & MBOX_CSR_MBOX_LOCK_LOCK_MASK) == 0) {
+                break;
+            } else if (ii == attempt_count) {
+                VPRINTF(FATAL, "FATAL: Failed to acquire lock after force unlock\n");
+                return 1;
+            }
         }
     }
+    // Disable notification interrupts for soc_req_lock, which will probably
+    // occur at a high frequency while sanitizing the mailbox and cause a slowdown
+    notif_intr_en = lsu_read_32(CLP_SOC_IFC_REG_INTR_BLOCK_RF_NOTIF_INTR_EN_R);
+    lsu_write_32(CLP_SOC_IFC_REG_INTR_BLOCK_RF_NOTIF_INTR_EN_R, notif_intr_en & ~(SOC_IFC_REG_INTR_BLOCK_RF_NOTIF_INTR_EN_R_NOTIF_SOC_REQ_LOCK_EN_MASK |
+                                                                                  SOC_IFC_REG_INTR_BLOCK_RF_NOTIF_INTR_EN_R_NOTIF_MBOX_ECC_COR_EN_MASK));
     for (uint32_t ii=0; ii < byte_count; ii+=4) {
         lsu_write_32(MBOX_DIR_BASE_ADDR+ii, 0x0);
     }
     lsu_write_32(CLP_MBOX_CSR_MBOX_UNLOCK, MBOX_CSR_MBOX_UNLOCK_UNLOCK_MASK);
+    lsu_write_32(CLP_SOC_IFC_REG_INTR_BLOCK_RF_NOTIF_INTR_EN_R, notif_intr_en);
     return 0;
 }
 

src/integration/uvmf_caliptra_top/uvmf_template_output/project_benches/caliptra_top/tb/sequences/src/caliptra_top_rand_sequence.svh

@@ -416,10 +416,17 @@ class caliptra_top_rand_sequence extends caliptra_top_bench_sequence_base;
             `uvm_fatal("CALIPTRA_TOP_RAND_TEST", $sformatf("caliptra_top_rand_sequence::body() - %s randomization failed", soc_ifc_env_seq_ii[ii].get_type_name()));
         if (override_mbox_sts_exp_error) begin
             soc_ifc_env_mbox_sequence_base_t mbox_seq;
+            soc_ifc_env_mbox_rand_multi_agent_sequence_t mbox_mult_agent_seq;
             if ($cast(mbox_seq,soc_ifc_env_seq_ii[ii])) begin
                 mbox_seq.mbox_sts_exp_error      = 1'b1;
                 mbox_seq.mbox_sts_exp_error_type = override_mbox_sts_exp_error_type;
             end
+            else if ($cast(mbox_mult_agent_seq,soc_ifc_env_seq_ii[ii])) begin
+                // Multi-agent sequences are complicated to add 'expected' error logic.
+                // Just wait for uC to start the sanitize operation and then kick off the sequence.
+                while(reg_model.mbox_csr_rm.mbox_lock.lock.get_mirrored_value() == 0)
+                    soc_ifc_subenv_soc_ifc_ctrl_agent_config.wait_for_num_clocks(1);
+            end
             // Non-mbox sequences (i.e. reset) also clear the error override condition
             else begin
                 override_mbox_sts_exp_error = 1'b0;
@@ -437,7 +444,7 @@ class caliptra_top_rand_sequence extends caliptra_top_bench_sequence_base;
             soc_ifc_env_mbox_sram_double_bit_flip_sequence_t mbox_2bit_seq;
             // At the end of the sequence, Caliptra is not sanitizing the mailbox, so
             // expect the next sequence to be interrupted with an unlock
-            if($cast(mbox_2bit_seq,obj) && mbox_2bit_seq.mbox_sts_exp_error) begin
+            if($cast(mbox_2bit_seq,soc_ifc_env_seq_ii[ii]) && mbox_2bit_seq.mbox_sts_exp_error) begin
                 override_mbox_sts_exp_error = 1'b1;
                 override_mbox_sts_exp_error_type = EXP_ERR_PROT;
             end

src/soc_ifc/uvmf_soc_ifc/uvmf_template_output/verification_ip/environment_packages/soc_ifc_env_pkg/sequences/mbox/soc_ifc/soc_ifc_env_mbox_reg_axs_invalid_small_sequence.svh

@@ -36,6 +36,7 @@ class soc_ifc_env_mbox_reg_axs_invalid_small_sequence extends soc_ifc_env_mbox_r
   constraint mbox_resp_dlen_max_small_c { mbox_resp_expected_dlen < 32'h0000_0200; }
   // Valid solution for the custom delay ruleset, to control random delays while
   // waiting to inject random error accesses
-  constraint custom_delay_c { rand_delay > 0; rand_delay < (mbox_op_rand.dlen * 25); }
+  constraint custom_delay_c { rand_delay > 0;
+                              rand_delay < ((mbox_op_rand.dlen+1) * 25); }
 
 endclass

src/soc_ifc/uvmf_soc_ifc/uvmf_template_output/verification_ip/environment_packages/soc_ifc_env_pkg/sequences/mbox/soc_ifc/soc_ifc_env_mbox_sequence_base.svh

@@ -39,11 +39,13 @@ class soc_ifc_env_mbox_sequence_base extends soc_ifc_env_sequence_base #(.CONFIG
   int sts_rsp_count;
   uvm_status_e reg_sts;
   uvm_event in_report_reg_sts;
+  process teardown_proc;
   rand bit do_apb_lock_check;
   rand bit retry_failed_reg_axs;
   bit mbox_sts_exp_error = 0; // Indicates this sequence will inject an error, which should manifest as a CMD_FAILURE response status
                               // TODO make this more comprehensive/intelligent about randomized error injection
   mbox_sts_exp_error_type_e mbox_sts_exp_error_type = EXP_ERR_NONE; // Known error types to expect/handle from test sequences
+  bit saw_mbox_unlock = 1'b0;
   int datain_ii = MBOX_SIZE_BYTES/4; // Initialize to max value. This iterator is reset in mbox_push_datain for loop, but is
                                      // evaluated against specific offsets for some error checking cases. So give it an
                                      // unambiguously invalid init value prior to use.
@@ -232,7 +234,7 @@ class soc_ifc_env_mbox_sequence_base extends soc_ifc_env_sequence_base #(.CONFIG
     sts_rsp_count = 0;
 
     fork
-        forever begin
+        while (teardown_proc == null) begin
             @(soc_ifc_status_agent_rsp_seq.new_rsp) sts_rsp_count++;
         end
     join_none
@@ -241,6 +243,12 @@ class soc_ifc_env_mbox_sequence_base extends soc_ifc_env_sequence_base #(.CONFIG
 
     mbox_setup();               if (rand_delay_en) do_rand_delay(1, step_delay);
     mbox_acquire_lock(op_sts);  if (rand_delay_en) do_rand_delay(1, step_delay);
+    fork
+        while ((teardown_proc == null) && (!saw_mbox_unlock)) begin
+            configuration.soc_ifc_ctrl_agent_config.wait_for_num_clocks(1);
+            saw_mbox_unlock = reg_model.mbox_csr_rm.mbox_unlock.unlock.get_mirrored_value();
+        end
+    join_none
     mbox_set_cmd(mbox_op_rand); if (rand_delay_en) do_rand_delay(1, step_delay);
     mbox_push_datain();         if (rand_delay_en) do_rand_delay(1, step_delay);
     mbox_execute();             if (rand_delay_en) do_rand_delay(1, step_delay);
@@ -519,6 +527,8 @@ task soc_ifc_env_mbox_sequence_base::mbox_read_resp_data();
                                          "soc_ifc_env_mbox_reg_axs_invalid_medium_sequence",
                                          "soc_ifc_env_mbox_reg_axs_invalid_large_sequence"})
             `uvm_info("MBOX_SEQ", $sformatf("SOC received response data with mbox_dlen [%0d] that does not match the expected data amount [%0d]! Not flagging err since this is an invalid reg-access sequence [%s]", dlen, mbox_resp_expected_dlen, this.get_type_name()), UVM_LOW)
+        else if (saw_mbox_unlock)
+            `uvm_info("MBOX_SEQ", $sformatf("SOC received response data with mbox_dlen [%0d] that does not match the expected data amount [%0d]! Not flagging err since mbox_unlock was observed", dlen, mbox_resp_expected_dlen), UVM_LOW)
         else
             `uvm_error("MBOX_SEQ", $sformatf("SOC received response data with mbox_dlen [%0d] that does not match the expected data amount [%0d]!", dlen, mbox_resp_expected_dlen))
     end
@@ -552,11 +562,14 @@ task soc_ifc_env_mbox_sequence_base::mbox_poll_status();
     do begin
         do_rand_delay(1, poll_delay);
         mbox_check_status(data, state);
-    end while (data == CMD_BUSY && state != MBOX_IDLE);
+    end while (data == CMD_BUSY && state != MBOX_IDLE && !saw_mbox_unlock);
 
     if (state == MBOX_IDLE) begin
         `uvm_info("MBOX_SEQ", "Detected mailbox state transition to IDLE - was mbox_unlock expected?", UVM_HIGH)
     end
+    else if (saw_mbox_unlock) begin
+        `uvm_info("MBOX_SEQ", "Detected mailbox unlock - was mbox_unlock expected?", UVM_HIGH)
+    end
     else if (data == DATA_READY) begin
         if (mbox_resp_expected_dlen == 0 && sts_rsp_count > 0 && soc_ifc_status_agent_rsp_seq.rsp.cptra_error_non_fatal_intr_pending) begin
             `uvm_info("MBOX_SEQ", $sformatf("Unexpected status [%p] likely is the result of a spurious reg access injection specifically intended to cause a protocol violation", data), UVM_HIGH)
@@ -635,6 +648,7 @@ endtask
 //              Currently just reports PAUSER violation count.
 //==========================================
 task soc_ifc_env_mbox_sequence_base::mbox_teardown();
+    this.teardown_proc = process::self();
     // Summary at sequence end
     `uvm_info("MBOX_SEQ", $sformatf("Count of mailbox accesses performed with invalid PAUSER: %0d", hit_invalid_pauser_count), UVM_MEDIUM)
 endtask

src/soc_ifc/uvmf_soc_ifc/uvmf_template_output/verification_ip/environment_packages/soc_ifc_env_pkg/sequences/mbox/soc_ifc/soc_ifc_env_mbox_sram_double_bit_flip_small_sequence.svh

@@ -37,8 +37,11 @@ class soc_ifc_env_mbox_sram_double_bit_flip_small_sequence extends soc_ifc_env_m
   // Valid solution for the custom delay ruleset, to control random delays while
   // waiting to inject random error accesses
   constraint custom_delay_c { rand_delay > 0;
-                              rand_delay dist {[1                   :mbox_op_rand.dlen*2 -1] :/ 250,
-                                               [mbox_op_rand.dlen*2 :mbox_op_rand.dlen*5 -1] :/ 100,
-                                               [mbox_op_rand.dlen*5 :mbox_op_rand.dlen*15-1] :/ 25}; }
+                              if (mbox_op_rand.dlen == 0)
+                                  rand_delay < 25;
+                              else
+                                  rand_delay dist {[1                   :mbox_op_rand.dlen*2 -1] :/ 250,
+                                                   [mbox_op_rand.dlen*2 :mbox_op_rand.dlen*5 -1] :/ 100,
+                                                   [mbox_op_rand.dlen*5 :mbox_op_rand.dlen*15-1] :/ 25}; }
 
 endclass