From 7529c9d24fbfe317047598b7667ed983cbf7753d Mon Sep 17 00:00:00 2001
From: vkjammala-arista <152394203+vkjammala-arista@users.noreply.github.com>
Date: Thu, 19 Sep 2024 09:14:05 +0530
Subject: [PATCH] [dualtor] Fix flakiness of route/test_static_route.py
 (#14202)

* [dualtor] Fix flakiness of route/test_static_route.py

Fixes:
1) Adding "setup_standby_ports_on_rand_unselected_tor" fixture to setup
   ports in standby mode in case of active-active topology. This is
   needed for packets not to go out of unexpected tor and cause test
   failures.
2) Test is performing "config_reload", this can cause switchover (active
   to standy and viceversa). But rand_selected_dut should be in active
   state for traffic verification to pass, so after config_reload we
   need to toggle ports to rand_selected_dut.

* Addressing review comments.

* Reverting minor unintended change.
---
 tests/route/test_static_route.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tests/route/test_static_route.py b/tests/route/test_static_route.py
index 4353b389b1..a6d8b61e8d 100644
--- a/tests/route/test_static_route.py
+++ b/tests/route/test_static_route.py
@@ -11,9 +11,11 @@
 from tests.common.fixtures.ptfhost_utils import change_mac_addresses, copy_arp_responder_py # noqa F811
 from tests.common.dualtor.dual_tor_utils import mux_cable_server_ip
 from tests.common.dualtor.mux_simulator_control import mux_server_url # noqa F811
+from tests.common.dualtor.dual_tor_utils import show_muxcable_status
 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811
 from tests.common.utilities import wait_until, get_intf_by_sub_intf
 from tests.common.utilities import get_neighbor_ptf_port_list
+from tests.common.helpers.assertions import pytest_assert
 from tests.common.helpers.assertions import pytest_require
 from tests.common.helpers.constants import UPSTREAM_NEIGHBOR_MAP
 from tests.common import config_reload
@@ -195,6 +197,12 @@ def _check_nh_in_output(nexthop):
     )
 
 
+def check_mux_status(duthost, expected_status):
+    show_mux_status_ret = show_muxcable_status(duthost)
+    status_values = set([intf_status['status'] for intf_status in show_mux_status_ret.values()])
+    return status_values == {expected_status}
+
+
 def run_static_route_test(duthost, unselected_duthost, ptfadapter, ptfhost, tbinfo,
                           prefix, nexthop_addrs, prefix_len, nexthop_devs, nexthop_interfaces,
                           is_route_flow_counter_supported, ipv6=False, config_reload_test=False): # noqa F811
@@ -248,6 +256,15 @@ def run_static_route_test(duthost, unselected_duthost, ptfadapter, ptfhost, tbin
                 config_reload(duthost, wait=500)
             else:
                 config_reload(duthost, wait=450)
+            # On dualtor, config_reload can result in a switchover (active tor can become standby and viceversa).
+            # So we need to make sure rand_selected_dut is in active state before verifying traffic.
+            if is_dual_tor:
+                duthost.shell("config mux mode active all")
+                unselected_duthost.shell("config mux mode standby all")
+                pytest_assert(wait_until(60, 5, 0, check_mux_status, duthost, 'active'),
+                              "Could not config ports to active on {}".format(duthost.hostname))
+                pytest_assert(wait_until(60, 5, 0, check_mux_status, unselected_duthost, 'standby'),
+                              "Could not config ports to standby on {}".format(unselected_duthost.hostname))
             # FIXME: We saw re-establishing BGP sessions can takes around 7 minutes
             # on some devices (like 4600) after config reload, so we need below patch
             wait_all_bgp_up(duthost)
@@ -339,6 +356,7 @@ def get_nexthops(duthost, tbinfo, ipv6=False, count=1):
 
 
 def test_static_route(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhost, tbinfo,
+                      setup_standby_ports_on_rand_unselected_tor, # noqa F811
                       toggle_all_simulator_ports_to_rand_selected_tor_m, is_route_flow_counter_supported): # noqa F811
     duthost = rand_selected_dut
     unselected_duthost = rand_unselected_dut
@@ -349,6 +367,7 @@ def test_static_route(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhos
 
 @pytest.mark.disable_loganalyzer
 def test_static_route_ecmp(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhost, tbinfo,
+                           setup_standby_ports_on_rand_unselected_tor, # noqa F811
                            toggle_all_simulator_ports_to_rand_selected_tor_m, is_route_flow_counter_supported): # noqa F811
     duthost = rand_selected_dut
     unselected_duthost = rand_unselected_dut
@@ -359,6 +378,7 @@ def test_static_route_ecmp(rand_selected_dut, rand_unselected_dut, ptfadapter, p
 
 
 def test_static_route_ipv6(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhost, tbinfo,
+                           setup_standby_ports_on_rand_unselected_tor, # noqa F811
                            toggle_all_simulator_ports_to_rand_selected_tor_m, is_route_flow_counter_supported): # noqa F811
     duthost = rand_selected_dut
     unselected_duthost = rand_unselected_dut
@@ -370,6 +390,7 @@ def test_static_route_ipv6(rand_selected_dut, rand_unselected_dut, ptfadapter, p
 
 @pytest.mark.disable_loganalyzer
 def test_static_route_ecmp_ipv6(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhost, tbinfo,
+                                setup_standby_ports_on_rand_unselected_tor, # noqa F811
                                 toggle_all_simulator_ports_to_rand_selected_tor_m, is_route_flow_counter_supported): # noqa F811
     duthost = rand_selected_dut
     unselected_duthost = rand_unselected_dut