From 56275fe1db096e2383d698d5d1e42d4fbc1fbac4 Mon Sep 17 00:00:00 2001 From: sanrajen Date: Thu, 13 Jun 2024 14:57:30 -0400 Subject: [PATCH 1/7] New tests for Reliable-TSA feature --- tests/bgp/test_reliable_tsa.py | 1834 +++++++++++++++++++++ tests/bgp/test_startup_tsa_tsb_service.py | 429 ++++- 2 files changed, 2261 insertions(+), 2 deletions(-) create mode 100644 tests/bgp/test_reliable_tsa.py diff --git a/tests/bgp/test_reliable_tsa.py b/tests/bgp/test_reliable_tsa.py new file mode 100644 index 0000000000..b6934c822a --- /dev/null +++ b/tests/bgp/test_reliable_tsa.py @@ -0,0 +1,1834 @@ +import logging +import pytest + +from tests.common import reboot, config_reload +from tests.common.reboot import wait_for_startup +from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.constants import DEFAULT_ASIC_ID +from tests.common.utilities import wait_until +from tests.common.platform.processes_utils import wait_critical_processes +from tests.common.platform.interface_utils import check_interface_status_of_up_ports +from tests.bgp.test_traffic_shift import get_traffic_shift_state, parse_routes_on_neighbors,\ + check_tsa_persistence_support, verify_current_routes_announced_to_neighs, check_and_log_routes_diff, \ + verify_only_loopback_routes_are_announced_to_neighs +from tests.bgp.test_startup_tsa_tsb_service import get_tsa_tsb_service_uptime, get_tsa_tsb_service_status, \ + get_startup_tsb_timer + +pytestmark = [ + pytest.mark.topology('t2') +] + +logger = logging.getLogger(__name__) + +TS_NORMAL = "System Mode: Normal" +TS_MAINTENANCE = "System Mode: Maintenance" +TS_INCONSISTENT = "System Mode: Not consistent" +TS_NO_NEIGHBORS = "System Mode: No external neighbors" +CONTAINER_CHECK_INTERVAL_SECS = 2 +CONTAINER_STOP_THRESHOLD_SECS = 60 +CONTAINER_RESTART_THRESHOLD_SECS = 300 +PROGRAM_STATUS = "RUNNING" +BGP_CRIT_PROCESS = "bgpcfgd" + + +@pytest.fixture(scope="module", autouse=True) +def enable_disable_startup_tsa_tsb_service(duthosts): + """ + @summary: enable/disable startup-tsa-tsb.service during OC run. + Args: + duthosts: Fixture returns a list of Ansible object DuT. + Returns: + None. + """ + for duthost in duthosts.frontend_nodes: + platform = duthost.facts['platform'] + startup_tsa_tsb_file_path = "/usr/share/sonic/device/{}/startup-tsa-tsb.conf".format(platform) + backup_tsa_tsb_file_path = "/usr/share/sonic/device/{}/backup-startup-tsa-tsb.bck".format(platform) + out = duthost.shell("cat {}".format(backup_tsa_tsb_file_path), module_ignore_errors=True)['rc'] + if not out: + duthost.shell("sudo mv {} {}".format(backup_tsa_tsb_file_path, startup_tsa_tsb_file_path), + module_ignore_errors=True) + yield + for duthost in duthosts.frontend_nodes: + platform = duthost.facts['platform'] + startup_tsa_tsb_file_path = "/usr/share/sonic/device/{}/startup-tsa-tsb.conf".format(platform) + backup_tsa_tsb_file_path = "/usr/share/sonic/device/{}/backup-startup-tsa-tsb.bck".format(platform) + out = duthost.shell("cat {}".format(startup_tsa_tsb_file_path), module_ignore_errors=True)['rc'] + if not out: + duthost.shell("sudo mv {} {}".format(startup_tsa_tsb_file_path, backup_tsa_tsb_file_path), + module_ignore_errors=True) + output = duthost.shell("TSB", module_ignore_errors=True) + pytest_assert(not output['rc'], "Failed TSB") + + +@pytest.fixture +def traffic_shift_community(duthost): + """ + @summary: Fetch device's traffic_shift_community string + """ + community = duthost.shell('sonic-cfggen -y /etc/sonic/constants.yml -v constants.bgp.traffic_shift_community')[ + 'stdout'] + return community + + +def nbrhosts_to_dut(duthost, nbrhosts): + """ + @summary: Fetch the neighbor hosts' details for duthost + @returns: dut_nbrhosts dict + """ + mg_facts = duthost.minigraph_facts(host=duthost.hostname)['ansible_facts'] + dut_nbrhosts = {} + for host in nbrhosts.keys(): + if host in mg_facts['minigraph_devices']: + new_nbrhost = {host: nbrhosts[host]} + dut_nbrhosts.update(new_nbrhost) + return dut_nbrhosts + + +def set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname): + """ + @summary: Common method to make sure the supervisor and line cards are in normal state before and after the test + """ + suphost = duthosts[enum_supervisor_dut_hostname] + # Initially make sure both supervisor and line cards are in BGP operational normal state + if get_tsa_chassisdb_config(suphost) != 'false' or get_sup_cfggen_tsa_value(suphost) != 'false': + suphost.shell('TSB') + suphost.shell('sudo config save -y') + pytest_assert('false' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + + for linecard in duthosts.frontend_nodes: + # Issue TSB on line card before proceeding further + if verify_dut_configdb_tsa_value(linecard) != 'false' or get_tsa_chassisdb_config(linecard) != 'false' or \ + get_traffic_shift_state(linecard, cmd='TSC no-stats') != TS_NORMAL: + linecard.shell('TSB') + linecard.shell('sudo config save -y') + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state") + + +def verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes): + """ + @summary: Verify all routes are announced to neighbors in TSB + """ + for linecard in duthosts.frontend_nodes: + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, dut_nbrhosts[linecard], + orig_v4_routes[linecard], cur_v4_routes, 4): + if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], + orig_v4_routes[linecard], cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, dut_nbrhosts[linecard], + orig_v6_routes[linecard], cur_v6_routes, 6): + if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], + orig_v6_routes[linecard], cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + +def get_tsa_chassisdb_config(duthost): + """ + @summary: Returns the dut's CHASSIS_APP_DB value for BGP_DEVICE_GLOBAL.STATE.tsa_enabled flag + """ + tsa_conf = duthost.shell('sonic-db-cli CHASSIS_APP_DB HGET \'BGP_DEVICE_GLOBAL|STATE\' tsa_enabled')['stdout'] + return tsa_conf + + +def get_sup_cfggen_tsa_value(suphost): + """ + @summary: Returns the supervisor sonic-cfggen value for BGP_DEVICE_GLOBAL.STATE.tsa_enabled flag + """ + tsa_conf = suphost.shell('sonic-cfggen -d -v BGP_DEVICE_GLOBAL.STATE.tsa_enabled')['stdout'] + return tsa_conf + + +def verify_dut_configdb_tsa_value(duthost): + """ + @summary: Returns the line cards' asic CONFIG_DB value for BGP_DEVICE_GLOBAL.STATE.tsa_enabled flag + """ + tsa_config = list() + tsa_enabled = 'false' + for asic_index in duthost.get_frontend_asic_ids(): + prefix = "-n asic{}".format(asic_index) if asic_index != DEFAULT_ASIC_ID else '' + output = duthost.shell('sonic-db-cli {} CONFIG_DB HGET \'BGP_DEVICE_GLOBAL|STATE\' \'tsa_enabled\''. + format(prefix))['stdout'] + tsa_config.append(output) + if 'true' in tsa_config: + tsa_enabled = 'true' + + return tsa_enabled + + +def kill_process_by_pid(duthost, container_name, program_name, program_pid): + """ + @summary: Kill a process in the specified container by its pid + """ + kill_cmd_result = duthost.shell("docker exec {} kill -SIGKILL {}".format(container_name, program_pid)) + + # Get the exit code of 'kill' command + exit_code = kill_cmd_result["rc"] + pytest_assert(exit_code == 0, "Failed to stop program '{}' before test".format(program_name)) + + logger.info("Program '{}' in container '{}' was stopped successfully" + .format(program_name, container_name)) + + +def get_program_info(duthost, container_name, program_name): + """ + @summary: Get program running status and its pid by analyzing the command + output of "docker exec supervisorctl status" + @return: Program running status and its pid + """ + program_status = None + program_pid = -1 + + program_list = duthost.shell("docker exec {} supervisorctl status". + format(container_name), module_ignore_errors=True) + for program_info in program_list["stdout_lines"]: + if program_info.find(program_name) != -1: + program_status = program_info.split()[1].strip() + if program_status == "RUNNING": + program_pid = int(program_info.split()[3].strip(',')) + break + + if program_pid != -1: + logger.info("Found program '{}' in the '{}' state with pid {}" + .format(program_name, program_status, program_pid)) + + return program_status, program_pid + + +def is_process_running(duthost, container_name, program_name): + """ + @summary: Determine whether a process under container is in the expected state (running/not running) + @returns: True if its running and false if its not + """ + global PROGRAM_STATUS + program_status, _ = get_program_info(duthost, container_name, program_name) + PROGRAM_STATUS = program_status + if program_status == "RUNNING": + return True + elif program_status in ["EXITED", "STOPPED", "STARTING"]: + return False + else: + pytest.fail("Failed to find program '{}' in container '{}'" + .format(program_name, container_name)) + + +def restart_bgp(duthost, container_name, service_name, program_name, program_pid): + """ + @summary: Kill a critical process in a container to verify whether the container + is stopped and restarted correctly + """ + global PROGRAM_STATUS + pytest_assert(wait_until(40, 3, 0, is_process_running, duthost, container_name, program_name), + "Program '{}' in container '{}' is in the '{}' state, expected 'RUNNING'" + .format(program_name, container_name, PROGRAM_STATUS)) + + kill_process_by_pid(duthost, container_name, program_name, program_pid) + logger.info("Waiting until container '{}' is stopped...".format(container_name)) + stopped = wait_until(CONTAINER_STOP_THRESHOLD_SECS, + CONTAINER_CHECK_INTERVAL_SECS, + 0, + check_container_state, duthost, container_name, False) + pytest_assert(stopped, "Failed to stop container '{}'".format(container_name)) + logger.info("Container '{}' was stopped".format(container_name)) + + logger.info("Waiting until container '{}' is restarted...".format(container_name)) + restarted = wait_until(CONTAINER_RESTART_THRESHOLD_SECS, + CONTAINER_CHECK_INTERVAL_SECS, + 0, + check_container_state, duthost, container_name, True) + if not restarted: + if is_hiting_start_limit(duthost, service_name): + clear_failed_flag_and_restart(duthost, service_name, container_name) + else: + pytest.fail("Failed to restart container '{}'".format(container_name)) + + logger.info("Container '{}' was restarted".format(container_name)) + + +def is_container_running(duthost, container_name): + """ + @summary: Decide whether the container is running or not + @return: Boolean value. True represents the container is running + """ + result = duthost.shell("docker inspect -f \{{\{{.State.Running\}}\}} {}".format(container_name)) # noqa: W605 + return result["stdout_lines"][0].strip() == "true" + + +def check_container_state(duthost, container_name, should_be_running): + """ + @summary: Determine whether a container is in the expected state (running/not running) + """ + is_running = is_container_running(duthost, container_name) + return is_running == should_be_running + + +def is_hiting_start_limit(duthost, service_name): + """ + @summary: Determine whether the service can not be restarted is due to + start-limit-hit or not + """ + service_status = duthost.shell("sudo systemctl status {}.service | grep 'Active'".format(service_name)) + for line in service_status["stdout_lines"]: + if "start-limit-hit" in line: + return True + + return False + + +def clear_failed_flag_and_restart(duthost, service_name, container_name): + """ + @summary: If a container hits the restart limitation, then we clear the failed flag and + restart it. + """ + logger.info("{} hits start limit and clear reset-failed flag".format(service_name)) + duthost.shell("sudo systemctl reset-failed {}.service".format(service_name)) + duthost.shell("sudo systemctl start {}.service".format(service_name)) + restarted = wait_until(300, 1, 0, check_container_state, duthost, container_name, True) + pytest_assert(restarted, "Failed to restart container '{}' after reset-failed was cleared".format(container_name)) + + +@pytest.mark.disable_loganalyzer +def test_sup_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test supervisor TSA action when supervisor and line cards are in TSB initially + Verify supervisor config state changes to TSA and Line card BGP TSA operational state changes to TSA from TSB + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Issue TSA from supervisor and verify line cards' BGP operational state changes to TSA + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + for linecard in duthosts.frontend_nodes: + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_sup_tsa_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test supervisor TSA action when supervisor is on TSB and line cards are in TSA initially + Verify supervisor config state changes to TSA and Line card BGP TSA operational state maintains TSA + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + dut_nbrhosts = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Convert line cards to BGP operational TSA state for the current test as initial config + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + + # Now Issue TSA from supervisor and make sure it changes from TSB->TSA + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + for linecard in duthosts.frontend_nodes: + # Verify DUT continues to be in maintenance state even with supervisor TSA action + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + "DUT is not in maintenance state with supervisor TSA action") + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_sup_tsb_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test supervisor TSB action when supervisor is on TSA and line cards are in TSB configuration initially but with + BGP operational TSA states + Verify supervisor config state changes to TSB and Line card BGP TSA operational state changes to TSB from TSA + Make sure only loopback routes are advertised to neighbors during line cards' TSA state and all routes are + announced back to neighbors when the line cards are back to TSB. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + dut_nbrhosts = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Keep supervisor in TSA mode to start with as part of the test + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + # Confirm all the line cards are in BGP operational TSA state due to supervisor TSA + for linecard in duthosts.frontend_nodes: + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + # Issue TSB on the supervisor + suphost.shell('TSB') + suphost.shell('sudo config save -y') + pytest_assert('false' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + + # Verify line cards change the state to TSB from TSA after supervisor TSB + for linecard in duthosts.frontend_nodes: + # Verify DUT changes to normal state with supervisor TSB action + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), + "DUT is not in normal state with supervisor TSB action") + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_sup_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test supervisor TSB action when supervisor and line cards are in TSA configuration initially + Verify supervisor config state changes to TSB and Line card BGP TSA operational state is maintained + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + dut_nbrhosts = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Keep supervisor in TSA mode to start with as part of the test + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + # Similarly keep line cards in TSA mode to start with as part of the test + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + + # Issue TSB on the supervisor + suphost.shell('TSB') + suphost.shell('sudo config save -y') + pytest_assert('false' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + + # Verify line cards maintains the BGP operational TSA state but with chassisdb tsa-enabled config as 'false' + # in sync with supervisor + for linecard in duthosts.frontend_nodes: + # Verify DUT continues to be in maintenance state even with supervisor TSB action + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_dut_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test line card TSA action when supervisor and line cards are in TSB initially + Verify line card config state changes to TSA and BGP TSA operational state changes to TSA from TSB + Verify supervisor card continues to be in TSB + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Issue TSA from line card and verify line cards' BGP operational state changes to TSA + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + # Verify supervisor still has tsa_enabled 'false' config + pytest_assert('false' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_dut_tsa_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test line card TSA action when supervisor is on TSA and line cards are in TSB initially + Verify line card config state changes to TSA and BGP TSA operational state maintains its TSA state + Verify supervisor card continues to be in TSA config + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Keep supervisor in TSA mode to start with as part of the test + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + # Confirm all the line cards are in BGP operational TSA state due to supervisor TSA + for linecard in duthosts.frontend_nodes: + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Verify line card config TSA enabled is still false + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + + # Issue TSA from line card and verify line cards' BGP operational state continues to be in TSA + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + # Verify supervisor still has tsa_enabled 'true' config + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_dut_tsb_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test line card TSB action when supervisor is on TSB and line cards are in TSA initially + Verify line card config state changes to TSB and BGP TSA operational state changes to TSB from TSA + Verify supervisor card continues to be in TSB config + Make sure only loopback routes are advertised to neighbors during line cards' TSA state and all routes are + announced back to neighbors when the line cards are back to TSB. + """ + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Keep supervisor in TSB mode to start with as part of the test + # And keep the line cards in TSA and verify line cards' BGP operational state changes to TSA + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + # Issue TSB from line card and verify line cards' BGP operational state changes to TSB + for linecard in duthosts.frontend_nodes: + linecard.shell('TSB') + linecard.shell('sudo config save -y') + # Verify line card config changed to tsa_enabled false + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + # Ensure that the DUT is in normal state + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + + # Make sure all routes are advertised back to neighbors after TSB on line cards + for linecard in duthosts.frontend_nodes: + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, + dut_nbrhosts[linecard], + orig_v4_routes[linecard], cur_v4_routes, 4): + if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], + orig_v4_routes[linecard], cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, + dut_nbrhosts[linecard], + orig_v6_routes[linecard], cur_v6_routes, 6): + if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], + orig_v6_routes[linecard], cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + finally: + # Bring back the supervisor and line cards to the normal state at the end of test + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + +@pytest.mark.disable_loganalyzer +def test_dut_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test line card TSB action when supervisor and line cards are in TSA configuration initially + Verify line card config state changes to TSB but the line card BGP TSA operational state is maintained + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + dut_nbrhosts = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Keep supervisor in TSA mode to start with as part of the test + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + # Similarly keep line cards in TSA mode to start with as part of the test + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + + # Issue TSB from line card and verify line cards' BGP operational state maintained at TSA + for linecard in duthosts.frontend_nodes: + linecard.shell('TSB') + linecard.shell('sudo config save -y') + # Verify line card config changed to tsa_enabled false + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + # Verify supervisor still has tsa_enabled 'true' config + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_sup_tsa_act_with_sup_reboot(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test supervisor TSA action when supervisor and line cards are in TSB initially + Verify supervisor config state changes to TSA and Line card BGP TSA operational state changes to TSA from TSB + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + Then, do 'config save' and reboot supervisor. + After reboot, make sure the BGP TSA operational states are same as before reboot. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + tsa_tsb_timer = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + for linecard in duthosts.frontend_nodes: + tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Issue TSA from supervisor and verify line cards' BGP operational state changes to TSA + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + for linecard in duthosts.frontend_nodes: + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + # Not verifying loopback routes here check since its been checked multiple times with previous test cases + + # Get a dut uptime before reboot + sup_uptime_before = suphost.get_up_time() + # Reboot supervisor and wait for startup_tsa_tsb service to start on line cards + logger.info("Cold reboot on supervisor node: %s", suphost.hostname) + reboot(suphost, localhost, wait=240) + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(suphost) + + sup_uptime = suphost.get_up_time() + logger.info('DUT {} up since {}'.format(suphost.hostname, sup_uptime)) + rebooted = float(sup_uptime_before.strftime("%s")) != float(sup_uptime.strftime("%s")) + assert rebooted, "Device {} did not reboot".format(suphost.hostname) + # verify chassisdb config is same as before reboot + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + for linecard in duthosts.frontend_nodes: + wait_for_startup(linecard, localhost, delay=10, timeout=300) + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = linecard.get_up_time() + logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(linecard) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + + # Verify DUT is in the same maintenance state like before supervisor reboot + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(linecard) + pytest_assert(wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard), + "Not all ports that are admin up on are operationally up") + + # Once all line cards are in maintenance state, proceed further + for linecard in duthosts.frontend_nodes: + # Verify startup_tsa_tsb service stopped after expected time + pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'exited'), + "startup_tsa_tsb service is not stopped even after configured timer expiry") + + # Ensure dut comes back to normal state after timer expiry + if not get_tsa_tsb_service_status(linecard, 'running'): + # Verify dut continues to be in TSA even after startup_tsa_tsb service is stopped + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state after startup_tsa_tsb service is stopped") + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + # Verify line card config changed to TSB after startup-tsa-tsb service expiry + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_sup_tsa_act_when_duts_on_tsa_with_sup_config_reload(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test supervisor TSA action when supervisor is on TSB and line cards are in TSA initially + Verify supervisor config state changes to TSA and Line card BGP TSA operational state maintained at TSA + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + Then, do config_reload on the supervisor. + After config_relaod, make sure the BGP TSA operational states are same as before. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + dut_nbrhosts = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Convert line cards to BGP operational TSA state for the current test as initial config + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + + # Now Issue TSA from supervisor and make sure it changes from TSB->TSA + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + for linecard in duthosts.frontend_nodes: + # Verify DUT continues to be in maintenance state even with supervisor TSA action + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + "DUT is not in maintenance state with supervisor TSA action") + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + + # Do config_reload on the supervisor and verify configs are same as before + config_reload(suphost, wait=300, safe_reload=True) + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + # Verify line cards traffic shift states are same as before config_reload + for linecard in duthosts.frontend_nodes: + # Verify DUT is in the same maintenance state like before supervisor config reload + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + "DUT is not in maintenance state after supervisor config reload") + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled chassisdb config is not enabled".format(linecard.hostname)) + + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_dut_tsa_act_with_reboot_when_sup_dut_on_tsb_init(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test line card TSA action when supervisor and line cards are in TSB initially + Verify line card config state changes to TSA and BGP TSA operational state changes to TSA from TSB + Verify supervisor card continues to be in TSB + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + Then, do 'config save' and reboot the line cards. + After reboot, make sure the BGP TSA operational states are same as before reboot on line cards. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + orig_v4_routes, orig_v6_routes = dict(), dict() + tsa_tsb_timer = dict() + dut_nbrhosts = dict() + for linecard in duthosts.frontend_nodes: + tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Issue TSA from line card and verify line cards' BGP operational state changes to TSA + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + + # Verify dut reboot scenario for one of the line card to make sure tsa config is in sync + for linecard in duthosts.frontend_nodes: + logger.info("Cold reboot on node: %s", linecard.hostname) + reboot(linecard, localhost, wait=240) + + for linecard in duthosts.frontend_nodes: + wait_for_startup(linecard, localhost, delay=10, timeout=300) + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = linecard.get_up_time() + logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(linecard) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + # Verify startup_tsa_tsb service is not started and in exited due to manual TSA + pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'exited'), + "startup_tsa_tsb service is in running state after dut reboot which is not expected") + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + # Verify line card config changed is still TSA enabled true after reboot + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + + # Make sure the ports, interfaces are UP and running after reboot + for linecard in duthosts.frontend_nodes: + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(linecard) + pytest_assert(wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard), + "Not all ports that are admin up on are operationally up") + + for linecard in duthosts.frontend_nodes: + # Verify only loopback routes are announced to neighbors when the linecards are in TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + # Verify supervisor still has tsa_enabled 'false' config + pytest_assert('false' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_dut_tsa_with_conf_reload_when_sup_on_tsa_dut_on_tsb_init(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test line card TSA action when supervisor is on TSA and line cards are in TSB initially + Verify line card config state changes to TSA and BGP TSA operational state maintains its TSA state + Verify supervisor card continues to be in TSA config + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + Then, do 'config save' and config_reload the line card + After config_reload, make sure the BGP TSA operational states are same as before config reload on line card. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Now Issue TSA from supervisor and make sure it changes from TSB->TSA + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + # Verify line cards' BGP operational state changes to TSA + for linecard in duthosts.frontend_nodes: + # Verify line card BGP operational state changes to TSA + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled chassisdb config is not enabled".format(linecard.hostname)) + + # Verify dut config_reload scenario for one of the line card to make sure tsa config is in sync + for linecard in duthosts.frontend_nodes: + linecard.shell('sudo config save -y') + config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state after config reload") + + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + break + + # Verify supervisor still has tsa_enabled 'true' config + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_user_init_tsa_on_dut_followed_by_sup_tsa(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test user initiated line card TSA action when supervisor and line cards are in TSB initially + Verify line card config state changes to TSA and BGP TSA operational state changes to TSA from TSB + Verify supervisor card continues to be in TSB + Then, issue TSA on supervisor card. + Verify supervisor and line card chassisdb config state changes to TSA and line card continues to be in + BGP operational TSA state. + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Issue TSA from line card and verify line cards' BGP operational state changes to TSA + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + + # Issue TSA from supervisor and verify line cards' BGP operational state continues to be in TSA + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + # Verify line cards' BGP operational state continues in mainternance state + for linecard in duthosts.frontend_nodes: + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + # Verify only loopback routes are announced with TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_user_init_tsa_on_dut_followed_by_sup_tsb(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test user initiated line card TSA action when supervisor and line cards are in TSB initially + Verify line card config state changes to TSA and BGP TSA operational state changes to TSA from TSB + Verify supervisor card continues to be in TSB + Then, issue TSB on supervisor card. + Verify supervisor and line card chassisdb config state changes to TSB and line card continues to be in + BGP operational TSA state. + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Issue TSA from line card and verify line cards' BGP operational state changes to TSA + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + + # Issue TSB from supervisor and verify line cards' BGP operational state continues to be in TSA + suphost.shell('TSB') + suphost.shell('sudo config save -y') + pytest_assert('false' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + + # Verify line cards' BGP operational state continues in mainternance state + for linecard in duthosts.frontend_nodes: + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + # Verify only loopback routes are announced with TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test supervisor TSA action when startup-tsa-tsb service is running on line cards after reboot + Verify line card BGP operational state continues to be in TSA after the supervisor TSA action + Verify supervisor card changes to TSA from TSB + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + tsa_tsb_timer = dict() + for linecard in duthosts.frontend_nodes: + tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Verify dut reboot scenario for line card to make sure tsa config is in sync + for linecard in duthosts.frontend_nodes: + logger.info("Cold reboot on node: %s", linecard.hostname) + reboot(linecard, localhost, wait=240) + wait_for_startup(linecard, localhost, delay=10, timeout=300) + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = linecard.get_up_time() + logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(linecard) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + # Verify startup_tsa_tsb service is started and running + pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'running'), + "startup_tsa_tsb service is not in running state after dut reboot") + # Now Issue TSA from supervisor and make sure it changes from TSB->TSA while the service is running + if get_tsa_tsb_service_status(linecard, 'running'): + # Now Issue TSA from supervisor and make sure it changes from TSB->TSA + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + # Verify line card config changed to tsa_enabled true during service run + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + + for linecard in duthosts.frontend_nodes: + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(linecard) + pytest_assert(wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard), + "Not all ports that are admin up on are operationally up") + + # Verify startup_tsa_tsb service stopped after expected time + pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'exited'), + "startup_tsa_tsb service is not stopped even after configured timer expiry") + + # Ensure dut is still in TSA even after startup-tsa-tsb timer expiry + if get_tsa_tsb_service_status(linecard, 'exited'): + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is in normal state after startup_tsa_tsb service is stopped") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + # Verify line card config changed to tsa_enabled false after timer expiry + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + # Verify only loopback routes are announced to neighbors at this state + for linecard in duthosts.frontend_nodes: + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_sup_tsb_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test supervisor TSB action when startup-tsa-tsb service is running on line cards after reboot + Verify line card BGP operational state changes to normal from maintenance after supervisor TSB with timer expiry + Make sure only loopback routes are advertised to neighbors during line cards' TSA state and make sure all routes + are advertised back once the line cards are in normal state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + tsa_tsb_timer = dict() + for linecard in duthosts.frontend_nodes: + tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Verify dut reboot scenario for one of the line card to make sure tsa config is in sync + for linecard in duthosts.frontend_nodes: + logger.info("Cold reboot on node: %s", linecard.hostname) + reboot(linecard, localhost, wait=240) + wait_for_startup(linecard, localhost, delay=10, timeout=300) + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = linecard.get_up_time() + logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(linecard) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + # Verify startup_tsa_tsb service is started and running + pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'running'), + "startup_tsa_tsb service is not in running state after dut reboot") + # Now Issue TSB from supervisor and make sure it changes from TSA->TSB + if get_tsa_tsb_service_status(linecard, 'running'): + # Now Issue TSB from supervisor + suphost.shell('TSB') + suphost.shell('sudo config save -y') + pytest_assert('false' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(linecard) + pytest_assert(wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard), + "Not all ports that are admin up on are operationally up") + + # Verify startup_tsa_tsb service stopped after expected time + pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'exited'), + "startup_tsa_tsb service is not stopped even after configured timer expiry") + + # Ensure dut gets back to normal state after startup-tsa-tsb timer expiry + if get_tsa_tsb_service_status(linecard, 'exited'): + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state after startup_tsa_tsb service is stopped") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + # Verify line card config changed to tsa_enabled false after timer expiry + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + break + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_on_tsa_duts_on_tsb( + duthosts, localhost, enum_supervisor_dut_hostname, nbrhosts, traffic_shift_community, tbinfo): + """ + Test supervisor TSB action when supervisor is on TSA and line cards are in TSB configuration initially but with + BGP operational TSA states + Verify supervisor config state changes to TSB and Line card BGP TSA operational state changes to TSB from TSA + Restart bgp on the line cards and make sure BGP TSA operational state is maintained after docker restart + Make sure only loopback routes are advertised to neighbors during line cards' TSA state and all routes are + announced back to neighbors when the line cards are back to TSB. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + dut_nbrhosts = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Keep supervisor in TSA mode to start with as part of the test + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + # Confirm all the line cards are in BGP operational TSA state due to supervisor TSA + for linecard in duthosts.frontend_nodes: + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + # Issue TSB on the supervisor + suphost.shell('TSB') + suphost.shell('sudo config save -y') + pytest_assert('false' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + + # Restart bgp on the line cards and check the status + for linecard in duthosts.frontend_nodes: + for asic in linecard.asics: + service_name = asic.get_service_name("bgp") + container_name = asic.get_docker_name("bgp") + logger.info("Restarting {} container on dut {}".format(container_name, linecard.hostname)) + process_status, program_pid = get_program_info(linecard, container_name, BGP_CRIT_PROCESS) + if process_status == "RUNNING": + restart_bgp(linecard, container_name, service_name, BGP_CRIT_PROCESS, program_pid) + + # Verify line cards continues to be in TSB state even after bgp restart + for linecard in duthosts.frontend_nodes: + # Verify DUT changes to normal state with supervisor TSB action + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state with supervisor TSB action") + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, localhost, enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test supervisor TSB action when supervisor and line cards are in TSA configuration initially + Verify supervisor config state changes to TSB and Line card BGP TSA operational state is maintained + Restart bgp on the line cards and make sure BGP TSA operational state is maintained after docker restart + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + dut_nbrhosts = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Keep supervisor in TSA mode to start with as part of the test + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + # Similarly keep line cards in TSA mode to start with as part of the test + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + + # Issue TSB on the supervisor + suphost.shell('TSB') + suphost.shell('sudo config save -y') + pytest_assert('false' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + + # Restart bgp on the line cards and check the status + for linecard in duthosts.frontend_nodes: + for asic in linecard.asics: + service_name = asic.get_service_name("bgp") + container_name = asic.get_docker_name("bgp") + logger.info("Restarting {} container on dut {}".format(container_name, linecard.hostname)) + process_status, program_pid = get_program_info(linecard, container_name, BGP_CRIT_PROCESS) + if process_status == "RUNNING": + restart_bgp(linecard, container_name, service_name, BGP_CRIT_PROCESS, program_pid) + + # Verify line cards maintains the BGP operational TSA state but with chassisdb tsa-enabled config as 'false' + # in sync with supervisor + for linecard in duthosts.frontend_nodes: + # Verify DUT continues to be in maintenance state even with supervisor TSB action + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) + + +@pytest.mark.disable_loganalyzer +def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_on_tsb_duts_on_tsa(duthosts, localhost, + enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test line card TSB action when supervisor is on TSB and line cards are in TSA initially + Verify line card config state changes to TSB and BGP TSA operational state changes to TSB from TSA + Restart bgp on the line cards and make sure BGP TSA operational state is maintained after docker restart + Verify supervisor card continues to be in TSB config + Make sure only loopback routes are advertised to neighbors during line cards' TSA state and all routes are + announced back to neighbors when the line cards are back to TSB. + """ + orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Keep supervisor in the current TSB mode to start with as part of the test + # And keep the line cards in TSA and verify line cards' BGP operational state changes to TSA + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + # Issue TSB from line card and verify line cards' BGP operational state changes to TSB + for linecard in duthosts.frontend_nodes: + linecard.shell('TSB') + linecard.shell('sudo config save -y') + # Verify line card config changed to tsa_enabled false + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + # Ensure that the DUT is in normal state + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + + # Restart bgp on the line cards and check the status + for linecard in duthosts.frontend_nodes: + for asic in linecard.asics: + service_name = asic.get_service_name("bgp") + container_name = asic.get_docker_name("bgp") + logger.info("Restarting {} container on dut {}".format(container_name, linecard.hostname)) + process_status, program_pid = get_program_info(linecard, container_name, BGP_CRIT_PROCESS) + if process_status == "RUNNING": + restart_bgp(linecard, container_name, service_name, BGP_CRIT_PROCESS, program_pid) + + # Verify line cards are in the same state as before docker restart + for linecard in duthosts.frontend_nodes: + # Verify line card config changed to tsa_enabled false + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is enabled".format(linecard.hostname)) + + for linecard in duthosts.frontend_nodes: + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, + dut_nbrhosts[linecard], + orig_v4_routes[linecard], cur_v4_routes, 4): + if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], + orig_v4_routes[linecard], cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, + dut_nbrhosts[linecard], + orig_v6_routes[linecard], cur_v6_routes, 6): + if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], + orig_v6_routes[linecard], cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + finally: + # Bring back the supervisor and line cards to the normal state at the end of test + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + +@pytest.mark.disable_loganalyzer +def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, localhost, + enum_supervisor_dut_hostname, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test line card TSB action when supervisor and line cards are in TSA configuration initially + Verify line card config state changes to TSB but the line card BGP TSA operational state is maintained + Restart bgp on the line cards and make sure BGP TSA operational state is continued after docker restart + Make sure only loopback routes are advertised to neighbors during line cards' TSA state. + """ + suphost = duthosts[enum_supervisor_dut_hostname] + dut_nbrhosts = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + for linecard in duthosts.frontend_nodes: + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + try: + # Get the original routes present on the neighbors for each line card + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Keep supervisor in TSA mode to start with as part of the test + suphost.shell('TSA') + suphost.shell('sudo config save -y') + pytest_assert('true' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) + + # Similarly keep line cards in TSA mode to start with as part of the test + for linecard in duthosts.frontend_nodes: + linecard.shell('TSA') + linecard.shell('sudo config save -y') + # Verify line card config changed to TSA enabled true + pytest_assert('true' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + + # Issue TSB from line card and verify line cards' BGP operational state maintained at TSA + for linecard in duthosts.frontend_nodes: + linecard.shell('TSB') + linecard.shell('sudo config save -y') + # Verify line card config changed to tsa_enabled false + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + + # Restart bgp on the line cards and check the status + for linecard in duthosts.frontend_nodes: + for asic in linecard.asics: + service_name = asic.get_service_name("bgp") + container_name = asic.get_docker_name("bgp") + logger.info("Restarting {} container on dut {}".format(container_name, linecard.hostname)) + process_status, program_pid = get_program_info(linecard, container_name, BGP_CRIT_PROCESS) + if process_status == "RUNNING": + restart_bgp(linecard, container_name, service_name, BGP_CRIT_PROCESS, program_pid) + + # Verify line cards are in the same state as before bgp restart + for linecard in duthosts.frontend_nodes: + # Verify line card config changed to tsa_enabled false + pytest_assert('false' == verify_dut_configdb_tsa_value(linecard), + "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + # Ensure that the DUT is in maintenance state + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('true' == get_tsa_chassisdb_config(linecard), + "{} tsa_enabled config is not enabled".format(linecard.hostname)) + # Verify only loopback routes are announced after TSA + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + finally: + # Bring back the supervisor and line cards to the normal state + set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + + # Verify all routes are advertised back to neighbors when duts are in TSB + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) diff --git a/tests/bgp/test_startup_tsa_tsb_service.py b/tests/bgp/test_startup_tsa_tsb_service.py index d52e45a5f3..2d8e04856b 100644 --- a/tests/bgp/test_startup_tsa_tsb_service.py +++ b/tests/bgp/test_startup_tsa_tsb_service.py @@ -102,8 +102,8 @@ def get_tsa_tsb_service_uptime(duthost): service_status = duthost.shell("sudo systemctl status startup_tsa_tsb.service | grep 'Active'") for line in service_status["stdout_lines"]: if 'active' in line: - tmp_time = line.split('since')[1].strip().encode('utf-8') - act_time = tmp_time.split('UTC')[0].strip().encode('utf-8') + tmp_time = line.split('since')[1].strip() + act_time = tmp_time.split('UTC')[0].strip() service_uptime = datetime.datetime.strptime(act_time[4:], '%Y-%m-%d %H:%M:%S') return service_uptime return service_uptime @@ -1155,3 +1155,428 @@ def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_f reboot_cause = get_reboot_cause(duthost) pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) + + +@pytest.mark.disable_loganalyzer +def test_user_init_tsa_while_service_run_on_dut(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, + nbrhosts, traffic_shift_community, tbinfo): + + """ + Test startup TSA_TSB service after DUT cold reboot + Verify startup_tsa_tsb.service started automatically when dut comes up + Verify this service configures TSA and starts a timer + Issue TSA while the service is running on dut, and make sure the TSA is configured + Make sure TSA_TSB service is stopped and dut continues to be in maintenance mode + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + tsa_tsb_timer = get_startup_tsb_timer(duthost) + if not tsa_tsb_timer: + pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) + dut_nbrhosts = nbrhosts_to_dut(duthost, nbrhosts) + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state") + if not check_tsa_persistence_support(duthost): + pytest.skip("TSA persistence not supported in the image") + + try: + # Get all routes on neighbors before doing reboot + orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) + orig_v6_routes = parse_routes_on_neighbors(duthost, nbrhosts, 6) + + # Reboot dut and wait for startup_tsa_tsb service to start + logger.info("Cold reboot on node: %s", duthost.hostname) + reboot(duthost, localhost, wait=240) + + logger.info('Cold reboot finished on {}'.format(duthost.hostname)) + dut_uptime = duthost.get_up_time() + logger.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + + # Ensure startup_tsa_tsb service is running after dut reboot + pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'running'), + "startup_tsa_tsb service is not started after reboot") + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = duthost.get_up_time() + logging.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(duthost) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + + # Issue TSA on DUT + duthost.shell("TSA") + duthost.shell('sudo config save -y') + + # Ensure startup_tsa_tsb service is in inactive state after user-initiated TSA + pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'inactive'), + "startup_tsa_tsb service is not in inactive state after user init TSA") + + # Verify DUT continues to be in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), + "DUT is not in maintenance state with saved TSA config after reboot") + + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(duthost) + pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), + "Not all ports that are admin up on are operationally up") + + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, duthost, dut_nbrhosts, traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + finally: + """ + Test TSB after config save and config reload + Verify all routes are announced back to neighbors + """ + # Recover to Normal state + duthost.shell("TSB") + duthost.shell('sudo config save -y') + + # Verify DUT comes back to normal state after TSB. + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v4_routes, cur_v4_routes, 4): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v4_routes, cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v6_routes, cur_v6_routes, 6): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + # Make sure the dut's reboot cause is as expected + logger.info("Check reboot cause of the dut") + reboot_cause = get_reboot_cause(duthost) + pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, + "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) + + +@pytest.mark.disable_loganalyzer +def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, + nbrhosts, traffic_shift_community, tbinfo): + + """ + Test startup TSA_TSB service after DUT cold reboot + Verify startup_tsa_tsb.service started automatically when dut comes up + Verify this service configures TSA and starts a timer + Issue TSB while the service is running on dut, and make sure the TSB is configured + Make sure TSA_TSB service is stopped and dut continues to be in normal mode + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + tsa_tsb_timer = get_startup_tsb_timer(duthost) + if not tsa_tsb_timer: + pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state") + if not check_tsa_persistence_support(duthost): + pytest.skip("TSA persistence not supported in the image") + + try: + # Get all routes on neighbors before doing reboot + orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) + orig_v6_routes = parse_routes_on_neighbors(duthost, nbrhosts, 6) + + # Reboot dut and wait for startup_tsa_tsb service to start + logger.info("Cold reboot on node: %s", duthost.hostname) + reboot(duthost, localhost, wait=240) + + logger.info('Cold reboot finished on {}'.format(duthost.hostname)) + dut_uptime = duthost.get_up_time() + logger.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + + # Ensure startup_tsa_tsb service is running after dut reboot + pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'running'), + "startup_tsa_tsb service is not started after reboot") + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = duthost.get_up_time() + logging.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(duthost) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + + # Issue TSB on DUT + duthost.shell("TSB") + duthost.shell('sudo config save -y') + + # Verify DUT comes back to normal state after TSB. + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") + + # Ensure startup_tsa_tsb service is in inactive state after user-initiated TSB + pytest_assert(wait_until(60, 5, 10, get_tsa_tsb_service_status, duthost, 'inactive'), + "startup_tsa_tsb service is not in inactive state after user init TSB") + + # Make sure DUT continues to be in good state after TSB + assert wait_until(300, 20, 2, duthost.critical_services_fully_started), \ + "Not all critical services are fully started on {}".format(duthost.hostname) + + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v4_routes, cur_v4_routes, 4): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v4_routes, cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v6_routes, cur_v6_routes, 6): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + finally: + + # Verify DUT is in normal state. + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state") + + # Make sure the dut's reboot cause is as expected + logger.info("Check reboot cause of the dut") + reboot_cause = get_reboot_cause(duthost) + pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, + "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) + + +@pytest.mark.disable_loganalyzer +def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, + enum_supervisor_dut_hostname, ptfhost, nbrhosts, + traffic_shift_community, creds, tbinfo): + """ + Test startup TSA_TSB service after DUT cold reboot + Verify startup_tsa_tsb.service started automatically when dut comes up + Verify this service configures TSA and starts a timer + Issue TSB from supervisor, while the service is running on dut, and make sure the TSB is configured on linecards + Make sure TSA_TSB service is stopped and dut changes from maintenance mode to normal mode + """ + suphost = duthosts[enum_supervisor_dut_hostname] + tsa_tsb_timer = dict() + dut_nbrhosts = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + for linecard in duthosts.frontend_nodes: + tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) + if not tsa_tsb_timer[linecard]: + pytest.skip("startup_tsa_tsb.service is not supported on the duts under {}".format(suphost.hostname)) + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), + "DUT is not in normal state") + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + + try: + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors before doing reboot + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Get a dut uptime before reboot + sup_uptime_before = suphost.get_up_time() + # Reboot dut and wait for startup_tsa_tsb service to start on linecards + logger.info("Cold reboot on supervisor node: %s", suphost.hostname) + reboot(suphost, localhost, wait=240) + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(suphost) + + sup_uptime = suphost.get_up_time() + logger.info('DUT {} up since {}'.format(suphost.hostname, sup_uptime)) + rebooted = float(sup_uptime_before.strftime("%s")) != float(sup_uptime.strftime("%s")) + assert rebooted, "Device {} did not reboot".format(suphost.hostname) + + for linecard in duthosts.frontend_nodes: + wait_for_startup(linecard, localhost, delay=10, timeout=300) + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = linecard.get_up_time() + logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(linecard) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(linecard) + + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + # Issue user initiated TSB on the supervisor + suphost.shell('TSB') + + for linecard in duthosts.frontend_nodes: + if get_tsa_tsb_service_status(linecard, 'running'): + # Verify DUT continues to be in maintenance state if the timer is running. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + else: + # Verify DUT continues came back to normal state after timer expiry. + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state when startup_tsa_tsb service is running") + + # Ensure startup_tsa_tsb service is in exited state after timer expiry + pytest_assert(wait_until(tsa_tsb_timer[linecard], 5, 0, get_tsa_tsb_service_status, linecard, 'exited'), + "startup_tsa_tsb service is not in exited state after user init TSB from supervisor") + pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, linecard), + "Not all ports that are admin up on are operationally up") + + for linecard in duthosts.frontend_nodes: + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, dut_nbrhosts[linecard], + orig_v4_routes[linecard], cur_v4_routes, 4): + if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], + orig_v4_routes[linecard], cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, dut_nbrhosts[linecard], + orig_v6_routes[linecard], cur_v6_routes, 6): + if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], + orig_v6_routes[linecard], cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + finally: + for linecard in duthosts.frontend_nodes: + # Make sure linecards are in Normal state and save the config to proceed further + linecard.shell("TSB") + linecard.shell('sudo config save -y') + # Verify DUT is in normal state. + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT {} is not in normal state".format(linecard)) + # Make sure the dut's reboot cause is as expected + logger.info("Check reboot cause of the dut {}".format(linecard)) + reboot_cause = get_reboot_cause(linecard) + pytest_assert(reboot_cause == SUP_REBOOT_CAUSE, + "Reboot cause {} did not match the trigger {}".format(reboot_cause, SUP_REBOOT_CAUSE)) + + # Make sure the Supervisor's reboot cause is as expected + logger.info("Check reboot cause of the supervisor") + reboot_cause = get_reboot_cause(suphost) + pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, + "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) + + +@pytest.mark.disable_loganalyzer +def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test startup TSA_TSB service after DUT cold reboot + Verify the configured tsa_tsb_timer is sufficient for system to be stable + Verify this service configures TSA and starts a timer and configures TSB once the timer is expired + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + tsa_tsb_timer = get_startup_tsb_timer(duthost) + if not tsa_tsb_timer: + pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state") + if not check_tsa_persistence_support(duthost): + pytest.skip("TSA persistence not supported in the image") + + try: + # Get all routes on neighbors before doing reboot + orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) + orig_v6_routes = parse_routes_on_neighbors(duthost, nbrhosts, 6) + + up_bgp_neighbors = duthost.get_bgp_neighbors_per_asic("established") + + # Reboot dut and wait for startup_tsa_tsb service to start + logger.info("Cold reboot on node: %s", duthost.hostname) + reboot(duthost, localhost, wait=240) + + logger.info('Cold reboot finished on {}'.format(duthost.hostname)) + dut_uptime = duthost.get_up_time() + logger.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + + # Ensure startup_tsa_tsb service is running after dut reboot + pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'running'), + "startup_tsa_tsb service is not started after reboot") + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = duthost.get_up_time() + logging.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(duthost) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + + logging.info("Wait until all critical services are fully started") + pytest_assert(wait_until(300, 20, 2, duthost.critical_services_fully_started)), \ + "Not all critical services are fully started on {}".format(duthost.hostname) + + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(duthost) + pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), + "Not all ports that are admin up on are operationally up") + + pytest_assert(wait_until(300, 10, 0, + duthost.check_bgp_session_state_all_asics, up_bgp_neighbors, "established")) + + stability_check_time = datetime.datetime.now() + time_to_stabilize = (stability_check_time - service_uptime).total_seconds() + logging.info("Time taken for system stability : {}".format(time_to_stabilize)) + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + + # Verify startup_tsa_tsb service stopped after expected time + pytest_assert(wait_until(tsa_tsb_timer, 20, 0, get_tsa_tsb_service_status, duthost, 'exited'), + "startup_tsa_tsb service is not stopped even after configured timer expiry") + + # Verify tsa_tsb_timer configured is sufficient + pytest_assert(time_to_stabilize < tsa_tsb_timer, + "Configured tsa_tsb_timer is not sufficient for the system to be stable") + + # Ensure dut comes back to normal state after timer expiry + if not get_tsa_tsb_service_status(duthost, 'running'): + # Verify TSB is configured on the dut after startup_tsa_tsb service is stopped + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state after startup_tsa_tsb service is stopped") + + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v4_routes, cur_v4_routes, 4): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v4_routes, cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v6_routes, cur_v6_routes, 6): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + finally: + + # Verify DUT is in normal state. + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state") + # Make sure the dut's reboot cause is as expected + logger.info("Check reboot cause of the dut") + reboot_cause = get_reboot_cause(duthost) + pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, + "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) From 388d1271ef9c3976ea10846783a2440115ef1ea7 Mon Sep 17 00:00:00 2001 From: sanrajen Date: Wed, 10 Jul 2024 17:08:22 -0400 Subject: [PATCH 2/7] Reliable-TSA: Addressing review comments - PR#13290 --- tests/bgp/test_reliable_tsa.py | 143 ++++++++++++++++----------------- 1 file changed, 68 insertions(+), 75 deletions(-) diff --git a/tests/bgp/test_reliable_tsa.py b/tests/bgp/test_reliable_tsa.py index b6934c822a..146993980e 100644 --- a/tests/bgp/test_reliable_tsa.py +++ b/tests/bgp/test_reliable_tsa.py @@ -8,11 +8,11 @@ from tests.common.utilities import wait_until from tests.common.platform.processes_utils import wait_critical_processes from tests.common.platform.interface_utils import check_interface_status_of_up_ports -from tests.bgp.test_traffic_shift import get_traffic_shift_state, parse_routes_on_neighbors,\ - check_tsa_persistence_support, verify_current_routes_announced_to_neighs, check_and_log_routes_diff, \ +from tests.bgp.test_traffic_shift import get_traffic_shift_state, parse_routes_on_neighbors, \ + verify_current_routes_announced_to_neighs, check_and_log_routes_diff, \ verify_only_loopback_routes_are_announced_to_neighs from tests.bgp.test_startup_tsa_tsb_service import get_tsa_tsb_service_uptime, get_tsa_tsb_service_status, \ - get_startup_tsb_timer + get_startup_tsb_timer, enable_disable_startup_tsa_tsb_service # noqa: F401 pytestmark = [ pytest.mark.topology('t2') @@ -29,36 +29,7 @@ CONTAINER_RESTART_THRESHOLD_SECS = 300 PROGRAM_STATUS = "RUNNING" BGP_CRIT_PROCESS = "bgpcfgd" - - -@pytest.fixture(scope="module", autouse=True) -def enable_disable_startup_tsa_tsb_service(duthosts): - """ - @summary: enable/disable startup-tsa-tsb.service during OC run. - Args: - duthosts: Fixture returns a list of Ansible object DuT. - Returns: - None. - """ - for duthost in duthosts.frontend_nodes: - platform = duthost.facts['platform'] - startup_tsa_tsb_file_path = "/usr/share/sonic/device/{}/startup-tsa-tsb.conf".format(platform) - backup_tsa_tsb_file_path = "/usr/share/sonic/device/{}/backup-startup-tsa-tsb.bck".format(platform) - out = duthost.shell("cat {}".format(backup_tsa_tsb_file_path), module_ignore_errors=True)['rc'] - if not out: - duthost.shell("sudo mv {} {}".format(backup_tsa_tsb_file_path, startup_tsa_tsb_file_path), - module_ignore_errors=True) - yield - for duthost in duthosts.frontend_nodes: - platform = duthost.facts['platform'] - startup_tsa_tsb_file_path = "/usr/share/sonic/device/{}/startup-tsa-tsb.conf".format(platform) - backup_tsa_tsb_file_path = "/usr/share/sonic/device/{}/backup-startup-tsa-tsb.bck".format(platform) - out = duthost.shell("cat {}".format(startup_tsa_tsb_file_path), module_ignore_errors=True)['rc'] - if not out: - duthost.shell("sudo mv {} {}".format(startup_tsa_tsb_file_path, backup_tsa_tsb_file_path), - module_ignore_errors=True) - output = duthost.shell("TSB", module_ignore_errors=True) - pytest_assert(not output['rc'], "Failed TSB") +supported_tsa_configs = ['false', 'true'] @pytest.fixture @@ -296,19 +267,20 @@ def clear_failed_flag_and_restart(duthost, service_name, container_name): @pytest.mark.disable_loganalyzer def test_sup_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_supervisor_dut_hostname, - nbrhosts, traffic_shift_community, tbinfo): + enable_disable_startup_tsa_tsb_service, nbrhosts, # noqa: F811 + traffic_shift_community, tbinfo): """ Test supervisor TSA action when supervisor and line cards are in TSB initially Verify supervisor config state changes to TSA and Line card BGP TSA operational state changes to TSA from TSB Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -345,6 +317,7 @@ def test_sup_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_su @pytest.mark.disable_loganalyzer def test_sup_tsa_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSA action when supervisor is on TSB and line cards are in TSA initially @@ -352,12 +325,12 @@ def test_sup_tsa_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -402,6 +375,7 @@ def test_sup_tsa_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, @pytest.mark.disable_loganalyzer def test_sup_tsb_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSB action when supervisor is on TSA and line cards are in TSB configuration initially but with @@ -411,12 +385,12 @@ def test_sup_tsb_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, announced back to neighbors when the line cards are back to TSB. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -467,6 +441,7 @@ def test_sup_tsb_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, @pytest.mark.disable_loganalyzer def test_sup_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSB action when supervisor and line cards are in TSA configuration initially @@ -474,12 +449,12 @@ def test_sup_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enu Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -535,6 +510,7 @@ def test_sup_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enu @pytest.mark.disable_loganalyzer def test_dut_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test line card TSA action when supervisor and line cards are in TSB initially @@ -543,12 +519,12 @@ def test_dut_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_su Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -589,6 +565,7 @@ def test_dut_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_su @pytest.mark.disable_loganalyzer def test_dut_tsa_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test line card TSA action when supervisor is on TSA and line cards are in TSB initially @@ -597,12 +574,12 @@ def test_dut_tsa_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -660,6 +637,7 @@ def test_dut_tsa_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, @pytest.mark.disable_loganalyzer def test_dut_tsb_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test line card TSB action when supervisor is on TSB and line cards are in TSA initially @@ -668,12 +646,13 @@ def test_dut_tsb_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, Make sure only loopback routes are advertised to neighbors during line cards' TSA state and all routes are announced back to neighbors when the line cards are back to TSB. """ + suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -743,6 +722,7 @@ def test_dut_tsb_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, @pytest.mark.disable_loganalyzer def test_dut_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test line card TSB action when supervisor and line cards are in TSA configuration initially @@ -750,12 +730,12 @@ def test_dut_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enu Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -815,6 +795,7 @@ def test_dut_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enu @pytest.mark.disable_loganalyzer def test_sup_tsa_act_with_sup_reboot(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSA action when supervisor and line cards are in TSB initially @@ -824,14 +805,14 @@ def test_sup_tsa_act_with_sup_reboot(duthosts, localhost, enum_supervisor_dut_ho After reboot, make sure the BGP TSA operational states are same as before reboot. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) tsa_tsb_timer = dict() orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -924,6 +905,7 @@ def test_sup_tsa_act_with_sup_reboot(duthosts, localhost, enum_supervisor_dut_ho @pytest.mark.disable_loganalyzer def test_sup_tsa_act_when_duts_on_tsa_with_sup_config_reload(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSA action when supervisor is on TSB and line cards are in TSA initially @@ -933,12 +915,12 @@ def test_sup_tsa_act_when_duts_on_tsa_with_sup_config_reload(duthosts, localhost After config_relaod, make sure the BGP TSA operational states are same as before. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -996,6 +978,7 @@ def test_sup_tsa_act_when_duts_on_tsa_with_sup_config_reload(duthosts, localhost @pytest.mark.disable_loganalyzer def test_dut_tsa_act_with_reboot_when_sup_dut_on_tsb_init(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test line card TSA action when supervisor and line cards are in TSB initially @@ -1006,14 +989,14 @@ def test_dut_tsa_act_with_reboot_when_sup_dut_on_tsb_init(duthosts, localhost, e After reboot, make sure the BGP TSA operational states are same as before reboot on line cards. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() tsa_tsb_timer = dict() dut_nbrhosts = dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1092,6 +1075,7 @@ def test_dut_tsa_act_with_reboot_when_sup_dut_on_tsb_init(duthosts, localhost, e @pytest.mark.disable_loganalyzer def test_dut_tsa_with_conf_reload_when_sup_on_tsa_dut_on_tsb_init(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811, E501 nbrhosts, traffic_shift_community, tbinfo): """ Test line card TSA action when supervisor is on TSA and line cards are in TSB initially @@ -1102,12 +1086,12 @@ def test_dut_tsa_with_conf_reload_when_sup_on_tsa_dut_on_tsb_init(duthosts, loca After config_reload, make sure the BGP TSA operational states are same as before config reload on line card. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1163,6 +1147,7 @@ def test_dut_tsa_with_conf_reload_when_sup_on_tsa_dut_on_tsb_init(duthosts, loca @pytest.mark.disable_loganalyzer def test_user_init_tsa_on_dut_followed_by_sup_tsa(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test user initiated line card TSA action when supervisor and line cards are in TSB initially @@ -1174,12 +1159,12 @@ def test_user_init_tsa_on_dut_followed_by_sup_tsa(duthosts, localhost, enum_supe Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1234,6 +1219,7 @@ def test_user_init_tsa_on_dut_followed_by_sup_tsa(duthosts, localhost, enum_supe @pytest.mark.disable_loganalyzer def test_user_init_tsa_on_dut_followed_by_sup_tsb(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test user initiated line card TSA action when supervisor and line cards are in TSB initially @@ -1245,12 +1231,12 @@ def test_user_init_tsa_on_dut_followed_by_sup_tsb(duthosts, localhost, enum_supe Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1305,6 +1291,7 @@ def test_user_init_tsa_on_dut_followed_by_sup_tsb(duthosts, localhost, enum_supe @pytest.mark.disable_loganalyzer def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSA action when startup-tsa-tsb service is running on line cards after reboot @@ -1313,14 +1300,14 @@ def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() tsa_tsb_timer = dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1398,6 +1385,7 @@ def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ @pytest.mark.disable_loganalyzer def test_sup_tsb_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811 nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSB action when startup-tsa-tsb service is running on line cards after reboot @@ -1406,14 +1394,14 @@ def test_sup_tsb_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ are advertised back once the line cards are in normal state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() tsa_tsb_timer = dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1482,7 +1470,8 @@ def test_sup_tsb_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ @pytest.mark.disable_loganalyzer def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_on_tsa_duts_on_tsb( - duthosts, localhost, enum_supervisor_dut_hostname, nbrhosts, traffic_shift_community, tbinfo): + duthosts, localhost, enum_supervisor_dut_hostname, enable_disable_startup_tsa_tsb_service, # noqa: F811 + nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSB action when supervisor is on TSA and line cards are in TSB configuration initially but with BGP operational TSA states @@ -1492,12 +1481,12 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_on_tsa_duts_on_tsb( announced back to neighbors when the line cards are back to TSB. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -1560,6 +1549,7 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_on_tsa_duts_on_tsb( @pytest.mark.disable_loganalyzer def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811, E501 nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSB action when supervisor and line cards are in TSA configuration initially @@ -1568,12 +1558,12 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -1640,6 +1630,7 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, @pytest.mark.disable_loganalyzer def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_on_tsb_duts_on_tsa(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811, E501 nbrhosts, traffic_shift_community, tbinfo): """ Test line card TSB action when supervisor is on TSB and line cards are in TSA initially @@ -1649,12 +1640,13 @@ def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_on_tsb_duts_on_tsa(duthost Make sure only loopback routes are advertised to neighbors during line cards' TSA state and all routes are announced back to neighbors when the line cards are back to TSB. """ + suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1746,6 +1738,7 @@ def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_on_tsb_duts_on_tsa(duthost @pytest.mark.disable_loganalyzer def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, localhost, enum_supervisor_dut_hostname, + enable_disable_startup_tsa_tsb_service, # noqa: F811, E501 nbrhosts, traffic_shift_community, tbinfo): """ Test line card TSB action when supervisor and line cards are in TSA configuration initially @@ -1754,12 +1747,12 @@ def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, Make sure only loopback routes are advertised to neighbors during line cards' TSA state. """ suphost = duthosts[enum_supervisor_dut_hostname] + if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: + pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: From 8661ffb66d9c73f33a7f629a0a4025ee52cdb4ba Mon Sep 17 00:00:00 2001 From: sanrajen Date: Wed, 10 Jul 2024 17:55:57 -0400 Subject: [PATCH 3/7] Fixing rebase conflicts internal --- tests/bgp/test_startup_tsa_tsb_service.py | 454 +--------------------- 1 file changed, 1 insertion(+), 453 deletions(-) diff --git a/tests/bgp/test_startup_tsa_tsb_service.py b/tests/bgp/test_startup_tsa_tsb_service.py index 2d8e04856b..f7f8e93d60 100644 --- a/tests/bgp/test_startup_tsa_tsb_service.py +++ b/tests/bgp/test_startup_tsa_tsb_service.py @@ -1,6 +1,6 @@ import logging import datetime -import pexpect + import pytest from tests.common import reboot, config_reload from tests.common.reboot import get_reboot_cause, SONIC_SSH_PORT, SONIC_SSH_REGEX, wait_for_startup @@ -166,37 +166,6 @@ def check_ssh_state(localhost, dut_ip, expected_state, timeout=60): return not res.is_failed and 'Timeout' not in res.get('msg', '') -def exec_tsa_tsb_cmd_on_supervisor(duthosts, enum_supervisor_dut_hostname, creds, tsa_tsb_cmd): - """ - @summary: Issue TSA/TSB command on supervisor card using user credentials - Verify command is executed on supervisor card - @returns: None - """ - try: - suphost = duthosts[enum_supervisor_dut_hostname] - sup_ip = suphost.mgmt_ip - sonic_username = creds['sonicadmin_user'] - sonic_password = creds['sonicadmin_password'] - logger.info('sonic-username: {}, sonic_password: {}'.format(sonic_username, sonic_password)) - ssh_cmd = "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no {}@{}".format(sonic_username, sup_ip) - connect = pexpect.spawn(ssh_cmd) - connect.expect('.*[Pp]assword:') - connect.sendline(sonic_password) - i = connect.expect('{}@{}:'.format(sonic_username, suphost.hostname), timeout=10) - pytest_assert(i == 0, "Failed to connect") - connect.sendline(tsa_tsb_cmd) - connect.expect('.*[Pp]assword for username \'{}\':'.format(sonic_username)) - connect.sendline(sonic_password) - j = connect.expect('{}@{}:'.format(sonic_username, suphost.hostname), timeout=10) - pytest_assert(j == 0, "Failed to connect") - except pexpect.exceptions.EOF: - pytest.fail("EOF reached") - except pexpect.exceptions.TIMEOUT: - pytest.fail("Timeout reached") - except Exception as e: - pytest.fail("Cannot connect to DUT {} host via SSH: {}".format(suphost.hostname, e)) - - @pytest.mark.disable_loganalyzer def test_tsa_tsb_service_with_dut_cold_reboot(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, nbrhosts, traffic_shift_community, tbinfo): @@ -932,427 +901,6 @@ def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_o "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) -@pytest.mark.disable_loganalyzer -def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, - enum_supervisor_dut_hostname, ptfhost, nbrhosts, - traffic_shift_community, creds, tbinfo): - """ - Test startup TSA_TSB service after DUT cold reboot - Verify startup_tsa_tsb.service started automatically when dut comes up - Verify this service configures TSA and starts a timer - Issue TSB from supervisor, while the service is running on dut, and make sure the TSB is configured on linecards - Make sure TSA_TSB service is stopped and dut changes from maintenance mode to normal mode - """ - tsa_tsb_cmd = 'sudo TSB' - suphost = duthosts[enum_supervisor_dut_hostname] - tsa_tsb_timer = dict() - dut_nbrhosts = dict() - orig_v4_routes, orig_v6_routes = dict(), dict() - for linecard in duthosts.frontend_nodes: - tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) - if not tsa_tsb_timer[linecard]: - pytest.skip("startup_tsa_tsb.service is not supported on the duts under {}".format(suphost.hostname)) - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), - "DUT is not in normal state") - if not check_tsa_persistence_support(linecard): - pytest.skip("TSA persistence not supported in the image") - - try: - for linecard in duthosts.frontend_nodes: - # Get all routes on neighbors before doing reboot - orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) - orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) - - # Get a dut uptime before reboot - sup_uptime_before = suphost.get_up_time() - # Reboot dut and wait for startup_tsa_tsb service to start on linecards - logger.info("Cold reboot on supervisor node: %s", suphost.hostname) - reboot(suphost, localhost, wait=240) - logging.info("Wait until all critical processes are fully started") - wait_critical_processes(suphost) - - sup_uptime = suphost.get_up_time() - logger.info('DUT {} up since {}'.format(suphost.hostname, sup_uptime)) - rebooted = float(sup_uptime_before.strftime("%s")) != float(sup_uptime.strftime("%s")) - assert rebooted, "Device {} did not reboot".format(suphost.hostname) - - for linecard in duthosts.frontend_nodes: - wait_for_startup(linecard, localhost, delay=10, timeout=300) - - # Ensure startup_tsa_tsb service started on expected time since dut rebooted - dut_uptime = linecard.get_up_time() - logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) - service_uptime = get_tsa_tsb_service_uptime(linecard) - time_diff = (service_uptime - dut_uptime).total_seconds() - pytest_assert(int(time_diff) < 120, - "startup_tsa_tsb service started much later than the expected time after dut reboot") - - # Verify DUT is in maintenance state. - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), - "DUT is not in maintenance state when startup_tsa_tsb service is running") - - logging.info("Wait until all critical processes are fully started") - wait_critical_processes(linecard) - - pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( - duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), - "Failed to verify routes on nbr in TSA") - - # Execute user initiated TSB from supervisor card - exec_tsa_tsb_cmd_on_supervisor(duthosts, enum_supervisor_dut_hostname, creds, tsa_tsb_cmd) - - for linecard in duthosts.frontend_nodes: - # Ensure dut comes back to normal state - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), - "DUT is not in normal state after TSB command from supervisor") - - # Ensure startup_tsa_tsb service is in inactive state after user-initiated TSB on supervisor - pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, linecard, 'inactive'), - "startup_tsa_tsb service is not in inactive state after user init TSB from supervisor") - - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, linecard), - "Not all ports that are admin up on are operationally up") - - # Wait until all routes are announced to neighbors - cur_v4_routes = {} - cur_v6_routes = {} - # Verify that all routes advertised to neighbor at the start of the test - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, dut_nbrhosts[linecard], - orig_v4_routes[linecard], cur_v4_routes, 4): - if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], - orig_v4_routes[linecard], cur_v4_routes, 4): - pytest.fail("Not all ipv4 routes are announced to neighbors") - - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, dut_nbrhosts[linecard], - orig_v6_routes[linecard], cur_v6_routes, 6): - if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], - orig_v6_routes[linecard], cur_v6_routes, 6): - pytest.fail("Not all ipv6 routes are announced to neighbors") - - finally: - for linecard in duthosts.frontend_nodes: - # Make sure linecards are in Normal state and save the config to proceed further - linecard.shell("TSB") - linecard.shell('sudo config save -y') - # Verify DUT is in normal state. - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), - "DUT {} is not in normal state".format(linecard)) - # Make sure the dut's reboot cause is as expected - logger.info("Check reboot cause of the dut {}".format(linecard)) - reboot_cause = get_reboot_cause(linecard) - pytest_assert(reboot_cause == SUP_REBOOT_CAUSE, - "Reboot cause {} did not match the trigger {}".format(reboot_cause, SUP_REBOOT_CAUSE)) - - # Make sure the Supervisor's reboot cause is as expected - logger.info("Check reboot cause of the supervisor") - reboot_cause = get_reboot_cause(suphost) - pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, - "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) - - -@pytest.mark.disable_loganalyzer -def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, - nbrhosts, traffic_shift_community, tbinfo): - """ - Test startup TSA_TSB service after DUT cold reboot - Verify the configured tsa_tsb_timer is sufficient for system to be stable - Verify this service configures TSA and starts a timer and configures TSB once the timer is expired - """ - duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - tsa_tsb_timer = get_startup_tsb_timer(duthost) - if not tsa_tsb_timer: - pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") - if not check_tsa_persistence_support(duthost): - pytest.skip("TSA persistence not supported in the image") - - try: - # Get all routes on neighbors before doing reboot - orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) - orig_v6_routes = parse_routes_on_neighbors(duthost, nbrhosts, 6) - - up_bgp_neighbors = duthost.get_bgp_neighbors_per_asic("established") - - # Reboot dut and wait for startup_tsa_tsb service to start - logger.info("Cold reboot on node: %s", duthost.hostname) - reboot(duthost, localhost, wait=240) - - logger.info('Cold reboot finished on {}'.format(duthost.hostname)) - dut_uptime = duthost.get_up_time() - logger.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) - - # Ensure startup_tsa_tsb service is running after dut reboot - pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'running'), - "startup_tsa_tsb service is not started after reboot") - - # Ensure startup_tsa_tsb service started on expected time since dut rebooted - dut_uptime = duthost.get_up_time() - logging.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) - service_uptime = get_tsa_tsb_service_uptime(duthost) - time_diff = (service_uptime - dut_uptime).total_seconds() - pytest_assert(int(time_diff) < 120, - "startup_tsa_tsb service started much later than the expected time after dut reboot") - - logging.info("Wait until all critical services are fully started") - pytest_assert(wait_until(300, 20, 2, duthost.critical_services_fully_started)), \ - "Not all critical services are fully started on {}".format(duthost.hostname) - - logging.info("Wait until all critical processes are fully started") - wait_critical_processes(duthost) - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), - "Not all ports that are admin up on are operationally up") - - pytest_assert(wait_until(300, 10, 0, - duthost.check_bgp_session_state_all_asics, up_bgp_neighbors, "established")) - - stability_check_time = datetime.datetime.now() - time_to_stabilize = (stability_check_time - service_uptime).total_seconds() - logging.info("Time taken for system stability : {}".format(time_to_stabilize)) - - # Verify DUT is in maintenance state. - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), - "DUT is not in maintenance state when startup_tsa_tsb service is running") - - # Verify startup_tsa_tsb service stopped after expected time - pytest_assert(wait_until(tsa_tsb_timer, 20, 0, get_tsa_tsb_service_status, duthost, 'exited'), - "startup_tsa_tsb service is not stopped even after configured timer expiry") - - # Verify tsa_tsb_timer configured is sufficient - pytest_assert(time_to_stabilize < tsa_tsb_timer, - "Configured tsa_tsb_timer is not sufficient for the system to be stable") - - # Ensure dut comes back to normal state after timer expiry - if not get_tsa_tsb_service_status(duthost, 'running'): - # Verify TSB is configured on the dut after startup_tsa_tsb service is stopped - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state after startup_tsa_tsb service is stopped") - - # Wait until all routes are announced to neighbors - cur_v4_routes = {} - cur_v6_routes = {} - # Verify that all routes advertised to neighbor at the start of the test - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, - orig_v4_routes, cur_v4_routes, 4): - if not check_and_log_routes_diff(duthost, nbrhosts, orig_v4_routes, cur_v4_routes, 4): - pytest.fail("Not all ipv4 routes are announced to neighbors") - - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, - orig_v6_routes, cur_v6_routes, 6): - if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): - pytest.fail("Not all ipv6 routes are announced to neighbors") - - finally: - - # Verify DUT is in normal state. - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") - # Make sure the dut's reboot cause is as expected - logger.info("Check reboot cause of the dut") - reboot_cause = get_reboot_cause(duthost) - pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, - "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) - - -@pytest.mark.disable_loganalyzer -def test_user_init_tsa_while_service_run_on_dut(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, - nbrhosts, traffic_shift_community, tbinfo): - - """ - Test startup TSA_TSB service after DUT cold reboot - Verify startup_tsa_tsb.service started automatically when dut comes up - Verify this service configures TSA and starts a timer - Issue TSA while the service is running on dut, and make sure the TSA is configured - Make sure TSA_TSB service is stopped and dut continues to be in maintenance mode - """ - duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - tsa_tsb_timer = get_startup_tsb_timer(duthost) - if not tsa_tsb_timer: - pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) - dut_nbrhosts = nbrhosts_to_dut(duthost, nbrhosts) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") - if not check_tsa_persistence_support(duthost): - pytest.skip("TSA persistence not supported in the image") - - try: - # Get all routes on neighbors before doing reboot - orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) - orig_v6_routes = parse_routes_on_neighbors(duthost, nbrhosts, 6) - - # Reboot dut and wait for startup_tsa_tsb service to start - logger.info("Cold reboot on node: %s", duthost.hostname) - reboot(duthost, localhost, wait=240) - - logger.info('Cold reboot finished on {}'.format(duthost.hostname)) - dut_uptime = duthost.get_up_time() - logger.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) - - # Ensure startup_tsa_tsb service is running after dut reboot - pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'running'), - "startup_tsa_tsb service is not started after reboot") - - # Ensure startup_tsa_tsb service started on expected time since dut rebooted - dut_uptime = duthost.get_up_time() - logging.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) - service_uptime = get_tsa_tsb_service_uptime(duthost) - time_diff = (service_uptime - dut_uptime).total_seconds() - pytest_assert(int(time_diff) < 120, - "startup_tsa_tsb service started much later than the expected time after dut reboot") - - # Verify DUT is in maintenance state. - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), - "DUT is not in maintenance state when startup_tsa_tsb service is running") - - # Issue TSA on DUT - duthost.shell("TSA") - duthost.shell('sudo config save -y') - - # Ensure startup_tsa_tsb service is in inactive state after user-initiated TSA - pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'inactive'), - "startup_tsa_tsb service is not in inactive state after user init TSA") - - # Verify DUT continues to be in maintenance state. - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), - "DUT is not in maintenance state with saved TSA config after reboot") - - logging.info("Wait until all critical processes are fully started") - wait_critical_processes(duthost) - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), - "Not all ports that are admin up on are operationally up") - - pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( - duthosts, duthost, dut_nbrhosts, traffic_shift_community), - "Failed to verify routes on nbr in TSA") - - finally: - """ - Test TSB after config save and config reload - Verify all routes are announced back to neighbors - """ - # Recover to Normal state - duthost.shell("TSB") - duthost.shell('sudo config save -y') - - # Verify DUT comes back to normal state after TSB. - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") - # Wait until all routes are announced to neighbors - cur_v4_routes = {} - cur_v6_routes = {} - # Verify that all routes advertised to neighbor at the start of the test - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, - orig_v4_routes, cur_v4_routes, 4): - if not check_and_log_routes_diff(duthost, nbrhosts, orig_v4_routes, cur_v4_routes, 4): - pytest.fail("Not all ipv4 routes are announced to neighbors") - - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, - orig_v6_routes, cur_v6_routes, 6): - if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): - pytest.fail("Not all ipv6 routes are announced to neighbors") - - # Make sure the dut's reboot cause is as expected - logger.info("Check reboot cause of the dut") - reboot_cause = get_reboot_cause(duthost) - pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, - "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) - - -@pytest.mark.disable_loganalyzer -def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, - nbrhosts, traffic_shift_community, tbinfo): - - """ - Test startup TSA_TSB service after DUT cold reboot - Verify startup_tsa_tsb.service started automatically when dut comes up - Verify this service configures TSA and starts a timer - Issue TSB while the service is running on dut, and make sure the TSB is configured - Make sure TSA_TSB service is stopped and dut continues to be in normal mode - """ - duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - tsa_tsb_timer = get_startup_tsb_timer(duthost) - if not tsa_tsb_timer: - pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") - if not check_tsa_persistence_support(duthost): - pytest.skip("TSA persistence not supported in the image") - - try: - # Get all routes on neighbors before doing reboot - orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) - orig_v6_routes = parse_routes_on_neighbors(duthost, nbrhosts, 6) - - # Reboot dut and wait for startup_tsa_tsb service to start - logger.info("Cold reboot on node: %s", duthost.hostname) - reboot(duthost, localhost, wait=240) - - logger.info('Cold reboot finished on {}'.format(duthost.hostname)) - dut_uptime = duthost.get_up_time() - logger.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) - - # Ensure startup_tsa_tsb service is running after dut reboot - pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'running'), - "startup_tsa_tsb service is not started after reboot") - - # Ensure startup_tsa_tsb service started on expected time since dut rebooted - dut_uptime = duthost.get_up_time() - logging.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) - service_uptime = get_tsa_tsb_service_uptime(duthost) - time_diff = (service_uptime - dut_uptime).total_seconds() - pytest_assert(int(time_diff) < 120, - "startup_tsa_tsb service started much later than the expected time after dut reboot") - - # Verify DUT is in maintenance state. - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), - "DUT is not in maintenance state when startup_tsa_tsb service is running") - - # Issue TSB on DUT - duthost.shell("TSB") - duthost.shell('sudo config save -y') - - # Verify DUT comes back to normal state after TSB. - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") - - # Ensure startup_tsa_tsb service is in inactive state after user-initiated TSB - pytest_assert(wait_until(60, 5, 10, get_tsa_tsb_service_status, duthost, 'inactive'), - "startup_tsa_tsb service is not in inactive state after user init TSB") - - # Make sure DUT continues to be in good state after TSB - assert wait_until(300, 20, 2, duthost.critical_services_fully_started), \ - "Not all critical services are fully started on {}".format(duthost.hostname) - - # Wait until all routes are announced to neighbors - cur_v4_routes = {} - cur_v6_routes = {} - # Verify that all routes advertised to neighbor at the start of the test - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, - orig_v4_routes, cur_v4_routes, 4): - if not check_and_log_routes_diff(duthost, nbrhosts, orig_v4_routes, cur_v4_routes, 4): - pytest.fail("Not all ipv4 routes are announced to neighbors") - - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, - orig_v6_routes, cur_v6_routes, 6): - if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): - pytest.fail("Not all ipv6 routes are announced to neighbors") - - finally: - - # Verify DUT is in normal state. - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") - - # Make sure the dut's reboot cause is as expected - logger.info("Check reboot cause of the dut") - reboot_cause = get_reboot_cause(duthost) - pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, - "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) - - @pytest.mark.disable_loganalyzer def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, enum_supervisor_dut_hostname, ptfhost, nbrhosts, From 0f5994ffd942732a78ad25ba7201e6c4d833ec44 Mon Sep 17 00:00:00 2001 From: sanrajen Date: Fri, 19 Jul 2024 18:00:17 -0400 Subject: [PATCH 4/7] Reliable-TSA: adjust autorestart state of bgp feature for bgp restart cases --- tests/bgp/test_reliable_tsa.py | 36 +++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/bgp/test_reliable_tsa.py b/tests/bgp/test_reliable_tsa.py index 146993980e..1685dc8c65 100644 --- a/tests/bgp/test_reliable_tsa.py +++ b/tests/bgp/test_reliable_tsa.py @@ -56,6 +56,37 @@ def nbrhosts_to_dut(duthost, nbrhosts): return dut_nbrhosts +@pytest.fixture +def enable_disable_bgp_autorestart_state(duthosts): + """ + @summary: enable/disable bgp feature autorestart state during OC run. + After test_pretest, autorestart status of bgp feature is disabled. This fixture + enables autorestart state of bgp before test start and disables once the test is done. + Args: + duthosts: Fixture returns a list of Ansible object DuT. + Returns: + None. + """ + # Enable autorestart status for bgp feature to overcome pretest changes + for duthost in duthosts.frontend_nodes: + feature_list, _ = duthost.get_feature_status() + bgp_autorestart_state = duthost.get_container_autorestart_states()['bgp'] + for feature, status in list(feature_list.items()): + if feature == 'bgp' and status == 'enabled' and bgp_autorestart_state == 'disabled': + duthost.shell("sudo config feature autorestart {} enabled".format(feature)) + break + yield + + # Disable autorestart status for bgp feature as in pretest + for duthost in duthosts.frontend_nodes: + feature_list, _ = duthost.get_feature_status() + bgp_autorestart_state = duthost.get_container_autorestart_states()['bgp'] + for feature, status in list(feature_list.items()): + if feature == 'bgp' and status == 'enabled' and bgp_autorestart_state == 'enabled': + duthost.shell("sudo config feature autorestart {} disabled".format(feature)) + break + + def set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname): """ @summary: Common method to make sure the supervisor and line cards are in normal state before and after the test @@ -1471,7 +1502,7 @@ def test_sup_tsb_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ @pytest.mark.disable_loganalyzer def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_on_tsa_duts_on_tsb( duthosts, localhost, enum_supervisor_dut_hostname, enable_disable_startup_tsa_tsb_service, # noqa: F811 - nbrhosts, traffic_shift_community, tbinfo): + enable_disable_bgp_autorestart_state, nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSB action when supervisor is on TSA and line cards are in TSB configuration initially but with BGP operational TSA states @@ -1550,6 +1581,7 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_on_tsa_duts_on_tsb( @pytest.mark.disable_loganalyzer def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, localhost, enum_supervisor_dut_hostname, enable_disable_startup_tsa_tsb_service, # noqa: F811, E501 + enable_disable_bgp_autorestart_state, nbrhosts, traffic_shift_community, tbinfo): """ Test supervisor TSB action when supervisor and line cards are in TSA configuration initially @@ -1631,6 +1663,7 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_on_tsb_duts_on_tsa(duthosts, localhost, enum_supervisor_dut_hostname, enable_disable_startup_tsa_tsb_service, # noqa: F811, E501 + enable_disable_bgp_autorestart_state, nbrhosts, traffic_shift_community, tbinfo): """ Test line card TSB action when supervisor is on TSB and line cards are in TSA initially @@ -1739,6 +1772,7 @@ def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_on_tsb_duts_on_tsa(duthost def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, localhost, enum_supervisor_dut_hostname, enable_disable_startup_tsa_tsb_service, # noqa: F811, E501 + enable_disable_bgp_autorestart_state, nbrhosts, traffic_shift_community, tbinfo): """ Test line card TSB action when supervisor and line cards are in TSA configuration initially From b5a78d9500c4f668def453e0de26fe7d0da90849 Mon Sep 17 00:00:00 2001 From: sanrajen Date: Thu, 25 Jul 2024 14:49:31 -0400 Subject: [PATCH 5/7] Recover dut for normal/abnormal reboot failures Recover dut in case of reboot failures or other issue flake8 fix flake8 blank line fix Include critical process check as well for dut recovery Typo fix intf check missing Fix for supervisor config reload test --- tests/bgp/test_reliable_tsa.py | 66 +++++++++--- tests/bgp/test_startup_tsa_tsb_service.py | 125 +++++++++++++--------- 2 files changed, 127 insertions(+), 64 deletions(-) diff --git a/tests/bgp/test_reliable_tsa.py b/tests/bgp/test_reliable_tsa.py index 1685dc8c65..670f5b5340 100644 --- a/tests/bgp/test_reliable_tsa.py +++ b/tests/bgp/test_reliable_tsa.py @@ -6,7 +6,7 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.constants import DEFAULT_ASIC_ID from tests.common.utilities import wait_until -from tests.common.platform.processes_utils import wait_critical_processes +from tests.common.platform.processes_utils import wait_critical_processes, _all_critical_processes_healthy from tests.common.platform.interface_utils import check_interface_status_of_up_ports from tests.bgp.test_traffic_shift import get_traffic_shift_state, parse_routes_on_neighbors, \ verify_current_routes_announced_to_neighs, check_and_log_routes_diff, \ @@ -841,8 +841,11 @@ def test_sup_tsa_act_with_sup_reboot(duthosts, localhost, enum_supervisor_dut_ho tsa_tsb_timer = dict() orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() + int_status_result, crit_process_check = dict(), dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) + int_status_result[linecard] = True + crit_process_check[linecard] = True dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -901,9 +904,8 @@ def test_sup_tsa_act_with_sup_reboot(duthosts, localhost, enum_supervisor_dut_ho "{} tsa_enabled config is not enabled".format(linecard.hostname)) logging.info("Wait until all critical processes are fully started") - wait_critical_processes(linecard) - pytest_assert(wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard), - "Not all ports that are admin up on are operationally up") + crit_process_check[linecard] = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) + int_status_result[linecard] = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) # Once all line cards are in maintenance state, proceed further for linecard in duthosts.frontend_nodes: @@ -930,6 +932,13 @@ def test_sup_tsa_act_with_sup_reboot(duthosts, localhost, enum_supervisor_dut_ho # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + for linecard in duthosts.frontend_nodes: + # Make sure linecards are in Normal state, if not do config-reload on the dut + if not (int_status_result[linecard] and crit_process_check[linecard] and + TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats')): + logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") + config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + # Verify all routes are advertised back to neighbors when duts are in TSB verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) @@ -948,7 +957,7 @@ def test_sup_tsa_act_when_duts_on_tsa_with_sup_config_reload(duthosts, localhost suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - dut_nbrhosts = dict() + dut_nbrhosts, up_bgp_neighbors = dict(), dict() orig_v4_routes, orig_v6_routes = dict(), dict() for linecard in duthosts.frontend_nodes: dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) @@ -981,6 +990,7 @@ def test_sup_tsa_act_when_duts_on_tsa_with_sup_config_reload(duthosts, localhost "DUT is not in maintenance state with supervisor TSA action") pytest_assert('true' == get_tsa_chassisdb_config(linecard), "{} tsa_enabled config is not enabled".format(linecard.hostname)) + up_bgp_neighbors[linecard] = linecard.get_bgp_neighbors_per_asic("established") # Do config_reload on the supervisor and verify configs are same as before config_reload(suphost, wait=300, safe_reload=True) @@ -994,6 +1004,9 @@ def test_sup_tsa_act_when_duts_on_tsa_with_sup_config_reload(duthosts, localhost "DUT is not in maintenance state after supervisor config reload") pytest_assert('true' == get_tsa_chassisdb_config(linecard), "{} tsa_enabled chassisdb config is not enabled".format(linecard.hostname)) + # Before verifying loopback address, make sure IBGP neighbors are in established state + pytest_assert(wait_until(300, 20, 0, linecard.check_bgp_session_state_all_asics, + up_bgp_neighbors[linecard], "established")) pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), @@ -1025,9 +1038,12 @@ def test_dut_tsa_act_with_reboot_when_sup_dut_on_tsb_init(duthosts, localhost, e orig_v4_routes, orig_v6_routes = dict(), dict() tsa_tsb_timer = dict() dut_nbrhosts = dict() + int_status_result, crit_process_check = dict(), dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + int_status_result[linecard] = True + crit_process_check[linecard] = True # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1082,9 +1098,8 @@ def test_dut_tsa_act_with_reboot_when_sup_dut_on_tsb_init(duthosts, localhost, e # Make sure the ports, interfaces are UP and running after reboot for linecard in duthosts.frontend_nodes: logging.info("Wait until all critical processes are fully started") - wait_critical_processes(linecard) - pytest_assert(wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard), - "Not all ports that are admin up on are operationally up") + crit_process_check[linecard] = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) + int_status_result[linecard] = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced to neighbors when the linecards are in TSA @@ -1100,6 +1115,13 @@ def test_dut_tsa_act_with_reboot_when_sup_dut_on_tsb_init(duthosts, localhost, e # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + for linecard in duthosts.frontend_nodes: + # Make sure linecards are in Normal state, if not do config-reload on the dut to recover + if not (int_status_result[linecard] and crit_process_check[linecard] and + TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats')): + logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") + config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + # Verify all routes are advertised back to neighbors when duts are in TSB verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) @@ -1336,9 +1358,12 @@ def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() tsa_tsb_timer = dict() + int_status_result, crit_process_check = dict(), dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + int_status_result[linecard] = True + crit_process_check[linecard] = True # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1383,9 +1408,8 @@ def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ for linecard in duthosts.frontend_nodes: logging.info("Wait until all critical processes are fully started") - wait_critical_processes(linecard) - pytest_assert(wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard), - "Not all ports that are admin up on are operationally up") + crit_process_check[linecard] = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) + int_status_result[linecard] = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) # Verify startup_tsa_tsb service stopped after expected time pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'exited'), @@ -1410,6 +1434,13 @@ def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + for linecard in duthosts.frontend_nodes: + # Make sure linecards are in Normal state, if not do config-reload on the dut to recover + if not (int_status_result[linecard] and crit_process_check[linecard] and + TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats')): + logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") + config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + # Verify all routes are advertised back to neighbors when duts are in TSB verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) @@ -1430,6 +1461,7 @@ def test_sup_tsb_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ orig_v4_routes, orig_v6_routes = dict(), dict() dut_nbrhosts = dict() tsa_tsb_timer = dict() + int_status_result, crit_process_check = True, True for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) @@ -1471,9 +1503,8 @@ def test_sup_tsb_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ "DUT is not in maintenance state when startup_tsa_tsb service is running") logging.info("Wait until all critical processes are fully started") - wait_critical_processes(linecard) - pytest_assert(wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard), - "Not all ports that are admin up on are operationally up") + crit_process_check = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) + int_status_result = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) # Verify startup_tsa_tsb service stopped after expected time pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'exited'), @@ -1495,6 +1526,13 @@ def test_sup_tsb_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + for linecard in duthosts.frontend_nodes: + # Make sure linecards are in Normal state, if not do config-reload on the dut to recover + if not (int_status_result and crit_process_check and + TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats')): + logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") + config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + # Verify all routes are advertised back to neighbors when duts are in TSB verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) diff --git a/tests/bgp/test_startup_tsa_tsb_service.py b/tests/bgp/test_startup_tsa_tsb_service.py index f7f8e93d60..e033d3ddf8 100644 --- a/tests/bgp/test_startup_tsa_tsb_service.py +++ b/tests/bgp/test_startup_tsa_tsb_service.py @@ -6,7 +6,7 @@ from tests.common.reboot import get_reboot_cause, SONIC_SSH_PORT, SONIC_SSH_REGEX, wait_for_startup from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until -from tests.common.platform.processes_utils import wait_critical_processes +from tests.common.platform.processes_utils import wait_critical_processes, _all_critical_processes_healthy from tests.common.platform.interface_utils import check_interface_status_of_up_ports from traffic_checker import get_traffic_shift_state, check_tsa_persistence_support from route_checker import parse_routes_on_neighbors, check_and_log_routes_diff, \ @@ -176,6 +176,7 @@ def test_tsa_tsb_service_with_dut_cold_reboot(duthosts, localhost, enum_rand_one """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] tsa_tsb_timer = get_startup_tsb_timer(duthost) + int_status_result, crit_process_check = True, True if not tsa_tsb_timer: pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) dut_nbrhosts = nbrhosts_to_dut(duthost, nbrhosts) @@ -215,9 +216,8 @@ def test_tsa_tsb_service_with_dut_cold_reboot(duthosts, localhost, enum_rand_one "DUT is not in maintenance state when startup_tsa_tsb service is running") logging.info("Wait until all critical processes are fully started") - wait_critical_processes(duthost) - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), - "Not all ports that are admin up on are operationally up") + crit_process_check = wait_until(600, 20, 0, _all_critical_processes_healthy, duthost) + int_status_result = wait_until(1200, 20, 0, check_interface_status_of_up_ports, duthost) pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, duthost, dut_nbrhosts, traffic_shift_community), "Failed to verify routes on nbr in TSA") @@ -247,10 +247,10 @@ def test_tsa_tsb_service_with_dut_cold_reboot(duthosts, localhost, enum_rand_one pytest.fail("Not all ipv6 routes are announced to neighbors") finally: - - # Verify DUT is in normal state. - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") + # Verify DUT is in normal state after cold reboot scenario. + if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): + logging.info("DUT is not in normal state after cold reboot, doing config-reload") + config_reload(duthost, safe_reload=True, check_intf_up_ports=True) # Make sure the dut's reboot cause is as expected logger.info("Check reboot cause of the dut") reboot_cause = get_reboot_cause(duthost) @@ -268,6 +268,7 @@ def test_tsa_tsb_service_with_dut_abnormal_reboot(duthosts, localhost, enum_rand """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] tsa_tsb_timer = get_startup_tsb_timer(duthost) + int_status_result, crit_process_check = True, True if not tsa_tsb_timer: pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) dut_nbrhosts = nbrhosts_to_dut(duthost, nbrhosts) @@ -324,9 +325,8 @@ def test_tsa_tsb_service_with_dut_abnormal_reboot(duthosts, localhost, enum_rand "DUT is not in maintenance state when startup_tsa_tsb service is running") logging.info("Wait until all critical processes are fully started") - wait_critical_processes(duthost) - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), - "Not all ports that are admin up on are operationally up") + crit_process_check = wait_until(600, 20, 0, _all_critical_processes_healthy, duthost) + int_status_result = wait_until(1200, 20, 0, check_interface_status_of_up_ports, duthost) pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, duthost, dut_nbrhosts, traffic_shift_community), "Failed to verify routes on nbr in TSA") @@ -356,10 +356,10 @@ def test_tsa_tsb_service_with_dut_abnormal_reboot(duthosts, localhost, enum_rand pytest.fail("Not all ipv6 routes are announced to neighbors") finally: - - # Verify DUT is in normal state. - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") + # Verify DUT is in normal state after abnormal reboot scenario. + if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): + logging.info("DUT is not in normal state after abnormal reboot, doing config-reload") + config_reload(duthost, safe_reload=True, check_intf_up_ports=True) # Make sure the dut's reboot cause is as expected logger.info("Check reboot cause of the dut") reboot_cause = get_reboot_cause(duthost) @@ -379,8 +379,11 @@ def test_tsa_tsb_service_with_supervisor_cold_reboot(duthosts, localhost, enum_s tsa_tsb_timer = dict() dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() + int_status_result, crit_process_check = dict(), dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) + int_status_result[linecard] = True + crit_process_check[linecard] = True if not tsa_tsb_timer[linecard]: pytest.skip("startup_tsa_tsb.service is not supported on the duts under {}".format(suphost.hostname)) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) @@ -425,9 +428,8 @@ def test_tsa_tsb_service_with_supervisor_cold_reboot(duthosts, localhost, enum_s "DUT is not in maintenance state when startup_tsa_tsb service is running") logging.info("Wait until all critical processes are fully started") - wait_critical_processes(linecard) - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, linecard), - "Not all ports that are admin up on are operationally up") + crit_process_check[linecard] = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) + int_status_result[linecard] = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), @@ -462,10 +464,14 @@ def test_tsa_tsb_service_with_supervisor_cold_reboot(duthosts, localhost, enum_s pytest.fail("Not all ipv6 routes are announced to neighbors") finally: + # Make sure DUT is in normal state after supervisor cold reboot + for linecard in duthosts.frontend_nodes: + if not (int_status_result[linecard] and crit_process_check[linecard] and + TS_NORMAL == get_traffic_shift_state(linecard)): + logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") + config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + for linecard in duthosts.frontend_nodes: - # Verify DUT is in normal state. - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), - "DUT {} is not in normal state".format(linecard)) # Make sure the dut's reboot cause is as expected logger.info("Check reboot cause of the dut {}".format(linecard)) reboot_cause = get_reboot_cause(linecard) @@ -492,8 +498,11 @@ def test_tsa_tsb_service_with_supervisor_abnormal_reboot(duthosts, localhost, en tsa_tsb_timer = dict() dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() + int_status_result, crit_process_check = dict(), dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) + int_status_result[linecard] = True + crit_process_check[linecard] = True if not tsa_tsb_timer[linecard]: pytest.skip("startup_tsa_tsb.service is not supported on the duts under {}".format(suphost.hostname)) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) @@ -550,14 +559,17 @@ def test_tsa_tsb_service_with_supervisor_abnormal_reboot(duthosts, localhost, en pytest_assert(int(time_diff) < 120, "startup_tsa_tsb service started much later than the expected time after dut reboot") + # Make sure BGP containers are running properly before verifying + pytest_assert(wait_until(90, 5, 0, check_tsc_command_error, linecard), + "TSC command still returns error even after startup_tsa_tsb service started") + # Verify DUT is in maintenance state. pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), "DUT is not in maintenance state when startup_tsa_tsb service is running") logging.info("Wait until all critical processes are fully started") - wait_critical_processes(linecard) - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, linecard), - "Not all ports that are admin up on are operationally up") + crit_process_check[linecard] = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) + int_status_result[linecard] = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), @@ -592,10 +604,14 @@ def test_tsa_tsb_service_with_supervisor_abnormal_reboot(duthosts, localhost, en pytest.fail("Not all ipv6 routes are announced to neighbors") finally: + # Make sure DUT is in normal state after supervisor abnormal reboot + for linecard in duthosts.frontend_nodes: + if not (int_status_result[linecard] and crit_process_check[linecard] and + TS_NORMAL == get_traffic_shift_state(linecard)): + logging.info("DUT is not in normal state after SUP abnormal reboot, doing config-reload") + config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + for linecard in duthosts.frontend_nodes: - # Verify DUT is in normal state. - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), - "DUT {} is not in normal state".format(linecard)) # Make sure the dut's reboot cause is as expected logger.info("Check reboot cause of the dut {}".format(linecard)) reboot_cause = get_reboot_cause(linecard) @@ -665,7 +681,7 @@ def test_tsa_tsb_service_with_user_init_tsa(duthosts, localhost, enum_rand_one_p logging.info("Wait until all critical processes are fully started") wait_critical_processes(duthost) - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), + pytest_assert(wait_until(1200, 20, 0, check_interface_status_of_up_ports, duthost), "Not all ports that are admin up on are operationally up") pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( @@ -718,6 +734,7 @@ def test_user_init_tsa_while_service_run_on_dut(duthosts, localhost, enum_rand_o """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] tsa_tsb_timer = get_startup_tsb_timer(duthost) + int_status_result, crit_process_check = True, True if not tsa_tsb_timer: pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) dut_nbrhosts = nbrhosts_to_dut(duthost, nbrhosts) @@ -769,9 +786,8 @@ def test_user_init_tsa_while_service_run_on_dut(duthosts, localhost, enum_rand_o "DUT is not in maintenance state with saved TSA config after reboot") logging.info("Wait until all critical processes are fully started") - wait_critical_processes(duthost) - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), - "Not all ports that are admin up on are operationally up") + crit_process_check = wait_until(600, 20, 0, _all_critical_processes_healthy, duthost) + int_status_result = wait_until(1200, 20, 0, check_interface_status_of_up_ports, duthost) pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, duthost, dut_nbrhosts, traffic_shift_community), @@ -786,8 +802,10 @@ def test_user_init_tsa_while_service_run_on_dut(duthosts, localhost, enum_rand_o duthost.shell("TSB") duthost.shell('sudo config save -y') - # Verify DUT comes back to normal state after TSB. - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") + # Verify DUT is in normal state after cold reboot scenario. + if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): + logging.info("DUT is not in normal state after cold reboot, doing config-reload") + config_reload(duthost, safe_reload=True, check_intf_up_ports=True) # Wait until all routes are announced to neighbors cur_v4_routes = {} cur_v6_routes = {} @@ -822,6 +840,7 @@ def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_o """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] tsa_tsb_timer = get_startup_tsb_timer(duthost) + int_status_result, crit_process_check = True, True if not tsa_tsb_timer: pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) # Ensure that the DUT is not in maintenance already before start of the test @@ -873,6 +892,8 @@ def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_o # Make sure DUT continues to be in good state after TSB assert wait_until(300, 20, 2, duthost.critical_services_fully_started), \ "Not all critical services are fully started on {}".format(duthost.hostname) + crit_process_check = wait_until(600, 20, 0, _all_critical_processes_healthy, duthost) + int_status_result = wait_until(1200, 20, 0, check_interface_status_of_up_ports, duthost) # Wait until all routes are announced to neighbors cur_v4_routes = {} @@ -889,10 +910,10 @@ def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_o pytest.fail("Not all ipv6 routes are announced to neighbors") finally: - - # Verify DUT is in normal state. - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") + # Verify DUT is in normal state after cold reboot scenario. + if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): + logging.info("DUT is not in normal state after cold reboot, doing config-reload") + config_reload(duthost, safe_reload=True, check_intf_up_ports=True) # Make sure the dut's reboot cause is as expected logger.info("Check reboot cause of the dut") @@ -913,11 +934,14 @@ def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, Make sure TSA_TSB service is stopped and dut changes from maintenance mode to normal mode """ suphost = duthosts[enum_supervisor_dut_hostname] + int_status_result, crit_process_check = dict(), dict() tsa_tsb_timer = dict() dut_nbrhosts = dict() orig_v4_routes, orig_v6_routes = dict(), dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) + int_status_result[linecard] = True + crit_process_check[linecard] = True if not tsa_tsb_timer[linecard]: pytest.skip("startup_tsa_tsb.service is not supported on the duts under {}".format(suphost.hostname)) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) @@ -962,7 +986,8 @@ def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, "DUT is not in maintenance state when startup_tsa_tsb service is running") logging.info("Wait until all critical processes are fully started") - wait_critical_processes(linecard) + crit_process_check[linecard] = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) + int_status_result[linecard] = wait_until(1200, 20, 10, check_interface_status_of_up_ports, linecard) pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), @@ -984,8 +1009,7 @@ def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, # Ensure startup_tsa_tsb service is in exited state after timer expiry pytest_assert(wait_until(tsa_tsb_timer[linecard], 5, 0, get_tsa_tsb_service_status, linecard, 'exited'), "startup_tsa_tsb service is not in exited state after user init TSB from supervisor") - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, linecard), - "Not all ports that are admin up on are operationally up") + int_status_result[linecard] = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) for linecard in duthosts.frontend_nodes: # Wait until all routes are announced to neighbors @@ -1009,9 +1033,10 @@ def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, # Make sure linecards are in Normal state and save the config to proceed further linecard.shell("TSB") linecard.shell('sudo config save -y') - # Verify DUT is in normal state. - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), - "DUT {} is not in normal state".format(linecard)) + if not (int_status_result[linecard] and crit_process_check[linecard] and + TS_NORMAL == get_traffic_shift_state(linecard)): + logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") + config_reload(linecard, safe_reload=True, check_intf_up_ports=True) # Make sure the dut's reboot cause is as expected logger.info("Check reboot cause of the dut {}".format(linecard)) reboot_cause = get_reboot_cause(linecard) @@ -1035,6 +1060,7 @@ def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_f """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] tsa_tsb_timer = get_startup_tsb_timer(duthost) + int_status_result, crit_process_check = True, True if not tsa_tsb_timer: pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) # Ensure that the DUT is not in maintenance already before start of the test @@ -1075,9 +1101,8 @@ def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_f "Not all critical services are fully started on {}".format(duthost.hostname) logging.info("Wait until all critical processes are fully started") - wait_critical_processes(duthost) - pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), - "Not all ports that are admin up on are operationally up") + crit_process_check = wait_until(600, 20, 0, _all_critical_processes_healthy, duthost) + int_status_result = wait_until(1200, 20, 0, check_interface_status_of_up_ports, duthost) pytest_assert(wait_until(300, 10, 0, duthost.check_bgp_session_state_all_asics, up_bgp_neighbors, "established")) @@ -1119,10 +1144,10 @@ def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_f pytest.fail("Not all ipv6 routes are announced to neighbors") finally: - - # Verify DUT is in normal state. - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") + # Verify DUT is in normal state after cold reboot scenario. + if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): + logging.info("DUT is not in normal state after cold reboot, doing config-reload") + config_reload(duthost, safe_reload=True, check_intf_up_ports=True) # Make sure the dut's reboot cause is as expected logger.info("Check reboot cause of the dut") reboot_cause = get_reboot_cause(duthost) From 7c393c111b9eddaea3203f8f2e52f3e3588e5a1d Mon Sep 17 00:00:00 2001 From: sanrajen Date: Wed, 11 Sep 2024 18:26:50 -0400 Subject: [PATCH 6/7] Keep supervisor/linecards tsa-enabled config in sync for startup-tsa-tsb tests Latest rebase changes --- tests/bgp/bgp_helpers.py | 36 +++++++- tests/bgp/test_reliable_tsa.py | 46 ++-------- tests/bgp/test_startup_tsa_tsb_service.py | 102 +++++++++++++++------- 3 files changed, 112 insertions(+), 72 deletions(-) diff --git a/tests/bgp/bgp_helpers.py b/tests/bgp/bgp_helpers.py index 72d408f2fb..fa71b4167a 100644 --- a/tests/bgp/bgp_helpers.py +++ b/tests/bgp/bgp_helpers.py @@ -13,7 +13,8 @@ import ipaddr as ipaddress from tests.common.helpers.assertions import pytest_require from tests.common.helpers.assertions import pytest_assert -from tests.common.helpers.constants import UPSTREAM_NEIGHBOR_MAP, DOWNSTREAM_NEIGHBOR_MAP, DEFAULT_NAMESPACE +from tests.common.helpers.constants import UPSTREAM_NEIGHBOR_MAP, DOWNSTREAM_NEIGHBOR_MAP, DEFAULT_NAMESPACE, \ + DEFAULT_ASIC_ID from tests.common.helpers.parallel import reset_ansible_local_tmp from tests.common.helpers.parallel import parallel_run from tests.common.utilities import wait_until @@ -845,3 +846,36 @@ def fetch_and_delete_pcap_file(bgp_pcap, log_dir, duthost, request): duthost.fetch(src=bgp_pcap, dest=local_pcap_filename, flat=True) duthost.file(path=bgp_pcap, state="absent") return local_pcap_filename + + +def get_tsa_chassisdb_config(duthost): + """ + @summary: Returns the dut's CHASSIS_APP_DB value for BGP_DEVICE_GLOBAL.STATE.tsa_enabled flag + """ + tsa_conf = duthost.shell('sonic-db-cli CHASSIS_APP_DB HGET \'BGP_DEVICE_GLOBAL|STATE\' tsa_enabled')['stdout'] + return tsa_conf + + +def get_sup_cfggen_tsa_value(suphost): + """ + @summary: Returns the supervisor sonic-cfggen value for BGP_DEVICE_GLOBAL.STATE.tsa_enabled flag + """ + tsa_conf = suphost.shell('sonic-cfggen -d -v BGP_DEVICE_GLOBAL.STATE.tsa_enabled')['stdout'] + return tsa_conf + + +def verify_dut_configdb_tsa_value(duthost): + """ + @summary: Returns the line cards' asic CONFIG_DB value for BGP_DEVICE_GLOBAL.STATE.tsa_enabled flag + """ + tsa_config = list() + tsa_enabled = 'false' + for asic_index in duthost.get_frontend_asic_ids(): + prefix = "-n asic{}".format(asic_index) if asic_index != DEFAULT_ASIC_ID else '' + output = duthost.shell('sonic-db-cli {} CONFIG_DB HGET \'BGP_DEVICE_GLOBAL|STATE\' \'tsa_enabled\''. + format(prefix))['stdout'] + tsa_config.append(output) + if 'true' in tsa_config: + tsa_enabled = 'true' + + return tsa_enabled diff --git a/tests/bgp/test_reliable_tsa.py b/tests/bgp/test_reliable_tsa.py index 670f5b5340..3f1cdaa8ca 100644 --- a/tests/bgp/test_reliable_tsa.py +++ b/tests/bgp/test_reliable_tsa.py @@ -4,13 +4,14 @@ from tests.common import reboot, config_reload from tests.common.reboot import wait_for_startup from tests.common.helpers.assertions import pytest_assert -from tests.common.helpers.constants import DEFAULT_ASIC_ID from tests.common.utilities import wait_until from tests.common.platform.processes_utils import wait_critical_processes, _all_critical_processes_healthy from tests.common.platform.interface_utils import check_interface_status_of_up_ports -from tests.bgp.test_traffic_shift import get_traffic_shift_state, parse_routes_on_neighbors, \ - verify_current_routes_announced_to_neighs, check_and_log_routes_diff, \ - verify_only_loopback_routes_are_announced_to_neighs +from tests.bgp.bgp_helpers import get_tsa_chassisdb_config, get_sup_cfggen_tsa_value, verify_dut_configdb_tsa_value +from traffic_checker import get_traffic_shift_state +from route_checker import parse_routes_on_neighbors, check_and_log_routes_diff, \ + verify_current_routes_announced_to_neighs, verify_only_loopback_routes_are_announced_to_neighs +from tests.bgp.constants import TS_NORMAL, TS_MAINTENANCE from tests.bgp.test_startup_tsa_tsb_service import get_tsa_tsb_service_uptime, get_tsa_tsb_service_status, \ get_startup_tsb_timer, enable_disable_startup_tsa_tsb_service # noqa: F401 @@ -20,10 +21,6 @@ logger = logging.getLogger(__name__) -TS_NORMAL = "System Mode: Normal" -TS_MAINTENANCE = "System Mode: Maintenance" -TS_INCONSISTENT = "System Mode: Not consistent" -TS_NO_NEIGHBORS = "System Mode: No external neighbors" CONTAINER_CHECK_INTERVAL_SECS = 2 CONTAINER_STOP_THRESHOLD_SECS = 60 CONTAINER_RESTART_THRESHOLD_SECS = 300 @@ -132,39 +129,6 @@ def verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_r pytest.fail("Not all ipv6 routes are announced to neighbors") -def get_tsa_chassisdb_config(duthost): - """ - @summary: Returns the dut's CHASSIS_APP_DB value for BGP_DEVICE_GLOBAL.STATE.tsa_enabled flag - """ - tsa_conf = duthost.shell('sonic-db-cli CHASSIS_APP_DB HGET \'BGP_DEVICE_GLOBAL|STATE\' tsa_enabled')['stdout'] - return tsa_conf - - -def get_sup_cfggen_tsa_value(suphost): - """ - @summary: Returns the supervisor sonic-cfggen value for BGP_DEVICE_GLOBAL.STATE.tsa_enabled flag - """ - tsa_conf = suphost.shell('sonic-cfggen -d -v BGP_DEVICE_GLOBAL.STATE.tsa_enabled')['stdout'] - return tsa_conf - - -def verify_dut_configdb_tsa_value(duthost): - """ - @summary: Returns the line cards' asic CONFIG_DB value for BGP_DEVICE_GLOBAL.STATE.tsa_enabled flag - """ - tsa_config = list() - tsa_enabled = 'false' - for asic_index in duthost.get_frontend_asic_ids(): - prefix = "-n asic{}".format(asic_index) if asic_index != DEFAULT_ASIC_ID else '' - output = duthost.shell('sonic-db-cli {} CONFIG_DB HGET \'BGP_DEVICE_GLOBAL|STATE\' \'tsa_enabled\''. - format(prefix))['stdout'] - tsa_config.append(output) - if 'true' in tsa_config: - tsa_enabled = 'true' - - return tsa_enabled - - def kill_process_by_pid(duthost, container_name, program_name, program_pid): """ @summary: Kill a process in the specified container by its pid diff --git a/tests/bgp/test_startup_tsa_tsb_service.py b/tests/bgp/test_startup_tsa_tsb_service.py index e033d3ddf8..f2d975a979 100644 --- a/tests/bgp/test_startup_tsa_tsb_service.py +++ b/tests/bgp/test_startup_tsa_tsb_service.py @@ -12,7 +12,7 @@ from route_checker import parse_routes_on_neighbors, check_and_log_routes_diff, \ verify_current_routes_announced_to_neighs, verify_only_loopback_routes_are_announced_to_neighs from tests.bgp.constants import TS_NORMAL, TS_MAINTENANCE - +from tests.bgp.bgp_helpers import get_tsa_chassisdb_config, get_sup_cfggen_tsa_value, verify_dut_configdb_tsa_value pytestmark = [ pytest.mark.topology('t2') @@ -145,6 +145,30 @@ def nbrhosts_to_dut(duthost, nbrhosts): return dut_nbrhosts +def initial_check_before_and_after_test(duthosts): + """ + @summary: Common method to make sure the supervisor and line cards are in normal state before and after the test + """ + for duthost in duthosts: + if duthost.is_supervisor_node(): + # Initially make sure both supervisor and line cards are in BGP operational normal state + if get_tsa_chassisdb_config(duthost) != 'false' or get_sup_cfggen_tsa_value(duthost) != 'false': + duthost.shell('TSB') + duthost.shell('sudo config save -y') + pytest_assert('false' == get_tsa_chassisdb_config(duthost), + "Supervisor {} tsa_enabled config is enabled".format(duthost.hostname)) + + for linecard in duthosts.frontend_nodes: + # Issue TSB on the line card before proceeding further + if verify_dut_configdb_tsa_value(linecard) != 'false' or get_tsa_chassisdb_config(linecard) != 'false' or \ + get_traffic_shift_state(linecard, cmd='TSC no-stats') != TS_NORMAL: + linecard.shell('TSB') + linecard.shell('sudo config save -y') + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state") + + def check_ssh_state(localhost, dut_ip, expected_state, timeout=60): """ Check the SSH state of DUT. @@ -180,12 +204,12 @@ def test_tsa_tsb_service_with_dut_cold_reboot(duthosts, localhost, enum_rand_one if not tsa_tsb_timer: pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) dut_nbrhosts = nbrhosts_to_dut(duthost, nbrhosts) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") if not check_tsa_persistence_support(duthost): pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_check_before_and_after_test(duthosts) + try: # Get all routes on neighbors before doing reboot orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) @@ -247,6 +271,8 @@ def test_tsa_tsb_service_with_dut_cold_reboot(duthosts, localhost, enum_rand_one pytest.fail("Not all ipv6 routes are announced to neighbors") finally: + # Bring back the supervisor and line cards to the BGP operational normal state + initial_check_before_and_after_test(duthosts) # Verify DUT is in normal state after cold reboot scenario. if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): logging.info("DUT is not in normal state after cold reboot, doing config-reload") @@ -273,12 +299,12 @@ def test_tsa_tsb_service_with_dut_abnormal_reboot(duthosts, localhost, enum_rand pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) dut_nbrhosts = nbrhosts_to_dut(duthost, nbrhosts) dut_ip = duthost.mgmt_ip - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") if not check_tsa_persistence_support(duthost): pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_check_before_and_after_test(duthosts) + try: # Get all routes on neighbors before doing reboot orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) @@ -356,6 +382,8 @@ def test_tsa_tsb_service_with_dut_abnormal_reboot(duthosts, localhost, enum_rand pytest.fail("Not all ipv6 routes are announced to neighbors") finally: + # Bring back the supervisor and line cards to the BGP operational normal state + initial_check_before_and_after_test(duthosts) # Verify DUT is in normal state after abnormal reboot scenario. if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): logging.info("DUT is not in normal state after abnormal reboot, doing config-reload") @@ -387,11 +415,10 @@ def test_tsa_tsb_service_with_supervisor_cold_reboot(duthosts, localhost, enum_s if not tsa_tsb_timer[linecard]: pytest.skip("startup_tsa_tsb.service is not supported on the duts under {}".format(suphost.hostname)) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), - "DUT is not in normal state") if not check_tsa_persistence_support(linecard): pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_check_before_and_after_test(duthosts) try: for linecard in duthosts.frontend_nodes: @@ -464,6 +491,9 @@ def test_tsa_tsb_service_with_supervisor_cold_reboot(duthosts, localhost, enum_s pytest.fail("Not all ipv6 routes are announced to neighbors") finally: + # Bring back the supervisor and line cards to the BGP operational normal state + initial_check_before_and_after_test(duthosts) + # Make sure DUT is in normal state after supervisor cold reboot for linecard in duthosts.frontend_nodes: if not (int_status_result[linecard] and crit_process_check[linecard] and @@ -506,12 +536,12 @@ def test_tsa_tsb_service_with_supervisor_abnormal_reboot(duthosts, localhost, en if not tsa_tsb_timer[linecard]: pytest.skip("startup_tsa_tsb.service is not supported on the duts under {}".format(suphost.hostname)) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), - "DUT is not in normal state") if not check_tsa_persistence_support(linecard): pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_check_before_and_after_test(duthosts) + try: for linecard in duthosts.frontend_nodes: # Get all routes on neighbors before doing reboot @@ -604,6 +634,9 @@ def test_tsa_tsb_service_with_supervisor_abnormal_reboot(duthosts, localhost, en pytest.fail("Not all ipv6 routes are announced to neighbors") finally: + # Bring back the supervisor and line cards to the BGP operational normal state + initial_check_before_and_after_test(duthosts) + # Make sure DUT is in normal state after supervisor abnormal reboot for linecard in duthosts.frontend_nodes: if not (int_status_result[linecard] and crit_process_check[linecard] and @@ -640,12 +673,12 @@ def test_tsa_tsb_service_with_user_init_tsa(duthosts, localhost, enum_rand_one_p pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) dut_nbrhosts = nbrhosts_to_dut(duthost, nbrhosts) orig_v4_routes, orig_v6_routes = {}, {} - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") if not check_tsa_persistence_support(duthost): pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_check_before_and_after_test(duthosts) + try: # Get all routes on neighbors before doing reboot orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) @@ -719,6 +752,8 @@ def test_tsa_tsb_service_with_user_init_tsa(duthosts, localhost, enum_rand_one_p reboot_cause = get_reboot_cause(duthost) pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) + # Bring back the supervisor and line cards to the BGP operational normal state + initial_check_before_and_after_test(duthosts) @pytest.mark.disable_loganalyzer @@ -738,12 +773,12 @@ def test_user_init_tsa_while_service_run_on_dut(duthosts, localhost, enum_rand_o if not tsa_tsb_timer: pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) dut_nbrhosts = nbrhosts_to_dut(duthost, nbrhosts) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") if not check_tsa_persistence_support(duthost): pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_check_before_and_after_test(duthosts) + try: # Get all routes on neighbors before doing reboot orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) @@ -825,6 +860,8 @@ def test_user_init_tsa_while_service_run_on_dut(duthosts, localhost, enum_rand_o reboot_cause = get_reboot_cause(duthost) pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) + # Bring back the supervisor and line cards to the BGP operational normal state + initial_check_before_and_after_test(duthosts) @pytest.mark.disable_loganalyzer @@ -843,12 +880,12 @@ def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_o int_status_result, crit_process_check = True, True if not tsa_tsb_timer: pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") if not check_tsa_persistence_support(duthost): pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_check_before_and_after_test(duthosts) + try: # Get all routes on neighbors before doing reboot orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) @@ -910,6 +947,8 @@ def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_o pytest.fail("Not all ipv6 routes are announced to neighbors") finally: + # Bring back the supervisor and line cards to the BGP operational normal state + initial_check_before_and_after_test(duthosts) # Verify DUT is in normal state after cold reboot scenario. if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): logging.info("DUT is not in normal state after cold reboot, doing config-reload") @@ -945,12 +984,12 @@ def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, if not tsa_tsb_timer[linecard]: pytest.skip("startup_tsa_tsb.service is not supported on the duts under {}".format(suphost.hostname)) dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), - "DUT is not in normal state") if not check_tsa_persistence_support(linecard): pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_check_before_and_after_test(duthosts) + try: for linecard in duthosts.frontend_nodes: # Get all routes on neighbors before doing reboot @@ -1029,10 +1068,11 @@ def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, pytest.fail("Not all ipv6 routes are announced to neighbors") finally: + # Bring back the supervisor and line cards to the BGP operational normal state + initial_check_before_and_after_test(duthosts) + for linecard in duthosts.frontend_nodes: # Make sure linecards are in Normal state and save the config to proceed further - linecard.shell("TSB") - linecard.shell('sudo config save -y') if not (int_status_result[linecard] and crit_process_check[linecard] and TS_NORMAL == get_traffic_shift_state(linecard)): logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") @@ -1063,12 +1103,12 @@ def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_f int_status_result, crit_process_check = True, True if not tsa_tsb_timer: pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), - "DUT is not in normal state") if not check_tsa_persistence_support(duthost): pytest.skip("TSA persistence not supported in the image") + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_check_before_and_after_test(duthosts) + try: # Get all routes on neighbors before doing reboot orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) @@ -1144,6 +1184,8 @@ def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_f pytest.fail("Not all ipv6 routes are announced to neighbors") finally: + # Bring back the supervisor and line cards to the BGP operational normal state + initial_check_before_and_after_test(duthosts) # Verify DUT is in normal state after cold reboot scenario. if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): logging.info("DUT is not in normal state after cold reboot, doing config-reload") From bb71883f41b96614e387177ce4f5c4d107b64e15 Mon Sep 17 00:00:00 2001 From: sanrajen Date: Thu, 19 Sep 2024 17:46:50 -0400 Subject: [PATCH 7/7] Refactor traffic_shift & traffic_shift_sup suites w.r.t reliable_tsa Do pre/post check only for t2 topo typo --- tests/bgp/bgp_helpers.py | 26 +++++++++ tests/bgp/route_checker.py | 2 +- tests/bgp/test_reliable_tsa.py | 4 +- tests/bgp/test_startup_tsa_tsb_service.py | 66 ++++++++--------------- tests/bgp/test_traffic_shift.py | 43 +++++++++++++-- tests/bgp/test_traffic_shift_sup.py | 31 +++++++---- 6 files changed, 111 insertions(+), 61 deletions(-) diff --git a/tests/bgp/bgp_helpers.py b/tests/bgp/bgp_helpers.py index fa71b4167a..1934c352ae 100644 --- a/tests/bgp/bgp_helpers.py +++ b/tests/bgp/bgp_helpers.py @@ -18,6 +18,8 @@ from tests.common.helpers.parallel import reset_ansible_local_tmp from tests.common.helpers.parallel import parallel_run from tests.common.utilities import wait_until +from tests.bgp.traffic_checker import get_traffic_shift_state +from tests.bgp.constants import TS_NORMAL BASE_DIR = os.path.dirname(os.path.realpath(__file__)) DUT_TMP_DIR = os.path.join('tmp', os.path.basename(BASE_DIR)) @@ -879,3 +881,27 @@ def verify_dut_configdb_tsa_value(duthost): tsa_enabled = 'true' return tsa_enabled + + +def initial_tsa_check_before_and_after_test(duthosts): + """ + @summary: Common method to make sure the supervisor and line cards are in normal state before and after the test + """ + for duthost in duthosts: + if duthost.is_supervisor_node(): + # Initially make sure both supervisor and line cards are in BGP operational normal state + if get_tsa_chassisdb_config(duthost) != 'false' or get_sup_cfggen_tsa_value(duthost) != 'false': + duthost.shell('TSB') + duthost.shell('sudo config save -y') + pytest_assert('false' == get_tsa_chassisdb_config(duthost), + "Supervisor {} tsa_enabled config is enabled".format(duthost.hostname)) + + for linecard in duthosts.frontend_nodes: + # Issue TSB on the line card before proceeding further + if verify_dut_configdb_tsa_value(linecard) != 'false' or get_tsa_chassisdb_config(linecard) != 'false' or \ + get_traffic_shift_state(linecard, cmd='TSC no-stats') != TS_NORMAL: + linecard.shell('TSB') + linecard.shell('sudo config save -y') + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state") diff --git a/tests/bgp/route_checker.py b/tests/bgp/route_checker.py index 322288718d..646dc0b86c 100644 --- a/tests/bgp/route_checker.py +++ b/tests/bgp/route_checker.py @@ -2,7 +2,7 @@ import ipaddr as ipaddress import re import json -from bgp_helpers import parse_rib +from tests.bgp.bgp_helpers import parse_rib from tests.common.devices.eos import EosHost from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.parallel import parallel_run diff --git a/tests/bgp/test_reliable_tsa.py b/tests/bgp/test_reliable_tsa.py index 3f1cdaa8ca..3fd88bdcc1 100644 --- a/tests/bgp/test_reliable_tsa.py +++ b/tests/bgp/test_reliable_tsa.py @@ -8,8 +8,8 @@ from tests.common.platform.processes_utils import wait_critical_processes, _all_critical_processes_healthy from tests.common.platform.interface_utils import check_interface_status_of_up_ports from tests.bgp.bgp_helpers import get_tsa_chassisdb_config, get_sup_cfggen_tsa_value, verify_dut_configdb_tsa_value -from traffic_checker import get_traffic_shift_state -from route_checker import parse_routes_on_neighbors, check_and_log_routes_diff, \ +from tests.bgp.traffic_checker import get_traffic_shift_state +from tests.bgp.route_checker import parse_routes_on_neighbors, check_and_log_routes_diff, \ verify_current_routes_announced_to_neighs, verify_only_loopback_routes_are_announced_to_neighs from tests.bgp.constants import TS_NORMAL, TS_MAINTENANCE from tests.bgp.test_startup_tsa_tsb_service import get_tsa_tsb_service_uptime, get_tsa_tsb_service_status, \ diff --git a/tests/bgp/test_startup_tsa_tsb_service.py b/tests/bgp/test_startup_tsa_tsb_service.py index f2d975a979..d29a4d4c78 100644 --- a/tests/bgp/test_startup_tsa_tsb_service.py +++ b/tests/bgp/test_startup_tsa_tsb_service.py @@ -8,11 +8,11 @@ from tests.common.utilities import wait_until from tests.common.platform.processes_utils import wait_critical_processes, _all_critical_processes_healthy from tests.common.platform.interface_utils import check_interface_status_of_up_ports -from traffic_checker import get_traffic_shift_state, check_tsa_persistence_support -from route_checker import parse_routes_on_neighbors, check_and_log_routes_diff, \ +from tests.bgp.bgp_helpers import initial_tsa_check_before_and_after_test +from tests.bgp.traffic_checker import get_traffic_shift_state, check_tsa_persistence_support +from tests.bgp.route_checker import parse_routes_on_neighbors, check_and_log_routes_diff, \ verify_current_routes_announced_to_neighs, verify_only_loopback_routes_are_announced_to_neighs from tests.bgp.constants import TS_NORMAL, TS_MAINTENANCE -from tests.bgp.bgp_helpers import get_tsa_chassisdb_config, get_sup_cfggen_tsa_value, verify_dut_configdb_tsa_value pytestmark = [ pytest.mark.topology('t2') @@ -145,30 +145,6 @@ def nbrhosts_to_dut(duthost, nbrhosts): return dut_nbrhosts -def initial_check_before_and_after_test(duthosts): - """ - @summary: Common method to make sure the supervisor and line cards are in normal state before and after the test - """ - for duthost in duthosts: - if duthost.is_supervisor_node(): - # Initially make sure both supervisor and line cards are in BGP operational normal state - if get_tsa_chassisdb_config(duthost) != 'false' or get_sup_cfggen_tsa_value(duthost) != 'false': - duthost.shell('TSB') - duthost.shell('sudo config save -y') - pytest_assert('false' == get_tsa_chassisdb_config(duthost), - "Supervisor {} tsa_enabled config is enabled".format(duthost.hostname)) - - for linecard in duthosts.frontend_nodes: - # Issue TSB on the line card before proceeding further - if verify_dut_configdb_tsa_value(linecard) != 'false' or get_tsa_chassisdb_config(linecard) != 'false' or \ - get_traffic_shift_state(linecard, cmd='TSC no-stats') != TS_NORMAL: - linecard.shell('TSB') - linecard.shell('sudo config save -y') - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), - "DUT is not in normal state") - - def check_ssh_state(localhost, dut_ip, expected_state, timeout=60): """ Check the SSH state of DUT. @@ -208,7 +184,7 @@ def test_tsa_tsb_service_with_dut_cold_reboot(duthosts, localhost, enum_rand_one pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) try: # Get all routes on neighbors before doing reboot @@ -272,7 +248,7 @@ def test_tsa_tsb_service_with_dut_cold_reboot(duthosts, localhost, enum_rand_one finally: # Bring back the supervisor and line cards to the BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) # Verify DUT is in normal state after cold reboot scenario. if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): logging.info("DUT is not in normal state after cold reboot, doing config-reload") @@ -303,7 +279,7 @@ def test_tsa_tsb_service_with_dut_abnormal_reboot(duthosts, localhost, enum_rand pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) try: # Get all routes on neighbors before doing reboot @@ -383,7 +359,7 @@ def test_tsa_tsb_service_with_dut_abnormal_reboot(duthosts, localhost, enum_rand finally: # Bring back the supervisor and line cards to the BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) # Verify DUT is in normal state after abnormal reboot scenario. if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): logging.info("DUT is not in normal state after abnormal reboot, doing config-reload") @@ -418,7 +394,7 @@ def test_tsa_tsb_service_with_supervisor_cold_reboot(duthosts, localhost, enum_s if not check_tsa_persistence_support(linecard): pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) try: for linecard in duthosts.frontend_nodes: @@ -492,7 +468,7 @@ def test_tsa_tsb_service_with_supervisor_cold_reboot(duthosts, localhost, enum_s finally: # Bring back the supervisor and line cards to the BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) # Make sure DUT is in normal state after supervisor cold reboot for linecard in duthosts.frontend_nodes: @@ -540,7 +516,7 @@ def test_tsa_tsb_service_with_supervisor_abnormal_reboot(duthosts, localhost, en pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) try: for linecard in duthosts.frontend_nodes: @@ -635,7 +611,7 @@ def test_tsa_tsb_service_with_supervisor_abnormal_reboot(duthosts, localhost, en finally: # Bring back the supervisor and line cards to the BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) # Make sure DUT is in normal state after supervisor abnormal reboot for linecard in duthosts.frontend_nodes: @@ -677,7 +653,7 @@ def test_tsa_tsb_service_with_user_init_tsa(duthosts, localhost, enum_rand_one_p pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) try: # Get all routes on neighbors before doing reboot @@ -753,7 +729,7 @@ def test_tsa_tsb_service_with_user_init_tsa(duthosts, localhost, enum_rand_one_p pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) # Bring back the supervisor and line cards to the BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) @pytest.mark.disable_loganalyzer @@ -777,7 +753,7 @@ def test_user_init_tsa_while_service_run_on_dut(duthosts, localhost, enum_rand_o pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) try: # Get all routes on neighbors before doing reboot @@ -861,7 +837,7 @@ def test_user_init_tsa_while_service_run_on_dut(duthosts, localhost, enum_rand_o pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) # Bring back the supervisor and line cards to the BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) @pytest.mark.disable_loganalyzer @@ -884,7 +860,7 @@ def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_o pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) try: # Get all routes on neighbors before doing reboot @@ -948,7 +924,7 @@ def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_o finally: # Bring back the supervisor and line cards to the BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) # Verify DUT is in normal state after cold reboot scenario. if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): logging.info("DUT is not in normal state after cold reboot, doing config-reload") @@ -988,7 +964,7 @@ def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) try: for linecard in duthosts.frontend_nodes: @@ -1069,7 +1045,7 @@ def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, finally: # Bring back the supervisor and line cards to the BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) for linecard in duthosts.frontend_nodes: # Make sure linecards are in Normal state and save the config to proceed further @@ -1107,7 +1083,7 @@ def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_f pytest.skip("TSA persistence not supported in the image") # Initially make sure both supervisor and line cards are in BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) try: # Get all routes on neighbors before doing reboot @@ -1185,7 +1161,7 @@ def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_f finally: # Bring back the supervisor and line cards to the BGP operational normal state - initial_check_before_and_after_test(duthosts) + initial_tsa_check_before_and_after_test(duthosts) # Verify DUT is in normal state after cold reboot scenario. if not (int_status_result and crit_process_check and TS_NORMAL == get_traffic_shift_state(duthost)): logging.info("DUT is not in normal state after cold reboot, doing config-reload") diff --git a/tests/bgp/test_traffic_shift.py b/tests/bgp/test_traffic_shift.py index e7f8bf6206..f13dc32381 100644 --- a/tests/bgp/test_traffic_shift.py +++ b/tests/bgp/test_traffic_shift.py @@ -2,15 +2,17 @@ import re import pytest from tests.common.devices.eos import EosHost -from bgp_helpers import get_routes_not_announced_to_bgpmon, remove_bgp_neighbors, restore_bgp_neighbors +from tests.bgp.bgp_helpers import get_routes_not_announced_to_bgpmon, remove_bgp_neighbors, restore_bgp_neighbors, \ + initial_tsa_check_before_and_after_test from tests.common import config_reload from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.constants import DEFAULT_ASIC_ID from tests.common.platform.processes_utils import wait_critical_processes from tests.common.utilities import wait_until -from route_checker import verify_only_loopback_routes_are_announced_to_neighs, parse_routes_on_neighbors, \ +from tests.bgp.route_checker import verify_only_loopback_routes_are_announced_to_neighs, parse_routes_on_neighbors, \ verify_current_routes_announced_to_neighs, check_and_log_routes_diff -from traffic_checker import get_traffic_shift_state, check_tsa_persistence_support, verify_traffic_shift_per_asic +from tests.bgp.traffic_checker import get_traffic_shift_state, check_tsa_persistence_support, \ + verify_traffic_shift_per_asic from tests.bgp.constants import TS_NORMAL, TS_MAINTENANCE, TS_NO_NEIGHBORS pytestmark = [ @@ -68,6 +70,9 @@ def test_TSA(duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, Verify all routes are announced to bgp monitor, and only loopback routes are announced to neighs """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + # Initially make sure both supervisor and line cards are in BGP operational normal state + if tbinfo['topo']['type'] == 't2': + initial_tsa_check_before_and_after_test(duthosts) try: # Issue TSA on DUT duthost.shell("TSA") @@ -89,6 +94,9 @@ def test_TSA(duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, finally: # Recover to Normal state duthost.shell("TSB") + # Bring back the supervisor and line cards to the BGP operational normal state + if tbinfo['topo']['type'] == 't2': + initial_tsa_check_before_and_after_test(duthosts) def test_TSB(duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, nbrhosts, bgpmon_setup_teardown, tbinfo): @@ -98,6 +106,9 @@ def test_TSB(duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, nbrho and all routes are announced to neighbors """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + # Initially make sure both supervisor and line cards are in BGP operational normal state + if tbinfo['topo']['type'] == 't2': + initial_tsa_check_before_and_after_test(duthosts) # Ensure that the DUT is not in maintenance already before start of the test pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") @@ -131,9 +142,13 @@ def test_TSB(duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, nbrho if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): pytest.fail("Not all ipv6 routes are announced to neighbors") + # Bring back the supervisor and line cards to the BGP operational normal state + if tbinfo['topo']['type'] == 't2': + initial_tsa_check_before_and_after_test(duthosts) + def test_TSA_B_C_with_no_neighbors(duthosts, enum_rand_one_per_hwsku_frontend_hostname, - bgpmon_setup_teardown, nbrhosts, core_dump_and_config_check): + bgpmon_setup_teardown, nbrhosts, core_dump_and_config_check, tbinfo): """ Test TSA, TSB, TSC with no neighbors on ASIC0 in case of multi-asic and single-asic. """ @@ -141,6 +156,9 @@ def test_TSA_B_C_with_no_neighbors(duthosts, enum_rand_one_per_hwsku_frontend_ho bgp_neighbors = {} duts_data = core_dump_and_config_check asic_index = 0 if duthost.is_multi_asic else DEFAULT_ASIC_ID + # Initially make sure both supervisor and line cards are in BGP operational normal state + if tbinfo['topo']['type'] == 't2': + initial_tsa_check_before_and_after_test(duthosts) # Ensure that the DUT is not in maintenance already before start of the test pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") @@ -193,6 +211,10 @@ def test_TSA_B_C_with_no_neighbors(duthosts, enum_rand_one_per_hwsku_frontend_ho if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): pytest.fail("Not all ipv6 routes are announced to neighbors") + # Bring back the supervisor and line cards to the BGP operational normal state + if tbinfo['topo']['type'] == 't2': + initial_tsa_check_before_and_after_test(duthosts) + @pytest.mark.disable_loganalyzer def test_TSA_TSB_with_config_reload(duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, nbrhosts, @@ -202,6 +224,9 @@ def test_TSA_TSB_with_config_reload(duthosts, enum_rand_one_per_hwsku_frontend_h Verify all routes are announced to bgp monitor, and only loopback routes are announced to neighs """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + # Initially make sure both supervisor and line cards are in BGP operational normal state + if tbinfo['topo']['type'] == 't2': + initial_tsa_check_before_and_after_test(duthosts) # Ensure that the DUT is not in maintenance already before start of the test pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") @@ -255,6 +280,9 @@ def test_TSA_TSB_with_config_reload(duthosts, enum_rand_one_per_hwsku_frontend_h duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): pytest.fail("Not all ipv6 routes are announced to neighbors") + # Bring back the supervisor and line cards to the BGP operational normal state + if tbinfo['topo']['type'] == 't2': + initial_tsa_check_before_and_after_test(duthosts) @pytest.mark.disable_loganalyzer @@ -266,6 +294,9 @@ def test_load_minigraph_with_traffic_shift_away(duthosts, enum_rand_one_per_hwsk Verify all routes are announced to bgp monitor, and only loopback routes are announced to neighs """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + # Initially make sure both supervisor and line cards are in BGP operational normal state + if tbinfo['topo']['type'] == 't2': + initial_tsa_check_before_and_after_test(duthosts) # Ensure that the DUT is not in maintenance already before start of the test pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") @@ -316,3 +347,7 @@ def test_load_minigraph_with_traffic_shift_away(duthosts, enum_rand_one_per_hwsk duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): pytest.fail("Not all ipv6 routes are announced to neighbors") + + # Bring back the supervisor and line cards to the BGP operational normal state + if tbinfo['topo']['type'] == 't2': + initial_tsa_check_before_and_after_test(duthosts) diff --git a/tests/bgp/test_traffic_shift_sup.py b/tests/bgp/test_traffic_shift_sup.py index 6a36c85d61..f512867686 100644 --- a/tests/bgp/test_traffic_shift_sup.py +++ b/tests/bgp/test_traffic_shift_sup.py @@ -4,7 +4,8 @@ import time from tests.common.helpers.assertions import pytest_assert from tests.common import config_reload -from traffic_checker import get_traffic_shift_state +from tests.bgp.traffic_checker import get_traffic_shift_state +from tests.bgp.bgp_helpers import initial_tsa_check_before_and_after_test pytestmark = [ pytest.mark.topology('t2') @@ -77,10 +78,12 @@ def test_TSA(self, duthosts, enum_supervisor_dut_hostname, check_support, creds) Test TSA Verify all linecards transition to maintenance state after TSA on supervisor """ + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_tsa_check_before_and_after_test(duthosts) self.setup_dutinfo(duthosts, enum_supervisor_dut_hostname, creds) try: # Issue TSA on DUT - self.run_cmd_on_sup("sudo TSA") + self.duthost.shell('TSA') # Verify DUT is in maintenance state. self.verify_traffic_shift_state_all_lcs(TS_MAINTENANCE, "maintenance") except Exception as e: @@ -88,23 +91,29 @@ def test_TSA(self, duthosts, enum_supervisor_dut_hostname, check_support, creds) logger.error("Exception caught in TSB test. Error message: {}".format(e)) finally: # Issue TSB on DUT to recover the chassis - self.run_cmd_on_sup("sudo TSB") + self.duthost.shell('TSB') + # Bring back the supervisor and line cards to the BGP operational normal state + initial_tsa_check_before_and_after_test(duthosts) def test_TSB(self, duthosts, enum_supervisor_dut_hostname, check_support, creds): """ Test TSB Verify all linecards transition back to normal state from maintenance after TSB on supervisor """ + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_tsa_check_before_and_after_test(duthosts) self.setup_dutinfo(duthosts, enum_supervisor_dut_hostname, creds) try: # Issue TSA on DUT to move chassis to maintenance - self.run_cmd_on_sup("sudo TSA") + self.duthost.shell('TSA') self.verify_traffic_shift_state_all_lcs(TS_MAINTENANCE, "maintenance") # Recover to Normal state - self.run_cmd_on_sup("sudo TSB") + self.duthost.shell('TSB') # Verify DUT is in normal state self.verify_traffic_shift_state_all_lcs(TS_NORMAL, "normal") + # Bring back the supervisor and line cards to the BGP operational normal state + initial_tsa_check_before_and_after_test(duthosts) except Exception as e: # Log exception logger.error("Exception caught in TSB test. Error message: {}".format(e)) @@ -116,27 +125,31 @@ def test_TSA_TSB_chassis_with_config_reload(self, duthosts, enum_supervisor_dut_ Verify all linecards remain in Maintenance state after TSA and config reload on supervisor Verify all linecards remain in Normal state after TSB and config reload on supervisor """ + # Initially make sure both supervisor and line cards are in BGP operational normal state + initial_tsa_check_before_and_after_test(duthosts) self.setup_dutinfo(duthosts, enum_supervisor_dut_hostname, creds) try: # Issue TSA on DUT to move chassis to maintenance - self.run_cmd_on_sup("sudo TSA") + self.duthost.shell('TSA') self.verify_traffic_shift_state_all_lcs(TS_MAINTENANCE, "maintenance") # Save config and perform config reload on all LCs - self.run_cmd_on_sup("rexec all -c 'sudo config save -y'") + self.duthost.shell('sudo config save -y') self.config_reload_all_lcs() # Verify DUT is still in maintenance state. self.verify_traffic_shift_state_all_lcs(TS_MAINTENANCE, "maintenance") finally: # Recover to Normal state - self.run_cmd_on_sup("sudo TSB") + self.duthost.shell('TSB') # Verify DUT is in normal state. self.verify_traffic_shift_state_all_lcs(TS_NORMAL, "normal") # Save config and perform config reload on all LCs - self.run_cmd_on_sup("rexec all -c 'sudo config save -y'") + self.duthost.shell('sudo config save -y') self.config_reload_all_lcs() # Verify DUT is in normal state. self.verify_traffic_shift_state_all_lcs(TS_NORMAL, "normal") + # Bring back the supervisor and line cards to the BGP operational normal state + initial_tsa_check_before_and_after_test(duthosts)