[TACACS] Improve TACACS UT test_accounting_tacacs_only_some_tacacs_se…

…rver_down (#10973)

## Description of PR
[TACACS] Improve TACACS UT test_accounting_tacacs_only_some_tacacs_server_down

Summary:
Fixes test_accounting_tacacs_only_some_tacacs_server_down randomly failed issue.

### Type of change

- [ ] Bug fix
- [ ] Testbed and Framework(new/improvement)
- [x] Test case(new/improvement)

## Approach
#### What is the motivation for this PR?
test_accounting_tacacs_only_some_tacacs_server_down randomly failed because TACACS config reload syslog not found.
Which is because multiple TACACS config command in short time will only trigger one TACAACS config reload, and UT code does not handle this case.

#### How did you do it?
Improve UT code to handle multiple TACACS config command case.

#### How did you verify/test it?
Pass all UT

tests/tacacs/test_accounting.py

@@ -10,7 +10,8 @@
         remove_all_tacacs_server
 from .utils import stop_tacacs_server, start_tacacs_server, \
         check_server_received, per_command_accounting_skip_versions, \
-        change_and_wait_aaa_config_update, ensure_tacacs_server_running_after_ut  # noqa: F401
+        change_and_wait_aaa_config_update, get_auditd_config_reload_timestamp, \
+        ensure_tacacs_server_running_after_ut  # noqa: F401
 from tests.common.errors import RunAnsibleModuleFail
 from tests.common.helpers.assertions import pytest_assert
 from tests.common.utilities import skip_release
@@ -219,11 +220,19 @@ def test_accounting_tacacs_only_some_tacacs_server_down(
     invalid_tacacs_server_ip = "127.0.0.1"
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
     tacacs_server_ip = ptfhost.mgmt_ip
+
+    # when tacacs config change multiple time in short time
+    # auditd service may been request reload during reloading
+    # when this happen, auditd will ignore request and only reload once
+    last_timestamp = get_auditd_config_reload_timestamp(duthost)
+
     duthost.shell("sudo config tacacs timeout 1")
     remove_all_tacacs_server(duthost)
     duthost.shell("sudo config tacacs add %s" % invalid_tacacs_server_ip)
     duthost.shell("sudo config tacacs add %s" % tacacs_server_ip)
-    change_and_wait_aaa_config_update(duthost, "sudo config aaa accounting tacacs+")
+    change_and_wait_aaa_config_update(duthost,
+                                      "sudo config aaa accounting tacacs+",
+                                      last_timestamp)
 
     cleanup_tacacs_log(ptfhost, rw_user_client)
 

tests/tacacs/utils.py

@@ -72,6 +72,12 @@ def setup_local_user(duthost, tacacs_creds):
 def setup_tacacs_client(duthost, tacacs_creds, tacacs_server_ip):
     """setup tacacs client"""
 
+    # UT should failed when set reachable TACACS server with this setup_tacacs_client
+    ping_result = duthost.shell("ping {} -c 1 -W 3".format(tacacs_server_ip))['stdout']
+    logger.info("TACACS server ping result: {}".format(ping_result))
+    if "100% packet loss" in ping_result:
+        pytest_assert(False, "TACACS server not reachable: {}".format(ping_result))
+
     # configure tacacs client
     default_tacacs_servers = []
     duthost.shell("sudo config tacacs passkey %s" % tacacs_creds[duthost.hostname]['tacacs_passkey'])
@@ -318,8 +324,10 @@ def get_auditd_config_reload_timestamp(duthost):
     return res["stdout_lines"][-1]
 
 
-def change_and_wait_aaa_config_update(duthost, command, timeout=10):
-    last_timestamp = get_auditd_config_reload_timestamp(duthost)
+def change_and_wait_aaa_config_update(duthost, command, last_timestamp=None, timeout=10):
+    if not last_timestamp:
+        last_timestamp = get_auditd_config_reload_timestamp(duthost)
+
     duthost.shell(command)
 
     # After AAA config update, hostcfgd will modify config file and notify auditd reload config