[crm]: Implement test for CRM (#473)

* [crm]: Implement test for CRM

Signed-off-by: Volodymyr Samotiy <volodymyrs@mellanox.com>

* [crm]: Implement test for CRM

* Add support of t0 and t1-lag topologies
* Add option to run commands with sudo privileges
* Fix problem with fetching ACL table ID

Signed-off-by: Volodymyr Samotiy <volodymyrs@mellanox.com>

* [crm]: Implement test for CRM

* Update FDB entry test case

Signed-off-by: Volodymyr Samotiy <volodymyrs@mellanox.com>

ansible/README.test.md

@@ -165,3 +165,7 @@ ansible-playbook test_sonic.yml -i inventory --limit {DUT_NAME} -e "testbed_name
 ```
 - Requires switch connected to a t0 testbed
 - Requires switch connected to fanout switch and fanout switch need support [QinQ](https://en.wikipedia.org/wiki/IEEE_802.1ad).
+
+### CRM test
+```
+ansible-playbook test_sonic.yml -i inventory --limit {DUT_NAME} --become --tags crm

ansible/roles/test/tasks/crm.yml

@@ -0,0 +1,57 @@
+- block:
+
+   - fail: msg="Information about tested missing"
+     when: (testbed_type is not defined)
+
+   - fail: msg="Invalid testbed_type value '{{testbed_type}}'"
+     when: testbed_type not in ['t1', 't1-lag', 't0']
+
+   - set_fact: crm_intf="{{minigraph_interfaces[0].attachto}}"
+     when: testbed_type == "t1"
+
+   - set_fact: crm_intf="{{minigraph_portchannel_interfaces[0].attachto}}"
+     when: (testbed_type == "t0") or (testbed_type == "t1-lag")
+
+   - name: Set polling interval
+     command: crm config polling interval 1
+
+   - name: Make sure CRM counters updated
+     pause: seconds=300
+
+   - name: Run test case "CRM IPv4 route resource"
+     include: roles/test/tasks/crm/crm_test_ipv4_route.yml
+
+   - name: Run test case "CRM IPv6 route resource"
+     include: roles/test/tasks/crm/crm_test_ipv6_route.yml
+
+   - name: Run test case "CRM IPv4 nexthop resource"
+     include: roles/test/tasks/crm/crm_test_ipv4_nexthop.yml
+
+   - name: Run test case "CRM IPv6 nexthop resource"
+     include: roles/test/tasks/crm/crm_test_ipv6_nexthop.yml
+
+   - name: Run test case "CRM IPv4 neighbor resource"
+     include: roles/test/tasks/crm/crm_test_ipv4_neighbor.yml
+
+   - name: Run test case "CRM IPv6 neighbor resource"
+     include: roles/test/tasks/crm/crm_test_ipv6_neighbor.yml
+
+   - name: Run test case "CRM nexthop group resource"
+     include: roles/test/tasks/crm/crm_test_nexthop_group.yml
+
+   - name: Run test case "CRM nexthop group member resource"
+     include: roles/test/tasks/crm/crm_test_nexthop_group_member.yml
+
+   - name: Run test case "CRM ACL entry resources"
+     include: roles/test/tasks/crm/crm_test_acl_entry.yml
+
+   - name: Run test case "CRM ACL counter resources"
+     include: roles/test/tasks/crm/crm_test_acl_counter.yml
+
+   - name: Run test case "CRM FDB entry resource"
+     include: roles/test/tasks/crm/crm_test_fdb_entry.yml
+
+  always:
+
+    - name: Restore polling interval
+      command: crm config polling interval 300

ansible/roles/test/tasks/crm/acl.json

@@ -0,0 +1,30 @@
+{
+    "acl": {
+        "acl-sets": {
+            "acl-set": {
+                "dataacl": {
+                    "acl-entries": {
+                        "acl-entry": {
+                            "1": {
+                                "actions": {
+                                    "config": {
+                                        "forwarding-action": "ACCEPT"
+                                    }
+                                },
+                                "config": {
+                                    "sequence-id": 1
+                                },
+                                "l2": {
+                                    "config": {
+                                        "ethertype": "2048"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+

ansible/roles/test/tasks/crm/crm_test_acl_counter.yml

@@ -0,0 +1,79 @@
+- block:
+
+    - set_fact: crm_stats_acl_counter_used=0
+    - set_fact: crm_stats_acl_counter_available=0
+
+    - name: Copy ACL JSON config to switch.
+      copy: src=roles/test/tasks/crm/acl.json dest=/tmp
+
+    - name: Add ACL
+      command: acl-loader update full /tmp/acl.json
+      become: yes
+
+    - name: Get ACL entry keys
+      command: bash -c "docker exec -i database redis-cli --raw -n 1 KEYS *SAI_OBJECT_TYPE_ACL_ENTRY*"
+      register: out
+    - set_fact: acl_tbl_keys={{out.stdout.split()}}
+
+    - name: Get ethertype for ACL entry in order to match ACL which was configured
+      command: bash -c "docker exec -i database redis-cli -n 1 HGET {{item}} SAI_ACL_ENTRY_ATTR_FIELD_ETHER_TYPE"
+      with_items: "{{acl_tbl_keys}}"
+      register: out
+
+    - name: Match ethertype value for ACL entry
+      set_fact: key={{item.item}}
+      with_items: "{{out.results}}"
+      when: item.stdout|search("2048")
+
+    - name: Get ACL table key
+      command: bash -c "docker exec -i database redis-cli -n 1 HGET {{key}} SAI_ACL_ENTRY_ATTR_TABLE_ID"
+      register: out
+    - set_fact: acl_tbl_key={{"CRM:ACL_TABLE_STATS:{0}".format(out.stdout|replace("oid:", ""))}}
+
+    - name: Make sure CRM counters updated
+      pause: seconds=1
+
+    - name: Get new "crm_stats_acl_counter_used" counter value
+      command: docker exec -i database redis-cli -n 2 HGET {{acl_tbl_key}} crm_stats_acl_counter_used
+      register: out
+    - set_fact: new_crm_stats_acl_counter_used={{out.stdout}}
+
+    - name: Get new "crm_stats_acl_counter_available" counter value
+      command: docker exec -i database redis-cli -n 2 HGET {{acl_tbl_key}} crm_stats_acl_counter_available
+      register: out
+    - set_fact: new_crm_stats_acl_counter_available={{out.stdout}}
+
+    - name: Verify "crm_stats_acl_counter_used" counter was incremented
+      assert: {that: "{{new_crm_stats_acl_counter_used|int - crm_stats_acl_counter_used|int == 2}}"}
+
+    - set_fact: crm_stats_acl_counter_available="{{new_crm_stats_acl_counter_available|int + new_crm_stats_acl_counter_used|int}}"
+
+    - name: Verify thresholds for "ACL entry" CRM resource
+      vars:
+        crm_cli_res: "acl group counter"
+        crm_used: "{{new_crm_stats_acl_counter_used}}"
+        crm_avail: "{{new_crm_stats_acl_counter_available}}"
+      include: roles/test/tasks/crm/crm_test_threshold.yml
+
+    - name: Remove ACL
+      command: acl-loader delete
+      become: yes
+
+    - name: Make sure CRM counters updated
+      pause: seconds=1
+
+    - name: Get new "crm_stats_acl_counter_used" counter value
+      command: docker exec -i database redis-cli -n 2 HGET {{acl_tbl_key}} crm_stats_acl_counter_used
+      register: out
+    - set_fact: new_crm_stats_acl_counter_used={{out.stdout}}
+
+    - name: Get new "crm_stats_acl_counter_available" counter value
+      command: docker exec -i database redis-cli -n 2 HGET {{acl_tbl_key}} crm_stats_acl_counter_available
+      register: out
+    - set_fact: new_crm_stats_acl_counter_available={{out.stdout}}
+
+    - name: Verify "crm_stats_acl_counter_used" counter was decremented
+      assert: {that: "{{new_crm_stats_acl_counter_used|int - crm_stats_acl_counter_used|int == 0}}"}
+
+    - name: Verify "crm_stats_acl_counter_available" counter was incremented
+      assert: {that: "{{new_crm_stats_acl_counter_available|int - crm_stats_acl_counter_available|int == 0}}"}

ansible/roles/test/tasks/crm/crm_test_acl_entry.yml

@@ -0,0 +1,82 @@
+- block:
+
+    - set_fact: crm_stats_acl_entry_used=0
+    - set_fact: crm_stats_acl_entry_available=0
+
+    - name: Copy ACL JSON config to switch.
+      copy: src=roles/test/tasks/crm/acl.json dest=/tmp
+
+    - name: Add ACL
+      command: acl-loader update full /tmp/acl.json
+      become: yes
+
+    - name: Make sure CRM counters updated
+      pause: seconds=1
+
+    - name: Get ACL entry keys
+      command: bash -c "docker exec -i database redis-cli --raw -n 1 KEYS *SAI_OBJECT_TYPE_ACL_ENTRY*"
+      register: out
+    - set_fact: acl_tbl_keys={{out.stdout.split()}}
+
+    - name: Get ethertype for ACL entry in order to match ACL which was configured
+      command: bash -c "docker exec -i database redis-cli -n 1 HGET {{item}} SAI_ACL_ENTRY_ATTR_FIELD_ETHER_TYPE"
+      with_items: "{{acl_tbl_keys}}"
+      register: out
+
+    - name: Match ethertype value for ACL entry
+      set_fact: key={{item.item}}
+      with_items: "{{out.results}}"
+      when: item.stdout|search("2048")
+
+    - name: Get ACL table key
+      command: bash -c "docker exec -i database redis-cli -n 1 HGET {{key}} SAI_ACL_ENTRY_ATTR_TABLE_ID"
+      register: out
+    - set_fact: acl_tbl_key={{"CRM:ACL_TABLE_STATS:{0}".format(out.stdout|replace("oid:", ""))}}
+
+    - name: Make sure CRM counters updated
+      pause: seconds=1
+
+    - name: Get new "crm_stats_acl_entry_used" counter value
+      command: docker exec -i database redis-cli -n 2 HGET {{acl_tbl_key}} crm_stats_acl_entry_used
+      register: out
+    - set_fact: new_crm_stats_acl_entry_used={{out.stdout}}
+
+    - name: Get new "crm_stats_acl_entry_available" counter value
+      command: docker exec -i database redis-cli -n 2 HGET {{acl_tbl_key}} crm_stats_acl_entry_available
+      register: out
+    - set_fact: new_crm_stats_acl_entry_available={{out.stdout}}
+
+    - name: Verify "crm_stats_acl_entry_used" counter was incremented
+      assert: {that: "{{new_crm_stats_acl_entry_used|int - crm_stats_acl_entry_used|int == 2}}"}
+
+    - set_fact: crm_stats_acl_entry_available="{{new_crm_stats_acl_entry_available|int + new_crm_stats_acl_entry_used|int}}"
+
+    - name: Verify thresholds for "ACL entry" CRM resource
+      vars:
+        crm_cli_res: "acl group entry"
+        crm_used: "{{new_crm_stats_acl_entry_used}}"
+        crm_avail: "{{new_crm_stats_acl_entry_available}}"
+      include: roles/test/tasks/crm/crm_test_threshold.yml
+
+    - name: Remove ACL
+      command: acl-loader delete
+      become: yes
+
+    - name: Make sure CRM counters updated
+      pause: seconds=1
+
+    - name: Get new "crm_stats_acl_entry_used" counter value
+      command: docker exec -i database redis-cli -n 2 HGET {{acl_tbl_key}} crm_stats_acl_entry_used
+      register: out
+    - set_fact: new_crm_stats_acl_entry_used={{out.stdout}}
+
+    - name: Get new "crm_stats_acl_entry_available" counter value
+      command: docker exec -i database redis-cli -n 2 HGET {{acl_tbl_key}} crm_stats_acl_entry_available
+      register: out
+    - set_fact: new_crm_stats_acl_entry_available={{out.stdout}}
+
+    - name: Verify "crm_stats_acl_entry_used" counter was decremented
+      assert: {that: "{{new_crm_stats_acl_entry_used|int - crm_stats_acl_entry_used|int == 0}}"}
+
+    - name: Verify "crm_stats_acl_entry_available" counter was incremented
+      assert: {that: "{{new_crm_stats_acl_entry_available|int - crm_stats_acl_entry_available|int == 0}}"}

ansible/roles/test/tasks/crm/crm_test_fdb_entry.yml

@@ -0,0 +1,95 @@
+- block:
+
+    - name: Get "crm_stats_fdb_entry_used" counter value
+      command: docker exec -i database redis-cli -n 2 HGET CRM:STATS crm_stats_fdb_entry_used
+      register: out
+    - set_fact: crm_stats_fdb_entry_used={{out.stdout}}
+
+    - name: Get "crm_stats_fdb_entry_available" counter value
+      command: docker exec -i database redis-cli -n 2 HGET CRM:STATS crm_stats_fdb_entry_available
+      register: out
+    - set_fact: crm_stats_fdb_entry_available={{out.stdout}}
+
+    - name: Copy FDB JSON config to switch.
+      copy: src=roles/test/tasks/crm/fdb.json dest=/tmp
+
+    - name: Copy FDB JSON config to SWSS container
+      command: docker cp /tmp/fdb.json swss:/
+
+    - name: Add FDB entry
+      command: docker exec -i swss swssconfig /fdb.json
+
+    - name: Add VLAN required for FDB entry
+      command: config vlan add 2
+      become: yes
+
+    - name: Add VLAN member required for FDB entry
+      command: config vlan member add 2 Ethernet0
+      become: yes
+
+    - name: Make sure CRM counters updated
+      pause: seconds=1
+
+    - name: Get new "crm_stats_fdb_entry_used" counter value
+      command: docker exec -i database redis-cli -n 2 HGET CRM:STATS crm_stats_fdb_entry_used
+      register: out
+    - set_fact: new_crm_stats_fdb_entry_used={{out.stdout}}
+
+    - name: Get new "crm_stats_fdb_entry_available" counter value
+      command: docker exec -i database redis-cli -n 2 HGET CRM:STATS crm_stats_fdb_entry_available
+      register: out
+    - set_fact: new_crm_stats_fdb_entry_available={{out.stdout}}
+
+    - name: Verify "crm_stats_fdb_entry_used" counter was incremented
+      assert: {that: "{{new_crm_stats_fdb_entry_used|int - crm_stats_fdb_entry_used|int == 1}}"}
+
+    - name: Verify "crm_stats_fdb_entry_available" counter was decremented
+      assert: {that: "{{crm_stats_fdb_entry_available|int - new_crm_stats_fdb_entry_available|int == 1}}"}
+
+    - name: Verify thresholds for "FDB entry" CRM resource
+      vars:
+        crm_cli_res: "fdb"
+        crm_used: "{{new_crm_stats_fdb_entry_used}}"
+        crm_avail: "{{new_crm_stats_fdb_entry_available}}"
+      include: roles/test/tasks/crm/crm_test_threshold.yml
+
+    - name: Remove FDB entry
+      command: fdbclear
+
+    - name: Make sure CRM counters updated
+      pause: seconds=1
+
+    - name: Get new "crm_stats_fdb_entry_used" counter value
+      command: docker exec -i database redis-cli -n 2 HGET CRM:STATS crm_stats_fdb_entry_used
+      register: out
+    - set_fact: new_crm_stats_fdb_entry_used={{out.stdout}}
+
+    - name: Get new "crm_stats_fdb_entry_available" counter value
+      command: docker exec -i database redis-cli -n 2 HGET CRM:STATS crm_stats_fdb_entry_available
+      register: out
+    - set_fact: new_crm_stats_fdb_entry_available={{out.stdout}}
+
+    - name: Verify "crm_stats_fdb_entry_used" counter was decremented
+      assert: {that: "{{new_crm_stats_fdb_entry_used|int - crm_stats_fdb_entry_used|int == 0}}"}
+
+    - name: Verify "crm_stats_fdb_entry_available" counter was incremented
+      assert: {that: "{{new_crm_stats_fdb_entry_available|int - crm_stats_fdb_entry_available|int == 0}}"}
+
+  always:
+
+    - name: Remove VLAN member required for FDB entry
+      command: config vlan member del 2 Ethernet0
+      become: yes
+
+    - name: Remove VLAN required for FDB entry
+      command: config vlan del 2
+      become: yes
+
+    - name: Remove FDB entry
+      command: fdbclear
+
+    - name: Remove FDB JSON config from switch.
+      command: rm /tmp/fdb.json
+
+    - name: Remove FDB JSON config from SWSS container
+      command: docker exec -i swss rm /fdb.json