[Mellanox] Support DSCP remapping in dual ToR topo on T0 switch (soni…

…c-net#12605) - Why I did it Support DSCP remapping in dual ToR topo on T0 switch for SKU Mellanox-SN4600c-C64, Mellanox-SN4600c-D48C40, Mellanox-SN2700, Mellanox-SN2700-D48C8. - How I did it Regarding buffer settings, originally, there are two lossless PGs and queues 3, 4. In dual ToR scenario, the lossless traffic from the leaf switch to the uplink of the ToR switch can be bounced back. To avoid PFC deadlock, we need to map the bounce-back lossless traffic to different PGs and queues. Therefore, 2 additional lossless PGs and queues are allocated on uplink ports on ToR switches. On uplink ports, map DSCP 2/6 to TC 2/6 respectively On downlink ports, both DSCP 2/6 are still mapped to TC 1 Buffer adjusted according to the ports information: Mellanox-SN4600c-C64: 56 downlinks 50G + 8 uplinks 100G Mellanox-SN4600c-D48C40, Mellanox-SN2700, Mellanox-SN2700-D48C8: 24 downlinks 50G + 8 uplinks 100G - How to verify it Unit test. Signed-off-by: Stephen Sun <stephens@nvidia.com>
dprital · Feb 7, 2023 · e3ff088 · e3ff088
1 parent 0704ff5
commit e3ff088
Show file tree

Hide file tree

Showing 18 changed files with 16,919 additions and 344 deletions.
diff --git a/device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700-D48C8/buffers_defaults_t0.j2 b/device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700-D48C8/buffers_defaults_t0.j2
@@ -1,5 +1,5 @@
 {#
-    Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES.
+    Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES.
     Apache-2.0
 
     Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +15,17 @@
     limitations under the License.
 #}
 {% set default_cable = '5m' %}
-{% set ingress_lossless_pool_size =  '7719936' %}
-{% set ingress_lossless_pool_xoff =  '1032192' %}
+{%- if ((SYSTEM_DEFAULTS is defined) and ('tunnel_qos_remap' in SYSTEM_DEFAULTS) and (SYSTEM_DEFAULTS['tunnel_qos_remap']['status'] == 'enabled')) -%}
+{% set ingress_lossless_pool_size =  '9047040' %}
+{% set ingress_lossless_pool_xoff  =  '851968' %}
 {% set egress_lossless_pool_size =  '13945824' %}
-{% set egress_lossy_pool_size =  '7719936' %}
+{% set egress_lossy_pool_size =  '9047040' %}
+{%- else -%}
+{% set ingress_lossless_pool_size =  '9595904' %}
+{% set ingress_lossless_pool_xoff =  '614400' %}
+{% set egress_lossless_pool_size =  '13945824' %}
+{% set egress_lossy_pool_size =  '9595904' %}
+{%- endif -%}
 
 {% import 'buffers_defaults_objects.j2' as defs with context %}
 
@@ -30,10 +37,18 @@
 {{ defs.generate_profile_lists(port_names_active, port_names_inactive) }}
 {%- endmacro %}
 
+{%- macro generate_queue_buffers_with_extra_lossless_queues_with_inactive_ports(port_names_active, port_names_extra_queues, port_names_inactive) %}
+{{ defs.generate_queue_buffers_with_extra_lossless_queues(port_names_active, port_names_extra_queues, port_names_inactive) }}
+{%- endmacro %}
+
 {%- macro generate_queue_buffers_with_inactive_ports(port_names_active, port_names_inactive) %}
 {{ defs.generate_queue_buffers(port_names_active, port_names_inactive) }}
 {%- endmacro %}
 
+{%- macro generate_pg_profiles_with_extra_lossless_pgs_with_inactive_ports(port_names_active, port_names_extra_pgs, port_names_inactive) %}
+{{ defs.generate_pg_profiles_with_extra_lossless_pgs(port_names_active, port_names_extra_pgs, port_names_inactive) }}
+{%- endmacro %}
+
 {%- macro generate_pg_profiles_with_inactive_ports(port_names_active, port_names_inactive) %}
 {{ defs.generate_pg_profiles(port_names_active, port_names_inactive) }}
 {%- endmacro %}
diff --git a/device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700-D48C8/qos.json.j2 b/device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700-D48C8/qos.json.j2
diff --git a/device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700-D48C8/qos.json.j2 b/device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700-D48C8/qos.json.j2
@@ -0,0 +1 @@
+../../x86_64-mlnx_msn4600c-r0/Mellanox-SN4600C-C64/qos.json.j2
diff --git a/device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700/buffers_defaults_t0.j2 b/device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700/buffers_defaults_t0.j2
@@ -1,5 +1,5 @@
 {#
-    Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES.
+    Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES.
     Apache-2.0
 
     Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +15,17 @@
     limitations under the License.
 #}
 {% set default_cable = '5m' %}
-{% set ingress_lossless_pool_size =  '10177536' %}
-{% set ingress_lossless_pool_xoff =  '688128' %}
+{%- if ((SYSTEM_DEFAULTS is defined) and ('tunnel_qos_remap' in SYSTEM_DEFAULTS) and (SYSTEM_DEFAULTS['tunnel_qos_remap']['status'] == 'enabled')) -%}
+{% set ingress_lossless_pool_size =  '9047040' %}
+{% set ingress_lossless_pool_xoff  =  '851968' %}
 {% set egress_lossless_pool_size =  '13945824' %}
-{% set egress_lossy_pool_size =  '10177536' %}
+{% set egress_lossy_pool_size =  '9047040' %}
+{%- else -%}
+{% set ingress_lossless_pool_size =  '9595904' %}
+{% set ingress_lossless_pool_xoff =  '614400' %}
+{% set egress_lossless_pool_size =  '13945824' %}
+{% set egress_lossy_pool_size =  '9595904' %}
+{%- endif -%}
 
 {% import 'buffers_defaults_objects.j2' as defs with context %}
 
@@ -30,10 +37,18 @@
 {{ defs.generate_profile_lists(port_names_active, port_names_inactive) }}
 {%- endmacro %}
 
+{%- macro generate_queue_buffers_with_extra_lossless_queues_with_inactive_ports(port_names_active, port_names_extra_queues, port_names_inactive) %}
+{{ defs.generate_queue_buffers_with_extra_lossless_queues(port_names_active, port_names_extra_queues, port_names_inactive) }}
+{%- endmacro %}
+
 {%- macro generate_queue_buffers_with_inactive_ports(port_names_active, port_names_inactive) %}
 {{ defs.generate_queue_buffers(port_names_active, port_names_inactive) }}
 {%- endmacro %}
 
+{%- macro generate_pg_profiles_with_extra_lossless_pgs_with_inactive_ports(port_names_active, port_names_extra_pgs, port_names_inactive) %}
+{{ defs.generate_pg_profiles_with_extra_lossless_pgs(port_names_active, port_names_extra_pgs, port_names_inactive) }}
+{%- endmacro %}
+
 {%- macro generate_pg_profiles_with_inactive_ports(port_names_active, port_names_inactive) %}
 {{ defs.generate_pg_profiles(port_names_active, port_names_inactive) }}
 {%- endmacro %}
diff --git a/device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700/qos.json.j2 b/device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700/qos.json.j2
@@ -1 +1 @@
-../ACS-MSN2700/qos.json.j2
+../../x86_64-mlnx_msn4600c-r0/Mellanox-SN4600C-C64/qos.json.j2
diff --git a/device/mellanox/x86_64-mlnx_msn4600c-r0/Mellanox-SN4600C-C64/buffers_defaults_t0.j2 b/device/mellanox/x86_64-mlnx_msn4600c-r0/Mellanox-SN4600C-C64/buffers_defaults_t0.j2
@@ -1,5 +1,5 @@
 {#
-    Copyright (c) 2018-2021 NVIDIA CORPORATION & AFFILIATES.
+    Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES.
     Apache-2.0
 
     Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +15,17 @@
     limitations under the License.
 #}
 {% set default_cable = '5m' %}
-{% set ingress_lossless_pool_size =  '49905664' %}
-{% set ingress_lossless_pool_xoff  =  '3702784' %}
+{%- if ((SYSTEM_DEFAULTS is defined) and ('tunnel_qos_remap' in SYSTEM_DEFAULTS) and (SYSTEM_DEFAULTS['tunnel_qos_remap']['status'] == 'enabled')) -%}
+{% set ingress_lossless_pool_size =  '50667520' %}
+{% set ingress_lossless_pool_xoff  =  '2629632' %}
 {% set egress_lossless_pool_size =  '60817392' %}
-{% set egress_lossy_pool_size =  '49905664' %}
+{% set egress_lossy_pool_size =  '50667520' %}
+{%- else -%}
+{% set ingress_lossless_pool_size =  '51511296' %}
+{% set ingress_lossless_pool_xoff  =  '2097152' %}
+{% set egress_lossless_pool_size =  '60817392' %}
+{% set egress_lossy_pool_size =  '51511296' %}
+{%- endif -%}
 
 {% import 'buffers_defaults_objects.j2' as defs with context %}
 
@@ -30,10 +37,18 @@
 {{ defs.generate_profile_lists(port_names_active, port_names_inactive) }}
 {%- endmacro %}
 
+{%- macro generate_queue_buffers_with_extra_lossless_queues_with_inactive_ports(port_names_active, port_names_extra_queues, port_names_inactive) %}
+{{ defs.generate_queue_buffers_with_extra_lossless_queues(port_names_active, port_names_extra_queues, port_names_inactive) }}
+{%- endmacro %}
+
 {%- macro generate_queue_buffers_with_inactive_ports(port_names_active, port_names_inactive) %}
 {{ defs.generate_queue_buffers(port_names_active, port_names_inactive) }}
 {%- endmacro %}
 
+{%- macro generate_pg_profiles_with_extra_lossless_pgs_with_inactive_ports(port_names_active, port_names_extra_pgs, port_names_inactive) %}
+{{ defs.generate_pg_profiles_with_extra_lossless_pgs(port_names_active, port_names_extra_pgs, port_names_inactive) }}
+{%- endmacro %}
+
 {%- macro generate_pg_profiles_with_inactive_ports(port_names_active, port_names_inactive) %}
 {{ defs.generate_pg_profiles(port_names_active, port_names_inactive) }}
 {%- endmacro %}
diff --git a/device/mellanox/x86_64-mlnx_msn4600c-r0/Mellanox-SN4600C-C64/qos.json.j2 b/device/mellanox/x86_64-mlnx_msn4600c-r0/Mellanox-SN4600C-C64/qos.json.j2
@@ -1,5 +1,5 @@
 {#
-    Copyright (c) 2020-2021 NVIDIA CORPORATION & AFFILIATES.
+    Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES.
     Apache-2.0
 
     Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,18 +14,20 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 #}
-{% if ('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'LeafRouter') %}
+
+{% if (('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'LeafRouter') or
+       ('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'ToRRouter') and
+       ('subtype' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['subtype'] == 'DualToR')) %}
 {% set different_dscp_to_tc_map = true %}
-{%- macro generate_dscp_to_tc_map() %}
-    "DSCP_TO_TC_MAP": {
-        "AZURE": {
+{%- macro  generate_normal_dscp_to_tc_map(tc_mapped_from_dscp_33) -%}
+        {
             "0" : "1",
             "1" : "1",
-            "2" : "2",
+            "2" : "1",
             "3" : "3",
             "4" : "4",
             "5" : "1",
-            "6" : "6",
+            "6" : "1",
             "7" : "1",
             "8" : "0",
             "9" : "1",
@@ -52,7 +54,7 @@
             "30": "1",
             "31": "1",
             "32": "1",
-            "33": "1",
+            "33": "{{tc_mapped_from_dscp_33}}",
             "34": "1",
             "35": "1",
             "36": "1",
@@ -83,15 +85,17 @@
             "61": "1",
             "62": "1",
             "63": "1"
-        },
-        "AZURE_UPLINK": {
+        }
+{%- endmacro -%}
+{%- macro generate_dscp_to_tc_map_with_addition_lossless_pgs_n_queues(tc_mapped_from_dscp_33) -%}
+        {
             "0" : "1",
             "1" : "1",
-            "2" : "1",
+            "2" : "2",
             "3" : "3",
             "4" : "4",
             "5" : "1",
-            "6" : "1",
+            "6" : "6",
             "7" : "1",
             "8" : "0",
             "9" : "1",
@@ -118,7 +122,7 @@
             "30": "1",
             "31": "1",
             "32": "1",
-            "33": "1",
+            "33": "{{tc_mapped_from_dscp_33}}",
             "34": "1",
             "35": "1",
             "36": "1",
@@ -150,10 +154,52 @@
             "62": "1",
             "63": "1"
         }
+{%- endmacro -%}
+{%- macro generate_dscp_to_tc_map() %}
+    "DSCP_TO_TC_MAP": {
+{% if ('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'LeafRouter') %}
+        "AZURE_UPLINK":
+        {{ generate_normal_dscp_to_tc_map("1") }},
+        "AZURE":
+        {{ generate_dscp_to_tc_map_with_addition_lossless_pgs_n_queues("1") }}
+{% endif %}
+{% if ('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'ToRRouter') %}
+        "AZURE_UPLINK":
+        {{ generate_dscp_to_tc_map_with_addition_lossless_pgs_n_queues("8") }},
+        "AZURE_TUNNEL":
+        {{ generate_normal_dscp_to_tc_map("8") }},
+        "AZURE":
+        {{ generate_normal_dscp_to_tc_map("8") }}
+{% endif %}
     },
 {%- endmacro %}
 {%- macro generate_tc_to_pg_map() %}
     "TC_TO_PRIORITY_GROUP_MAP": {
+{% if (('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'ToRRouter') and
+       ('subtype' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['subtype'] == 'DualToR')) %}
+        "AZURE_TUNNEL": {
+            "0": "0",
+            "1": "0",
+            "2": "0",
+            "3": "2",
+            "4": "6",
+            "5": "0",
+            "6": "0",
+            "7": "0",
+            "8": "0"
+        },
+        "AZURE": {
+            "0": "0",
+            "1": "0",
+            "2": "2",
+            "3": "3",
+            "4": "4",
+            "5": "0",
+            "6": "6",
+            "7": "0",
+            "8": "0"
+        }
+{% else %}
         "AZURE": {
             "0": "0",
             "1": "0",
@@ -164,8 +210,67 @@
             "6": "6",
             "7": "0"
         }
+{% endif %}
+    },
+{%- endmacro %}
+{%- macro generate_tc_to_queue_map() %}
+    "TC_TO_QUEUE_MAP": {
+{% if (('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'ToRRouter') and
+       ('subtype' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['subtype'] == 'DualToR')) %}
+        "AZURE_TUNNEL": {
+            "0": "0",
+            "1": "1",
+            "2": "2",
+            "3": "2",
+            "4": "6",
+            "5": "5",
+            "6": "6",
+            "7": "7",
+            "8": "1"
+        },
+        "AZURE": {
+            "0": "0",
+            "1": "1",
+            "2": "2",
+            "3": "3",
+            "4": "4",
+            "5": "5",
+            "6": "6",
+            "7": "7",
+            "8": "1"
+        }
+{% else %}
+        "AZURE": {
+            "0": "0",
+            "1": "1",
+            "2": "2",
+            "3": "3",
+            "4": "4",
+            "5": "5",
+            "6": "6",
+            "7": "7"
+        }
+{% endif %}
+    },
+{%- endmacro %}
+{% if (('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'ToRRouter') and
+       ('subtype' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['subtype'] == 'DualToR')) %}
+{%- macro generate_tc_to_dscp_map() %}
+    "TC_TO_DSCP_MAP": {
+        "AZURE_TUNNEL": {
+            "0": "8",
+            "1": "0",
+            "2": "2",
+            "3": "2",
+            "4": "6",
+            "5": "46",
+            "6": "6",
+            "7": "48",
+            "8": "33"
+        }
     },
 {%- endmacro %}
+{%- endif %}
 {%- macro generate_global_dscp_to_tc_map() %}
 {# This is an empty macro since the global DSCP_TO_TC map is not required #}
 {%- endmacro %}