Skip to content

Commit

Permalink
Add failure notification for orchagent (#1665)
Browse files Browse the repository at this point in the history 
What I did
Add SAI failure handling functions in aclorch, bufferorch, copporch, dtelorch, fdborch, fgnhgorch, intfsorch, mirrororch, natorch, policerorch, macsecorch, portsorch, qosorch, sfloworch, switchorch, tunneldecaporch, vrforch.

Why I did it
Failure notification mechanism to ensure enough notifications in the presence of SAI failures and avoid running switches with unhandled failures.
  • Loading branch information
@shi-su
shi-su authored and Shi Su committed Aug 17, 2021
1 parent fc8e43f commit db9238f
Show file tree
Hide file tree
Showing 20 changed files with 1,218 additions and 224 deletions.
22 changes: 17 additions & 5 deletions orchagent/aclorch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3626,7 +3626,10 @@ sai_status_t AclOrch::createDTelWatchListTables()
if (status != SAI_STATUS_SUCCESS)
{
SWSS_LOG_ERROR("Failed to create table %s", flowWLTable.description.c_str());
return status;
if (handleSaiCreateStatus(SAI_API_ACL, status) != task_success)
{
return status;
}
}

gCrmOrch->incCrmAclUsedCounter(CrmResourceType::CRM_ACL_TABLE, SAI_ACL_STAGE_INGRESS, SAI_ACL_BIND_POINT_TYPE_SWITCH);
Expand Down Expand Up @@ -3687,14 +3690,17 @@ sai_status_t AclOrch::createDTelWatchListTables()
if (status != SAI_STATUS_SUCCESS)
{
SWSS_LOG_ERROR("Failed to create table %s", dropWLTable.description.c_str());
return status;
if (handleSaiCreateStatus(SAI_API_ACL, status) != task_success)
{
return status;
}
}

gCrmOrch->incCrmAclUsedCounter(CrmResourceType::CRM_ACL_TABLE, SAI_ACL_STAGE_INGRESS, SAI_ACL_BIND_POINT_TYPE_SWITCH);
m_AclTables[table_oid] = dropWLTable;
SWSS_LOG_INFO("Successfully created ACL table %s, oid: %" PRIx64, dropWLTable.description.c_str(), table_oid);

return status;
return SAI_STATUS_SUCCESS;
}

sai_status_t AclOrch::deleteDTelWatchListTables()
Expand All @@ -3719,7 +3725,10 @@ sai_status_t AclOrch::deleteDTelWatchListTables()
if (status != SAI_STATUS_SUCCESS)
{
SWSS_LOG_ERROR("Failed to delete table %s", table_id.c_str());
return status;
if (handleSaiRemoveStatus(SAI_API_ACL, status) != task_success)
{
return status;
}
}

gCrmOrch->decCrmAclUsedCounter(CrmResourceType::CRM_ACL_TABLE, SAI_ACL_STAGE_INGRESS, SAI_ACL_BIND_POINT_TYPE_SWITCH, table_oid);
Expand All @@ -3739,7 +3748,10 @@ sai_status_t AclOrch::deleteDTelWatchListTables()
if (status != SAI_STATUS_SUCCESS)
{
SWSS_LOG_ERROR("Failed to delete table %s", table_id.c_str());
return status;
if (handleSaiRemoveStatus(SAI_API_ACL, status) != task_success)
{
return status;
}
}

gCrmOrch->decCrmAclUsedCounter(CrmResourceType::CRM_ACL_TABLE, SAI_ACL_STAGE_INGRESS, SAI_ACL_BIND_POINT_TYPE_SWITCH, table_oid);
Expand Down
60 changes: 50 additions & 10 deletions orchagent/bufferorch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,11 @@ task_process_status BufferOrch::processBufferPool(KeyOpFieldsValuesTuple &tuple)
else if (SAI_STATUS_SUCCESS != sai_status)
{
SWSS_LOG_ERROR("Failed to modify buffer pool, name:%s, sai object:%" PRIx64 ", status:%d", object_name.c_str(), sai_object, sai_status);
return task_process_status::task_failed;
task_process_status handle_status = handleSaiSetStatus(SAI_API_BUFFER, sai_status);
if (handle_status != task_process_status::task_success)
{
return handle_status;
}
}
}
SWSS_LOG_DEBUG("Modified existing pool:%" PRIx64 ", type:%s name:%s ", sai_object, map_type_name.c_str(), object_name.c_str());
Expand All @@ -418,7 +422,11 @@ task_process_status BufferOrch::processBufferPool(KeyOpFieldsValuesTuple &tuple)
if (SAI_STATUS_SUCCESS != sai_status)
{
SWSS_LOG_ERROR("Failed to create buffer pool %s with type %s, rv:%d", object_name.c_str(), map_type_name.c_str(), sai_status);
return task_process_status::task_failed;
task_process_status handle_status = handleSaiCreateStatus(SAI_API_BUFFER, sai_status);
if (handle_status != task_process_status::task_success)
{
return handle_status;
}
}
(*(m_buffer_type_maps[map_type_name]))[object_name].m_saiObjectId = sai_object;
SWSS_LOG_NOTICE("Created buffer pool %s with type %s", object_name.c_str(), map_type_name.c_str());
Expand All @@ -444,7 +452,11 @@ task_process_status BufferOrch::processBufferPool(KeyOpFieldsValuesTuple &tuple)
if (SAI_STATUS_SUCCESS != sai_status)
{
SWSS_LOG_ERROR("Failed to remove buffer pool %s with type %s, rv:%d", object_name.c_str(), map_type_name.c_str(), sai_status);
return task_process_status::task_failed;
task_process_status handle_status = handleSaiRemoveStatus(SAI_API_BUFFER, sai_status);
if (handle_status != task_process_status::task_success)
{
return handle_status;
}
}
SWSS_LOG_NOTICE("Removed buffer pool %s with type %s", object_name.c_str(), map_type_name.c_str());
auto it_to_delete = (m_buffer_type_maps[map_type_name])->find(object_name);
Expand Down Expand Up @@ -598,7 +610,11 @@ task_process_status BufferOrch::processBufferProfile(KeyOpFieldsValuesTuple &tup
else if (SAI_STATUS_SUCCESS != sai_status)
{
SWSS_LOG_ERROR("Failed to modify buffer profile, name:%s, sai object:%" PRIx64 ", status:%d", object_name.c_str(), sai_object, sai_status);
return task_process_status::task_failed;
task_process_status handle_status = handleSaiSetStatus(SAI_API_BUFFER, sai_status);
if (handle_status != task_process_status::task_success)
{
return handle_status;
}
}
}
}
Expand All @@ -608,7 +624,11 @@ task_process_status BufferOrch::processBufferProfile(KeyOpFieldsValuesTuple &tup
if (SAI_STATUS_SUCCESS != sai_status)
{
SWSS_LOG_ERROR("Failed to create buffer profile %s with type %s, rv:%d", object_name.c_str(), map_type_name.c_str(), sai_status);
return task_process_status::task_failed;
task_process_status handle_status = handleSaiCreateStatus(SAI_API_BUFFER, sai_status);
if (handle_status != task_process_status::task_success)
{
return handle_status;
}
}
(*(m_buffer_type_maps[map_type_name]))[object_name].m_saiObjectId = sai_object;
SWSS_LOG_NOTICE("Created buffer profile %s with type %s", object_name.c_str(), map_type_name.c_str());
Expand All @@ -631,7 +651,11 @@ task_process_status BufferOrch::processBufferProfile(KeyOpFieldsValuesTuple &tup
if (SAI_STATUS_SUCCESS != sai_status)
{
SWSS_LOG_ERROR("Failed to remove buffer profile %s with type %s, rv:%d", object_name.c_str(), map_type_name.c_str(), sai_status);
return task_process_status::task_failed;
task_process_status handle_status = handleSaiRemoveStatus(SAI_API_BUFFER, sai_status);
if (handle_status != task_process_status::task_success)
{
return handle_status;
}
}

SWSS_LOG_NOTICE("Remove buffer profile %s with type %s", object_name.c_str(), map_type_name.c_str());
Expand Down Expand Up @@ -734,7 +758,11 @@ task_process_status BufferOrch::processQueue(KeyOpFieldsValuesTuple &tuple)
if (sai_status != SAI_STATUS_SUCCESS)
{
SWSS_LOG_ERROR("Failed to set queue's buffer profile attribute, status:%d", sai_status);
return task_process_status::task_failed;
task_process_status handle_status = handleSaiSetStatus(SAI_API_QUEUE, sai_status);
if (handle_status != task_process_status::task_success)
{
return handle_status;
}
}
}
}
Expand Down Expand Up @@ -857,7 +885,11 @@ task_process_status BufferOrch::processPriorityGroup(KeyOpFieldsValuesTuple &tup
if (sai_status != SAI_STATUS_SUCCESS)
{
SWSS_LOG_ERROR("Failed to set port:%s pg:%zd buffer profile attribute, status:%d", port_name.c_str(), ind, sai_status);
return task_process_status::task_failed;
task_process_status handle_status = handleSaiSetStatus(SAI_API_BUFFER, sai_status);
if (handle_status != task_process_status::task_success)
{
return handle_status;
}
}
}
}
Expand Down Expand Up @@ -936,7 +968,11 @@ task_process_status BufferOrch::processIngressBufferProfileList(KeyOpFieldsValue
if (sai_status != SAI_STATUS_SUCCESS)
{
SWSS_LOG_ERROR("Failed to set ingress buffer profile list on port, status:%d, key:%s", sai_status, port_name.c_str());
return task_process_status::task_failed;
task_process_status handle_status = handleSaiSetStatus(SAI_API_PORT, sai_status);
if (handle_status != task_process_status::task_success)
{
return handle_status;
}
}
}

Expand Down Expand Up @@ -986,7 +1022,11 @@ task_process_status BufferOrch::processEgressBufferProfileList(KeyOpFieldsValues
if (sai_status != SAI_STATUS_SUCCESS)
{
SWSS_LOG_ERROR("Failed to set egress buffer profile list on port, status:%d, key:%s", sai_status, port_name.c_str());
return task_process_status::task_failed;
task_process_status handle_status = handleSaiSetStatus(SAI_API_PORT, sai_status);
if (handle_status != task_process_status::task_success)
{
return handle_status;
}
}
}

Expand Down
Loading

0 comments on commit db9238f

Please sign in to comment.