Skip to content
This repository has been archived by the owner on Dec 8, 2022. It is now read-only.

Retry using exponential backoff upon connection failure with GGD system tests #1892

Merged
merged 14 commits into from
Apr 23, 2020
Merged

Retry using exponential backoff upon connection failure with GGD system tests #1892

Changes from 13 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
8b3efd3
feat: Add RETRY_EXPONENTIAL for GGD_JSONRequestStart
yourslab Apr 6, 2020
0a2d45c
docs: Rename _ggdJSONRequestStart to prvGGD_JSONRequestStart
yourslab Apr 6, 2020
2c40ec1
fix: Add missing semi-colon
yourslab Apr 6, 2020
4156600
feat: Add request retry logic in GGD_GetGGCIPandCertificate
yourslab Apr 6, 2020
2a7ba6d
revert: Remove connection retry in ggd library
yourslab Apr 7, 2020
f4d9d43
feat: Transfer retry logic to GetGGCIPandCertificate
yourslab Apr 7, 2020
0dbc8e6
style: Add period to end of comment
yourslab Apr 9, 2020
5830b1a
revert: Apply no retry when passing in NULL params
yourslab Apr 9, 2020
55c2e13
revert: Remove retry in GetGGCIPandCertificate test
yourslab Apr 10, 2020
a22465b
fix: Remove loop and add retry to GetGGCIPandCertificate
yourslab Apr 10, 2020
ce4a05f
fix: Remove unused variables
yourslab Apr 10, 2020
e5e9f33
fix: Remove nBufferLength unused variable
yourslab Apr 10, 2020
b5dc9c6
docs: Clarify comment on IOT_TEST_GGD_CONNECT_RETRY_COUNT
yourslab Apr 10, 2020
375dbb6
docs: Change wording of num retries macro comment
yourslab Apr 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 79 additions & 46 deletions libraries/freertos_plus/aws/greengrass/test/aws_test_ggd_system.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,34 @@
#include "jsmn.h"
#include "unity_fixture.h"
#include "unity.h"
#include "aws_test_utils.h"
#include "aws_greengrass_discovery_test_access_declare.h"
#include "aws_test_runner.h"
#include "iot_mqtt.h"
#include "iot_init.h"

#define ggdJSON_FILE "{\"GGGroups\":[{\"GGGroupId\":\"myGroupID\",\"Cores\":[{\"thingArn\":\"myGreenGrassCoreArn\",\"Connectivity\":[{\"Id\":\"AUTOIP_10.60.212.138_0\",\"HostAddress\":\"44.44.44.44\",\"PortNumber\":1234,\"Metadata\":\"\"},{\"Id\":\"AUTOIP_127.0.0.1_1\",\"HostAddress\":\"127.0.0.1\",\"PortNumber\":8883,\"Metadata\":\"\"},{\"Id\":\"AUTOIP_192.168.2.2_2\",\"HostAddress\":\"01.23.456.789\",\"PortNumber\":4321,\"Metadata\":\"\"},{\"Id\":\"AUTOIP_::1_3\",\"HostAddress\":\"::1\",\"PortNumber\":8883,\"Metadata\":\"\"},{\"Id\":\"AUTOIP_fe80::bfda:8f62:7b4b:f358_4\",\"HostAddress\":\"fe80::bfda:8f62:7b4b:f358\",\"PortNumber\":8883,\"Metadata\":\"\"},{\"Id\":\"AUTOIP_fe80::e234:cff9:f53f:6216_5\",\"HostAddress\":\"fe80::e234:cff9:f53f:6216\",\"PortNumber\":8883,\"Metadata\":\"\"}]}],\"CAs\":[\"-----BEGIN CERTIFICATE-----\\nMIIEFTCCAv2gAwIBAgIVAPRru+NqCDr0r6oD6PnTG05rWuY+MA0GCSqGSIb3DQEB\\nCwUAMIGoMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPQW1hem9uLmNvbSBJbmMuMRww\\nGgYDVQQLDBNBbWF6b24gV2ViIFNlcnZpY2VzMRMwEQYDVQQIDApXYXNoaW5ndG9u\\nMRAwDgYDVQQHDAdTZWF0dGxlMTowOAYDVQQDDDE5NDI5MjczNzY5NjU6ZDk3ZmZl\\nZmUtNTI4MS00ZWM5LTk4NDYtYjNlZTQxMDRjMjAxMCAXDTE3MDcwNjIwMDczOFoY\\nDzIwOTcwNzA2MjAwNzM3WjCBqDELMAkGA1UEBhMCVVMxGDAWBgNVBAoMD0FtYXpv\\nbi5jb20gSW5jLjEcMBoGA1UECwwTQW1hem9uIFdlYiBTZXJ2aWNlczETMBEGA1UE\\nCAwKV2FzaGluZ3RvbjEQMA4GA1UEBwwHU2VhdHRsZTE6MDgGA1UEAwwxOTQyOTI3\\nMzc2OTY1OmQ5N2ZmZWZlLTUyODEtNGVjOS05ODQ2LWIzZWU0MTA0YzIwMTCCASIw\\nDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKxzJpXU2DZDEglh/FT01epAWby6\\np4Ymw76icyMzBUJzafibABJ3cTyjDQE6ZqbSl1ryBxGwQBsveIgj8SVVtv927wk7\\nlncgD+EghfTZgSfscND653AJeVFQlCeHipZI32wzXyPmwglFrWp9vsrY/8BO1Kjk\\nSAs4o8fDVVMAaZCJDMuc5csc3CQ2OJYLOl+SZisGNM1h0xHpWieM38KDDrp99x8Q\\nTwDmgaMjtdIJR7Y9Nzm0N78gTf3gTazEO9iUKojVCNubxK/lQ6KjJ0JcvsljPpVp\\nuzjOmn91xmNoHEQCboa7YoYNNbdAbftGeUl16wFdTgbuUS9vakk5idVoC2ECAwEA\\nAaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUmcz4OlH9+mlpnTKG3taI\\nw+6FSk0wDQYJKoZIhvcNAQELBQADggEBACeiQ6MxiktsU0sLNmP1cNbiuBuutjoq\\nymk476Bhr4E2WSE0B9W1TFOSLIYx9oN63T3lXzsGHP/MznueIbqbwFf/o5aXI7th\\n+J+i9LgBrViNvzkze7G0GiPuEQ7ox4XnPBJAFtTZxa8gXL95QfcypERpQs28lg7W\\nQpdNhiBN+c4o1aSOzJ474sjXnjtI1G2jRTKucm0buYYeAeVT7kpBq9YL7gGfOcyj\\nsPxQEgyQV2Mk+b1q7lYDS4tnzoRkUfNLgAtDKSh8S8iVhAR6wRR2G3aMySKrOxbg\\nalghO3OqfeuTwIj9w17JTAyYAME22RJQ6oxEJ8rHp/9PaYnOmiSkP7M=\\n-----END CERTIFICATE-----\\n\"]}]}"
#define ggdTestLOOP_NUMBER 10
#define ggdTestMAX_REQUEST_LOOP_COUNT 3

/**
* @brief The initial delay in milliseconds that is doubled each retry of request.
*/
#ifndef IOT_TEST_GGD_INITIAL_CONNECTION_RETRY_DELAY
#define IOT_TEST_GGD_INITIAL_CONNECTION_RETRY_DELAY ( ( uint32_t ) 1100 )
#endif

/**
* @brief The amount of times to retry a connection if it fails with 1 being a single attempt.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wording still seems a bit off to me. How would you feel about something like:

The number of times to try a connection. Values greater than 1 will retry on failure with an exponential backoff.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good

*/
#ifndef IOT_TEST_GGD_CONNECT_RETRY_COUNT
#define IOT_TEST_GGD_CONNECT_RETRY_COUNT ( 1 )
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this is 1, does that mean there is no retry, but rather a single attempt?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1 means a single attempt. I shall comment that out. Thank you!

#endif
/** @endcond */

#if IOT_TEST_GGD_CONNECT_RETRY_COUNT < 1
#error "IOT_TEST_GGD_CONNECT_RETRY_COUNT must be at least 1."
#endif

static const char cJSON_FILE[] = ggdJSON_FILE;
static const char cCERTIFICATE[] = "-----BEGIN CERTIFICATE-----\nMIIEFTCCAv2gAwIBAgIVAPRru+NqCDr0r6oD6PnTG05rWuY+MA0GCSqGSIb3DQEB\nCwUAMIGoMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPQW1hem9uLmNvbSBJbmMuMRww\nGgYDVQQLDBNBbWF6b24gV2ViIFNlcnZpY2VzMRMwEQYDVQQIDApXYXNoaW5ndG9u\nMRAwDgYDVQQHDAdTZWF0dGxlMTowOAYDVQQDDDE5NDI5MjczNzY5NjU6ZDk3ZmZl\nZmUtNTI4MS00ZWM5LTk4NDYtYjNlZTQxMDRjMjAxMCAXDTE3MDcwNjIwMDczOFoY\nDzIwOTcwNzA2MjAwNzM3WjCBqDELMAkGA1UEBhMCVVMxGDAWBgNVBAoMD0FtYXpv\nbi5jb20gSW5jLjEcMBoGA1UECwwTQW1hem9uIFdlYiBTZXJ2aWNlczETMBEGA1UE\nCAwKV2FzaGluZ3RvbjEQMA4GA1UEBwwHU2VhdHRsZTE6MDgGA1UEAwwxOTQyOTI3\nMzc2OTY1OmQ5N2ZmZWZlLTUyODEtNGVjOS05ODQ2LWIzZWU0MTA0YzIwMTCCASIw\nDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKxzJpXU2DZDEglh/FT01epAWby6\np4Ymw76icyMzBUJzafibABJ3cTyjDQE6ZqbSl1ryBxGwQBsveIgj8SVVtv927wk7\nlncgD+EghfTZgSfscND653AJeVFQlCeHipZI32wzXyPmwglFrWp9vsrY/8BO1Kjk\nSAs4o8fDVVMAaZCJDMuc5csc3CQ2OJYLOl+SZisGNM1h0xHpWieM38KDDrp99x8Q\nTwDmgaMjtdIJR7Y9Nzm0N78gTf3gTazEO9iUKojVCNubxK/lQ6KjJ0JcvsljPpVp\nuzjOmn91xmNoHEQCboa7YoYNNbdAbftGeUl16wFdTgbuUS9vakk5idVoC2ECAwEA\nAaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUmcz4OlH9+mlpnTKG3taI\nw+6FSk0wDQYJKoZIhvcNAQELBQADggEBACeiQ6MxiktsU0sLNmP1cNbiuBuutjoq\nymk476Bhr4E2WSE0B9W1TFOSLIYx9oN63T3lXzsGHP/MznueIbqbwFf/o5aXI7th\n+J+i9LgBrViNvzkze7G0GiPuEQ7ox4XnPBJAFtTZxa8gXL95QfcypERpQs28lg7W\nQpdNhiBN+c4o1aSOzJ474sjXnjtI1G2jRTKucm0buYYeAeVT7kpBq9YL7gGfOcyj\nsPxQEgyQV2Mk+b1q7lYDS4tnzoRkUfNLgAtDKSh8S8iVhAR6wRR2G3aMySKrOxbg\nalghO3OqfeuTwIj9w17JTAyYAME22RJQ6oxEJ8rHp/9PaYnOmiSkP7M=\n-----END CERTIFICATE-----\n";
static const char cMyGroupID[] = "myGroupID";
Expand All @@ -55,6 +74,12 @@ static BaseType_t prvGGD_JSONRequestGetFileLoop( uint32_t ulBufferSize,
BaseType_t * pxJSONFileRetrieveCompleted,
uint32_t ulJSONFileSize );

/* Wrapper function to retry request in case of failure with exponential backoff. */
static BaseType_t prvGGD_JSONRequestStart( const char * pcHostAddress,
uint16_t usGGDPort,
const char * pcThingName,
Socket_t * pxSocket );

TEST_GROUP( GGD_System );

TEST_SETUP( GGD_System )
Expand Down Expand Up @@ -108,6 +133,20 @@ static BaseType_t prvGGD_JSONRequestGetFileLoop( uint32_t ulBufferSize,
return xStatus;
}

static BaseType_t prvGGD_JSONRequestStart( const char * pcHostAddress,
uint16_t usGGDPort,
const char * pcThingName,
Socket_t * pxSocket )
{
BaseType_t xStatus;

RETRY_EXPONENTIAL( xStatus = GGD_JSONRequestStart( pcHostAddress, usGGDPort, pcThingName, pxSocket ),
pdPASS,
IOT_TEST_GGD_INITIAL_CONNECTION_RETRY_DELAY,
IOT_TEST_GGD_CONNECT_RETRY_COUNT );
return xStatus;
}

TEST( GGD_System, JSONRequestAbort )
{
/** @brief check for stability for all meaningfull values of socket.
Expand All @@ -121,10 +160,10 @@ TEST( GGD_System, JSONRequestAbort )
xSocket = SOCKETS_INVALID_SOCKET;
GGD_JSONRequestAbort( &xSocket );

GGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );
prvGGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );
GGD_JSONRequestAbort( &xSocket );
}
else
Expand All @@ -144,11 +183,8 @@ TEST( GGD_System, JSONRequestAbort )

TEST( GGD_System, GetGGCIPandCertificate )
{
BaseType_t i;
BaseType_t xStatus;
char cMsgBuffer[ 128 ];
GGD_HostAddressData_t xHostAddressData;
int16_t nBufferLength = 128;
uint32_t ulBufferSize = testrunnerBUFFER_SIZE;

if( TEST_PROTECT() )
Expand All @@ -158,20 +194,17 @@ TEST( GGD_System, GetGGCIPandCertificate )
* Check with auto-search flag set to false and true
* @{
*/
for( i = 0; i < ggdTestLOOP_NUMBER; i++ )
{
xStatus = GGD_GetGGCIPandCertificate( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
cBuffer, /*lint !e971 can use char without signed/unsigned. */
ulBufferSize,
&xHostAddressData );

snprintf( cMsgBuffer, nBufferLength,
"GGD_GetGGCIPandCertificate returned %d on iteration %d",
( int ) xStatus, ( int ) i );
TEST_ASSERT_EQUAL_INT32_MESSAGE( pdPASS, xStatus, cBuffer );
}
RETRY_EXPONENTIAL( xStatus = GGD_GetGGCIPandCertificate( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
cBuffer, /*lint !e971 can use char without signed/unsigned. */
ulBufferSize,
&xHostAddressData ),
pdPASS,
IOT_TEST_GGD_INITIAL_CONNECTION_RETRY_DELAY,
IOT_TEST_GGD_CONNECT_RETRY_COUNT );

TEST_ASSERT_EQUAL_INT32_MESSAGE( pdPASS, xStatus, cBuffer );

/** @}*/

Expand Down Expand Up @@ -285,10 +318,10 @@ TEST( GGD_System, GetIPandCertificateFromJSON )
TEST_ASSERT_EQUAL_INT32( strlen( cCERTIFICATE ) + 1, xHostAddressData.ulCertificateSize );

xAutoSearchFlag = pdTRUE;
xStatus = GGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );
xStatus = prvGGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );

if( xStatus == pdPASS )
{
Expand Down Expand Up @@ -417,10 +450,10 @@ TEST( GGD_System, JSONRequestGetFile )
/** @brief Check return status and value in ideal case.
* @{
*/
xStatus = GGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );
xStatus = prvGGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );

if( xStatus == pdPASS )
{
Expand All @@ -446,10 +479,10 @@ TEST( GGD_System, JSONRequestGetFile )
/** @brief Retrieve the JSON file in separate chunks.
* @{
*/
xStatus = GGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );
xStatus = prvGGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );

if( xStatus == pdPASS )
{
Expand Down Expand Up @@ -484,10 +517,10 @@ TEST( GGD_System, JSONRequestGetFile )
/** @brief Check fail if we receive more bytes than expected.
* @{
*/
xStatus = GGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );
xStatus = prvGGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );

if( xStatus == pdPASS )
{
Expand Down Expand Up @@ -564,10 +597,10 @@ TEST( GGD_System, JSONRequestGetSize )
/** @brief Check return status and value in ideal case
* @{
*/
xStatus = GGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );
xStatus = prvGGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );
TEST_ASSERT_EQUAL_INT32( pdPASS, xStatus );

xStatus = GGD_JSONRequestGetSize( &xSocket, &ulJSONFileSize );
Expand Down Expand Up @@ -603,10 +636,10 @@ TEST( GGD_System, JSONRequestStart )
/** @brief Check return status and value in ideal case
* @{
*/
xStatus = GGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );
xStatus = prvGGD_JSONRequestStart( clientcredentialMQTT_BROKER_ENDPOINT,
clientcredentialGREENGRASS_DISCOVERY_PORT,
clientcredentialIOT_THING_NAME,
&xSocket );
TEST_ASSERT_EQUAL_INT32( pdPASS, xStatus );
GGD_SecureConnect_Disconnect( &xSocket );
/** @}*/
Expand Down