From 5c646960940a2294993f8dff3ef964493fb48f94 Mon Sep 17 00:00:00 2001 From: Firmino Changani Date: Mon, 25 Dec 2023 23:07:02 +0000 Subject: [PATCH] fix(#111): handle timeouts in an optimistic manner --- .env | 2 ++ .env.local | 2 ++ .github/workflows/deploy-production.yml | 2 +- cmd/demo/main.go | 13 -------- cmd/endpoint_simulator/main.go | 4 +++ cmd/service/main.go | 23 +++++++------- cmd/worker/main.go | 15 +++++----- .../endpoint_checkers/http_checker.go | 30 +++++++++++++++++-- .../endpoint_checkers/http_checker_test.go | 29 ++++++++++++++---- .../app/command/bulk_check_endpoints_test.go | 2 +- misc/deploy/service.fly.toml | 1 + misc/deploy/worker.fly.toml | 1 + 12 files changed, 82 insertions(+), 42 deletions(-) delete mode 100644 cmd/demo/main.go diff --git a/.env b/.env index e53889a..a1dfa55 100644 --- a/.env +++ b/.env @@ -20,3 +20,5 @@ SENT_FROM_EMAIL_ADDRESS=noreply@transactional.dobermann.dev WORKER_REGION=europe PRODUCTION_MODE=false + +ENDPOINT_CHECK_TIMEOUT_IN_SECONDS=30 \ No newline at end of file diff --git a/.env.local b/.env.local index 09e4d09..2a88aa3 100644 --- a/.env.local +++ b/.env.local @@ -20,3 +20,5 @@ SENT_FROM_EMAIL_ADDRESS=noreply@transactional.dobermann.dev WORKER_REGION=europe PRODUCTION_MODE=false + +ENDPOINT_CHECK_TIMEOUT_IN_SECONDS=30 \ No newline at end of file diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 54c63d6..f140f57 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -3,7 +3,7 @@ on: workflow_dispatch: inputs: version: - description: 0.0.0 (semver) + description: v0.0.0 (semver) required: true jobs: diff --git a/cmd/demo/main.go b/cmd/demo/main.go deleted file mode 100644 index 33174f5..0000000 --- a/cmd/demo/main.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "fmt" - "time" -) - -func main() { - s := time.Now() - time.Sleep(time.Second * 5) - e := time.Since(s) - fmt.Println(e.Milliseconds()) -} diff --git a/cmd/endpoint_simulator/main.go b/cmd/endpoint_simulator/main.go index 34d9eb5..18b5d53 100644 --- a/cmd/endpoint_simulator/main.go +++ b/cmd/endpoint_simulator/main.go @@ -66,6 +66,10 @@ func main() { signal.Notify(done, syscall.SIGINT, syscall.SIGTERM) router.GET("/", func(c echo.Context) error { + if c.QueryParam("timeout") == "true" { + time.Sleep(time.Second * 15) + } + if c.QueryParam("is_up") == "true" { return c.NoContent(http.StatusOK) } diff --git a/cmd/service/main.go b/cmd/service/main.go index 8df5162..40c29e7 100644 --- a/cmd/service/main.go +++ b/cmd/service/main.go @@ -40,16 +40,17 @@ import ( var Version = "development" type Config struct { - AmqpUrl string `envconfig:"AMQP_URL"` - Port int `envconfig:"HTTP_PORT"` - JwtSecret string `envconfig:"JWT_SECRET"` - DebugMode string `envconfig:"DEBUG_MODE"` - DatabaseURL string `envconfig:"DATABASE_URL"` - ResendApiKey string `envconfig:"RESEND_API_KEY"` - HostnameForNotifications string `envconfig:"HOSTNAME_NOTIFICATION"` - SentFromEmailAddress string `envconfig:"SENT_FROM_EMAIL_ADDRESS"` - IsProductionMode bool `envconfig:"PRODUCTION_MODE"` - Region string `envconfig:"WORKER_REGION"` + AmqpUrl string `envconfig:"AMQP_URL"` + Port int `envconfig:"HTTP_PORT"` + JwtSecret string `envconfig:"JWT_SECRET"` + DebugMode string `envconfig:"DEBUG_MODE"` + DatabaseURL string `envconfig:"DATABASE_URL"` + ResendApiKey string `envconfig:"RESEND_API_KEY"` + HostnameForNotifications string `envconfig:"HOSTNAME_NOTIFICATION"` + SentFromEmailAddress string `envconfig:"SENT_FROM_EMAIL_ADDRESS"` + IsProductionMode bool `envconfig:"PRODUCTION_MODE"` + Region string `envconfig:"WORKER_REGION"` + EndpointCheckTimeoutInSeconds int `envconfig:"ENDPOINT_CHECK_TIMEOUT_IN_SECONDS" required:"true"` } func (c Config) IsDebugMode() bool { @@ -138,7 +139,7 @@ func main() { logger.Info("Connected successfully to RabbitMQ") - httpChecker, err := endpoint_checkers.NewHttpChecker(config.Region) + httpChecker, err := endpoint_checkers.NewHttpChecker(config.Region, config.EndpointCheckTimeoutInSeconds) if err != nil { logger.Fatal(err) } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index c8f0622..dd67d05 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -25,12 +25,13 @@ import ( var Version = "development" type Config struct { - AmqpUrl string `envconfig:"AMQP_URL"` - Port int `envconfig:"HTTP_PORT"` - DebugMode string `envconfig:"DEBUG_MODE"` - DatabaseURL string `envconfig:"DATABASE_URL"` - Region string `envconfig:"WORKER_REGION" required:"true"` - IsProductionMode bool `envconfig:"PRODUCTION_MODE"` + AmqpUrl string `envconfig:"AMQP_URL"` + Port int `envconfig:"HTTP_PORT"` + DebugMode string `envconfig:"DEBUG_MODE"` + DatabaseURL string `envconfig:"DATABASE_URL"` + Region string `envconfig:"WORKER_REGION" required:"true"` + IsProductionMode bool `envconfig:"PRODUCTION_MODE"` + EndpointCheckTimeoutInSeconds int `envconfig:"ENDPOINT_CHECK_TIMEOUT_IN_SECONDS" required:"true"` } func (c Config) IsDebugMode() bool { @@ -68,7 +69,7 @@ func main() { logger.Fatal(err) } - httpChecker, err := endpoint_checkers.NewHttpChecker(config.Region) + httpChecker, err := endpoint_checkers.NewHttpChecker(config.Region, config.EndpointCheckTimeoutInSeconds) if err != nil { logger.Fatal(err) } diff --git a/internal/adapters/endpoint_checkers/http_checker.go b/internal/adapters/endpoint_checkers/http_checker.go index 57be800..2d8f8f4 100644 --- a/internal/adapters/endpoint_checkers/http_checker.go +++ b/internal/adapters/endpoint_checkers/http_checker.go @@ -2,6 +2,7 @@ package endpoint_checkers import ( "context" + "errors" "fmt" "net/http" "time" @@ -14,15 +15,19 @@ type HttpChecker struct { region monitor.Region } -func NewHttpChecker(region string) (HttpChecker, error) { +func NewHttpChecker(region string, timeoutInSeconds int) (HttpChecker, error) { reg, err := monitor.NewRegion(region) if err != nil { return HttpChecker{}, err } + if timeoutInSeconds < 0 || timeoutInSeconds > 30 { + return HttpChecker{}, fmt.Errorf("the timeout should be within the range of 1 and 30") + } + return HttpChecker{ client: &http.Client{ - Timeout: time.Second * 5, + Timeout: time.Second * time.Duration(timeoutInSeconds), }, region: reg, }, nil @@ -36,13 +41,32 @@ func (h HttpChecker) Check(ctx context.Context, endpointUrl string) (*monitor.Ch startedAt := time.Now() resp, err := h.client.Do(req) + if errors.Is(err, errors.Unwrap(err)) { + return h.createCheckResults(startedAt, nil, true) + } + if err != nil { return nil, fmt.Errorf("unable to check endpoint %s: %v", endpointUrl, err) } defer func() { _ = resp.Body.Close() }() + return h.createCheckResults(startedAt, &resp.StatusCode, false) +} + +func (h HttpChecker) createCheckResults(startedAt time.Time, statusCode *int, isForcedTimeout bool) (*monitor.CheckResult, error) { checkDuration := time.Since(startedAt) - checkResult, err := monitor.NewCheckResult(int16(resp.StatusCode), h.region, time.Now(), int16(checkDuration.Milliseconds())) + + var sCode int16 + + if statusCode != nil { + sCode = int16(*statusCode) + } + + if isForcedTimeout { + sCode = int16(http.StatusRequestTimeout) + } + + checkResult, err := monitor.NewCheckResult(sCode, h.region, time.Now(), int16(checkDuration.Milliseconds())) if err != nil { return nil, fmt.Errorf("unable to create CheckResult: %v", err) } diff --git a/internal/adapters/endpoint_checkers/http_checker_test.go b/internal/adapters/endpoint_checkers/http_checker_test.go index bf6a7d6..c6109af 100644 --- a/internal/adapters/endpoint_checkers/http_checker_test.go +++ b/internal/adapters/endpoint_checkers/http_checker_test.go @@ -3,6 +3,7 @@ package endpoint_checkers_test import ( "context" "fmt" + "net/http" "os" "testing" @@ -10,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/flowck/dobermann/backend/internal/adapters/endpoint_checkers" + "github.com/flowck/dobermann/backend/internal/domain/monitor" ) func TestHttpChecker(t *testing.T) { @@ -18,12 +20,27 @@ func TestHttpChecker(t *testing.T) { simulatorEndpointUrl := os.Getenv("SIMULATOR_ENDPOINT_URL") - httpChecker, err := endpoint_checkers.NewHttpChecker("europe") + httpChecker, err := endpoint_checkers.NewHttpChecker(monitor.RegionEurope.String(), 2) require.NoError(t, err) - _, err = httpChecker.Check(ctx, fmt.Sprintf("%s?is_up=true", simulatorEndpointUrl)) - assert.NoError(t, err) - checkResult, err := httpChecker.Check(ctx, fmt.Sprintf("%s?is_up=false", simulatorEndpointUrl)) - require.NoError(t, err) - assert.True(t, checkResult.IsEndpointDown()) + t.Run("is_up", func(t *testing.T) { + t.Parallel() + _, err = httpChecker.Check(ctx, fmt.Sprintf("%s?is_up=true", simulatorEndpointUrl)) + assert.NoError(t, err) + }) + + t.Run("is_down", func(t *testing.T) { + t.Parallel() + checkResult, err := httpChecker.Check(ctx, fmt.Sprintf("%s?is_up=false", simulatorEndpointUrl)) + require.NoError(t, err) + assert.True(t, checkResult.IsEndpointDown()) + }) + + t.Run("error_endpoint_timeouts", func(t *testing.T) { + t.Parallel() + + result, err := httpChecker.Check(ctx, fmt.Sprintf("%s?is_up=true&timeout=true", simulatorEndpointUrl)) + require.NoError(t, err) + assert.Equal(t, http.StatusRequestTimeout, int(result.StatusCode())) + }) } diff --git a/internal/app/command/bulk_check_endpoints_test.go b/internal/app/command/bulk_check_endpoints_test.go index 87afff9..3c76287 100644 --- a/internal/app/command/bulk_check_endpoints_test.go +++ b/internal/app/command/bulk_check_endpoints_test.go @@ -35,7 +35,7 @@ func (p mockTxProvider) Transact(ctx context.Context, fn command.TransactFunc) e } func TestNewBulkCheckEndpointsHandler(t *testing.T) { - endpointsChecker, err := endpoint_checkers.NewHttpChecker("europe") + endpointsChecker, err := endpoint_checkers.NewHttpChecker("europe", 5) require.NoError(t, err) txProvider := mockTxProvider{ EventPublisher: events.NewPublisherMock(), diff --git a/misc/deploy/service.fly.toml b/misc/deploy/service.fly.toml index 4e29ee9..cd0ddca 100644 --- a/misc/deploy/service.fly.toml +++ b/misc/deploy/service.fly.toml @@ -15,6 +15,7 @@ primary_region = "lhr" PRODUCTION_MODE=true HOSTNAME_NOTIFICATION='https://dobermann.dev' SENT_FROM_EMAIL_ADDRESS='noreply@transactional.dobermann.dev' + ENDPOINT_CHECK_TIMEOUT_IN_SECONDS=30 [http_service] diff --git a/misc/deploy/worker.fly.toml b/misc/deploy/worker.fly.toml index fa3ac32..fda82e3 100644 --- a/misc/deploy/worker.fly.toml +++ b/misc/deploy/worker.fly.toml @@ -11,6 +11,7 @@ primary_region = "lhr" [env] PRODUCTION_MODE=true + ENDPOINT_CHECK_TIMEOUT_IN_SECONDS=30 [http_service] internal_port = 8080