Merge pull request #242 from bugsnag/rewrite_features

Rewrite test features to new steps
bugsnag · Aug 21, 2024 · 1ea6317 · 1ea6317
2 parents bf878a4 + e983d36
commit 1ea6317
Show file tree

Hide file tree

Showing 29 changed files with 350 additions and 635 deletions.
diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows]
+        os: [ubuntu, windows]
         go-version: ['1.11', '1.12', '1.13', '1.14', '1.15', '1.16', '1.17', '1.18', '1.19', '1.20', '1.21', '1.22']
 
     steps:

diff --git a/features/apptype.feature b/features/apptype.feature
@@ -1,22 +1,18 @@
 Feature: Configuring app type
 
 Background:
-  Given I set environment variable "API_KEY" to "a35a2a72bd230ac0aa0f52715bbdc6aa"
-  And I configure the bugsnag endpoint
-  And I set environment variable "APP_TYPE" to "background-queue"
-  And I have built the service "app"
+  Given I set environment variable "BUGSNAG_APP_TYPE" to "background-queue"
 
 Scenario: An error report contains the configured app type when running a go app
-  Given I set environment variable "AUTO_CAPTURE_SESSIONS" to "false"
-  When I run the go service "app" with the test case "handled"
-  Then I wait to receive a request
-  And the request is a valid error report with api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
+  Given I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "0"
+  When I start the service "app"
+  And I run "HandledErrorScenario"
+  And I wait to receive an error
   And the event "app.type" equals "background-queue"
 
 Scenario: An session report contains the configured app type when running a go app
-  Given I set environment variable "AUTO_CAPTURE_SESSIONS" to "true"
-  When I run the go service "app" with the test case "session"
-  Then I wait to receive a request after the start up session
-  And the request is a valid session report with api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
-  And the payload field "app.type" equals "background-queue"
-
+  Given I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "1"
+  When I start the service "app"
+  And I run "SendSessionScenario"
+  And I wait to receive 2 sessions
+  And the session payload field "app.type" equals "background-queue"
diff --git a/features/appversion.feature b/features/appversion.feature
@@ -1,21 +1,18 @@
 Feature: Configuring app version
 
 Background:
-  Given I set environment variable "API_KEY" to "a35a2a72bd230ac0aa0f52715bbdc6aa"
-  And I configure the bugsnag endpoint
-  And I set environment variable "APP_VERSION" to "3.1.2"
-  And I have built the service "app"
+  And I set environment variable "BUGSNAG_APP_VERSION" to "3.1.2"
 
 Scenario: An error report contains the configured app type when running a go app
-  Given I set environment variable "AUTO_CAPTURE_SESSIONS" to "false"
-  When I run the go service "app" with the test case "handled"
-  Then I wait to receive a request
-  And the request is a valid error report with api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
+  Given I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "0"
+  When I start the service "app"
+  And I run "HandledErrorScenario"
+  And I wait to receive an error
   And the event "app.version" equals "3.1.2"
 
-Scenario: An session report contains the configured app type when running a go app
-  Given I set environment variable "AUTO_CAPTURE_SESSIONS" to "true"
-  When I run the go service "app" with the test case "session"
-  Then I wait to receive a request after the start up session
-  And the request is a valid session report with api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
-  And the payload field "app.version" equals "3.1.2"
+Scenario: A session report contains the configured app type when running a go app
+  Given I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "1"
+  When I start the service "app"
+  And I run "SendSessionScenario"
+  And I wait to receive 2 sessions
+  And the session payload field "app.version" equals "3.1.2"
diff --git a/features/autonotify.feature b/features/autonotify.feature
@@ -1,13 +1,11 @@
 Feature: Using auto notify
 
-Background:
-  Given I set environment variable "API_KEY" to "a35a2a72bd230ac0aa0f52715bbdc6aa"
-  And I configure the bugsnag endpoint
-  And I have built the service "app"
-
 Scenario: An error report is sent when an AutoNotified crash occurs which later gets recovered
-  When I run the go service "app" with the test case "autonotify"
-  Then I wait for 3 seconds
-  And the request 1 is a valid error report with api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
-  And the exception "errorClass" equals "*errors.errorString" for request 1
-  And the exception "message" equals "Go routine killed with auto notify" for request 1
+  When I start the service "app"
+  And I run "AutonotifyPanicScenario"
+  And I wait to receive 2 errors
+  And the exception "errorClass" equals "*errors.errorString"
+  And the exception "message" equals "Go routine killed with auto notify"
+  And I discard the oldest error
+  And the exception "errorClass" equals "panic"
+  And the exception "message" equals "Go routine killed with auto notify [recovered]"
diff --git a/features/configuration.feature b/features/configuration.feature
@@ -5,92 +5,96 @@ Feature: Configure integration with environment variables
 
     Background:
         Given I set environment variable "BUGSNAG_API_KEY" to "a35a2a72bd230ac0aa0f52715bbdc6aa"
-        And I set environment variable "BUGSNAG_NOTIFY_ENDPOINT" to the notify endpoint
-        And I set environment variable "BUGSNAG_SESSIONS_ENDPOINT" to the sessions endpoint
-        And I have built the service "autoconfigure"
 
     Scenario Outline: Adding content to handled events through env variables
         Given I set environment variable "<variable>" to "<value>"
         And I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "0"
-        When I run the go service "autoconfigure" with the test case "<testcase>"
-        Then I wait to receive a request
-        And the request is a valid error report with api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
+        When I start the service "app"
+        And I run "<testcase>"
+        And I wait to receive an error
         And the event "<field>" equals "<value>"
 
         Examples:
             | testcase | variable                              | value           | field                         |
-            | panic    | BUGSNAG_APP_VERSION                   | 1.4.34          | app.version                   |
-            | panic    | BUGSNAG_APP_TYPE                      | mailer-daemon   | app.type                      |
-            | panic    | BUGSNAG_RELEASE_STAGE                 | beta1           | app.releaseStage              |
-            | panic    | BUGSNAG_HOSTNAME                      | dream-machine-2 | device.hostname               |
-            | panic    | BUGSNAG_METADATA_device_instance      | kube2-33-A      | metaData.device.instance      |
-            | panic    | BUGSNAG_METADATA_framework_version    | v3.1.0          | metaData.framework.version    |
-            | panic    | BUGSNAG_METADATA_device_runtime_level | 1C              | metaData.device.runtime_level |
-            | panic    | BUGSNAG_METADATA_Carrot               | orange          | metaData.custom.Carrot        |
+            | AutoconfigPanicScenario    | BUGSNAG_APP_VERSION                   | 1.4.34          | app.version                   |
+            | AutoconfigPanicScenario    | BUGSNAG_APP_TYPE                      | mailer-daemon   | app.type                      |
+            | AutoconfigPanicScenario    | BUGSNAG_RELEASE_STAGE                 | beta1           | app.releaseStage              |
+            | AutoconfigPanicScenario    | BUGSNAG_HOSTNAME                      | dream-machine-2 | device.hostname               |
+            | AutoconfigPanicScenario    | BUGSNAG_METADATA_device_instance      | kube2-33-A      | metaData.device.instance      |
+            | AutoconfigPanicScenario    | BUGSNAG_METADATA_framework_version    | v3.1.0          | metaData.framework.version    |
+            | AutoconfigPanicScenario    | BUGSNAG_METADATA_device_runtime_level | 1C              | metaData.device.runtime_level |
+            | AutoconfigPanicScenario    | BUGSNAG_METADATA_Carrot               | orange          | metaData.custom.Carrot        |
 
-            | handled  | BUGSNAG_APP_VERSION                   | 1.4.34          | app.version                   |
-            | handled  | BUGSNAG_APP_TYPE                      | mailer-daemon   | app.type                      |
-            | handled  | BUGSNAG_RELEASE_STAGE                 | beta1           | app.releaseStage              |
-            | handled  | BUGSNAG_HOSTNAME                      | dream-machine-2 | device.hostname               |
-            | handled  | BUGSNAG_METADATA_device_instance      | kube2-33-A      | metaData.device.instance      |
-            | handled  | BUGSNAG_METADATA_framework_version    | v3.1.0          | metaData.framework.version    |
-            | handled  | BUGSNAG_METADATA_device_runtime_level | 1C              | metaData.device.runtime_level |
-            | handled  | BUGSNAG_METADATA_Carrot               | orange          | metaData.custom.Carrot        |
+            | AutoconfigHandledScenario  | BUGSNAG_APP_VERSION                   | 1.4.34          | app.version                   |
+            | AutoconfigHandledScenario  | BUGSNAG_APP_TYPE                      | mailer-daemon   | app.type                      |
+            | AutoconfigHandledScenario  | BUGSNAG_RELEASE_STAGE                 | beta1           | app.releaseStage              |
+            | AutoconfigHandledScenario  | BUGSNAG_HOSTNAME                      | dream-machine-2 | device.hostname               |
+            | AutoconfigHandledScenario  | BUGSNAG_METADATA_device_instance      | kube2-33-A      | metaData.device.instance      |
+            | AutoconfigHandledScenario  | BUGSNAG_METADATA_framework_version    | v3.1.0          | metaData.framework.version    |
+            | AutoconfigHandledScenario  | BUGSNAG_METADATA_device_runtime_level | 1C              | metaData.device.runtime_level |
+            | AutoconfigHandledScenario  | BUGSNAG_METADATA_Carrot               | orange          | metaData.custom.Carrot        |
 
     Scenario: Configuring project packages
         Given I set environment variable "BUGSNAG_PROJECT_PACKAGES" to "main,test"
         And I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "0"
-        When I run the go service "autoconfigure" with the test case "panic"
-        Then I wait to receive a request
-        And the request is a valid error report with api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
-        And the in-project frames of the stacktrace are:
-            | file     | method        | lineNumber |
-            | cases.go | explicitPanic | 22         |
-            | main.go  | main          | 11         |
+        When I start the service "app"
+        And I run "AutoconfigPanicScenario"
+        And I wait to receive an error
+        And the "file" of stack frame 0 equals "features/fixtures/app/autoconfig_scenario.go"
+        And the "method" of stack frame 0 equals "AutoconfigPanicScenario.func1"
+        And the "lineNumber" of stack frame 0 equals 11
+        And the "file" of stack frame 1 equals "features/fixtures/app/main.go"
+        And the "method" of stack frame 1 equals "main"
+        And the "lineNumber" of stack frame 1 equals 65
 
     Scenario: Configuring source root
-        Given I set environment variable "BUGSNAG_SOURCE_ROOT" to the app directory
+        Given I set environment variable "BUGSNAG_SOURCE_ROOT" to "/app/src/features/fixtures/app/"
         And I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "0"
-        And I run the go service "autoconfigure" with the test case "panic"
-        Then I wait to receive a request
-        And the request is a valid error report with api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
-        And the in-project frames of the stacktrace are:
-            | file     | method        | lineNumber |
-            | cases.go | explicitPanic | 22         |
-            | main.go  | main          | 11         |
+        When I start the service "app"
+        And I run "AutoconfigPanicScenario"
+        And I wait to receive an error
+        And the "file" of stack frame 0 equals "autoconfig_scenario.go"
+        And the "method" of stack frame 0 equals "AutoconfigPanicScenario.func1"
+        And the "lineNumber" of stack frame 0 equals 11
+        And the "file" of stack frame 1 equals "main.go"
+        And the "method" of stack frame 1 equals "main"
+        And the "lineNumber" of stack frame 1 equals 65
 
     Scenario: Delivering events filtering through notify release stages
         Given I set environment variable "BUGSNAG_NOTIFY_RELEASE_STAGES" to "prod,beta"
         And I set environment variable "BUGSNAG_RELEASE_STAGE" to "beta"
         And I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "0"
-        And I run the go service "autoconfigure" with the test case "panic"
-        Then I wait to receive a request
-        And the request is a valid error report with api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
+        When I start the service "app"
+        And I run "AutoconfigPanicScenario"
+        And I wait to receive an error
 
     Scenario: Suppressing events through notify release stages
         Given I set environment variable "BUGSNAG_NOTIFY_RELEASE_STAGES" to "prod,beta"
         And I set environment variable "BUGSNAG_RELEASE_STAGE" to "dev"
         And I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "0"
-        And I run the go service "autoconfigure" with the test case "panic"
-        Then 0 requests were received
+        When I start the service "app"
+        And I run "AutoconfigPanicScenario"
+        Then I should receive no errors
 
     Scenario: Suppressing events using panic handler
         Given I set environment variable "BUGSNAG_DISABLE_PANIC_HANDLER" to "1"
         And I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "0"
-        And I run the go service "autoconfigure" with the test case "panic"
-        And I wait for 2 seconds
-        Then 0 requests were received
+        When I start the service "app"
+        And I run "AutoconfigPanicScenario"
+        Then I should receive no errors
 
     Scenario: Enabling synchronous event delivery
         Given I set environment variable "BUGSNAG_SYNCHRONOUS" to "1"
         And I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "0"
-        When I run the go service "autoconfigure" with the test case "handled"
-        Then 1 request was received
+        When I start the service "app"
+        And I run "AutoconfigHandledScenario"
+        And I wait to receive an error
 
     Scenario: Filtering metadata
         Given I set environment variable "BUGSNAG_PARAMS_FILTERS" to "tomato,pears"
         And I set environment variable "BUGSNAG_AUTO_CAPTURE_SESSIONS" to "0"
-        When I run the go service "autoconfigure" with the test case "handled-metadata"
-        Then I wait to receive a request
+        When I start the service "app"
+        And I run "AutoconfigMetadataScenario"
+        And I wait to receive an error
         And the event "metaData.fruit.Tomato" equals "[FILTERED]"
-        And the event "metaData.snacks.Carrot" equals "4"
+        And the event "metaData.snacks.Carrot" equals "4"
diff --git a/features/endpoint.feature b/features/endpoint.feature
diff --git a/features/fixtures/app/main.go b/features/fixtures/app/main.go
@@ -9,25 +9,25 @@ import (
 )
 
 var scenariosMap = map[string]func(Command) func(){
-	"UnhandledScenario":            UnhandledCrashScenario,
-	"HandledScenario":              HandledErrorScenario,
-	"MultipleUnhandledScenario":    MultipleUnhandledErrorsScenario,
-	"MultipleHandledScenario":      MultipleHandledErrorsScenario,
-	"NestedErrorScenario":          NestedHandledErrorScenario,
-	"MetadataScenario":             MetadataScenario,
-	"FilteredMetadataScenario":     FilteredMetadataScenario,
-	"HandledCallbackErrorScenario": HandledCallbackErrorScenario,
-	"SendSessionScenario":          SendSessionScenario,
-	"HandledToUnhandledScenario":   HandledToUnhandledScenario,
-	"SetUserScenario":              SetUserScenario,
-	"RecoverAfterPanicScenario":    RecoverAfterPanicScenario,
-	"AutonotifyPanicScenario":      AutonotifyPanicScenario,
-	"SessionAndErrorScenario":      SessionAndErrorScenario,
-	"OnBeforeNotifyScenario":       OnBeforeNotifyScenario,
-	"AutoconfigPanicScenario":      AutoconfigPanicScenario,
-	"AutoconfigHandledScenario":    AutoconfigHandledScenario,
-	"AutoconfigMetadataScenario":   AutoconfigMetadataScenario,
-	"HttpServerScenario":           HttpServerScenario,
+	"UnhandledCrashScenario":          UnhandledCrashScenario,
+	"HandledErrorScenario":            HandledErrorScenario,
+	"MultipleUnhandledErrorsScenario": MultipleUnhandledErrorsScenario,
+	"MultipleHandledErrorsScenario":   MultipleHandledErrorsScenario,
+	"NestedHandledErrorScenario":      NestedHandledErrorScenario,
+	"MetadataScenario":                MetadataScenario,
+	"FilteredMetadataScenario":        FilteredMetadataScenario,
+	"HandledCallbackErrorScenario":    HandledCallbackErrorScenario,
+	"SendSessionScenario":             SendSessionScenario,
+	"HandledToUnhandledScenario":      HandledToUnhandledScenario,
+	"SetUserScenario":                 SetUserScenario,
+	"RecoverAfterPanicScenario":       RecoverAfterPanicScenario,
+	"AutonotifyPanicScenario":         AutonotifyPanicScenario,
+	"SessionAndErrorScenario":         SessionAndErrorScenario,
+	"OnBeforeNotifyScenario":          OnBeforeNotifyScenario,
+	"AutoconfigPanicScenario":         AutoconfigPanicScenario,
+	"AutoconfigHandledScenario":       AutoconfigHandledScenario,
+	"AutoconfigMetadataScenario":      AutoconfigMetadataScenario,
+	"HttpServerScenario":              HttpServerScenario,
 }
 
 func main() {