feat(verification): add testing.T to VerifyProvider for better test r…

…eporting

BREAKING CHANGE: implements feature request pact-foundation#58

README.md

@@ -255,24 +255,22 @@ Read more about [flexible matching](https://github.com/realestate-com-au/pact/wi
 	satisfy the requirements of each of your known consumers:
 
 	```go
-	response := pact.VerifyProvider(types.VerifyRequest{
-		ProviderBaseURL:        "http://localhost:8000",
-		PactURLs:               []string{"./pacts/my_consumer-my_provider.json"},		
-		ProviderStatesSetupURL: "http://localhost:8000/setup",
-	})
-
-	if err != nil {
-		t.Fatal("Error:", err)
-	}
+    pact.VerifyProvider(t, types.VerifyRequest{
+      ProviderBaseURL:        "http://localhost:8000",
+      PactURLs:               []string{"./pacts/my_consumer-my_provider.json"},		
+      ProviderStatesSetupURL: "http://localhost:8000/setup",
+    })
 	```
 
-	Note that `PactURLs` is a list of local pact files or remote based
-	urls (e.g. from a
-	[Pact Broker](http://docs.pact.io/documentation/sharings_pacts.html)).
+  The `VerifyProvider` will handle all verifications, treating them as subtests
+  and giving you granular test reporting. If you don't like this behaviour, you may call `VerifyProviderRaw` directly and handle the errors manually.
 
+  Note that `PactURLs` may a list of local pact files or remote based
+  urls (e.g. from a
+  [Pact Broker](http://docs.pact.io/documentation/sharings_pacts.html)).
 
-	See the `Skip()'ed` [integration tests](https://github.com/pact-foundation/pact-go/blob/master/dsl/pact_test.go)
-	for a more complete E2E example.
+  See the `Skip()'ed` [integration tests](https://github.com/pact-foundation/pact-go/blob/master/dsl/pact_test.go)
+  for a more complete E2E example.
 
 #### Provider Verification
 
@@ -281,7 +279,7 @@ When validating a Provider, you have 3 options to provide the Pact files:
 1. Use `PactURLs` to specify the exact set of pacts to be replayed:
 
 	```go
-	response = pact.VerifyProvider(types.VerifyRequest{
+	pact.VerifyProvider(t, types.VerifyRequest{
 		ProviderBaseURL:        "http://myproviderhost",
 		PactURLs:               []string{"http://broker/pacts/provider/them/consumer/me/latest/dev"},		
 		ProviderStatesSetupURL: "http://myproviderhost/setup",
@@ -292,7 +290,7 @@ When validating a Provider, you have 3 options to provide the Pact files:
 1. Use `PactBroker` to automatically find all of the latest consumers:
 
 	```go
-	response = pact.VerifyProvider(types.VerifyRequest{
+	pact.VerifyProvider(t, types.VerifyRequest{
 		ProviderBaseURL:        "http://myproviderhost",
 		BrokerURL:              "http://brokerHost",		
 		ProviderStatesSetupURL: "http://myproviderhost/setup",
@@ -303,7 +301,7 @@ When validating a Provider, you have 3 options to provide the Pact files:
 1. Use `PactBroker` and `Tags` to automatically find all of the latest consumers:
 
 	```go
-	response = pact.VerifyProvider(types.VerifyRequest{
+	pact.VerifyProvider(t, types.VerifyRequest{
 		ProviderBaseURL:        "http://myproviderhost",
 		BrokerURL:              "http://brokerHost",
 		Tags:                   []string{"latest", "sit4"},		
@@ -319,7 +317,6 @@ in development.
 
 See this [article](http://rea.tech/enter-the-pact-matrix-or-how-to-decouple-the-release-cycles-of-your-microservices/)
 for more on this strategy.
-e
 
 
 See the examples or read more at http://docs.pact.io/documentation/provider_states.html.

daemon/daemon_test.go

@@ -255,7 +255,7 @@ func TestVerifyProvider_MissingProviderBaseURL(t *testing.T) {
 		t.Fatal("Expected an error")
 	}
 
-	if !strings.Contains(err.Error(), "ProviderBaseURL is mandatory") {
+	if !strings.Contains(err.Error(), "Provider base URL is mandatory") {
 		t.Fatalf("Expected error message but got '%s'", err.Error())
 	}
 }
@@ -273,7 +273,7 @@ func TestVerifyProvider_MissingPactURLs(t *testing.T) {
 		t.Fatal("Expected an error")
 	}
 
-	if !strings.Contains(err.Error(), "PactURLs is mandatory") {
+	if !strings.Contains(err.Error(), "Pact URLs is mandatory") {
 		t.Fatalf("Expected error message but got '%s'", err.Error())
 	}
 }

daemon/verification_service.go

@@ -24,12 +24,12 @@ type VerificationService struct {
 // 		--broker-password
 //    --publish_verification_results
 //    --provider_app_version
-func (m *VerificationService) NewService(args []string) Service {
+func (v *VerificationService) NewService(args []string) Service {
 	log.Printf("[DEBUG] starting verification service with args: %v\n", args)
 
-	m.Args = args
-	m.Cmd = getVerifierCommandPath()
-	return m
+	v.Args = args
+	v.Cmd = getVerifierCommandPath()
+	return v
 }
 
 func getVerifierCommandPath() string {

daemon/verification_service_test.go

@@ -1,6 +1,9 @@
 package daemon
 
-import "testing"
+import (
+	"fmt"
+	"testing"
+)
 
 func TestVerificationService_NewService(t *testing.T) {
 	s := &VerificationService{}
@@ -11,6 +14,6 @@ func TestVerificationService_NewService(t *testing.T) {
 	}
 
 	if s.Args[0] != "--foo bar" {
-		t.Fatalf("Expected '--foo bar' argument to be passed")
+		t.Fatalf(fmt.Sprintf(`Expected "--foo bar" argument to be passed, got "%s"`, s.Args[0]))
 	}
 }

doc.go

@@ -130,24 +130,18 @@ A typical Provider side test would like something like:
 		}
 		go startMyAPI("http://localhost:8000")
 
-		response, err := pact.VerifyProvider(types.VerifyRequest{
+		pact.VerifyProvider(types.VerifyRequest{
 			ProviderBaseURL:        "http://localhost:8000",
 			PactURLs:               []string{"./pacts/my_consumer-my_provider.json"},
 			ProviderStatesSetupURL: "http://localhost:8000/setup",
 		})
-
-		if err != nil {
-			t.Fatal("Error:", err)
-		}
-
-		for _, example := range response.Examples {
-			if example.Status != "passed" {
-				t.Errorf("%s\n%s\n", example.FullDescription, example.Exception.Message)
-			}
-		}
 	}
 
-Note that `PactURLs` can be a list of local pact files or remote based
+The `VerifyProvider` will handle all verifications, treating them as subtests
+and giving you granular test reporting. If you don't like this behaviour, you may
+call `VerifyProviderRaw` directly and handle the errors manually.
+
+Note that `PactURLs` may be a list of local pact files or remote based
 urls (possibly from a Pact Broker
 - http://docs.pact.io/documentation/sharings_pacts.html).
 

dsl/client_test.go

@@ -262,7 +262,7 @@ func TestClient_VerifyProviderFailValidation(t *testing.T) {
 		t.Fatal("Expected a error but got none")
 	}
 
-	if !strings.Contains(err.Error(), "PactURLs is mandatory") {
+	if !strings.Contains(err.Error(), "Pact URLs is mandatory") {
 		t.Fatalf("Expected a proper error message but got '%s'", err.Error())
 	}
 }

dsl/pact.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"testing"
 
 	"github.com/hashicorp/logutils"
 	"github.com/pact-foundation/pact-go/types"
@@ -234,9 +235,9 @@ func (p *Pact) WritePact() error {
 	return nil
 }
 
-// VerifyProvider reads the provided pact files and runs verification against
-// a running Provider API.
-func (p *Pact) VerifyProvider(request types.VerifyRequest) (types.ProviderVerifierResponse, error) {
+// VerifyProviderRaw reads the provided pact files and runs verification against
+// a running Provider API, providing raw response from the Verification process.
+func (p *Pact) VerifyProviderRaw(request types.VerifyRequest) (types.ProviderVerifierResponse, error) {
 	p.Setup(false)
 
 	// If we provide a Broker, we go to it to find consumers
@@ -252,3 +253,26 @@ func (p *Pact) VerifyProvider(request types.VerifyRequest) (types.ProviderVerifi
 
 	return p.pactClient.VerifyProvider(request)
 }
+
+// VerifyProvider accepts an instance of `*testing.T`
+// running the provider verification with granular test reporting and
+// automatic failure reporting for nice, simple tests.
+func (p *Pact) VerifyProvider(t *testing.T, request types.VerifyRequest) (types.ProviderVerifierResponse, error) {
+	res, err := p.VerifyProviderRaw(request)
+
+	if err != nil {
+		t.Fatal("Error:", err)
+		return res, err
+	}
+
+	for _, example := range res.Examples {
+		t.Run(example.Description, func(st *testing.T) {
+			st.Log(example.FullDescription)
+			if example.Status != "passed" {
+				st.Errorf("%s\n", example.Exception.Message)
+			}
+		})
+	}
+
+	return res, err
+}

dsl/pact_test.go

@@ -271,7 +271,7 @@ func TestPact_VerifyProvider(t *testing.T) {
 	waitForPortInTest(port, t)
 
 	pact := &Pact{Port: port, LogLevel: "DEBUG", pactClient: &PactClient{Port: port}}
-	_, err := pact.VerifyProvider(types.VerifyRequest{
+	_, err := pact.VerifyProviderRaw(types.VerifyRequest{
 		ProviderBaseURL: "http://www.foo.com",
 		PactURLs:        []string{"foo.json", "bar.json"},
 	})
@@ -294,7 +294,7 @@ func TestPact_VerifyProviderBroker(t *testing.T) {
 	waitForPortInTest(port, t)
 
 	pact := &Pact{Port: port, LogLevel: "DEBUG", pactClient: &PactClient{Port: port}, Provider: "bobby"}
-	_, err := pact.VerifyProvider(types.VerifyRequest{
+	_, err := pact.VerifyProviderRaw(types.VerifyRequest{
 		ProviderBaseURL:            "http://www.foo.com",
 		BrokerURL:                  s.URL,
 		PublishVerificationResults: true,
@@ -319,7 +319,7 @@ func TestPact_VerifyProviderBrokerNoConsumers(t *testing.T) {
 	waitForPortInTest(port, t)
 
 	pact := &Pact{Port: port, LogLevel: "DEBUG", pactClient: &PactClient{Port: port}, Provider: "providernotexist"}
-	_, err := pact.VerifyProvider(types.VerifyRequest{
+	_, err := pact.VerifyProviderRaw(types.VerifyRequest{
 		ProviderBaseURL: "http://www.foo.com",
 		BrokerURL:       s.URL,
 	})
@@ -340,7 +340,7 @@ func TestPact_VerifyProviderFail(t *testing.T) {
 	waitForPortInTest(port, t)
 
 	pact := &Pact{Port: port, LogLevel: "DEBUG", pactClient: &PactClient{Port: port}}
-	_, err := pact.VerifyProvider(types.VerifyRequest{
+	_, err := pact.VerifyProviderRaw(types.VerifyRequest{
 		ProviderBaseURL: "http://www.foo.com",
 		PactURLs:        []string{"foo.json", "bar.json"},
 	})

examples/gin/provider/user_service_test.go

@@ -20,23 +20,18 @@ func TestPact_Provider(t *testing.T) {
 	pact := createPact()
 
 	// Verify the Provider with local Pact Files
-	res, err := pact.VerifyProvider(types.VerifyRequest{
+	pact.VerifyProvider(t, types.VerifyRequest{
 		ProviderBaseURL:        fmt.Sprintf("http://localhost:%d", port),
 		PactURLs:               []string{filepath.ToSlash(fmt.Sprintf("%s/billy-bobby.json", pactDir))},
 		ProviderStatesSetupURL: fmt.Sprintf("http://localhost:%d/setup", port),
 	})
 
-	if err != nil {
-		t.Fatal("Error:", err)
-	}
-	assertExamples(t, res)
-
 	// Pull from pact broker, used in e2e/integrated tests for pact-go release
 	if os.Getenv("PACT_INTEGRATED_TESTS") != "" {
 		var brokerHost = os.Getenv("PACT_BROKER_HOST")
 
 		// Verify the Provider - Specific Published Pacts
-		res, err = pact.VerifyProvider(types.VerifyRequest{
+		pact.VerifyProvider(t, types.VerifyRequest{
 			ProviderBaseURL:            fmt.Sprintf("http://127.0.0.1:%d", port),
 			PactURLs:                   []string{fmt.Sprintf("%s/pacts/provider/bobby/consumer/billy/latest/sit4", brokerHost)},
 			ProviderStatesSetupURL:     fmt.Sprintf("http://127.0.0.1:%d/setup", port),
@@ -46,13 +41,8 @@ func TestPact_Provider(t *testing.T) {
 			ProviderVersion:            "1.0.0",
 		})
 
-		if err != nil {
-			t.Fatal("Error:", err)
-		}
-		assertExamples(t, res)
-
 		// Verify the Provider - Latest Published Pacts for any known consumers
-		res, err = pact.VerifyProvider(types.VerifyRequest{
+		pact.VerifyProvider(t, types.VerifyRequest{
 			ProviderBaseURL:            fmt.Sprintf("http://127.0.0.1:%d", port),
 			BrokerURL:                  brokerHost,
 			ProviderStatesSetupURL:     fmt.Sprintf("http://127.0.0.1:%d/setup", port),
@@ -62,13 +52,8 @@ func TestPact_Provider(t *testing.T) {
 			ProviderVersion:            "1.0.0",
 		})
 
-		if err != nil {
-			t.Fatal("Error:", err)
-		}
-		assertExamples(t, res)
-
 		// Verify the Provider - Tag-based Published Pacts for any known consumers
-		res, err = pact.VerifyProvider(types.VerifyRequest{
+		pact.VerifyProvider(t, types.VerifyRequest{
 			ProviderBaseURL:            fmt.Sprintf("http://127.0.0.1:%d", port),
 			ProviderStatesSetupURL:     fmt.Sprintf("http://127.0.0.1:%d/setup", port),
 			BrokerURL:                  brokerHost,
@@ -79,11 +64,6 @@ func TestPact_Provider(t *testing.T) {
 			ProviderVersion:            "1.0.0",
 		})
 
-		if err != nil {
-			t.Fatal("Error:", err)
-		}
-		assertExamples(t, res)
-
 	} else {
 		t.Log("Skipping pulling from broker as PACT_INTEGRATED_TESTS is not set")
 	}

examples/go-kit/provider/user_service_test.go

@@ -58,23 +58,18 @@ func TestPact_Provider(t *testing.T) {
 	pact := createPact()
 
 	// Verify the Provider with local Pact Files
-	res, err := pact.VerifyProvider(types.VerifyRequest{
+	pact.VerifyProvider(t, types.VerifyRequest{
 		ProviderBaseURL:        fmt.Sprintf("http://localhost:%d", port),
 		PactURLs:               []string{filepath.ToSlash(fmt.Sprintf("%s/billy-bobby.json", pactDir))},
 		ProviderStatesSetupURL: fmt.Sprintf("http://localhost:%d/setup", port),
 	})
 
-	if err != nil {
-		t.Fatal("Error:", err)
-	}
-	assertExamples(t, res)
-
 	// Pull from pact broker, used in e2e/integrated tests for pact-go release
 	if os.Getenv("PACT_INTEGRATED_TESTS") != "" {
 		var brokerHost = os.Getenv("PACT_BROKER_HOST")
 
 		// Verify the Provider - Specific Published Pacts
-		res, err = pact.VerifyProvider(types.VerifyRequest{
+		pact.VerifyProvider(t, types.VerifyRequest{
 			ProviderBaseURL:            fmt.Sprintf("http://127.0.0.1:%d", port),
 			PactURLs:                   []string{fmt.Sprintf("%s/pacts/provider/bobby/consumer/billy/latest/sit4", brokerHost)},
 			ProviderStatesSetupURL:     fmt.Sprintf("http://127.0.0.1:%d/setup", port),
@@ -84,13 +79,8 @@ func TestPact_Provider(t *testing.T) {
 			ProviderVersion:            "1.0.0",
 		})
 
-		if err != nil {
-			t.Fatal("Error:", err)
-		}
-		assertExamples(t, res)
-
 		// Verify the Provider - Latest Published Pacts for any known consumers
-		res, err = pact.VerifyProvider(types.VerifyRequest{
+		pact.VerifyProvider(t, types.VerifyRequest{
 			ProviderBaseURL:            fmt.Sprintf("http://127.0.0.1:%d", port),
 			BrokerURL:                  brokerHost,
 			ProviderStatesSetupURL:     fmt.Sprintf("http://127.0.0.1:%d/setup", port),
@@ -100,13 +90,8 @@ func TestPact_Provider(t *testing.T) {
 			ProviderVersion:            "1.0.0",
 		})
 
-		if err != nil {
-			t.Fatal("Error:", err)
-		}
-		assertExamples(t, res)
-
 		// Verify the Provider - Tag-based Published Pacts for any known consumers
-		res, err = pact.VerifyProvider(types.VerifyRequest{
+		pact.VerifyProvider(t, types.VerifyRequest{
 			ProviderBaseURL:            fmt.Sprintf("http://127.0.0.1:%d", port),
 			ProviderStatesSetupURL:     fmt.Sprintf("http://127.0.0.1:%d/setup", port),
 			BrokerURL:                  brokerHost,
@@ -117,10 +102,6 @@ func TestPact_Provider(t *testing.T) {
 			ProviderVersion:            "1.0.0",
 		})
 
-		if err != nil {
-			t.Fatal("Error:", err)
-		}
-		assertExamples(t, res)
 	} else {
 		t.Log("Skipping pulling from broker as PACT_INTEGRATED_TESTS is not set")
 	}
@@ -179,7 +160,6 @@ func startInstrumentedProvider() {
 		if err != nil {
 			return
 		}
-
 		json.Unmarshal(body, &state)
 
 		svc := s.(*loggingMiddleware).next.(*userService)