diff --git a/internal/jose/jose.go b/internal/jose/jose.go
index 654d7973a..be16b66fc 100644
--- a/internal/jose/jose.go
+++ b/internal/jose/jose.go
@@ -1,6 +1,7 @@
 package jose
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"fmt"
@@ -82,6 +83,41 @@ func RunJoseCommand(ctx context.Context, t *testing.T, args []string, outw, errw
 	return nil
 }
 
+type AlgorithmSet struct {
+	data map[string]struct{}
+}
+
+func NewAlgorithmSet() *AlgorithmSet {
+	return &AlgorithmSet{
+		data: make(map[string]struct{}),
+	}
+}
+
+func (set *AlgorithmSet) Add(s string) {
+	set.data[s] = struct{}{}
+}
+
+func (set *AlgorithmSet) Has(s string) bool {
+	_, ok := set.data[s]
+	return ok
+}
+
+func Algorithms(ctx context.Context, t *testing.T) (*AlgorithmSet, error) {
+	var buf bytes.Buffer
+	if err := RunJoseCommand(ctx, t, []string{"alg"}, &buf, nil); err != nil {
+		return nil, fmt.Errorf(`failed to generate jose tool's supported algorithms: %w`, err)
+	}
+
+	set := NewAlgorithmSet()
+
+	scanner := bufio.NewScanner(&buf)
+	for scanner.Scan() {
+		alg := scanner.Text()
+		set.Add(alg)
+	}
+	return set, nil
+}
+
 // GenerateJwk creates a new key using the jose tool, and returns its filename and
 // a cleanup function.
 // The caller is responsible for calling the cleanup
diff --git a/jwx_test.go b/jwx_test.go
index 06c746c99..b74243404 100644
--- a/jwx_test.go
+++ b/jwx_test.go
@@ -17,6 +17,7 @@ import (
 	"github.com/lestrrat-go/jwx/v2/jwe"
 	"github.com/lestrrat-go/jwx/v2/jwk"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestShowBuildInfo(t *testing.T) {
@@ -173,43 +174,57 @@ func TestJoseCompatibility(t *testing.T) {
 		}
 	})
 	t.Run("jwe", func(t *testing.T) {
+		// For some reason "jose" does not come with RSA-OAEP on some platforms.
+		// In order to avoid doing this in an ad-hoc way, we're just going to
+		// ask our jose package for the algorithms that it supports, and generate
+		// the list dynamically
+
 		t.Parallel()
-		tests := []interopTest{
-			{jwa.RSA1_5, jwa.A128GCM},
-			{jwa.RSA1_5, jwa.A128CBC_HS256},
-			{jwa.RSA1_5, jwa.A256CBC_HS512},
-			{jwa.RSA_OAEP, jwa.A128GCM},
-			{jwa.RSA_OAEP, jwa.A128CBC_HS256},
-			{jwa.RSA_OAEP, jwa.A256CBC_HS512},
-			{jwa.RSA_OAEP_256, jwa.A128GCM},
-			{jwa.RSA_OAEP_256, jwa.A128CBC_HS256},
-			{jwa.RSA_OAEP_256, jwa.A256CBC_HS512},
-			{jwa.ECDH_ES, jwa.A128GCM},
-			{jwa.ECDH_ES, jwa.A256GCM},
-			{jwa.ECDH_ES, jwa.A128CBC_HS256},
-			{jwa.ECDH_ES, jwa.A256CBC_HS512},
-			{jwa.ECDH_ES_A128KW, jwa.A128GCM},
-			{jwa.ECDH_ES_A128KW, jwa.A128CBC_HS256},
-			{jwa.ECDH_ES_A256KW, jwa.A256GCM},
-			{jwa.ECDH_ES_A256KW, jwa.A256CBC_HS512},
-			{jwa.A128KW, jwa.A128GCM},
-			{jwa.A128KW, jwa.A128CBC_HS256},
-			{jwa.A256KW, jwa.A256GCM},
-			{jwa.A256KW, jwa.A256CBC_HS512},
-			{jwa.A128GCMKW, jwa.A128GCM},
-			{jwa.A128GCMKW, jwa.A128CBC_HS256},
-			{jwa.A256GCMKW, jwa.A256GCM},
-			{jwa.A256GCMKW, jwa.A256CBC_HS512},
-			{jwa.PBES2_HS256_A128KW, jwa.A128GCM},
-			{jwa.PBES2_HS256_A128KW, jwa.A128CBC_HS256},
-			{jwa.PBES2_HS384_A192KW, jwa.A192GCM},
-			{jwa.PBES2_HS384_A192KW, jwa.A192CBC_HS384},
-			{jwa.PBES2_HS512_A256KW, jwa.A256GCM},
-			{jwa.PBES2_HS512_A256KW, jwa.A256CBC_HS512},
-			{jwa.DIRECT, jwa.A128GCM},
-			{jwa.DIRECT, jwa.A128CBC_HS256},
-			{jwa.DIRECT, jwa.A256GCM},
-			{jwa.DIRECT, jwa.A256CBC_HS512},
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		set, err := jose.Algorithms(ctx, t)
+		require.NoError(t, err)
+
+		var tests []interopTest
+
+		for _, keyenc := range []jwa.KeyEncryptionAlgorithm{jwa.RSA1_5, jwa.RSA_OAEP, jwa.RSA_OAEP_256} {
+			if !set.Has(keyenc.String()) {
+				t.Logf("jose does not support key encryption algorithm %q: skipping", keyenc)
+				continue
+			}
+			for _, contentenc := range []jwa.ContentEncryptionAlgorithm{jwa.A128GCM, jwa.A128CBC_HS256, jwa.A256CBC_HS512} {
+				tests = append(tests, interopTest{keyenc, contentenc})
+			}
+		}
+
+		for _, keyenc := range []jwa.KeyEncryptionAlgorithm{jwa.ECDH_ES, jwa.ECDH_ES_A128KW, jwa.A128KW, jwa.A128GCMKW, jwa.A256KW, jwa.A256GCMKW, jwa.PBES2_HS256_A128KW, jwa.DIRECT} {
+			if !set.Has(keyenc.String()) {
+				t.Logf("jose does not support key encryption algorithm %q: skipping", keyenc)
+				continue
+			}
+			for _, contentenc := range []jwa.ContentEncryptionAlgorithm{jwa.A128GCM, jwa.A128CBC_HS256} {
+				tests = append(tests, interopTest{keyenc, contentenc})
+			}
+		}
+
+		for _, keyenc := range []jwa.KeyEncryptionAlgorithm{jwa.ECDH_ES, jwa.ECDH_ES_A256KW, jwa.A256KW, jwa.A256GCMKW, jwa.PBES2_HS512_A256KW, jwa.DIRECT} {
+			if !set.Has(keyenc.String()) {
+				t.Logf("jose does not support key encryption algorithm %q: skipping", keyenc)
+				continue
+			}
+			for _, contentenc := range []jwa.ContentEncryptionAlgorithm{jwa.A256GCM, jwa.A256CBC_HS512} {
+				tests = append(tests, interopTest{keyenc, contentenc})
+			}
+		}
+
+		for _, keyenc := range []jwa.KeyEncryptionAlgorithm{jwa.PBES2_HS384_A192KW} {
+			if !set.Has(keyenc.String()) {
+				t.Logf("jose does not support key encryption algorithm %q: skipping", keyenc)
+				continue
+			}
+			for _, contentenc := range []jwa.ContentEncryptionAlgorithm{jwa.A192GCM, jwa.A192CBC_HS384} {
+				tests = append(tests, interopTest{keyenc, contentenc})
+			}
 		}
 
 		for _, test := range tests {
@@ -426,7 +441,7 @@ func TestGuessFormat(t *testing.T) {
 	}{
 		{
 			Name:     "Raw String",
-			Expected: jwx.UnknownFormat,
+			Expected: jwx.InvalidFormat,
 			Source:   []byte(`Hello, World`),
 		},
 		{
@@ -436,7 +451,7 @@ func TestGuessFormat(t *testing.T) {
 		},
 		{
 			Name:     "Random JSON Array",
-			Expected: jwx.UnknownFormat,
+			Expected: jwx.InvalidFormat,
 			Source:   []byte(`["random", "JSON"]`),
 		},
 		{
diff --git a/tools/test.sh b/tools/test.sh
index b8236fab9..905a6fe0a 100755
--- a/tools/test.sh
+++ b/tools/test.sh
@@ -19,13 +19,24 @@ case "$MODE" in
 		;;
 esac
 
+failures=0
 echo "mode: atomic" > "$DST"
 for dir in . ./examples ./bench/performance ./cmd/jwx; do
+	testout=$(mktemp /tmp/jwx-test.XXXXX)
 	pushd "$dir" > /dev/null
-	go test -race -json ${testopts[@]} ./... | tparse
+	go test -race -json ${testopts[@]} ./... > $testout
+	if [[ "$?" != "0" ]]; then
+		failures=$((failures+1))
+	fi
+	tparse -file="$testout"
+	rm "$testout"
 	if [[ -e "$tmpfile" ]]; then
 		cat "$tmpfile" | tail -n +2 | grep -v "internal/jose" | grep -v "internal/jwxtest" | grep -v "internal/cmd" >> "$DST"
 		rm "$tmpfile"
 	fi
 	popd > /dev/null
 done
+
+if [[ "$failures" != "0" ]]; then
+	exit 1
+fi