Decode path-encoded URL components (#2332)

* Decode path-encoded URL components

This change causes pct-encoded characters passed via path parameters to
be correctly decoded as described in google.api.http (see: path template
syntax) and as implemented in grpc-http-json-transcoding.

A new configuration option is introduced, `WithDecodeMode()`, which
understands several modes. Backwards compatibility is maintained, with
the hope of UnescapingModeAllExceptReserved becoming the default mode in
V3.

* Add tests for URL path unescaping

* Improve error handling for unescaping

* docs: path parameter unescaping

docs/docs/faq.md

@@ -39,6 +39,10 @@ When a message is added which happens to conflict with another message (e.g. by
 
 The gRPC-Gateway is a generator that generates a Go implementation of a JSON/HTTP-gRPC reverse proxy based on annotations in your proto file, while the [grpc-httpjson-transcoding](https://github.com/grpc-ecosystem/grpc-httpjson-transcoding) library doesn't require the generation step, it uses protobuf descriptors as config. It can be used as a component of an existing proxy. Google Cloud Endpoints and the gRPC-JSON transcoder filter in Envoy are using this.
 
+<!-- TODO(v3): remove this note when default behavior matches Envoy/Cloud Endpoints -->
+**Behavior differences:**
+- By default, gRPC-Gateway does not escape path parameters in the same way. [This can be configured.](../mapping/customizing_your_gateway.md#Controlling-path-parameter-unescaping)
+
 ## What is the difference between the gRPC-Gateway and gRPC-web?
 
 ### Usage

docs/docs/mapping/customizing_your_gateway.md

@@ -345,6 +345,30 @@ func handleStreamError(ctx context.Context, err error) *status.Status {
 
 If no custom handler is provided, the default stream error handler will include any gRPC error attributes (code, message, detail messages), if the error being reported includes them. If the error does not have these attributes, a gRPC code of `Unknown` (2) is reported.
 
+## Controlling path parameter unescaping
+
+<!-- TODO(v3): Remove comments about default behavior -->
+
+By default, gRPC-Gateway unescapes the entire URL path string attempting to route a request. This causes routing errors when the path parameter contains an illegal character such as `/`.
+
+To replicate the behavior described in [google.api.http](https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L224), use [runtime.WithUnescapingMode()](https://pkg.go.dev/github.com/grpc-ecosystem/grpc-gateway/runtime?tab=doc#WithUnescapingMode) to configure the unescaping behavior, as in the example below:
+
+```go
+mux := runtime.NewServeMux(
+	runtime.WithUnescapingMode(runtime.UnescapingModeAllExceptReserved),
+)
+```
+
+For multi-segment parameters (e.g. `{id=**}`) [RFC 6570](https://tools.ietf.org/html/rfc6570) Reserved Expansion characters are left escaped and the gRPC API will need to unescape them.
+
+To replicate the default V2 escaping behavior but also allow passing pct-encoded `/` characters, the ServeMux can be configured as in the example below:
+
+```go
+mux := runtime.NewServeMux(
+	runtime.WithUnescapingMode(runtime.UnescapingModeAllCharacters),
+)
+```
+
 ## Routing Error handler
 
 To override the error behavior when `*runtime.ServeMux` was not able to serve the request due to routing issues, use the `runtime.WithRoutingErrorHandler` option.

runtime/mux.go

@@ -2,6 +2,7 @@ package runtime
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/textproto"
@@ -14,6 +15,31 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+// UnescapingMode defines the behavior of ServeMux when unescaping path parameters.
+type UnescapingMode int
+
+const (
+	// UnescapingModeLegacy is the default V2 behavior, which escapes the entire
+	// path string before doing any routing.
+	UnescapingModeLegacy UnescapingMode = iota
+
+	// EscapingTypeExceptReserved unescapes all path parameters except RFC 6570
+	// reserved characters.
+	UnescapingModeAllExceptReserved
+
+	// EscapingTypeExceptSlash unescapes URL path parameters except path
+	// seperators, which will be left as "%2F".
+	UnescapingModeAllExceptSlash
+
+	// URL path parameters will be fully decoded.
+	UnescapingModeAllCharacters
+
+	// UnescapingModeDefault is the default escaping type.
+	// TODO(v3): default this to UnescapingModeAllExceptReserved per grpc-httpjson-transcoding's
+	// reference implementation
+	UnescapingModeDefault = UnescapingModeLegacy
+)
+
 // A HandlerFunc handles a specific pair of path pattern and HTTP method.
 type HandlerFunc func(w http.ResponseWriter, r *http.Request, pathParams map[string]string)
 
@@ -31,6 +57,7 @@ type ServeMux struct {
 	streamErrorHandler        StreamErrorHandlerFunc
 	routingErrorHandler       RoutingErrorHandlerFunc
 	disablePathLengthFallback bool
+	unescapingMode            UnescapingMode
 }
 
 // ServeMuxOption is an option that can be given to a ServeMux on construction.
@@ -48,6 +75,14 @@ func WithForwardResponseOption(forwardResponseOption func(context.Context, http.
 	}
 }
 
+// WithEscapingType sets the escaping type. See the definitions of UnescapingMode
+// for more information.
+func WithUnescapingMode(mode UnescapingMode) ServeMuxOption {
+	return func(serveMux *ServeMux) {
+		serveMux.unescapingMode = mode
+	}
+}
+
 // SetQueryParameterParser sets the query parameter parser, used to populate message from query parameters.
 // Configuring this will mean the generated OpenAPI output is no longer correct, and it should be
 // done with careful consideration.
@@ -153,6 +188,7 @@ func NewServeMux(opts ...ServeMuxOption) *ServeMux {
 		errorHandler:           DefaultHTTPErrorHandler,
 		streamErrorHandler:     DefaultStreamErrorHandler,
 		routingErrorHandler:    DefaultRoutingErrorHandler,
+		unescapingMode:         UnescapingModeDefault,
 	}
 
 	for _, opt := range opts {
@@ -204,6 +240,11 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// TODO(v3): remove UnescapingModeLegacy
+	if s.unescapingMode != UnescapingModeLegacy && r.URL.RawPath != "" {
+		path = r.URL.RawPath
+	}
+
 	components := strings.Split(path[1:], "/")
 
 	if override := r.Header.Get("X-HTTP-Method-Override"); override != "" && s.isPathLengthFallback(r) {
@@ -244,8 +285,16 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			components[l-1], verb = lastComponent[:idx], lastComponent[idx+1:]
 		}
 
-		pathParams, err := h.pat.Match(components, verb)
+		pathParams, err := h.pat.MatchAndEscape(components, verb, s.unescapingMode)
 		if err != nil {
+			var mse MalformedSequenceError
+			if ok := errors.As(err, &mse); ok {
+				_, outboundMarshaler := MarshalerForRequest(s, r)
+				s.errorHandler(ctx, s, outboundMarshaler, w, r, &HTTPStatusError{
+					HTTPStatus: http.StatusBadRequest,
+					Err:        mse,
+				})
+			}
 			continue
 		}
 		h.h(w, r, pathParams)
@@ -259,8 +308,16 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 		for _, h := range handlers {
-			pathParams, err := h.pat.Match(components, verb)
+			pathParams, err := h.pat.MatchAndEscape(components, verb, s.unescapingMode)
 			if err != nil {
+				var mse MalformedSequenceError
+				if ok := errors.As(err, &mse); ok {
+					_, outboundMarshaler := MarshalerForRequest(s, r)
+					s.errorHandler(ctx, s, outboundMarshaler, w, r, &HTTPStatusError{
+						HTTPStatus: http.StatusBadRequest,
+						Err:        mse,
+					})
+				}
 				continue
 			}
 			// X-HTTP-Method-Override is optional. Always allow fallback to POST.

runtime/mux_test.go

@@ -30,6 +30,7 @@ func TestMuxServeHTTP(t *testing.T) {
 		respContent string
 
 		disablePathLengthFallback bool
+		unescapingMode            runtime.UnescapingMode
 	}{
 		{
 			patterns:   nil,
@@ -330,11 +331,74 @@ func TestMuxServeHTTP(t *testing.T) {
 			respStatus:  http.StatusOK,
 			respContent: "POST /foo/{id=*}:verb:subverb",
 		},
+		{
+			patterns: []stubPattern{
+				{
+					method: "GET",
+					ops:    []int{int(utilities.OpLitPush), 0, int(utilities.OpPush), 1, int(utilities.OpCapture), 1, int(utilities.OpLitPush), 2},
+					pool:   []string{"foo", "id", "bar"},
+				},
+			},
+			reqMethod: "POST",
+			reqPath:   "/foo/404%2fwith%2Fspace/bar",
+			headers: map[string]string{
+				"Content-Type": "application/json",
+			},
+			respStatus:     http.StatusNotFound,
+			unescapingMode: runtime.UnescapingModeLegacy,
+		},
+		{
+			patterns: []stubPattern{
+				{
+					method: "GET",
+					ops: []int{
+						int(utilities.OpLitPush), 0,
+						int(utilities.OpPush), 0,
+						int(utilities.OpConcatN), 1,
+						int(utilities.OpCapture), 1,
+						int(utilities.OpLitPush), 2},
+					pool: []string{"foo", "id", "bar"},
+				},
+			},
+			reqMethod: "GET",
+			reqPath:   "/foo/success%2fwith%2Fspace/bar",
+			headers: map[string]string{
+				"Content-Type": "application/json",
+			},
+			respStatus:     http.StatusOK,
+			unescapingMode: runtime.UnescapingModeAllExceptReserved,
+			respContent:    "GET /foo/{id=*}/bar",
+		},
+		{
+			patterns: []stubPattern{
+				{
+					method: "GET",
+					ops: []int{
+						int(utilities.OpLitPush), 0,
+						int(utilities.OpPushM), 0,
+						int(utilities.OpConcatN), 1,
+						int(utilities.OpCapture), 1,
+					},
+					pool: []string{"foo", "id", "bar"},
+				},
+			},
+			reqMethod: "GET",
+			reqPath:   "/foo/success%2fwith%2Fspace",
+			headers: map[string]string{
+				"Content-Type": "application/json",
+			},
+			respStatus:     http.StatusOK,
+			unescapingMode: runtime.UnescapingModeAllExceptReserved,
+			respContent:    "GET /foo/{id=**}",
+		},
 	} {
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			var opts []runtime.ServeMuxOption
+			opts = append(opts, runtime.WithUnescapingMode(spec.unescapingMode))
 			if spec.disablePathLengthFallback {
-				opts = append(opts, runtime.WithDisablePathLengthFallback())
+				opts = append(opts,
+					runtime.WithDisablePathLengthFallback(),
+				)
 			}
 			mux := runtime.NewServeMux(opts...)
 			for _, p := range spec.patterns {