Add --http3 flag to buf curl (#3127)

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/klauspost/pgzip v1.2.6
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
 	github.com/pkg/profile v1.7.0
+	github.com/quic-go/quic-go v0.45.1
 	github.com/rs/cors v1.11.0
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
@@ -74,6 +75,7 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
@@ -91,11 +93,13 @@ require (
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/opencontainers/runtime-spec v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
@@ -104,6 +108,7 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
 	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.uber.org/mock v0.4.0 // indirect
 	golang.org/x/sys v0.24.0 // indirect
 	golang.org/x/text v0.17.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect

go.sum

@@ -99,6 +99,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
 github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
@@ -188,6 +190,10 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
+github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
+github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
@@ -206,8 +212,12 @@ github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDj
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
-github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
+github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
+github.com/quic-go/quic-go v0.45.1 h1:tPfeYCk+uZHjmDRwHHQmvHRYL2t44ROTujLeFVBmjCA=
+github.com/quic-go/quic-go v0.45.1/go.mod h1:1dLehS7TIR64+vxGR70GDcatWTOtMX2PUtnKsjbTurI=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po=
@@ -225,6 +235,7 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
@@ -259,6 +270,8 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
@@ -315,8 +328,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
 golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
-golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

private/buf/bufcurl/tls.go

@@ -38,6 +38,12 @@ type TLSSettings struct {
 	// "h2", and exclude "http/1.1". If the server does not pick a
 	// protocol, "h2" is assumed as the default.
 	HTTP2PriorKnowledge bool
+	// If true, the server is known to support HTTP/3. When set, the
+	// ALPN protocols sent during the TLS handshake will include
+	// only "h3", and exclude the other versions. Since HTTP/3 is
+	// based on UDP, it is the only version to expect using the UDP
+	// port right now.
+	HTTP3 bool
 }
 
 // MakeVerboseTLSConfig constructs a *tls.Config that logs information to the
@@ -46,6 +52,8 @@ func MakeVerboseTLSConfig(settings *TLSSettings, authority string, printer verbo
 	var conf tls.Config
 	if settings.HTTP2PriorKnowledge {
 		conf.NextProtos = []string{"h2"}
+	} else if settings.HTTP3 {
+		conf.NextProtos = []string{"h3"}
 	} else {
 		conf.NextProtos = []string{"h2", "http/1.1"}
 	}
@@ -59,6 +67,8 @@ func MakeVerboseTLSConfig(settings *TLSSettings, authority string, printer verbo
 		if state.NegotiatedProtocol == "" {
 			if settings.HTTP2PriorKnowledge {
 				printer.Printf("* ALPN: server did not agree on a protocol. Using default h2")
+			} else if settings.HTTP3 {
+				printer.Printf("* ALPN: server did not agree on a protocol. Still attempting h3")
 			} else {
 				printer.Printf("* ALPN: server did not agree on a protocol. Using default http/1.1.")
 			}

private/buf/cmd/buf/command/curl/curl.go

@@ -41,6 +41,8 @@ import (
 	"github.com/bufbuild/buf/private/pkg/netrc"
 	"github.com/bufbuild/buf/private/pkg/stringutil"
 	"github.com/bufbuild/buf/private/pkg/verbose"
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
 	"github.com/spf13/pflag"
 	"go.uber.org/multierr"
 	"golang.org/x/net/http2"
@@ -60,6 +62,7 @@ const (
 	protocolFlagName            = "protocol"
 	unixSocketFlagName          = "unix-socket"
 	http2PriorKnowledgeFlagName = "http2-prior-knowledge"
+	http3FlagName               = "http3"
 
 	// TLS flags
 	keyFlagName           = "key"
@@ -207,6 +210,7 @@ type flags struct {
 	Protocol            string
 	UnixSocket          string
 	HTTP2PriorKnowledge bool
+	HTTP3               bool
 
 	// TLS
 	Key, Cert, CACert, ServerName string
@@ -324,6 +328,16 @@ choose either HTTP 1.1 or HTTP/2 for URLs with an https scheme. With this flag s
 HTTP/2 is always used, even over plain-text.`,
 	)
 
+	flagSet.BoolVar(
+		&f.HTTP3,
+		http3FlagName,
+		false,
+		`This flag can be used to indicate that HTTP/3 should be used. Without this, HTTP 1.1
+will be used with URLs with an http scheme, and protocol negotiation will be used to
+choose either HTTP 1.1 or HTTP/2 for URLs with an https scheme. With this flag set,
+HTTP/3 is always used.`,
+	)
+
 	flagSet.BoolVar(
 		&f.NoKeepAlive,
 		noKeepAliveFlagName,
@@ -544,6 +558,19 @@ func (f *flags) validate(hasURL, isSecure bool) error {
 		return fmt.Errorf("grpc protocol cannot be used with plain-text URLs (http) unless --%s flag is set", http2PriorKnowledgeFlagName)
 	}
 
+	if !isSecure && f.HTTP3 {
+		return fmt.Errorf("--%s cannot be used with plain-text URLs (http)", http3FlagName)
+	}
+
+	if f.UnixSocket != "" && f.HTTP3 {
+		return fmt.Errorf("--%s cannot be used with --%s", unixSocketFlagName, http3FlagName)
+	}
+
+	// NOTE: This can be removed once trailer support lands for quic-go: https://github.com/quic-go/quic-go/issues/2266
+	if f.Protocol == connect.ProtocolGRPC && f.HTTP3 {
+		return fmt.Errorf("--%s cannot be used with --%s=%s", http3FlagName, protocolFlagName, connect.ProtocolGRPC)
+	}
+
 	if f.Netrc && f.NetrcFile != "" {
 		return fmt.Errorf("--%s and --%s flags are mutually exclusive; they may not both be specified", netrcFlagName, netrcFileFlagName)
 	}
@@ -691,6 +718,18 @@ func (f *flags) determineCredentials(
 	return basicAuth(username, password), nil
 }
 
+func (f *flags) getTLSConfig(authority string, printer verbose.Printer) (*tls.Config, error) {
+	return bufcurl.MakeVerboseTLSConfig(&bufcurl.TLSSettings{
+		KeyFile:             f.Key,
+		CertFile:            f.Cert,
+		CACertFile:          f.CACert,
+		ServerName:          f.ServerName,
+		Insecure:            f.Insecure,
+		HTTP2PriorKnowledge: f.HTTP2PriorKnowledge,
+		HTTP3:               f.HTTP3,
+	}, authority, printer)
+}
+
 func promptForPassword(ctx context.Context, container app.Container, prompt string) (string, error) {
 	// NB: The comments below and the mechanism of handling I/O async was
 	// copied from the "registry login" command.
@@ -916,7 +955,11 @@ func run(ctx context.Context, container appext.Container, f *flags) (err error)
 			// This shouldn't be possible since we check in flags.validate, but just in case
 			return nil, errors.New("URL positional argument is missing")
 		}
-		return makeHTTPClient(f, isSecure, bufcurl.GetAuthority(host, requestHeaders), container.VerbosePrinter())
+		roundTripper, err := makeHTTPRoundTripper(f, isSecure, bufcurl.GetAuthority(host, requestHeaders), container.VerbosePrinter())
+		if err != nil {
+			return nil, err
+		}
+		return bufcurl.NewVerboseHTTPClient(roundTripper, container.VerbosePrinter()), nil
 	})
 
 	output := container.Stdout()
@@ -1025,7 +1068,10 @@ func run(ctx context.Context, container appext.Container, f *flags) (err error)
 	}
 }
 
-func makeHTTPClient(f *flags, isSecure bool, authority string, printer verbose.Printer) (connect.HTTPClient, error) {
+func makeHTTPRoundTripper(f *flags, isSecure bool, authority string, printer verbose.Printer) (http.RoundTripper, error) {
+	if f.HTTP3 {
+		return makeHTTP3RoundTripper(f, authority, printer)
+	}
 	var dialer net.Dialer
 	if f.ConnectTimeoutSeconds != 0 {
 		dialer.Timeout = secondsToDuration(f.ConnectTimeoutSeconds)
@@ -1035,6 +1081,7 @@ func makeHTTPClient(f *flags, isSecure bool, authority string, printer verbose.P
 	} else {
 		dialer.KeepAlive = secondsToDuration(f.KeepAliveTimeSeconds)
 	}
+
 	var dialFunc func(ctx context.Context, network, address string) (net.Conn, error)
 	if f.UnixSocket != "" {
 		dialFunc = func(ctx context.Context, network, addr string) (net.Conn, error) {
@@ -1055,14 +1102,7 @@ func makeHTTPClient(f *flags, isSecure bool, authority string, printer verbose.P
 
 	var dialTLSFunc func(ctx context.Context, network, address string) (net.Conn, error)
 	if isSecure {
-		tlsConfig, err := bufcurl.MakeVerboseTLSConfig(&bufcurl.TLSSettings{
-			KeyFile:             f.Key,
-			CertFile:            f.Cert,
-			CACertFile:          f.CACert,
-			ServerName:          f.ServerName,
-			Insecure:            f.Insecure,
-			HTTP2PriorKnowledge: f.HTTP2PriorKnowledge,
-		}, authority, printer)
+		tlsConfig, err := f.getTLSConfig(authority, printer)
 		if err != nil {
 			return nil, err
 		}
@@ -1104,7 +1144,53 @@ func makeHTTPClient(f *flags, isSecure bool, authority string, printer verbose.P
 			MaxIdleConns:      1,
 		}
 	}
-	return bufcurl.NewVerboseHTTPClient(transport, printer), nil
+	return transport, nil
+}
+
+func makeHTTP3RoundTripper(f *flags, authority string, printer verbose.Printer) (http.RoundTripper, error) {
+	quicCfg := &quic.Config{
+		KeepAlivePeriod: -1,
+	}
+	if f.ConnectTimeoutSeconds != 0 {
+		quicCfg.HandshakeIdleTimeout = secondsToDuration(f.ConnectTimeoutSeconds)
+	}
+	if !f.NoKeepAlive {
+		quicCfg.KeepAlivePeriod = secondsToDuration(f.KeepAliveTimeSeconds)
+	}
+
+	tlsConfig, err := f.getTLSConfig(authority, printer)
+	if err != nil {
+		return nil, err
+	}
+
+	udpConn, err := net.ListenUDP("udp", nil)
+	if err != nil {
+		return nil, err
+	}
+	transport := &quic.Transport{Conn: udpConn}
+	roundTripper := &http3.RoundTripper{
+		TLSClientConfig: tlsConfig,
+		QUICConfig:      quicCfg,
+		Dial: func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) {
+			printer.Printf("* Dialing (udp) %s...", addr)
+			udpAddr, err := net.ResolveUDPAddr("udp", addr)
+			if err != nil {
+				return nil, err
+			}
+			printer.Printf("* ALPN: offering %s", strings.Join(tlsCfg.NextProtos, ","))
+			conn, err := transport.DialEarly(ctx, udpAddr, tlsCfg, cfg)
+			if err != nil {
+				return nil, err
+			}
+			printer.Printf("* Connected to %s", conn.RemoteAddr().String())
+			return conn, err
+		},
+		EnableDatagrams:        false,
+		AdditionalSettings:     map[uint64]uint64{},
+		MaxResponseHeaderBytes: 0,
+		DisableCompression:     false,
+	}
+	return roundTripper, nil
 }
 
 func secondsToDuration(secs float64) time.Duration {