Trace flyctl deploy path (#2986)

* init tracing in flyctl deploy path

* downgrade buildkit moby/buildkit#4382

* instrument image resolving

* export traces into jaegar for testing

* authenticate all spans we send to honeycomb

* better way to auth, i think

* use dev & prod collectors accordingly

* instrument all flaps calls

* instrument all web calls

* lowercase values and keys, remove tracing dependency

* rip out duplicated tracing in flaps

* attach user id to spana

* linter wahala

* use ctx to set request type

* use snake case, address code review comments

* pass original context

* fix merge conflicts

* attach trace context to http requests

api/.golangci.yml

@@ -49,6 +49,10 @@ linters-settings:
         - github.com/Khan/genqlient
         - github.com/PuerkitoBio/rehttp
         - github.com/superfly/graphql
+        - go.opentelemetry.io/otel
+        - go.opentelemetry.io/otel/attribute
+        - go.opentelemetry.io/otel/codes
+        - go.opentelemetry.io/otel/trace
       domains:
         - golang.org
 

api/client.go

@@ -15,6 +15,10 @@ import (
 	genq "github.com/Khan/genqlient/graphql"
 	"github.com/superfly/flyctl/api/tokens"
 	"github.com/superfly/graphql"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 )
 
 var (
@@ -24,6 +28,20 @@ var (
 	defaultTransport http.RoundTripper = http.DefaultTransport
 )
 
+var contextKeyAction = contextKey("gql_action")
+
+func ctxWithAction(ctx context.Context, action string) context.Context {
+	return context.WithValue(ctx, contextKeyAction, action)
+}
+
+func actionFromCtx(ctx context.Context) string {
+	action := ctx.Value(contextKeyAction)
+	if action != nil {
+		return action.(string)
+	}
+	return "unknown_actiom"
+}
+
 // SetBaseURL - Sets the base URL for the API
 func SetBaseURL(url string) {
 	baseURL = url
@@ -139,8 +157,36 @@ func (c *Client) Run(req *graphql.Request) (Query, error) {
 
 func (c *Client) Logger() Logger { return c.logger }
 
+func (c *Client) getRequestType(r *graphql.Request) string {
+	query := r.Query()
+
+	if strings.Contains(query, "mutation") {
+		return "mutation"
+	}
+
+	if strings.Contains(query, "query") {
+		return "query"
+	}
+	return "unknown"
+}
+func (c *Client) getErrorFromErrors(errors Errors) string {
+	errs := []string{}
+	for _, err := range errors {
+		errs = append(errs, err.Message)
+	}
+
+	return strings.Join(errs, ",")
+}
+
 // RunWithContext - Runs a GraphQL request within a Go context
 func (c *Client) RunWithContext(ctx context.Context, req *graphql.Request) (Query, error) {
+	tracer := otel.GetTracerProvider().Tracer("github.com/superfly/flyctl/api")
+	ctx, span := tracer.Start(ctx, fmt.Sprintf("web.%s", actionFromCtx(ctx)), trace.WithAttributes(
+		attribute.String("request.action", actionFromCtx(ctx)),
+		attribute.String("request.type", c.getRequestType(req)),
+	))
+	defer span.End()
+
 	if instrumenter != nil {
 		start := time.Now()
 		defer func() {
@@ -151,6 +197,11 @@ func (c *Client) RunWithContext(ctx context.Context, req *graphql.Request) (Quer
 	var resp Query
 	err := c.client.Run(ctx, req, &resp)
 
+	if resp.Errors != nil {
+		span.RecordError(fmt.Errorf(c.getErrorFromErrors(resp.Errors)))
+		span.SetStatus(codes.Error, "failed to do grapqhl request")
+	}
+
 	if resp.Errors != nil && errorLog {
 		fmt.Fprintf(os.Stderr, "Error: %+v\n", resp.Errors)
 	}

api/go.mod

@@ -6,12 +6,17 @@ require (
 	github.com/Khan/genqlient v0.6.0
 	github.com/PuerkitoBio/rehttp v1.3.0
 	github.com/superfly/graphql v0.2.4
+	go.opentelemetry.io/otel v1.19.0
+	go.opentelemetry.io/otel/trace v1.19.0
 	golang.org/x/crypto v0.16.0
 )
 
 require (
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/vektah/gqlparser/v2 v2.5.1 // indirect
+	go.opentelemetry.io/otel/metric v1.19.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
 )

api/go.sum

@@ -13,6 +13,12 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -26,11 +32,17 @@ github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/superfly/graphql v0.2.4 h1:Av8hSk4x8WvKJ6MTnEwrLknSVSGPc7DWpgT3z/kt3PU=
 github.com/superfly/graphql v0.2.4/go.mod h1:CVfDl31srm8HnJ9udwLu6hFNUW/P6GUM2dKcG1YQ8jc=
 github.com/vektah/gqlparser/v2 v2.5.1 h1:ZGu+bquAY23jsxDRcYpWjttRZrUz07LbiY77gUOHcr4=
 github.com/vektah/gqlparser/v2 v2.5.1/go.mod h1:mPgqFBu/woKTVYWyNk8cO3kh4S/f4aRFZrvOnp3hmCs=
+go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
+go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
+go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
+go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
+go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
+go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
 golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
 golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=

api/resource_activity.go

@@ -29,6 +29,7 @@ func (c *Client) GetAppChanges(ctx context.Context, appName string) ([]AppChange
 	req := c.NewRequest(query)
 
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "get_app_changes")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {

api/resource_allocs.go

@@ -38,6 +38,7 @@ func (c *Client) GetAllocations(ctx context.Context, appName string, showComplet
 	req := c.NewRequest(query)
 	req.Var("appName", appName)
 	req.Var("showCompleted", showCompleted)
+	ctx = ctxWithAction(ctx, "get_allocations")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {
@@ -62,6 +63,7 @@ func (c *Client) GetAllocationTaskNames(ctx context.Context, appName string) (ma
 	req := c.NewRequest(query)
 	req.Var("appName", appName)
 	req.Var("showCompleted", false)
+	ctx = ctxWithAction(ctx, "get_allocations_task_names")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {

api/resource_apps.go

@@ -71,6 +71,7 @@ func (client *Client) getAppsPage(ctx context.Context, orgID *string, role *stri
 		`
 
 	req := client.NewRequest(query)
+	ctx = ctxWithAction(ctx, "get_apps_page")
 	if orgID != nil {
 		req.Var("org", *orgID)
 	}
@@ -100,6 +101,7 @@ func (client *Client) GetAppID(ctx context.Context, appName string) (string, err
 
 	req := client.NewRequest(query)
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "get_app_id")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -199,6 +201,7 @@ func (client *Client) GetApp(ctx context.Context, appName string) (*App, error)
 
 	req := client.NewRequest(query)
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "get_app")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -237,6 +240,7 @@ func (client *Client) GetAppCompact(ctx context.Context, appName string) (*AppCo
 
 	req := client.NewRequest(query)
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "get_app_compact")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -305,6 +309,7 @@ func (client *Client) GetAppInfo(ctx context.Context, appName string) (*AppInfo,
 
 	req := client.NewRequest(query)
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "get_app_info")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -333,6 +338,7 @@ func (client *Client) GetAppBasic(ctx context.Context, appName string) (*AppBasi
 
 	req := client.NewRequest(query)
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "get_app_basic")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -359,6 +365,7 @@ func (client *Client) GetAppMonitoring(ctx context.Context, appName string) (*Ap
 
 	req := client.NewRequest(query)
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "get_app_monitoring")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -393,6 +400,7 @@ func (client *Client) GetAppPostgres(ctx context.Context, appName string) (*AppP
 
 	req := client.NewRequest(query)
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "get_app_postgres")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -427,6 +435,7 @@ func (client *Client) CreateApp(ctx context.Context, input CreateAppInput) (*App
 	req := client.NewRequest(query)
 
 	req.Var("input", input)
+	ctx = ctxWithAction(ctx, "create_app")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -450,6 +459,7 @@ func (client *Client) DeleteApp(ctx context.Context, appName string) error {
 	req := client.NewRequest(query)
 
 	req.Var("appId", appName)
+	ctx = ctxWithAction(ctx, "delete_app")
 
 	_, err := client.RunWithContext(ctx, req)
 	return err
@@ -476,6 +486,7 @@ func (client *Client) MoveApp(ctx context.Context, appName string, orgID string)
 		"appId":          appName,
 		"organizationId": orgID,
 	})
+	ctx = ctxWithAction(ctx, "move_app")
 
 	data, err := client.RunWithContext(ctx, req)
 	return &data.App, err
@@ -502,6 +513,7 @@ func (client *Client) SuspendApp(ctx context.Context, appName string) (*App, err
 	req.Var("input", map[string]string{
 		"appId": appName,
 	})
+	ctx = ctxWithAction(ctx, "suspend_app")
 
 	data, err := client.RunWithContext(ctx, req)
 	return &data.SuspendApp.App, err
@@ -529,6 +541,8 @@ func (client *Client) ResumeApp(ctx context.Context, appName string) (*AppCompac
 		"appId": appName,
 	})
 
+	ctx = ctxWithAction(ctx, "resume_app")
+
 	data, err := client.RunWithContext(ctx, req)
 	return &data.ResumeApp.App, err
 }
@@ -551,6 +565,7 @@ func (client *Client) RestartApp(ctx context.Context, appName string) (*App, err
 	req.Var("input", map[string]string{
 		"appId": appName,
 	})
+	ctx = ctxWithAction(ctx, "restart_app")
 
 	data, err := client.RunWithContext(ctx, req)
 	return &data.RestartApp.App, err
@@ -574,6 +589,7 @@ func (client *Client) ResolveImageForApp(ctx context.Context, appName, imageRef
 	req := client.NewRequest(query)
 	req.Var("appName", appName)
 	req.Var("imageRef", imageRef)
+	ctx = ctxWithAction(ctx, "resolve_image")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {

api/resource_certificates.go

@@ -20,6 +20,7 @@ func (c *Client) GetAppCertificates(ctx context.Context, appName string) ([]AppC
 	req := c.NewRequest(query)
 
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "get_app_certificates")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {
@@ -75,6 +76,7 @@ func (c *Client) CheckAppCertificate(ctx context.Context, appName, hostname stri
 		"appId":    appName,
 		"hostname": hostname,
 	})
+	ctx = ctxWithAction(ctx, "check_app_certificates")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {
@@ -128,6 +130,7 @@ func (c *Client) AddCertificate(ctx context.Context, appName, hostname string) (
 
 	req.Var("appId", appName)
 	req.Var("hostname", hostname)
+	ctx = ctxWithAction(ctx, "add_certificates")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {
@@ -156,6 +159,7 @@ func (c *Client) DeleteCertificate(ctx context.Context, appName, hostname string
 
 	req.Var("appId", appName)
 	req.Var("hostname", hostname)
+	ctx = ctxWithAction(ctx, "delete_certificates")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {

api/resource_config.go

@@ -15,6 +15,7 @@ func (client *Client) GetConfig(ctx context.Context, appName string) (*AppConfig
 
 	req := client.NewRequest(query)
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "get_config")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -42,6 +43,7 @@ func (client *Client) ParseConfig(ctx context.Context, appName string, definitio
 	req := client.NewRequest(query)
 	req.Var("appName", appName)
 	req.Var("definition", definition)
+	ctx = ctxWithAction(ctx, "parse_config")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -64,6 +66,7 @@ func (client *Client) ValidateConfig(ctx context.Context, appName string, defini
 
 	req := client.NewRequest(query)
 	req.Var("definition", definition)
+	ctx = ctxWithAction(ctx, "validate_config")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {

api/resource_deploy.go

@@ -32,6 +32,7 @@ func (client *Client) DeployImage(ctx context.Context, input DeployImageInput) (
 	req := client.NewRequest(query)
 
 	req.Var("input", input)
+	ctx = ctxWithAction(ctx, "deploy_image")
 
 	data, err := client.RunWithContext(ctx, req)
 	if err != nil {
@@ -82,6 +83,7 @@ func (c *Client) GetDeploymentStatus(ctx context.Context, appName string, deploy
 	req.Var("appName", appName)
 	req.Var("deploymentId", deploymentID)
 	req.Var("evaluationId", evaluationID)
+	ctx = ctxWithAction(ctx, "get_deployment_status")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {
@@ -113,6 +115,7 @@ func (c *Client) GetReleaseCommand(ctx context.Context, id string) (*ReleaseComm
 	req := c.NewRequest(query)
 
 	req.Var("id", id)
+	ctx = ctxWithAction(ctx, "get_release_command")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {
@@ -132,6 +135,7 @@ func (c *Client) CanPerformBluegreenDeployment(ctx context.Context, appName stri
 	req := c.NewRequest(query)
 
 	req.Var("appName", appName)
+	ctx = ctxWithAction(ctx, "can_perform_bluegreen_deployment")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {

api/resource_dns.go

@@ -28,6 +28,7 @@ func (c *Client) GetDNSRecords(ctx context.Context, domainName string) ([]*DNSRe
 	req := c.NewRequest(query)
 
 	req.Var("domainName", domainName)
+	ctx = ctxWithAction(ctx, "get_dns_records")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {
@@ -55,6 +56,7 @@ func (c *Client) ExportDNSRecords(ctx context.Context, domainId string) (string,
 	req.Var("input", map[string]interface{}{
 		"domainId": domainId,
 	})
+	ctx = ctxWithAction(ctx, "export_dns_records")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {
@@ -93,6 +95,7 @@ func (c *Client) ImportDNSRecords(ctx context.Context, domainId string, zonefile
 		"domainId": domainId,
 		"zonefile": zonefile,
 	})
+	ctx = ctxWithAction(ctx, "import_dns_records")
 
 	data, err := c.RunWithContext(ctx, req)
 	if err != nil {