From d43aa6145316192cc2b7eb53a13a693545944608 Mon Sep 17 00:00:00 2001 From: Kim Tsao Date: Mon, 17 Oct 2022 13:59:59 -0400 Subject: [PATCH] backport proxy and custom timeout support to console v4.11 Signed-off-by: Kim Tsao --- README.md | 84 ++++++++++++++++++++----------------- pkg/devfile/parser/parse.go | 14 +++++-- pkg/util/util.go | 35 +++++++++++----- 3 files changed, 82 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 6d365edb..0d0cc3b5 100644 --- a/README.md +++ b/README.md @@ -20,20 +20,28 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g ```go // ParserArgs is the struct to pass into parser functions which contains required info for parsing devfile. parserArgs := parser.ParserArgs{ - Path: path, - FlattenedDevfile: &flattenedDevfile, - RegistryURLs: registryURLs, - DefaultNamespace: defaultNamespace, - Context: context, - K8sClient: client, - } + Path: path, + FlattenedDevfile: &flattenedDevfile, + RegistryURLs: registryURLs, + DefaultNamespace: defaultNamespace, + Context: context, + K8sClient: client, + } // Parses the devfile and validates the devfile data // if top-level variables are not substituted successfully, the warnings can be logged by parsing variableWarning devfile, variableWarning, err := devfilePkg.ParseDevfileAndValidate(parserArgs) ``` +2. To override the HTTP request and response timeouts for a devfile with a parent reference from a registry URL, specify the HTTPTimeout value in the parser arguments + ```go + // specify the timeout in seconds + httpTimeout := 20 + parserArgs := parser.ParserArgs{ + HTTPTimeout: &httpTimeout + } + ``` -2. To get specific content from devfile +3. To get specific content from devfile ```go // To get all the components from the devfile components, err := devfile.Data.GetComponents(DevfileOptions{}) @@ -42,55 +50,55 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g // & import: {strategy: Dockerfile} components, err := devfile.Data.GetComponents(DevfileOptions{ Filter: map[string]interface{}{ - "tool": "console-import", - "import": map[string]interface{}{ - "strategy": "Dockerfile", - }, - }, + "tool": "console-import", + "import": map[string]interface{}{ + "strategy": "Dockerfile", + }, + }, }) // To get all the volume components components, err := devfile.Data.GetComponents(DevfileOptions{ - ComponentOptions: ComponentOptions{ - ComponentType: v1.VolumeComponentType, - }, + ComponentOptions: ComponentOptions{ + ComponentType: v1.VolumeComponentType, + }, }) // To get all the exec commands that belong to the build group commands, err := devfile.Data.GetCommands(DevfileOptions{ - CommandOptions: CommandOptions{ - CommandType: v1.ExecCommandType, - CommandGroupKind: v1.BuildCommandGroupKind, - }, + CommandOptions: CommandOptions{ + CommandType: v1.ExecCommandType, + CommandGroupKind: v1.BuildCommandGroupKind, + }, }) ``` -3. To get the Kubernetes objects from the devfile, visit [generators.go source file](pkg/devfile/generator/generators.go) +4. To get the Kubernetes objects from the devfile, visit [generators.go source file](pkg/devfile/generator/generators.go) ```go // To get a slice of Kubernetes containers of type corev1.Container from the devfile component containers containers, err := generator.GetContainers(devfile) // To generate a Kubernetes deployment of type v1.Deployment deployParams := generator.DeploymentParams{ - TypeMeta: generator.GetTypeMeta(deploymentKind, deploymentAPIVersion), - ObjectMeta: generator.GetObjectMeta(name, namespace, labels, annotations), - InitContainers: initContainers, - Containers: containers, - Volumes: volumes, - PodSelectorLabels: labels, - } - deployment := generator.GetDeployment(deployParams) + TypeMeta: generator.GetTypeMeta(deploymentKind, deploymentAPIVersion), + ObjectMeta: generator.GetObjectMeta(name, namespace, labels, annotations), + InitContainers: initContainers, + Containers: containers, + Volumes: volumes, + PodSelectorLabels: labels, + } + deployment := generator.GetDeployment(deployParams) ``` -4. To update devfile content +5. To update devfile content ```go // To update an existing component in devfile object err := devfile.Data.UpdateComponent(v1.Component{ - Name: "component1", - ComponentUnion: v1.ComponentUnion{ - Container: &v1.ContainerComponent{ - Container: v1.Container{ - Image: "image1", + Name: "component1", + ComponentUnion: v1.ComponentUnion{ + Container: &v1.ContainerComponent{ + Container: v1.Container{ + Image: "image1", }, }, }, @@ -114,7 +122,7 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g err := devfile.Data.DeleteComponent(componentName) ``` -5. To write to a devfile, visit [writer.go source file](pkg/devfile/parser/writer.go) +6. To write to a devfile, visit [writer.go source file](pkg/devfile/parser/writer.go) ```go // If the devfile object has been created with devfile path already set, can simply call WriteYamlDevfile to writes the devfile err := devfile.WriteYamlDevfile() @@ -138,8 +146,8 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g // create devfile object with the new DevfileCtx and DevfileData devfile := parser.DevfileObj{ - Ctx: ctx, - Data: devfileData, + Ctx: ctx, + Data: devfileData, } // write to the devfile on disk diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index 066e14a1..18b4b83f 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -81,6 +81,8 @@ type ParserArgs struct { Context context.Context // K8sClient is the Kubernetes client instance used for interacting with a cluster K8sClient client.Client + // HTTPTimeout overrides the request and response timeout values for reading a parent devfile reference from the registry. If a negative value is specified, the default timeout will be used. + HTTPTimeout *int } // ParseDevfile func populates the devfile data, parses and validates the devfile integrity. @@ -104,6 +106,7 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { registryURLs: args.RegistryURLs, context: args.Context, k8sClient: args.K8sClient, + httpTimeout: args.HTTPTimeout, } flattenedDevfile := true @@ -138,6 +141,8 @@ type resolverTools struct { context context.Context // K8sClient is the Kubernetes client instance used for interacting with a cluster k8sClient client.Client + // httpTimeout is the timeout value in seconds passed in from the client. + httpTimeout *int } func populateAndParseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, tool resolverTools, flattenedDevfile bool) (DevfileObj, error) { @@ -378,7 +383,7 @@ func parseFromRegistry(importReference v1.ImportReference, resolveCtx *resolutio id := importReference.Id registryURL := importReference.RegistryUrl if registryURL != "" { - devfileContent, err := getDevfileFromRegistry(id, registryURL) + devfileContent, err := getDevfileFromRegistry(id, registryURL, tool.httpTimeout) if err != nil { return DevfileObj{}, err } @@ -392,7 +397,7 @@ func parseFromRegistry(importReference v1.ImportReference, resolveCtx *resolutio } else if tool.registryURLs != nil { for _, registryURL := range tool.registryURLs { - devfileContent, err := getDevfileFromRegistry(id, registryURL) + devfileContent, err := getDevfileFromRegistry(id, registryURL, tool.httpTimeout) if devfileContent != nil && err == nil { d.Ctx, err = devfileCtx.NewByteContentDevfileCtx(devfileContent) if err != nil { @@ -411,13 +416,16 @@ func parseFromRegistry(importReference v1.ImportReference, resolveCtx *resolutio return DevfileObj{}, fmt.Errorf("failed to get id: %s from registry URLs provided", id) } -func getDevfileFromRegistry(id, registryURL string) ([]byte, error) { +func getDevfileFromRegistry(id, registryURL string, httpTimeout *int) ([]byte, error) { if !strings.HasPrefix(registryURL, "http://") && !strings.HasPrefix(registryURL, "https://") { return nil, fmt.Errorf("the provided registryURL: %s is not a valid URL", registryURL) } param := util.HTTPRequestParams{ URL: fmt.Sprintf("%s/devfiles/%s", registryURL, id), } + + param.Timeout = httpTimeout + return util.HTTPGetRequest(param, 0) } diff --git a/pkg/util/util.go b/pkg/util/util.go index 7d10a9b7..fc03c8f4 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -40,10 +40,9 @@ import ( ) const ( - HTTPRequestTimeout = 30 * time.Second // HTTPRequestTimeout configures timeout of all HTTP requests - ResponseHeaderTimeout = 30 * time.Second // ResponseHeaderTimeout is the timeout to retrieve the server's response headers - ModeReadWriteFile = 0600 // default Permission for a file - CredentialPrefix = "odo-" // CredentialPrefix is the prefix of the credential that uses to access secure registry + HTTPRequestResponseTimeout = 30 * time.Second // HTTPRequestTimeout configures timeout of all HTTP requests + ModeReadWriteFile = 0600 // default Permission for a file + CredentialPrefix = "odo-" // CredentialPrefix is the prefix of the credential that uses to access secure registry ) // httpCacheDir determines directory where odo will cache HTTP respones @@ -70,8 +69,9 @@ type ResourceRequirementInfo struct { // HTTPRequestParams holds parameters of forming http request type HTTPRequestParams struct { - URL string - Token string + URL string + Token string + Timeout *int } // DownloadParams holds parameters of forming file download request @@ -723,11 +723,26 @@ func HTTPGetRequest(request HTTPRequestParams, cacheFor int) ([]byte, error) { req.Header.Add("Authorization", bearer) } + overriddenTimeout := HTTPRequestResponseTimeout + timeout := request.Timeout + if timeout != nil { + //if value is invalid, the default will be used + if *timeout > 0 { + //convert timeout to seconds + overriddenTimeout = time.Duration(*timeout) * time.Second + klog.V(4).Infof("HTTP request and response timeout overridden value is %v ", overriddenTimeout) + } else { + klog.V(4).Infof("Invalid httpTimeout is passed in, using default value") + } + + } + httpClient := &http.Client{ Transport: &http.Transport{ - ResponseHeaderTimeout: ResponseHeaderTimeout, + Proxy: http.ProxyFromEnvironment, + ResponseHeaderTimeout: overriddenTimeout, }, - Timeout: HTTPRequestTimeout, + Timeout: overriddenTimeout, } klog.V(4).Infof("HTTPGetRequest: %s", req.URL.String()) @@ -1022,8 +1037,8 @@ func DownloadFile(params DownloadParams) error { // DownloadFileInMemory uses the url to download the file and return bytes func DownloadFileInMemory(url string) ([]byte, error) { var httpClient = &http.Client{Transport: &http.Transport{ - ResponseHeaderTimeout: ResponseHeaderTimeout, - }, Timeout: HTTPRequestTimeout} + ResponseHeaderTimeout: HTTPRequestResponseTimeout, + }, Timeout: HTTPRequestResponseTimeout} resp, err := httpClient.Get(url) if err != nil { return nil, err