From 3f85c85b2650d998d71bab5542da5bc20a0942f3 Mon Sep 17 00:00:00 2001 From: Kim Tsao <84398375+kim-tsao@users.noreply.github.com> Date: Mon, 19 Sep 2022 13:55:47 -0400 Subject: [PATCH] backport proxy support. Includes fixes for both devfile/api#926 and devfile/api#897 (#139) Signed-off-by: Kim Tsao Signed-off-by: Kim Tsao --- registry-library/README.md | 37 +++++++----- registry-library/library/library.go | 77 +++++++++++++++--------- registry-library/library/library_test.go | 26 ++++++++ 3 files changed, 98 insertions(+), 42 deletions(-) diff --git a/registry-library/README.md b/registry-library/README.md index a9a7192a..cdca53a0 100644 --- a/registry-library/README.md +++ b/registry-library/README.md @@ -12,19 +12,18 @@ Devfile registry library is used for interacting with devfile registry, consumer ## How to use it 1. Import devfile registry library -```go -import ( - registryLibrary "github.com/devfile/registry-support/registry-library/library" -) -``` + ```go + import ( + registryLibrary "github.com/devfile/registry-support/registry-library/library" + ) + ``` 2. Invoke devfile registry library - a. Get the index of devfile registry for various devfile types ```go registryIndex, err := registryLibrary.GetRegistryIndex(registryURL, options, StackDevfileType) - if err != nil { - return err - } + if err != nil { + return err + } ``` b. Get the indices of multiple devfile registries for various devfile types ```go @@ -32,17 +31,17 @@ import ( ``` c. Download the stack devfile from devfile registry ```go - err := registryLibrary.PullStackByMediaTypesFromRegistry(registry, stack, registryLibrary.DevfileMediaTypeList, destDir, options) - if err != nil { - return err - } + err := registryLibrary.PullStackByMediaTypesFromRegistry(registry, stack, registryLibrary.DevfileMediaTypeList, destDir, options) + if err != nil { + return err + } ``` d. Download the whole stack from devfile registry ```go err := registryLibrary.PullStackFromRegistry(registry, stack, destDir, options) if err != nil { - return err - } + return err + } ``` e. Specify Options ```go @@ -54,3 +53,11 @@ import ( }, } ``` + f. Override the HTTP request and response timeout values + ```go + customTimeout := 20 + options := registryLibrary.RegistryOptions{ + HTTPTimeout: &customTimeout + } + ``` + diff --git a/registry-library/library/library.go b/registry-library/library/library.go index faeb597d..cc8c66d4 100644 --- a/registry-library/library/library.go +++ b/registry-library/library/library.go @@ -1,13 +1,17 @@ -// -// Copyright (c) 2020 Red Hat, Inc. -// This program and the accompanying materials are made -// available under the terms of the Eclipse Public License 2.0 -// which is available at https://www.eclipse.org/legal/epl-2.0/ -// -// SPDX-License-Identifier: EPL-2.0 -// -// Contributors: -// Red Hat, Inc. - initial API and implementation +/* Copyright 2020-2022 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package library @@ -47,8 +51,7 @@ const ( DevfilePNGLogoMediaType = "image/png" DevfileArchiveMediaType = "application/x-tar" - httpRequestTimeout = 30 * time.Second // httpRequestTimeout configures timeout of all HTTP requests - responseHeaderTimeout = 30 * time.Second // responseHeaderTimeout is the timeout to retrieve the server's response headers + httpRequestResponseTimeout = 30 * time.Second // httpRequestTimeout configures timeout of all HTTP requests ) var ( @@ -73,9 +76,15 @@ type TelemetryData struct { } type RegistryOptions struct { + // SkipTLSVerify is false by default which is the recommended setting for a devfile registry deployed in production. SkipTLSVerify should only be set to true + // if you are testing a devfile registry or proxy server that is set up with self-signed certificates in a pre-production environment. SkipTLSVerify bool - Telemetry TelemetryData - Filter RegistryFilter + // Telemetry allows clients to send telemetry data to the community Devfile Registry + Telemetry TelemetryData + // Filter allows clients to specify which architectures they want to filter their devfiles on + Filter RegistryFilter + // HTTPTimeout overrides the request and response timeout values for the custom HTTP clients set by the registry library. If unset or a negative value is specified, the default timeout of 30s will be used. + HTTPTimeout *int } type RegistryFilter struct { @@ -140,13 +149,8 @@ func GetRegistryIndex(registryURL string, options RegistryOptions, devfileTypes setHeaders(&req.Header, options) - httpClient := &http.Client{ - Transport: &http.Transport{ - ResponseHeaderTimeout: responseHeaderTimeout, - TLSClientConfig: &tls.Config{InsecureSkipVerify: options.SkipTLSVerify}, - }, - Timeout: httpRequestTimeout, - } + httpClient := getHTTPClient(options) + resp, err := httpClient.Do(req) if err != nil { return nil, err @@ -162,7 +166,7 @@ func GetRegistryIndex(registryURL string, options RegistryOptions, devfileTypes return registryIndex, nil } -// GetMultipleRegistryIndices returns returns the list of stacks and/or samples of multiple registries +// GetMultipleRegistryIndices returns the list of stacks and/or samples of multiple registries func GetMultipleRegistryIndices(registryURLs []string, options RegistryOptions, devfileTypes ...indexSchema.DevfileType) []Registry { registryList := make([]Registry, len(registryURLs)) registryContentsChannel := make(chan []indexSchema.Schema) @@ -242,11 +246,9 @@ func PullStackByMediaTypesFromRegistry(registry string, stack string, allowedMed if urlObj.Scheme == "https" { plainHTTP = false } - httpClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: options.SkipTLSVerify}, - }, - } + + httpClient := getHTTPClient(options) + headers := make(http.Header) setHeaders(&headers, options) @@ -344,3 +346,24 @@ func setHeaders(headers *http.Header, options RegistryOptions) { headers.Add("Locale", t.Locale) } } + +//getHTTPClient returns a new http client object +func getHTTPClient(options RegistryOptions) *http.Client { + + overriddenTimeout := httpRequestResponseTimeout + timeout := options.HTTPTimeout + //if value is invalid or unspecified, the default will be used + if timeout != nil && *timeout > 0 { + //convert timeout to seconds + overriddenTimeout = time.Duration(*timeout) * time.Second + } + + return &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ResponseHeaderTimeout: overriddenTimeout, + TLSClientConfig: &tls.Config{InsecureSkipVerify: options.SkipTLSVerify}, + }, + Timeout: overriddenTimeout, + } +} diff --git a/registry-library/library/library_test.go b/registry-library/library/library_test.go index fd581816..0791baa8 100644 --- a/registry-library/library/library_test.go +++ b/registry-library/library/library_test.go @@ -14,6 +14,8 @@ import ( func TestGetRegistryIndex(t *testing.T) { const serverIP = "127.0.0.1:8080" + invalidHTTPTimeout := -1 + validHTTPTimeout := 10 archFilteredIndex := []indexSchema.Schema{ { Name: "archindex1", @@ -135,6 +137,30 @@ func TestGetRegistryIndex(t *testing.T) { url: serverIP, wantErr: true, }, + { + name: "Get Registry Index with invalid httpTimeout value", + url: "http://" + serverIP, + options: RegistryOptions{ + HTTPTimeout: &invalidHTTPTimeout, + Filter: RegistryFilter{ + Architectures: []string{"amd64", "arm64"}, + }, + }, + devfileTypes: []indexSchema.DevfileType{indexSchema.SampleDevfileType}, + wantSchemas: archFilteredIndex, + }, + { + name: "Get Registry Index with valid httpTimeout value", + url: "http://" + serverIP, + options: RegistryOptions{ + HTTPTimeout: &validHTTPTimeout, + Filter: RegistryFilter{ + Architectures: []string{"amd64", "arm64"}, + }, + }, + devfileTypes: []indexSchema.DevfileType{indexSchema.SampleDevfileType}, + wantSchemas: archFilteredIndex, + }, } for _, test := range tests {