From c7b9767d774bc6175d23a8224c0bd2555cb57a05 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 21 Oct 2024 13:06:04 +0200 Subject: [PATCH 1/4] Added support for pip options in environment dependencies --- bundle/config/mutator/translate_paths_test.go | 4 ++++ bundle/libraries/local_path.go | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index c03cee73e0..1495d43286 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -699,6 +699,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) { "../dist/env2.whl", "simplejson", "/Workspace/Users/foo@bar.com/test.whl", + "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple bridgecraft", + "https://foo@bar.com/packages/pypi/simple", }, }, }, @@ -719,6 +721,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, strings.Join([]string{".", "dist", "env2.whl"}, string(os.PathSeparator)), b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) assert.Equal(t, "simplejson", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[2]) assert.Equal(t, "/Workspace/Users/foo@bar.com/test.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[3]) + assert.Equal(t, "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple bridgecraft", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[4]) + assert.Equal(t, "https://foo@bar.com/packages/pypi/simple", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[5]) } func TestTranslatePathWithComplexVariables(t *testing.T) { diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index 6d60d56bc6..69dca981b1 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -57,6 +57,12 @@ func IsLibraryLocal(dep string) bool { } } + // If the dependency starts with --, it's a pip flag option which is a valid + // entry for environment dependencies but not a local path + if strings.HasPrefix(dep, "--") { + return false + } + // If the dependency is a requirements file, it's not a valid local path if strings.HasPrefix(dep, "-r") { return false From f60707a47a7a33506aa49fe9be2c5eb9aa6f7abe Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 21 Oct 2024 13:24:21 +0200 Subject: [PATCH 2/4] Update bundle/config/mutator/translate_paths_test.go Co-authored-by: Pieter Noordhuis --- bundle/config/mutator/translate_paths_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 1495d43286..de7e6213dc 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -699,7 +699,7 @@ func TestTranslatePathJobEnvironments(t *testing.T) { "../dist/env2.whl", "simplejson", "/Workspace/Users/foo@bar.com/test.whl", - "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple bridgecraft", + "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple foobar", "https://foo@bar.com/packages/pypi/simple", }, }, From 10636312f0a0a47555a519905b443038931806fb Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 21 Oct 2024 13:24:27 +0200 Subject: [PATCH 3/4] Update bundle/config/mutator/translate_paths_test.go Co-authored-by: Pieter Noordhuis --- bundle/config/mutator/translate_paths_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index de7e6213dc..00ea1d0733 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -721,7 +721,7 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, strings.Join([]string{".", "dist", "env2.whl"}, string(os.PathSeparator)), b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) assert.Equal(t, "simplejson", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[2]) assert.Equal(t, "/Workspace/Users/foo@bar.com/test.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[3]) - assert.Equal(t, "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple bridgecraft", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[4]) + assert.Equal(t, "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple foobar", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[4]) assert.Equal(t, "https://foo@bar.com/packages/pypi/simple", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[5]) } From d6d557164b004e09b4198427a0d63a33ae5c9483 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 21 Oct 2024 13:39:35 +0200 Subject: [PATCH 4/4] flag can be anywhere --- bundle/config/mutator/translate_paths_test.go | 4 +++- bundle/libraries/local_path.go | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 00ea1d0733..9d655b27bf 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -700,6 +700,7 @@ func TestTranslatePathJobEnvironments(t *testing.T) { "simplejson", "/Workspace/Users/foo@bar.com/test.whl", "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple foobar", + "foobar --extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple", "https://foo@bar.com/packages/pypi/simple", }, }, @@ -722,7 +723,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, "simplejson", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[2]) assert.Equal(t, "/Workspace/Users/foo@bar.com/test.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[3]) assert.Equal(t, "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple foobar", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[4]) - assert.Equal(t, "https://foo@bar.com/packages/pypi/simple", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[5]) + assert.Equal(t, "foobar --extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[5]) + assert.Equal(t, "https://foo@bar.com/packages/pypi/simple", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[6]) } func TestTranslatePathWithComplexVariables(t *testing.T) { diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index 69dca981b1..53b7141044 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -59,7 +59,7 @@ func IsLibraryLocal(dep string) bool { // If the dependency starts with --, it's a pip flag option which is a valid // entry for environment dependencies but not a local path - if strings.HasPrefix(dep, "--") { + if containsPipFlag(dep) { return false } @@ -76,6 +76,11 @@ func IsLibraryLocal(dep string) bool { return IsLocalPath(dep) } +func containsPipFlag(input string) bool { + re := regexp.MustCompile(`--[a-zA-Z0-9-]+`) + return re.MatchString(input) +} + // ^[a-zA-Z0-9\-_]+: Matches the package name, allowing alphanumeric characters, dashes (-), and underscores (_). // \[.*\])?: Optionally matches any extras specified in square brackets, e.g., [security]. // ((==|!=|<=|>=|~=|>|<)\d+(\.\d+){0,2}(\.\*)?): Optionally matches version specifiers, supporting various operators (==, !=, etc.) followed by a version number (e.g., 2.25.1).