Skip to content

Commit

Permalink
Fixed using repo files as pipeline libraries (#847)
Browse files Browse the repository at this point in the history
## Changes
Fixed using repo files as pipeline libraries

## Tests
Added regression test
  • Loading branch information
andrewnester authored Oct 9, 2023
1 parent 054df2b commit 8d8de3f
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 3 deletions.
8 changes: 7 additions & 1 deletion bundle/config/mutator/expand_pipeline_glob_paths_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
Path: "dbfs:/me@company.com/test.ipynb",
},
},
{
Notebook: &pipelines.NotebookLibrary{
Path: "/Repos/somerepo/test.ipynb",
},
},
},
},
},
Expand All @@ -93,7 +98,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
require.NoError(t, err)

libraries := b.Config.Resources.Pipelines["pipeline"].Libraries
require.Len(t, libraries, 9)
require.Len(t, libraries, 10)

// Making sure glob patterns are expanded correctly
require.True(t, containsNotebook(libraries, filepath.Join("test", "test2.ipynb")))
Expand All @@ -107,6 +112,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
// Making sure absolute pass to remote FS file references work as well
require.True(t, containsNotebook(libraries, "/Workspace/Users/me@company.com/test.ipynb"))
require.True(t, containsNotebook(libraries, "dbfs:/me@company.com/test.ipynb"))
require.True(t, containsNotebook(libraries, "/Repos/somerepo/test.ipynb"))

// Making sure other libraries are not replaced
require.True(t, containsJar(libraries, "./*.jar"))
Expand Down
8 changes: 7 additions & 1 deletion bundle/libraries/libraries.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net/url"
"path"
"path/filepath"
"strings"

Expand Down Expand Up @@ -173,7 +174,7 @@ func IsLocalPath(path string) bool {
return false
}

return !isWorkspacePath(path)
return !isAbsoluteRemotePath(path)
}

func isExplicitFileScheme(path string) bool {
Expand All @@ -200,3 +201,8 @@ func isWorkspacePath(path string) bool {
strings.HasPrefix(path, "/Users/") ||
strings.HasPrefix(path, "/Shared/")
}

func isAbsoluteRemotePath(p string) bool {
// If path for library starts with /, it's a remote absolute path
return path.IsAbs(p)
}
2 changes: 1 addition & 1 deletion bundle/libraries/libraries_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (

var testCases map[string]bool = map[string]bool{
"./some/local/path": true,
"/some/full/path": true,
"/some/full/path": false,
"/Workspace/path/to/package": false,
"/Users/path/to/package": false,
"file://path/to/package": true,
Expand Down

0 comments on commit 8d8de3f

Please sign in to comment.