From 6814c615060246adc1f56f827c82587a95d85f97 Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Tue, 5 Mar 2024 14:11:18 +0100 Subject: [PATCH] [full-ci] enhancement: allow ocis to provide custom web applications (#8523) * enhancement: allow ocis to provide custom web applications * enhancement: add an option to disable web apps * test: add default logger tests * test: add app loading tests * test: add asset server tests * enhancement: make use of dedicated app conf file and app asset paths * enhancement: adjust asset locations and deprecate WEB_ASSET_PATH * enhancement: get rid of default logger and use the service level logger instead * Apply suggestions from code review Co-authored-by: Benedikt Kulmann Co-authored-by: kobergj * enhancement: use basename as app id * Apply suggestions from code review Co-authored-by: Martin * enhancement: use afero as fs abstraction * enhancement: simplify logo upload * enhancement: make use of introductionVersion field annotations --------- Co-authored-by: Benedikt Kulmann Co-authored-by: kobergj Co-authored-by: Martin --- .../unreleased/enhancement-web-app-loading.md | 11 + go.mod | 5 +- go.sum | 2 + internal/testenv/test.go | 34 ++ internal/testenv/test_test.go | 23 + ocis-pkg/assetsfs/assetsfs.go | 63 --- ocis-pkg/config/helpers.go | 4 +- ocis-pkg/log/log.go | 10 +- ocis-pkg/log/log_test.go | 24 + ocis-pkg/x/io/fsx/fallback.go | 37 ++ ocis-pkg/x/io/fsx/fallback_test.go | 83 ++++ ocis-pkg/x/io/fsx/fsx.go | 59 +++ ocis-pkg/x/io/fsx/fsx_test.go | 76 +++ ocis-pkg/x/path/filepathx/path.go | 12 + ocis-pkg/x/path/filepathx/path_test.go | 65 +++ services/antivirus/pkg/config/parser/parse.go | 4 +- services/idp/pkg/assets/option.go | 14 +- services/search/pkg/engine/bleve.go | 1 + services/web/Makefile | 2 +- services/web/README.md | 134 ++++- services/web/assets/{ => apps}/.keep | 0 services/web/assets/core/.keep | 0 services/web/pkg/apps/apps.go | 154 ++++++ services/web/pkg/apps/apps_test.go | 181 +++++++ services/web/pkg/assets/server.go | 29 +- services/web/pkg/assets/server_test.go | 125 +++++ services/web/pkg/config/config.go | 11 +- .../web/pkg/config/defaults/defaultconfig.go | 3 +- services/web/pkg/config/parser/parse.go | 25 +- services/web/pkg/server/http/server.go | 32 +- services/web/pkg/service/v0/branding.go | 21 +- services/web/pkg/service/v0/option.go | 41 +- services/web/pkg/service/v0/service.go | 32 +- services/web/web.go | 3 +- sonar-project.properties | 15 +- vendor/github.com/spf13/afero/.gitignore | 2 + vendor/github.com/spf13/afero/LICENSE.txt | 174 +++++++ vendor/github.com/spf13/afero/README.md | 442 +++++++++++++++++ vendor/github.com/spf13/afero/afero.go | 111 +++++ vendor/github.com/spf13/afero/appveyor.yml | 10 + vendor/github.com/spf13/afero/basepath.go | 222 +++++++++ .../github.com/spf13/afero/cacheOnReadFs.go | 315 ++++++++++++ vendor/github.com/spf13/afero/const_bsds.go | 23 + .../github.com/spf13/afero/const_win_unix.go | 22 + .../github.com/spf13/afero/copyOnWriteFs.go | 327 ++++++++++++ vendor/github.com/spf13/afero/httpFs.go | 114 +++++ .../spf13/afero/internal/common/adapters.go | 27 + vendor/github.com/spf13/afero/iofs.go | 298 +++++++++++ vendor/github.com/spf13/afero/ioutil.go | 243 +++++++++ vendor/github.com/spf13/afero/lstater.go | 27 + vendor/github.com/spf13/afero/match.go | 110 +++++ vendor/github.com/spf13/afero/mem/dir.go | 37 ++ vendor/github.com/spf13/afero/mem/dirmap.go | 43 ++ vendor/github.com/spf13/afero/mem/file.go | 359 ++++++++++++++ vendor/github.com/spf13/afero/memmap.go | 465 ++++++++++++++++++ vendor/github.com/spf13/afero/os.go | 113 +++++ vendor/github.com/spf13/afero/path.go | 106 ++++ vendor/github.com/spf13/afero/readonlyfs.go | 96 ++++ vendor/github.com/spf13/afero/regexpfs.go | 223 +++++++++ vendor/github.com/spf13/afero/symlink.go | 55 +++ vendor/github.com/spf13/afero/unionFile.go | 330 +++++++++++++ vendor/github.com/spf13/afero/util.go | 329 +++++++++++++ vendor/modules.txt | 5 + 63 files changed, 5834 insertions(+), 129 deletions(-) create mode 100644 changelog/unreleased/enhancement-web-app-loading.md create mode 100644 internal/testenv/test.go create mode 100644 internal/testenv/test_test.go delete mode 100644 ocis-pkg/assetsfs/assetsfs.go create mode 100644 ocis-pkg/log/log_test.go create mode 100644 ocis-pkg/x/io/fsx/fallback.go create mode 100644 ocis-pkg/x/io/fsx/fallback_test.go create mode 100644 ocis-pkg/x/io/fsx/fsx.go create mode 100644 ocis-pkg/x/io/fsx/fsx_test.go create mode 100644 ocis-pkg/x/path/filepathx/path.go create mode 100644 ocis-pkg/x/path/filepathx/path_test.go rename services/web/assets/{ => apps}/.keep (100%) create mode 100644 services/web/assets/core/.keep create mode 100644 services/web/pkg/apps/apps.go create mode 100644 services/web/pkg/apps/apps_test.go create mode 100644 services/web/pkg/assets/server_test.go create mode 100644 vendor/github.com/spf13/afero/.gitignore create mode 100644 vendor/github.com/spf13/afero/LICENSE.txt create mode 100644 vendor/github.com/spf13/afero/README.md create mode 100644 vendor/github.com/spf13/afero/afero.go create mode 100644 vendor/github.com/spf13/afero/appveyor.yml create mode 100644 vendor/github.com/spf13/afero/basepath.go create mode 100644 vendor/github.com/spf13/afero/cacheOnReadFs.go create mode 100644 vendor/github.com/spf13/afero/const_bsds.go create mode 100644 vendor/github.com/spf13/afero/const_win_unix.go create mode 100644 vendor/github.com/spf13/afero/copyOnWriteFs.go create mode 100644 vendor/github.com/spf13/afero/httpFs.go create mode 100644 vendor/github.com/spf13/afero/internal/common/adapters.go create mode 100644 vendor/github.com/spf13/afero/iofs.go create mode 100644 vendor/github.com/spf13/afero/ioutil.go create mode 100644 vendor/github.com/spf13/afero/lstater.go create mode 100644 vendor/github.com/spf13/afero/match.go create mode 100644 vendor/github.com/spf13/afero/mem/dir.go create mode 100644 vendor/github.com/spf13/afero/mem/dirmap.go create mode 100644 vendor/github.com/spf13/afero/mem/file.go create mode 100644 vendor/github.com/spf13/afero/memmap.go create mode 100644 vendor/github.com/spf13/afero/os.go create mode 100644 vendor/github.com/spf13/afero/path.go create mode 100644 vendor/github.com/spf13/afero/readonlyfs.go create mode 100644 vendor/github.com/spf13/afero/regexpfs.go create mode 100644 vendor/github.com/spf13/afero/symlink.go create mode 100644 vendor/github.com/spf13/afero/unionFile.go create mode 100644 vendor/github.com/spf13/afero/util.go diff --git a/changelog/unreleased/enhancement-web-app-loading.md b/changelog/unreleased/enhancement-web-app-loading.md new file mode 100644 index 00000000000..7850fa08d90 --- /dev/null +++ b/changelog/unreleased/enhancement-web-app-loading.md @@ -0,0 +1,11 @@ +Enhancement: Custom WEB App Loading + +We've added a new feature which allows the administrator of the environment to provide custom web applications to the +users. This feature is useful for organizations that have specific web applications that they want to provide to their +users. + +The users will then be able to access these custom web applications from the web ui. +For a detailed description of the feature, please read the WEB service README.md file. + +https://github.com/owncloud/ocis/pull/8523 +https://github.com/owncloud/ocis/issues/8392 diff --git a/go.mod b/go.mod index f0611d4d1c7..6a0523130f2 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/owncloud/ocis/v2 go 1.21 require ( + dario.cat/mergo v1.0.0 github.com/CiscoM31/godata v1.0.10 github.com/KimMachineGun/automemlimit v0.5.0 github.com/Masterminds/semver v1.5.0 @@ -79,6 +80,7 @@ require ( github.com/rs/zerolog v1.32.0 github.com/shamaton/msgpack/v2 v2.1.1 github.com/sirupsen/logrus v1.9.3 + github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 github.com/thejerf/suture/v4 v4.0.2 @@ -96,6 +98,7 @@ require ( go.opentelemetry.io/otel/sdk v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 golang.org/x/crypto v0.19.0 + golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 golang.org/x/image v0.15.0 golang.org/x/net v0.21.0 golang.org/x/oauth2 v0.17.0 @@ -114,7 +117,6 @@ require ( require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect - dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -333,7 +335,6 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.23.0 // indirect - golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/go.sum b/go.sum index 43edb7bafec..0f2268b466b 100644 --- a/go.sum +++ b/go.sum @@ -1968,6 +1968,8 @@ github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY52 github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= diff --git a/internal/testenv/test.go b/internal/testenv/test.go new file mode 100644 index 00000000000..b9f321edd40 --- /dev/null +++ b/internal/testenv/test.go @@ -0,0 +1,34 @@ +package testenv + +import ( + "fmt" + "os" + "os/exec" +) + +// CMDTest spawns a new independent test environment +type CMDTest struct { + n string + f func() +} + +// NewCMDTest creates a new CMDTest instance +func NewCMDTest(name string) CMDTest { + return CMDTest{ + n: name, + } +} + +// Run runs the cmd subtest +func (t CMDTest) Run(envs ...string) ([]byte, error) { + cmd := exec.Command(os.Args[0], fmt.Sprintf("-test.run=%s", t.n)) + cmd.Env = append(os.Environ(), "RUN_CMD_TEST=1") + cmd.Env = append(cmd.Env, envs...) + + return cmd.CombinedOutput() +} + +// ShouldRun checks if the cmd subtest should run +func (CMDTest) ShouldRun() bool { + return os.Getenv("RUN_CMD_TEST") == "1" +} diff --git a/internal/testenv/test_test.go b/internal/testenv/test_test.go new file mode 100644 index 00000000000..f4ca30a036b --- /dev/null +++ b/internal/testenv/test_test.go @@ -0,0 +1,23 @@ +package testenv + +import ( + "fmt" + "testing" + + "github.com/onsi/gomega" +) + +func TestNewSubTest(t *testing.T) { + testString := "this is a sub-test" + cmdTest := NewCMDTest(t.Name()) + if cmdTest.ShouldRun() { + fmt.Println(testString) + return + } + + out, err := cmdTest.Run() + + g := gomega.NewWithT(t) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(string(out)).To(gomega.ContainSubstring(testString)) +} diff --git a/ocis-pkg/assetsfs/assetsfs.go b/ocis-pkg/assetsfs/assetsfs.go deleted file mode 100644 index 826acff4ab1..00000000000 --- a/ocis-pkg/assetsfs/assetsfs.go +++ /dev/null @@ -1,63 +0,0 @@ -package assetsfs - -import ( - "fmt" - "io/fs" - "net/http" - "os" - "path/filepath" - - "github.com/owncloud/ocis/v2/ocis-pkg/log" -) - -// FileSystem customized to load assets -type FileSystem struct { - fs http.FileSystem - assetPath string - log log.Logger -} - -// Open checks if assetPath is set and tries to load from there. Falls back to fs if that is not possible -func (f *FileSystem) Open(original string) (http.File, error) { - if f.assetPath != "" { - file, err := os.Open(filepath.Join(f.assetPath, original)) - if err == nil { - return file, nil - } - } - return f.fs.Open(original) -} - -func (f *FileSystem) OpenEmbedded(name string) (http.File, error) { - return f.fs.Open(name) -} - -// Create creates a new file in the assetPath -func (f *FileSystem) Create(name string) (*os.File, error) { - fullPath := f.jailPath(name) - if err := os.MkdirAll(filepath.Dir(fullPath), 0770); err != nil { - return nil, err - } - return os.Create(fullPath) -} - -// jailPath returns the fullPath `/`. It makes sure that the path is -// always under `` to prevent directory traversal. -func (f *FileSystem) jailPath(name string) string { - return filepath.Join(f.assetPath, filepath.Join("/", name)) -} - -// New initializes a new FileSystem. Quits on error -func New(embedFS fs.FS, assetPath string, logger log.Logger) *FileSystem { - f, err := fs.Sub(embedFS, "assets") - if err != nil { - fmt.Println("Cannot load subtree fs:", err.Error()) - os.Exit(1) - } - - return &FileSystem{ - fs: http.FS(f), - assetPath: assetPath, - log: logger, - } -} diff --git a/ocis-pkg/config/helpers.go b/ocis-pkg/config/helpers.go index c0ec6f1f4cb..a7042dccb52 100644 --- a/ocis-pkg/config/helpers.go +++ b/ocis-pkg/config/helpers.go @@ -5,6 +5,7 @@ import ( gofig "github.com/gookit/config/v2" gooyaml "github.com/gookit/config/v2/yaml" + "github.com/owncloud/ocis/v2/ocis-pkg/config/defaults" ) @@ -25,8 +26,7 @@ func BindSourcesToStructs(service string, dst interface{}) (*gofig.Config, error }) cnf.AddDriver(gooyaml.Driver) - cfgFile := path.Join(defaults.BaseConfigPath(), service+".yaml") - _ = cnf.LoadFiles([]string{cfgFile}...) + _ = cnf.LoadFiles(path.Join(defaults.BaseConfigPath(), service+".yaml")) err := cnf.BindStruct("", &dst) if err != nil { diff --git a/ocis-pkg/log/log.go b/ocis-pkg/log/log.go index fda1fa050eb..24567189cc0 100644 --- a/ocis-pkg/log/log.go +++ b/ocis-pkg/log/log.go @@ -10,10 +10,11 @@ import ( chimiddleware "github.com/go-chi/chi/v5/middleware" mzlog "github.com/go-micro/plugins/v4/logger/zerolog" - "github.com/owncloud/ocis/v2/ocis-pkg/shared" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "go-micro.dev/v4/logger" + + "github.com/owncloud/ocis/v2/ocis-pkg/shared" ) var ( @@ -148,3 +149,10 @@ func (l Logger) SubloggerWithRequestID(c context.Context) Logger { l.With().Str(RequestIDString, chimiddleware.GetReqID(c)).Logger(), } } + +// Deprecation logs a deprecation message, +// it is used to inform the user that a certain feature is deprecated and will be removed in the future. +// Do not use a logger here because the message MUST be visible independent of the log level. +func Deprecation(a ...any) { + fmt.Printf("\033[1;31mDEPRECATION: %s\033[0m\n", a...) +} diff --git a/ocis-pkg/log/log_test.go b/ocis-pkg/log/log_test.go new file mode 100644 index 00000000000..ff43cb8e97b --- /dev/null +++ b/ocis-pkg/log/log_test.go @@ -0,0 +1,24 @@ +package log_test + +import ( + "testing" + + "github.com/onsi/gomega" + + "github.com/owncloud/ocis/v2/internal/testenv" + "github.com/owncloud/ocis/v2/ocis-pkg/log" +) + +func TestDeprecation(t *testing.T) { + cmdTest := testenv.NewCMDTest(t.Name()) + if cmdTest.ShouldRun() { + log.Deprecation("this is a deprecation") + return + } + + out, err := cmdTest.Run() + + g := gomega.NewWithT(t) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(string(out)).To(gomega.HavePrefix("\033[1;31mDEPRECATION: this is a deprecation")) +} diff --git a/ocis-pkg/x/io/fsx/fallback.go b/ocis-pkg/x/io/fsx/fallback.go new file mode 100644 index 00000000000..a45ac15524b --- /dev/null +++ b/ocis-pkg/x/io/fsx/fallback.go @@ -0,0 +1,37 @@ +package fsx + +import ( + "github.com/spf13/afero" +) + +var ( + // assert interfaces implemented + _ afero.Fs = (*FallbackFS)(nil) + _ FS = (*FallbackFS)(nil) +) + +// FallbackFS is a filesystem that layers a primary filesystem on top of a secondary filesystem. +type FallbackFS struct { + FS + primary *BaseFS + secondary *BaseFS +} + +// Primary returns the primary filesystem. +func (d *FallbackFS) Primary() *BaseFS { + return d.primary +} + +// Secondary returns the secondary filesystem. +func (d *FallbackFS) Secondary() *BaseFS { + return d.secondary +} + +// NewFallbackFS returns a new FallbackFS instance. +func NewFallbackFS(primary, secondary FS) *FallbackFS { + return &FallbackFS{ + FS: FromAfero(afero.NewCopyOnWriteFs(secondary, primary)), + primary: &BaseFS{Fs: primary}, + secondary: &BaseFS{Fs: secondary}, + } +} diff --git a/ocis-pkg/x/io/fsx/fallback_test.go b/ocis-pkg/x/io/fsx/fallback_test.go new file mode 100644 index 00000000000..cc61bf74b88 --- /dev/null +++ b/ocis-pkg/x/io/fsx/fallback_test.go @@ -0,0 +1,83 @@ +package fsx_test + +import ( + "io" + "testing" + + "github.com/onsi/gomega" + "github.com/spf13/afero" + + "github.com/owncloud/ocis/v2/ocis-pkg/x/io/fsx" +) + +func TestLayeredFS(t *testing.T) { + g := gomega.NewWithT(t) + + read := func(fs fsx.FS, name string) (string, error) { + f, err := fs.Open(name) + if err != nil { + return "", err + } + + b, err := io.ReadAll(f) + if err != nil { + return "", err + } + + return string(b), nil + } + + mustRead := func(fs fsx.FS, name string) string { + s, err := read(fs, name) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + return s + } + + create := func(fs fsx.FS, name, content string) { + err := afero.WriteFile(fs, name, []byte(content), 0644) + g.Expect(err).ToNot(gomega.HaveOccurred()) + } + + primary := fsx.NewMemMapFs() + create(primary, "both.txt", "primary") + g.Expect(mustRead(primary, "both.txt")).To(gomega.Equal("primary")) + create(primary, "primary.txt", "primary") + g.Expect(mustRead(primary, "primary.txt")).To(gomega.Equal("primary")) + + secondary := fsx.NewMemMapFs() + create(secondary, "both.txt", "secondary") + g.Expect(mustRead(secondary, "both.txt")).To(gomega.Equal("secondary")) + create(secondary, "secondary.txt", "secondary") + g.Expect(mustRead(secondary, "secondary.txt")).To(gomega.Equal("secondary")) + + fs := fsx.NewFallbackFS(primary, fsx.NewReadOnlyFs(secondary)) + g.Expect(mustRead(fs, "both.txt")).To(gomega.Equal("primary")) + g.Expect(mustRead(fs, "primary.txt")).To(gomega.Equal("primary")) + g.Expect(mustRead(fs, "secondary.txt")).To(gomega.Equal("secondary")) + + create(fs, "fallback-fs.txt", "fallback-fs") + g.Expect(mustRead(fs, "fallback-fs.txt")).To(gomega.Equal("fallback-fs")) + g.Expect(mustRead(primary, "fallback-fs.txt")).To(gomega.Equal("fallback-fs")) + g.Expect(mustRead(fs.Primary(), "fallback-fs.txt")).To(gomega.Equal("fallback-fs")) + _, err := read(secondary, "fallback-fs.txt") + g.Expect(err).To(gomega.HaveOccurred()) + _, err = read(fs.Secondary(), "fallback-fs.txt") + g.Expect(err).To(gomega.HaveOccurred()) +} + +func TestLayeredFS_Primary(t *testing.T) { + g := gomega.NewWithT(t) + primary := fsx.NewMemMapFs() + fs := fsx.NewFallbackFS(primary, fsx.NewMemMapFs()) + + g.Expect(primary).To(gomega.BeIdenticalTo(fs.Primary().Fs)) +} + +func TestLayeredFS_Secondary(t *testing.T) { + g := gomega.NewWithT(t) + secondary := fsx.NewMemMapFs() + fs := fsx.NewFallbackFS(fsx.NewMemMapFs(), secondary) + + g.Expect(secondary).To(gomega.BeIdenticalTo(fs.Secondary().Fs)) +} diff --git a/ocis-pkg/x/io/fsx/fsx.go b/ocis-pkg/x/io/fsx/fsx.go new file mode 100644 index 00000000000..45b14aeec55 --- /dev/null +++ b/ocis-pkg/x/io/fsx/fsx.go @@ -0,0 +1,59 @@ +package fsx + +import ( + "io/fs" + + "github.com/spf13/afero" +) + +var ( + // assert interfaces implemented + _ afero.Fs = (*BaseFS)(nil) + _ FS = (*BaseFS)(nil) +) + +// FS is our default interface for filesystems. +type FS interface { + afero.Fs + IOFS() fs.FS +} + +// BaseFS is our default implementation of the FS interface. +type BaseFS struct { + afero.Fs +} + +// IOFS returns the filesystem as an io/fs.FS. +func (b *BaseFS) IOFS() fs.FS { + return afero.NewIOFS(b) +} + +// FromAfero returns a new BaseFS instance from an afero.Fs. +func FromAfero(fSys afero.Fs) *BaseFS { + return &BaseFS{Fs: fSys} +} + +// FromIOFS returns a new BaseFS instance from an io/fs.FS. +func FromIOFS(fSys fs.FS) *BaseFS { + return FromAfero(&afero.FromIOFS{FS: fSys}) +} + +// NewBasePathFs returns a new BaseFS which wraps the given filesystem with a base path. +func NewBasePathFs(fSys FS, basePath string) *BaseFS { + return FromAfero(afero.NewBasePathFs(fSys, basePath)) +} + +// NewOsFs returns a new BaseFS which wraps the OS filesystem. +func NewOsFs() *BaseFS { + return FromAfero(afero.NewOsFs()) +} + +// NewReadOnlyFs returns a new BaseFS which wraps the given filesystem with a read-only filesystem. +func NewReadOnlyFs(FfSys FS) *BaseFS { + return FromAfero(afero.NewReadOnlyFs(FfSys)) +} + +// NewMemMapFs returns a new BaseFS which wraps the memory filesystem. +func NewMemMapFs() *BaseFS { + return FromAfero(afero.NewMemMapFs()) +} diff --git a/ocis-pkg/x/io/fsx/fsx_test.go b/ocis-pkg/x/io/fsx/fsx_test.go new file mode 100644 index 00000000000..238fbd5f647 --- /dev/null +++ b/ocis-pkg/x/io/fsx/fsx_test.go @@ -0,0 +1,76 @@ +package fsx_test + +import ( + "os" + "reflect" + "testing" + + "github.com/onsi/gomega" + + "github.com/owncloud/ocis/v2/ocis-pkg/x/io/fsx" +) + +func TestBase(t *testing.T) { + g := gomega.NewWithT(t) + wrapped := fsx.NewMemMapFs() + fs := fsx.BaseFS{Fs: wrapped.Fs} + + g.Expect(wrapped.Fs).Should(gomega.BeIdenticalTo(fs.Fs)) +} + +func TestBase_IOFS(t *testing.T) { + g := gomega.NewWithT(t) + fs := fsx.BaseFS{Fs: fsx.NewMemMapFs()} + g.Expect(reflect.TypeOf(fs.IOFS()).Name()).Should(gomega.Equal("IOFS")) +} + +func TestFromIOFS(t *testing.T) { + g := gomega.NewWithT(t) + fs := fsx.FromIOFS(os.DirFS(".")) + + g.Expect(reflect.TypeOf(fs.Fs).String()).Should(gomega.Equal("*afero.FromIOFS")) +} + +func TestNewOsFs(t *testing.T) { + g := gomega.NewWithT(t) + fs := fsx.NewOsFs() + + g.Expect(reflect.TypeOf(fs.Fs).String()).Should(gomega.Equal("*afero.OsFs")) +} + +func TestNewBasePathFs(t *testing.T) { + g := gomega.NewWithT(t) + base := fsx.NewMemMapFs() + + err := base.MkdirAll("first/foo/bar", 0755) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + err = base.MkdirAll("second/foo/baz", 0755) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + fs := fsx.NewBasePathFs(base, "first") + + info, err := fs.Stat("/foo/bar") + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(info.IsDir()).To(gomega.BeTrue()) + + info, err = fs.Stat("/foo/baz") + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(info).To(gomega.BeNil()) + + info, err = fs.Stat("../second/foo/baz") + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(info).To(gomega.BeNil()) +} + +func TestNewReadOnlyFs(t *testing.T) { + g := gomega.NewWithT(t) + base := fsx.NewMemMapFs() + + err := base.MkdirAll("first/foo/bar", 0755) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + fs := fsx.NewReadOnlyFs(base) + err = fs.MkdirAll("first/foo/bay", 0755) + g.Expect(err).To(gomega.HaveOccurred()) +} diff --git a/ocis-pkg/x/path/filepathx/path.go b/ocis-pkg/x/path/filepathx/path.go new file mode 100644 index 00000000000..8101e893316 --- /dev/null +++ b/ocis-pkg/x/path/filepathx/path.go @@ -0,0 +1,12 @@ +package filepathx + +import ( + "path/filepath" +) + +// JailJoin joins any number of path elements into a single path, +// it protects against directory traversal by removing any "../" elements +// and ensuring that the path is always under the jail. +func JailJoin(jail string, elem ...string) string { + return filepath.Join(jail, filepath.Join(append([]string{"/"}, elem...)...)) +} diff --git a/ocis-pkg/x/path/filepathx/path_test.go b/ocis-pkg/x/path/filepathx/path_test.go new file mode 100644 index 00000000000..fdd0640c5f2 --- /dev/null +++ b/ocis-pkg/x/path/filepathx/path_test.go @@ -0,0 +1,65 @@ +package filepathx_test + +import ( + "testing" + + "github.com/owncloud/ocis/v2/ocis-pkg/x/path/filepathx" +) + +func TestJailJoin(t *testing.T) { + type args struct { + jail string + elem []string + } + + tests := []struct { + name string + args args + want string + }{ + { + name: "regular use case", + args: args{ + jail: "/", + elem: []string{"a", "b", "c"}, + }, + want: "/a/b/c", + }, + { + name: "access parent directory", + args: args{ + jail: "/", + elem: []string{"a", "b", "c", ".."}, + }, + want: "/a/b", + }, + { + name: "restrict breaking out of jail", + args: args{ + jail: "/", + elem: []string{"a", "b", "c", "..", "..", "..", "..", "..", "..", ".."}, + }, + want: "/", + }, + { + name: "restrict to child of jail", + args: args{ + jail: "/a/b", + elem: []string{"a", "b", "c", "..", "..", "..", "..", "..", "..", ".."}, + }, + want: "/a/b", + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := filepathx.JailJoin(tt.args.jail, tt.args.elem...); got != tt.want { + t.Errorf("JailJoin() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/services/antivirus/pkg/config/parser/parse.go b/services/antivirus/pkg/config/parser/parse.go index cacf144acde..be14dc7b451 100644 --- a/services/antivirus/pkg/config/parser/parse.go +++ b/services/antivirus/pkg/config/parser/parse.go @@ -2,10 +2,10 @@ package parser import ( "errors" - "fmt" "time" ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/antivirus/pkg/config" "github.com/owncloud/ocis/v2/services/antivirus/pkg/config/defaults" @@ -38,7 +38,7 @@ func ParseConfig(cfg *config.Config) error { func Validate(cfg *config.Config) error { if cfg.Scanner.ICAP.DeprecatedTimeout != 0 { cfg.Scanner.ICAP.Timeout = time.Duration(cfg.Scanner.ICAP.DeprecatedTimeout) * time.Second - fmt.Println("ANTIVIRUS_ICAP_TIMEOUT is deprecated, use ANTIVIRUS_ICAP_SCAN_TIMEOUT instead") + log.Deprecation("ANTIVIRUS_ICAP_TIMEOUT is deprecated, use ANTIVIRUS_ICAP_SCAN_TIMEOUT instead") } return nil diff --git a/services/idp/pkg/assets/option.go b/services/idp/pkg/assets/option.go index fe8dbf575e8..76899e6553e 100644 --- a/services/idp/pkg/assets/option.go +++ b/services/idp/pkg/assets/option.go @@ -3,8 +3,8 @@ package assets import ( "net/http" - "github.com/owncloud/ocis/v2/ocis-pkg/assetsfs" "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/ocis-pkg/x/io/fsx" "github.com/owncloud/ocis/v2/services/idp" "github.com/owncloud/ocis/v2/services/idp/pkg/config" ) @@ -12,13 +12,21 @@ import ( // New returns a new http filesystem to serve assets. func New(opts ...Option) http.FileSystem { options := newOptions(opts...) - return assetsfs.New(idp.Assets, options.Config.Asset.Path, options.Logger) + + var assetFS fsx.FS = fsx.NewBasePathFs(fsx.FromIOFS(idp.Assets), "assets") + + // only use a fsx.NewFallbackFS and fsx.OsFs if a path is set, use the embedded fs only otherwise + if options.Config.Asset.Path != "" { + assetFS = fsx.NewFallbackFS(fsx.NewBasePathFs(fsx.NewOsFs(), options.Config.Asset.Path), assetFS) + } + + return http.FS(assetFS.IOFS()) } // Option defines a single option function. type Option func(o *Options) -// Options defines the available options for this package. +// Options define the available options for this package. type Options struct { Logger log.Logger Config *config.Config diff --git a/services/search/pkg/engine/bleve.go b/services/search/pkg/engine/bleve.go index 54375d99250..b473260a59a 100644 --- a/services/search/pkg/engine/bleve.go +++ b/services/search/pkg/engine/bleve.go @@ -27,6 +27,7 @@ import ( "github.com/cs3org/reva/v2/pkg/utils" libregraph "github.com/owncloud/libre-graph-api-go" + searchMessage "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/search/v0" searchService "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0" "github.com/owncloud/ocis/v2/services/search/pkg/content" diff --git a/services/web/Makefile b/services/web/Makefile index f0649750e08..2937b1ccdd5 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -33,7 +33,7 @@ ci-node-generate: pull-assets .PHONY: pull-assets pull-assets: git clean -xfd assets - curl --fail -slL -o- https://github.com/owncloud/web/releases/download/$(WEB_ASSETS_VERSION)/web.tar.gz | tar xzf - -C assets/ + curl --fail -slL -o- https://github.com/owncloud/web/releases/download/$(WEB_ASSETS_VERSION)/web.tar.gz | tar xzf - -C assets/core/ ############ licenses ############ .PHONY: ci-node-check-licenses diff --git a/services/web/README.md b/services/web/README.md index ad459c31c0a..0fad3d18240 100644 --- a/services/web/README.md +++ b/services/web/README.md @@ -1,26 +1,148 @@ # Web -The web service embeds and serves the static files for the [Infinite Scale Web Client](https://github.com/owncloud/web). +The web service embeds and serves the static files for the [Infinite Scale Web Client](https://github.com/owncloud/web). Note that clients will respond with a connection error if the web service is not available. The web service also provides a minimal API for branding functionality like changing the logo shown. ## Custom Compiled Web Assets -If you want to use your custom compiled web client assets instead of the embedded ones, then you can do that by setting the `WEB_ASSET_PATH` variable to point to your compiled files. See [ownCloud Web / Getting Started](https://owncloud.dev/clients/web/getting-started/) and [ownCloud Web / Setup with oCIS](https://owncloud.dev/clients/web/backend-ocis/) for more details. +If you want to use your custom compiled web client assets instead of the embedded ones, +then you can do that by setting the `WEB_ASSET_PATH` variable to point to your compiled files. +See [ownCloud Web / Getting Started](https://owncloud.dev/clients/web/getting-started/) and [ownCloud Web / Setup with oCIS](https://owncloud.dev/clients/web/backend-ocis/) for more details. ## Web UI Configuration -Note that single configuration settings of the embedded web UI can be defined via `WEB_OPTION_xxx` environment variables. If a json based configuration file is used via the `WEB_UI_CONFIG_FILE` environment variable, these configurations take precedence over single options set. +Note that single configuration settings of the embedded web UI can be defined via `WEB_OPTION_xxx` environment variables. + +If a json based configuration file is used via the `WEB_UI_CONFIG_FILE` environment variable, these configurations take +precedence over single options set. ### Web UI Options -Beside theming, the behavior of the web UI can be configured via options. See the environment variables `WEB_OPTION_xxx` for more details. +Besides theming, the behavior of the web UI can be configured via options. See the environment variables `WEB_OPTION_xxx` +for more details. ### Web UI Config File -When defined via the `WEB_UI_CONFIG_FILE` environment variable, the configuration of the web UI can be made with a [json based](https://github.com/owncloud/web/tree/master/config) file. +When defined via the `WEB_UI_CONFIG_FILE` environment variable, the configuration of the web UI can be made +with a [json based](https://github.com/owncloud/web/tree/master/config) file. ### Embedding Web -Web can be consumed by another application in a stripped down version called “Embed mode”. This mode is supposed to be used in the context of selecting or sharing resources. For more details see the developer documentation [ownCloud Web / Embed Mode](https://owncloud.dev/clients/web/embed-mode/). See the environment variables: `WEB_OPTION_MODE` and `WEB_OPTION_EMBED_TARGET` to configure the embedded mode. +Web can be consumed by another application in a stripped down version called “Embed mode”. +This mode is supposed to be used in the context of selecting or sharing resources. + +For more details see the developer documentation [ownCloud Web / Embed Mode](https://owncloud.dev/clients/web/embed-mode/). +See the environment variables: `WEB_OPTION_MODE` and `WEB_OPTION_EMBED_TARGET` to configure the embedded mode. + +# Web Apps + +The administrator of the environment is capable of providing custom web applications to the users. +This feature is useful for organizations that want to provide third party or custom apps to their users. + +It's important to note that the feature at the moment is only capable of providing static (js, mjs, e.g.) web applications +and does not support injection of dynamic web applications (custom dynamic backends). + +## Loading Applications + +Web applications are loaded from the buildin FS in the ocis binary, e.g. `ocis_src_path/services/web/assets/apps` +this cannot be manipulated at runtime only at build-time. + +Additionally, the administrator can provide custom applications in one of the following ways: + +* By loading a web application from the default ocis base directory, e.g. `$OCIS_BASE_DATA_PATH/web/assets/apps` (default behavior). +* By loading a web application from a user-provided path, by setting the `WEB_ASSET_APPS_PATH` environment variable. + +The list of available applications is composed of the build in extensions and the custom applications +provided by the administrator, e.g. `WEB_ASSET_APPS_PATH` or `$OCIS_BASE_DATA_PATH/web/assets/apps`. + +For example, if ocis would contain a build in extension named `image-viewer-dfx` and the administrator provides a custom +application named `image-viewer-obj` in the `WEB_ASSET_APPS_PATH` directory, the user will be able to access both +applications from the web ui. + +## Application Structure + +Applications always have to follow a strict structure, which is as follows: + +* each application must be in its own directory +* each application directory must contain a `manifest.json` file + +Everything else is skipped and not considered as an application. + +The `manifest.json` file contains the following fields: + +* `entrypoint` - required - the entrypoint of the application, e.g. `index.js`, the path is relative to the parent directory +* `config` - optional - a list of key-value pairs that are passed to the global web application configuration + +## Application Configuration + +It's important to note that an application manifest should never be changed manually; +if a custom configuration is needed, the administrator should provide the required configuration inside the +`$OCIS_BASE_DATA_PATH/config/apps.yaml` file. + +The `apps.yaml` file must contain a list of key-value pairs which gets merged with the `config` field. +For example, if the `image-viewer-obj` application contains the following configuration: + +```json +{ + "entrypoint": "index.js", + "config": { + "maxWith": 1280, + "maxHeight": 1280 + } +} +``` + +and the `apps.yaml` file contains the following configuration: + +```yaml +image-viewer-obj: + config: + maxHeight: 640 + maxSize: 512 +``` + +the final configuration for web will be: + +```json +{ + "external_apps": [ + { + "id": "image-viewer-obj", + "path": "index.js", + "config": { + "maxWith": 1280, + "maxHeight": 640, + "maxSize": 512 + } + } + ] +} +``` + +besides the configuration from the `manifest.json` file, the `apps.yaml` file can also contain the following fields: + +* `disabled` - optional - defaults to `false` - if set to `true`, the application will not be loaded + +The local provided configuration yaml will always override the shipped application manifest configuration. + +## Fallback Mechanism + +Besides the configuration and application registration, there is one further important aspect to know; +in the process of loading the application assets, the system uses a fallback mechanism to load the assets. + +This is incredibly useful for cases where just a single asset should be overwritten, e.g., a logo or similar. + +Consider the following, ocis is shipped with a default extension named `image-viewer-dfx` which contains a logo, +but the administrator wants to provide a custom logo for the `image-viewer-dfx` application. + +This can be achieved by providing a custom logo in the `WEB_ASSET_APPS_PATH` directory, +e.g. `WEB_ASSET_APPS_PATH/image-viewer-dfx/logo.png`. +Every other asset is loaded from the build in extension, but the logo is loaded from the custom directory. + +The same applies for the `manifest.json` file, if the administrator wants to provide a custom `manifest.json` file. + +## Miscellaneous + +Please note that ocis needs a restart to load new applications or changes to the `apps.yaml` file. diff --git a/services/web/assets/.keep b/services/web/assets/apps/.keep similarity index 100% rename from services/web/assets/.keep rename to services/web/assets/apps/.keep diff --git a/services/web/assets/core/.keep b/services/web/assets/core/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/web/pkg/apps/apps.go b/services/web/pkg/apps/apps.go new file mode 100644 index 00000000000..b00cc84f791 --- /dev/null +++ b/services/web/pkg/apps/apps.go @@ -0,0 +1,154 @@ +package apps + +import ( + "encoding/json" + "errors" + "io/fs" + "path" + + "dario.cat/mergo" + "github.com/go-playground/validator/v10" + "golang.org/x/exp/maps" + + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/ocis-pkg/x/path/filepathx" + "github.com/owncloud/ocis/v2/services/web/pkg/config" +) + +var ( + validate *validator.Validate + + // ErrInvalidApp is the error when an app is invalid + ErrInvalidApp = errors.New("invalid app") + + // ErrMissingManifest is the error when the manifest is missing + ErrMissingManifest = errors.New("missing manifest") + + // ErrInvalidManifest is the error when the manifest is invalid + ErrInvalidManifest = errors.New("invalid manifest") + + // ErrEntrypointDoesNotExist is the error when the entrypoint does not exist or is not a file + ErrEntrypointDoesNotExist = errors.New("entrypoint does not exist") +) + +const ( + // _manifest is the name of the manifest file for an application + _manifest = "manifest.json" +) + +func init() { + validate = validator.New(validator.WithRequiredStructEnabled()) +} + +// Application contains the metadata of an application +type Application struct { + // ID is the unique identifier of the application + ID string + + // Entrypoint is the entrypoint of the application within the bundle + Entrypoint string `json:"entrypoint" validate:"required"` + + // Config contains the application-specific configuration + Config map[string]interface{} `json:"config,omitempty"` +} + +// ToExternal converts an Application to an ExternalApp configuration +func (a Application) ToExternal(entrypoint string) config.ExternalApp { + return config.ExternalApp{ + ID: a.ID, + Path: filepathx.JailJoin(entrypoint, a.Entrypoint), + Config: a.Config, + } +} + +// List returns a list of applications from the given filesystems, +// individual filesystems are searched for applications, and the list is merged. +// Last finding gets priority in case of conflicts, so the order of the filesystems is important. +func List(logger log.Logger, data map[string]config.App, fSystems ...fs.FS) []Application { + registry := map[string]Application{} + + for _, fSystem := range fSystems { + if fSystem == nil { + continue + } + + entries, err := fs.ReadDir(fSystem, ".") + if err != nil { + // skip non-directory listings, every app needs to be contained inside a directory + continue + } + + for _, entry := range entries { + var appData config.App + name := entry.Name() + + // configuration for the application is optional, if it is not present, the default configuration is used + if data, ok := data[name]; ok { + appData = data + } + + if appData.Disabled { + // if the app is disabled, skip it + continue + } + + application, err := Build(fSystem, name, appData.Config) + if err != nil { + // if app creation fails, log the error and continue with the next app + logger.Debug().Err(err).Str("path", entry.Name()).Msg("failed to load application") + continue + } + + // everything is fine, add the application to the list of applications + registry[name] = application + } + } + + return maps.Values(registry) +} + +func Build(fSystem fs.FS, id string, conf map[string]any) (Application, error) { + // skip non-directory listings, every app needs to be contained inside a directory + entry, err := fs.Stat(fSystem, id) + if err != nil || !entry.IsDir() { + return Application{}, ErrInvalidApp + } + + // read the manifest.json from the app directory. + manifest := path.Join(id, _manifest) + reader, err := fSystem.Open(manifest) + if err != nil { + // manifest.json is required + return Application{}, errors.Join(err, ErrMissingManifest) + } + defer reader.Close() + + var application Application + if json.NewDecoder(reader).Decode(&application) != nil { + // a valid manifest.json is required + return Application{}, errors.Join(err, ErrInvalidManifest) + } + + if err := validate.Struct(application); err != nil { + // the application is required to be valid + return Application{}, errors.Join(err, ErrInvalidManifest) + } + + // overload the default configuration with the application-specific configuration, + // the application-specific configuration has priority, and failing is fine here + _ = mergo.Merge(&application.Config, conf, mergo.WithOverride) + + // the entrypoint is jailed to the app directory + application.Entrypoint = filepathx.JailJoin(id, application.Entrypoint) + info, err := fs.Stat(fSystem, application.Entrypoint) + switch { + case err != nil: + return Application{}, errors.Join(err, ErrEntrypointDoesNotExist) + case info.IsDir(): + return Application{}, ErrEntrypointDoesNotExist + } + + application.ID = id + + return application, nil +} diff --git a/services/web/pkg/apps/apps_test.go b/services/web/pkg/apps/apps_test.go new file mode 100644 index 00000000000..0f226ed9b70 --- /dev/null +++ b/services/web/pkg/apps/apps_test.go @@ -0,0 +1,181 @@ +package apps_test + +import ( + "io/fs" + "testing" + "testing/fstest" + + "github.com/onsi/gomega" + + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/services/web/pkg/apps" + "github.com/owncloud/ocis/v2/services/web/pkg/config" +) + +func TestApplication_ToExternal(t *testing.T) { + g := gomega.NewWithT(t) + app := apps.Application{ + ID: "app", + Entrypoint: "entrypoint.js", + Config: map[string]interface{}{ + "foo": "bar", + }, + } + + externalApp := app.ToExternal("path") + + g.Expect(externalApp.ID).To(gomega.Equal("app")) + g.Expect(externalApp.Path).To(gomega.Equal("path/entrypoint.js")) + g.Expect(externalApp.Config).To(gomega.Equal(app.Config)) +} + +func TestBuild(t *testing.T) { + g := gomega.NewWithT(t) + dir := &fstest.MapFile{ + Mode: fs.ModeDir, + } + + _, err := apps.Build(fstest.MapFS{ + "app": &fstest.MapFile{}, + }, "app", map[string]any{}) + g.Expect(err).To(gomega.MatchError(apps.ErrInvalidApp)) + + _, err = apps.Build(fstest.MapFS{ + "app": dir, + }, "app", map[string]any{}) + g.Expect(err).To(gomega.MatchError(apps.ErrMissingManifest)) + + _, err = apps.Build(fstest.MapFS{ + "app": dir, + "app/manifest.json": dir, + }, "app", map[string]any{}) + g.Expect(err).To(gomega.MatchError(apps.ErrInvalidManifest)) + + _, err = apps.Build(fstest.MapFS{ + "app": dir, + "app/manifest.json": &fstest.MapFile{ + Data: []byte("{}"), + }, + }, "app", map[string]any{}) + g.Expect(err).To(gomega.MatchError(apps.ErrInvalidManifest)) + + _, err = apps.Build(fstest.MapFS{ + "app": dir, + "app/entrypoint.js": &fstest.MapFile{}, + "app/manifest.json": &fstest.MapFile{ + Data: []byte(`{"id":"app", "entrypoint":"entrypoint.js"}`), + }, + }, "app", map[string]any{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + _, err = apps.Build(fstest.MapFS{ + "app": dir, + "app/entrypoint.js": dir, + "app/manifest.json": &fstest.MapFile{ + Data: []byte(`{"id":"app", "entrypoint":"entrypoint.js"}`), + }, + }, "app", map[string]any{}) + g.Expect(err).To(gomega.MatchError(apps.ErrEntrypointDoesNotExist)) + + _, err = apps.Build(fstest.MapFS{ + "app": dir, + "app/manifest.json": &fstest.MapFile{ + Data: []byte(`{"id":"app", "entrypoint":"entrypoint.js"}`), + }, + }, "app", map[string]any{}) + g.Expect(err).To(gomega.MatchError(apps.ErrEntrypointDoesNotExist)) + + application, err := apps.Build(fstest.MapFS{ + "app": dir, + "app/entrypoint.js": &fstest.MapFile{}, + "app/manifest.json": &fstest.MapFile{ + Data: []byte(`{"id":"app", "entrypoint":"entrypoint.js", "config": {"foo": "1", "bar": "2"}}`), + }, + }, "app", map[string]any{"foo": "overwritten-1", "baz": "injected-1"}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + g.Expect(application.Entrypoint).To(gomega.Equal("app/entrypoint.js")) + g.Expect(application.Config).To(gomega.Equal(map[string]interface{}{ + "foo": "overwritten-1", "baz": "injected-1", "bar": "2", + })) +} + +func TestList(t *testing.T) { + g := gomega.NewWithT(t) + + applications := apps.List(log.NopLogger(), map[string]config.App{}) + g.Expect(applications).To(gomega.BeEmpty()) + + applications = apps.List(log.NopLogger(), map[string]config.App{}, nil) + g.Expect(applications).To(gomega.BeEmpty()) + + applications = apps.List(log.NopLogger(), map[string]config.App{}, fstest.MapFS{}) + g.Expect(applications).To(gomega.BeEmpty()) + + dir := &fstest.MapFile{ + Mode: fs.ModeDir, + } + + applications = apps.List(log.NopLogger(), map[string]config.App{ + "app": { + Disabled: true, + }, + }, fstest.MapFS{ + "app": dir, + }) + g.Expect(applications).To(gomega.BeEmpty()) + + applications = apps.List(log.NopLogger(), map[string]config.App{ + "app": {}, + }, fstest.MapFS{ + "app": dir, + }) + g.Expect(applications).To(gomega.BeEmpty()) + + applications = apps.List(log.NopLogger(), map[string]config.App{ + "app-3": { + Config: map[string]any{ + "foo": "local conf 1", + "bar": "local conf 2", + }, + }, + }, fstest.MapFS{ + "app-1": dir, + "app-1/entrypoint.js": &fstest.MapFile{}, + "app-1/manifest.json": &fstest.MapFile{ + Data: []byte(`{"id":"app-1", "entrypoint":"entrypoint.js", "config": {"foo": "fs1"}}`), + }, + "app-2": dir, + "app-2/entrypoint.js": &fstest.MapFile{}, + "app-2/manifest.json": &fstest.MapFile{ + Data: []byte(`{"id":"app-2", "entrypoint":"entrypoint.js", "config": {"foo": "fs1"}}`), + }, + }, fstest.MapFS{ + "app-1": dir, + "app-1/entrypoint.js": &fstest.MapFile{}, + "app-1/manifest.json": &fstest.MapFile{ + Data: []byte(`{"id":"app-1", "entrypoint":"entrypoint.js", "config": {"foo": "fs2"}}`), + }, + "app-3": dir, + "app-3/entrypoint.js": &fstest.MapFile{}, + "app-3/manifest.json": &fstest.MapFile{ + Data: []byte(`{"id":"app-3", "entrypoint":"entrypoint.js", "config": {"foo": "fs2"}}`), + }, + }) + g.Expect(len(applications)).To(gomega.Equal(3)) + + for _, application := range applications { + switch { + case application.Entrypoint == "app-1/entrypoint.js": + g.Expect(application.Config["foo"]).To(gomega.Equal("fs2")) + case application.Entrypoint == "app-2/entrypoint.js": + g.Expect(application.Config["foo"]).To(gomega.Equal("fs1")) + case application.Entrypoint == "app-3/entrypoint.js": + g.Expect(application.Config["foo"]).To(gomega.Equal("local conf 1")) + g.Expect(application.Config["bar"]).To(gomega.Equal("local conf 2")) + default: + t.Fatalf("unexpected application %s", application.Entrypoint) + } + } + +} diff --git a/services/web/pkg/assets/server.go b/services/web/pkg/assets/server.go index 0e487cc29dd..ab5c90fb8ad 100644 --- a/services/web/pkg/assets/server.go +++ b/services/web/pkg/assets/server.go @@ -3,6 +3,7 @@ package assets import ( "bytes" "io" + "io/fs" "mime" "net/http" "path" @@ -12,32 +13,42 @@ import ( ) type fileServer struct { - root http.FileSystem + fsys http.FileSystem } -func FileServer(root http.FileSystem) http.Handler { - return &fileServer{root} +func FileServer(fsys fs.FS) http.Handler { + return &fileServer{http.FS(fsys)} } func (f *fileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { - upath := path.Clean(path.Join("/", r.URL.Path)) - r.URL.Path = upath + uPath := path.Clean(path.Join("/", r.URL.Path)) + r.URL.Path = uPath - fallbackIndex := func() { + tryIndex := func() { r.URL.Path = "/index.html" + + // not every fs contains a file named index.html, + // therefore, we need to check if the file exists and stop the recursion if it doesn't + file, err := f.fsys.Open(r.URL.Path) + if err != nil { + http.NotFound(w, r) + return + } + defer file.Close() + f.ServeHTTP(w, r) } - asset, err := f.root.Open(upath) + asset, err := f.fsys.Open(uPath) if err != nil { - fallbackIndex() + tryIndex() return } defer asset.Close() s, _ := asset.Stat() if s.IsDir() { - fallbackIndex() + tryIndex() return } diff --git a/services/web/pkg/assets/server_test.go b/services/web/pkg/assets/server_test.go new file mode 100644 index 00000000000..de6e42565b6 --- /dev/null +++ b/services/web/pkg/assets/server_test.go @@ -0,0 +1,125 @@ +package assets_test + +import ( + "fmt" + "io" + "io/fs" + "net/http" + "net/http/httptest" + "testing" + "testing/fstest" + + "github.com/onsi/gomega" + + "github.com/owncloud/ocis/v2/services/web/pkg/assets" +) + +func TestFileServer(t *testing.T) { + g := gomega.NewWithT(t) + recorderStatus := func(s int) string { + return fmt.Sprintf("%03d %s", s, http.StatusText(s)) + } + + { + s := assets.FileServer(fstest.MapFS{}) + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/foo", nil) + //defer req.Body.Close() + s.ServeHTTP(w, req) + res := w.Result() + defer res.Body.Close() + + g.Expect(res.Status).To(gomega.Equal(recorderStatus(http.StatusNotFound))) + } + + for _, tt := range []struct { + name string + url string + fs fstest.MapFS + expected string + }{ + { + name: "not found fallback", + url: "/index.txt", + fs: fstest.MapFS{ + "index.html": &fstest.MapFile{ + Data: []byte("index file content"), + }, + }, + expected: `index file content`, + }, + { + name: "directory fallback", + url: "/some-folder", + fs: fstest.MapFS{ + "some-folder": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + "index.html": &fstest.MapFile{ + Data: []byte("index file content"), + }, + }, + expected: `index file content`, + }, + { + name: "index.html", + url: "/index.html", + fs: fstest.MapFS{ + "index.html": &fstest.MapFile{ + Data: []byte("index file content"), + }, + }, + expected: `index file content`, + }, + { + name: "oidc-callback.html", + url: "/oidc-callback.html", + fs: fstest.MapFS{ + "index.html": &fstest.MapFile{ + Data: []byte("oidc-callback file content"), + }, + }, + expected: `oidc-callback file content`, + }, + { + name: "oidc-silent-redirect.html", + url: "/oidc-silent-redirect.html", + fs: fstest.MapFS{ + "index.html": &fstest.MapFile{ + Data: []byte("oidc-silent-redirect file content"), + }, + }, + expected: `oidc-silent-redirect file content`, + }, + { + name: "some-file.txt", + url: "/some-file.txt", + fs: fstest.MapFS{ + "some-file.txt": &fstest.MapFile{ + Data: []byte("some file content"), + }, + }, + expected: "some file content", + }, + } { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", tt.url, nil) + assets.FileServer(tt.fs).ServeHTTP(w, req) + res := w.Result() + defer res.Body.Close() + + g.Expect(res.Status).To(gomega.Equal(recorderStatus(http.StatusOK))) + + data, err := io.ReadAll(res.Body) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(string(data)).To(gomega.Equal(tt.expected)) + + }) + } + +} diff --git a/services/web/pkg/config/config.go b/services/web/pkg/config/config.go index a244fb55312..7e90e63960f 100644 --- a/services/web/pkg/config/config.go +++ b/services/web/pkg/config/config.go @@ -21,6 +21,7 @@ type Config struct { Asset Asset `yaml:"asset"` File string `yaml:"file" env:"WEB_UI_CONFIG_FILE" desc:"Read the ownCloud Web json based configuration from this path/file. The config file takes precedence over WEB_OPTION_xxx environment variables. See the text description for more details."` Web Web `yaml:"web"` + Apps map[string]App TokenManager *TokenManager `yaml:"token_manager"` @@ -30,7 +31,9 @@ type Config struct { // Asset defines the available asset configuration. type Asset struct { - Path string `yaml:"path" env:"WEB_ASSET_PATH" desc:"Serve ownCloud Web assets from a path on the filesystem instead of the builtin assets."` + DeprecatedPath string `yaml:"path" env:"WEB_ASSET_PATH" desc:"Serve ownCloud Web assets from a path on the filesystem instead of the builtin assets." deprecationVersion:"5.1.0" removalVersion:"6.0.0" deprecationInfo:"The WEB_ASSET_PATH is deprecated and will be removed in the future." deprecationReplacement:"Use WEB_ASSET_CORE_PATH instead."` + CorePath string `yaml:"core_path" env:"WEB_ASSET_CORE_PATH" desc:"Serve ownCloud Web assets from a path on the filesystem instead of the builtin assets." introductionVersion:"5.1"` + AppsPath string `yaml:"apps_path" env:"WEB_ASSET_APPS_PATH" desc:"Serve ownCloud Web apps assets from a path on the filesystem instead of the builtin assets." introductionVersion:"5.1"` } // CustomStyle references additional css to be loaded into ownCloud Web. @@ -110,6 +113,12 @@ type Web struct { Config WebConfig `yaml:"config"` } +// App defines the individual app configuration. +type App struct { + Disabled bool `yaml:"disabled"` + Config map[string]any `yaml:"config"` +} + // TokenManager is the config for using the reva token manager type TokenManager struct { JWTSecret string `yaml:"jwt_secret" env:"OCIS_JWT_SECRET;WEB_JWT_SECRET" desc:"The secret to mint and validate jwt tokens."` diff --git a/services/web/pkg/config/defaults/defaultconfig.go b/services/web/pkg/config/defaults/defaultconfig.go index c9aa3ddcbf9..166cb1efd40 100644 --- a/services/web/pkg/config/defaults/defaultconfig.go +++ b/services/web/pkg/config/defaults/defaultconfig.go @@ -80,7 +80,8 @@ func DefaultConfig() *config.Config { Name: "web", }, Asset: config.Asset{ - Path: filepath.Join(defaults.BaseDataPath(), "web/assets"), + CorePath: filepath.Join(defaults.BaseDataPath(), "web/assets/core"), + AppsPath: filepath.Join(defaults.BaseDataPath(), "web/assets/apps"), }, GatewayAddress: "com.owncloud.api.gateway", Web: config.Web{ diff --git a/services/web/pkg/config/parser/parse.go b/services/web/pkg/config/parser/parse.go index f90db175646..c5bb6f03f50 100644 --- a/services/web/pkg/config/parser/parse.go +++ b/services/web/pkg/config/parser/parse.go @@ -4,11 +4,11 @@ import ( "errors" ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis-pkg/config/envdecode" + "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/ocis-pkg/shared" "github.com/owncloud/ocis/v2/services/web/pkg/config" "github.com/owncloud/ocis/v2/services/web/pkg/config/defaults" - - "github.com/owncloud/ocis/v2/ocis-pkg/config/envdecode" ) // ParseConfig loads configuration from known paths. @@ -28,6 +28,12 @@ func ParseConfig(cfg *config.Config) error { } } + // apps are a special case, as they are not part of the main config, but are loaded from a separate config file + _, err = ociscfg.BindSourcesToStructs("apps", &cfg.Apps) + if err != nil { + return err + } + defaults.Sanitize(cfg) return Validate(cfg) @@ -37,5 +43,20 @@ func Validate(cfg *config.Config) error { if cfg.TokenManager.JWTSecret == "" { return shared.MissingJWTTokenError(cfg.Service.Name) } + + // deprecation: migration requested + // check if the config still uses the deprecated asset path, if so, + // log a warning and copy the value to the setting that is actually used + // this is to ensure a smooth transition from the old to the new core asset path (pre 5.1 to 5.1) + if cfg.Asset.DeprecatedPath != "" { + if cfg.Asset.CorePath == "" { + cfg.Asset.CorePath = cfg.Asset.DeprecatedPath + } + + // message should be logged to the console, + // do not use a logger here because the message MUST be visible independent of the log level + log.Deprecation("WEB_ASSET_PATH is deprecated and will be removed in the future. Use WEB_ASSET_CORE_PATH instead.") + } + return nil } diff --git a/services/web/pkg/server/http/server.go b/services/web/pkg/server/http/server.go index 80145b8d5d8..417d17849c4 100644 --- a/services/web/pkg/server/http/server.go +++ b/services/web/pkg/server/http/server.go @@ -2,17 +2,27 @@ package http import ( "fmt" + "path" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" chimiddleware "github.com/go-chi/chi/v5/middleware" + "go-micro.dev/v4" + "github.com/owncloud/ocis/v2/ocis-pkg/cors" "github.com/owncloud/ocis/v2/ocis-pkg/middleware" "github.com/owncloud/ocis/v2/ocis-pkg/registry" "github.com/owncloud/ocis/v2/ocis-pkg/service/http" "github.com/owncloud/ocis/v2/ocis-pkg/version" + "github.com/owncloud/ocis/v2/ocis-pkg/x/io/fsx" + "github.com/owncloud/ocis/v2/services/web" + "github.com/owncloud/ocis/v2/services/web/pkg/apps" webmid "github.com/owncloud/ocis/v2/services/web/pkg/middleware" svc "github.com/owncloud/ocis/v2/services/web/pkg/service/v0" - "go-micro.dev/v4" +) + +var ( + // _customAppsEndpoint path is used to make app artifacts available by the web service. + _customAppsEndpoint = "/assets/apps" ) // Server initializes the http service and server. @@ -46,8 +56,28 @@ func Server(opts ...Option) (http.Service, error) { return http.Service{}, err } + coreFS := fsx.NewFallbackFS( + fsx.NewBasePathFs(fsx.NewOsFs(), options.Config.Asset.CorePath), + fsx.NewBasePathFs(fsx.FromIOFS(web.Assets), "assets/core"), + ) + appsFS := fsx.NewFallbackFS( + fsx.NewReadOnlyFs(fsx.NewBasePathFs(fsx.NewOsFs(), options.Config.Asset.AppsPath)), + fsx.NewBasePathFs(fsx.FromIOFS(web.Assets), "assets/apps"), + ) + + // build and inject the list of applications into the config + for _, application := range apps.List(options.Logger, options.Config.Apps, appsFS.Secondary().IOFS(), appsFS.Primary().IOFS()) { + options.Config.Web.Config.ExternalApps = append( + options.Config.Web.Config.ExternalApps, + application.ToExternal(path.Join(options.Config.HTTP.Root, _customAppsEndpoint)), + ) + } + handle := svc.NewService( svc.Logger(options.Logger), + svc.CoreFS(coreFS), + svc.AppFS(appsFS.IOFS()), + svc.AppsHTTPEndpoint(_customAppsEndpoint), svc.Config(options.Config), svc.GatewaySelector(gatewaySelector), svc.Middleware( diff --git a/services/web/pkg/service/v0/branding.go b/services/web/pkg/service/v0/branding.go index 06606387b91..47da4306080 100644 --- a/services/web/pkg/service/v0/branding.go +++ b/services/web/pkg/service/v0/branding.go @@ -11,6 +11,7 @@ import ( permissionsapi "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" revactx "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/spf13/afero" ) var ( @@ -67,7 +68,7 @@ func (p Web) UploadLogo(w http.ResponseWriter, r *http.Request) { } fp := filepath.Join("branding", filepath.Join("/", fileHeader.Filename)) - err = p.storeAsset(fp, file) + err = afero.WriteReader(p.coreFS, fp, file) if err != nil { w.WriteHeader(http.StatusInternalServerError) return @@ -109,7 +110,7 @@ func (p Web) ResetLogo(w http.ResponseWriter, r *http.Request) { return } - f, err := p.fs.OpenEmbedded(_themesConfigPath) + f, err := p.coreFS.Secondary().Open(_themesConfigPath) if err != nil { w.WriteHeader(http.StatusInternalServerError) return @@ -128,17 +129,6 @@ func (p Web) ResetLogo(w http.ResponseWriter, r *http.Request) { } } -func (p Web) storeAsset(name string, asset io.Reader) error { - dst, err := p.fs.Create(name) - if err != nil { - return err - } - defer dst.Close() - - _, err = io.Copy(dst, asset) - return err -} - func (p Web) getLogoPath(r io.Reader) (string, error) { // This decoding of the themes.json file is not optimal. If we need to decode it for other // usecases as well we should consider decoding to a struct. @@ -159,7 +149,7 @@ func (p Web) getLogoPath(r io.Reader) (string, error) { } func (p Web) updateLogoThemeConfig(logoPath string) error { - f, err := p.fs.Open(_themesConfigPath) + f, err := p.coreFS.Open(_themesConfigPath) if err == nil { defer f.Close() } @@ -184,10 +174,11 @@ func (p Web) updateLogoThemeConfig(logoPath string) error { logoCfg["login"] = logoPath logoCfg["topbar"] = logoPath - dst, err := p.fs.Create(_themesConfigPath) + dst, err := p.coreFS.Create(_themesConfigPath) if err != nil { return err } + defer dst.Close() return json.NewEncoder(dst).Encode(m) } diff --git a/services/web/pkg/service/v0/option.go b/services/web/pkg/service/v0/option.go index 96b626c5d9f..066897c1594 100644 --- a/services/web/pkg/service/v0/option.go +++ b/services/web/pkg/service/v0/option.go @@ -1,25 +1,31 @@ package svc import ( + "io/fs" "net/http" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "go.opentelemetry.io/otel/trace" + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/ocis-pkg/x/io/fsx" "github.com/owncloud/ocis/v2/services/web/pkg/config" - "go.opentelemetry.io/otel/trace" ) // Option defines a single option function. type Option func(o *Options) -// Options defines the available options for this package. +// Options define the available options for this package. type Options struct { - Logger log.Logger - Config *config.Config - Middleware []func(http.Handler) http.Handler - GatewaySelector pool.Selectable[gateway.GatewayAPIClient] - TraceProvider trace.TracerProvider + Logger log.Logger + Config *config.Config + Middleware []func(http.Handler) http.Handler + GatewaySelector pool.Selectable[gateway.GatewayAPIClient] + TraceProvider trace.TracerProvider + AppFS fs.FS + AppsHTTPEndpoint string + CoreFS *fsx.FallbackFS } // newOptions initializes the available default options. @@ -67,3 +73,24 @@ func TraceProvider(val trace.TracerProvider) Option { o.TraceProvider = val } } + +// AppFS provides a function to set the appFS option. +func AppFS(val fs.FS) Option { + return func(o *Options) { + o.AppFS = val + } +} + +// AppsHTTPEndpoint provides a function to set the appsHTTPEndpoint option. +func AppsHTTPEndpoint(val string) Option { + return func(o *Options) { + o.AppsHTTPEndpoint = val + } +} + +// CoreFS provides a function to set the coreFS option. +func CoreFS(val *fsx.FallbackFS) Option { + return func(o *Options) { + o.CoreFS = val + } +} diff --git a/services/web/pkg/service/v0/service.go b/services/web/pkg/service/v0/service.go index 0681eba03b5..191b2e239cf 100644 --- a/services/web/pkg/service/v0/service.go +++ b/services/web/pkg/service/v0/service.go @@ -3,8 +3,10 @@ package svc import ( "encoding/json" "fmt" + "io/fs" "net/http" "net/url" + "path" "strconv" "strings" "time" @@ -12,15 +14,15 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" "github.com/go-chi/chi/v5" + "github.com/riandyrn/otelchi" + "github.com/owncloud/ocis/v2/ocis-pkg/account" - "github.com/owncloud/ocis/v2/ocis-pkg/assetsfs" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/ocis-pkg/middleware" "github.com/owncloud/ocis/v2/ocis-pkg/tracing" - "github.com/owncloud/ocis/v2/services/web" + "github.com/owncloud/ocis/v2/ocis-pkg/x/io/fsx" "github.com/owncloud/ocis/v2/services/web/pkg/assets" "github.com/owncloud/ocis/v2/services/web/pkg/config" - "github.com/riandyrn/otelchi" ) // ErrConfigInvalid is returned when the config parse is invalid. @@ -49,11 +51,12 @@ func NewService(opts ...Option) Service { otelchi.WithPropagators(tracing.GetPropagator()), ), ) + svc := Web{ logger: options.Logger, config: options.Config, mux: m, - fs: assetsfs.New(web.Assets, options.Config.Asset.Path, options.Logger), + coreFS: options.CoreFS, gatewaySelector: options.GatewaySelector, } @@ -67,7 +70,16 @@ func NewService(opts ...Option) Service { r.Post("/", svc.UploadLogo) r.Delete("/", svc.ResetLogo) }) - r.Mount("/", svc.Static(options.Config.HTTP.CacheTTL)) + r.Mount(options.AppsHTTPEndpoint, svc.Static( + options.AppFS, + path.Join(svc.config.HTTP.Root, options.AppsHTTPEndpoint), + options.Config.HTTP.CacheTTL, + )) + r.Mount("/", svc.Static( + svc.coreFS.IOFS(), + svc.config.HTTP.Root, + options.Config.HTTP.CacheTTL, + )) }) _ = chi.Walk(m, func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error { @@ -78,12 +90,12 @@ func NewService(opts ...Option) Service { return svc } -// Web defines implements the business logic for Service. +// Web defines the handlers for the web service. type Web struct { logger log.Logger config *config.Config mux *chi.Mux - fs *assetsfs.FileSystem + coreFS *fsx.FallbackFS gatewaySelector pool.Selectable[gateway.GatewayAPIClient] } @@ -127,8 +139,8 @@ func (p Web) Config(w http.ResponseWriter, _ *http.Request) { } // Static simply serves all static files. -func (p Web) Static(ttl int) http.HandlerFunc { - rootWithSlash := p.config.HTTP.Root +func (p Web) Static(f fs.FS, root string, ttl int) http.HandlerFunc { + rootWithSlash := root if !strings.HasSuffix(rootWithSlash, "/") { rootWithSlash = rootWithSlash + "/" @@ -136,7 +148,7 @@ func (p Web) Static(ttl int) http.HandlerFunc { static := http.StripPrefix( rootWithSlash, - assets.FileServer(p.fs), + assets.FileServer(f), ) lastModified := time.Now().UTC().Format(http.TimeFormat) diff --git a/services/web/web.go b/services/web/web.go index 6830e279e8c..4472b2f0e4e 100644 --- a/services/web/web.go +++ b/services/web/web.go @@ -5,5 +5,6 @@ import ( ) //go:generate make generate -//go:embed all:assets/* + +//go:embed all:assets var Assets embed.FS diff --git a/sonar-project.properties b/sonar-project.properties index c4ada13e438..d4514ead902 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -33,6 +33,17 @@ sonar.go.coverage.reportPaths=cache/coverage/* sonar.go.golangci-lint.reportPaths=cache/checkstyle/checkstyle.xml # Exclude files -sonar.exclusions=**/third_party,docs/**,changelog/**,**/package.json,**/rollup.config.js,CHANGELOG.md,deployments/**,tests/**,vendor/**,vendor-bin/**,README.md,**/mocks/**,/protogen/**,services/search/pkg/query/kql/dictionary_gen.go -sonar.coverage.exclusions=**/*_test.go,**mocks/**,/protogen/**,services/search/pkg/query/kql/dictionary_gen.go +sonar.exclusions=**/third_party,docs/**,changelog/**,**/package.json,**/rollup.config.js,CHANGELOG.md,deployments/**,tests/**,vendor/**,vendor-bin/**,README.md,**/mocks/**,/protogen/**,**/*_gen.go +sonar.coverage.exclusions=**/*_test.go,**mocks/**,/protogen/**,**/*_gen.go sonar.cpd.exclusions=**/*_test.go,**/revaconfig/**,services/settings/pkg/store/defaults/defaults.go + +# Rule exclusions +sonar.issue.ignore.multicriteria=g1,g2 + +# Ignore "Define a constant instead of duplicating this literal" rule for tests +sonar.issue.ignore.multicriteria.g1.ruleKey=go:S1192 +sonar.issue.ignore.multicriteria.g1.resourceKey=**/*_test.go + +# Ignore "Rename function XXX to match the regular expression ^(_|[a-zA-Z0-9]+)$" rule for tests +sonar.issue.ignore.multicriteria.g2.ruleKey=go:S100 +sonar.issue.ignore.multicriteria.g2.resourceKey=**/*_test.go diff --git a/vendor/github.com/spf13/afero/.gitignore b/vendor/github.com/spf13/afero/.gitignore new file mode 100644 index 00000000000..9c1d9861181 --- /dev/null +++ b/vendor/github.com/spf13/afero/.gitignore @@ -0,0 +1,2 @@ +sftpfs/file1 +sftpfs/test/ diff --git a/vendor/github.com/spf13/afero/LICENSE.txt b/vendor/github.com/spf13/afero/LICENSE.txt new file mode 100644 index 00000000000..298f0e2665e --- /dev/null +++ b/vendor/github.com/spf13/afero/LICENSE.txt @@ -0,0 +1,174 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/vendor/github.com/spf13/afero/README.md b/vendor/github.com/spf13/afero/README.md new file mode 100644 index 00000000000..3bafbfdfcaf --- /dev/null +++ b/vendor/github.com/spf13/afero/README.md @@ -0,0 +1,442 @@ +![afero logo-sm](https://cloud.githubusercontent.com/assets/173412/11490338/d50e16dc-97a5-11e5-8b12-019a300d0fcb.png) + +A FileSystem Abstraction System for Go + +[![Test](https://github.com/spf13/afero/actions/workflows/test.yml/badge.svg)](https://github.com/spf13/afero/actions/workflows/test.yml) [![GoDoc](https://godoc.org/github.com/spf13/afero?status.svg)](https://godoc.org/github.com/spf13/afero) [![Join the chat at https://gitter.im/spf13/afero](https://badges.gitter.im/Dev%20Chat.svg)](https://gitter.im/spf13/afero?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +# Overview + +Afero is a filesystem framework providing a simple, uniform and universal API +interacting with any filesystem, as an abstraction layer providing interfaces, +types and methods. Afero has an exceptionally clean interface and simple design +without needless constructors or initialization methods. + +Afero is also a library providing a base set of interoperable backend +filesystems that make it easy to work with afero while retaining all the power +and benefit of the os and ioutil packages. + +Afero provides significant improvements over using the os package alone, most +notably the ability to create mock and testing filesystems without relying on the disk. + +It is suitable for use in any situation where you would consider using the OS +package as it provides an additional abstraction that makes it easy to use a +memory backed file system during testing. It also adds support for the http +filesystem for full interoperability. + + +## Afero Features + +* A single consistent API for accessing a variety of filesystems +* Interoperation between a variety of file system types +* A set of interfaces to encourage and enforce interoperability between backends +* An atomic cross platform memory backed file system +* Support for compositional (union) file systems by combining multiple file systems acting as one +* Specialized backends which modify existing filesystems (Read Only, Regexp filtered) +* A set of utility functions ported from io, ioutil & hugo to be afero aware +* Wrapper for go 1.16 filesystem abstraction `io/fs.FS` + +# Using Afero + +Afero is easy to use and easier to adopt. + +A few different ways you could use Afero: + +* Use the interfaces alone to define your own file system. +* Wrapper for the OS packages. +* Define different filesystems for different parts of your application. +* Use Afero for mock filesystems while testing + +## Step 1: Install Afero + +First use go get to install the latest version of the library. + + $ go get github.com/spf13/afero + +Next include Afero in your application. +```go +import "github.com/spf13/afero" +``` + +## Step 2: Declare a backend + +First define a package variable and set it to a pointer to a filesystem. +```go +var AppFs = afero.NewMemMapFs() + +or + +var AppFs = afero.NewOsFs() +``` +It is important to note that if you repeat the composite literal you +will be using a completely new and isolated filesystem. In the case of +OsFs it will still use the same underlying filesystem but will reduce +the ability to drop in other filesystems as desired. + +## Step 3: Use it like you would the OS package + +Throughout your application use any function and method like you normally +would. + +So if my application before had: +```go +os.Open("/tmp/foo") +``` +We would replace it with: +```go +AppFs.Open("/tmp/foo") +``` + +`AppFs` being the variable we defined above. + + +## List of all available functions + +File System Methods Available: +```go +Chmod(name string, mode os.FileMode) : error +Chown(name string, uid, gid int) : error +Chtimes(name string, atime time.Time, mtime time.Time) : error +Create(name string) : File, error +Mkdir(name string, perm os.FileMode) : error +MkdirAll(path string, perm os.FileMode) : error +Name() : string +Open(name string) : File, error +OpenFile(name string, flag int, perm os.FileMode) : File, error +Remove(name string) : error +RemoveAll(path string) : error +Rename(oldname, newname string) : error +Stat(name string) : os.FileInfo, error +``` +File Interfaces and Methods Available: +```go +io.Closer +io.Reader +io.ReaderAt +io.Seeker +io.Writer +io.WriterAt + +Name() : string +Readdir(count int) : []os.FileInfo, error +Readdirnames(n int) : []string, error +Stat() : os.FileInfo, error +Sync() : error +Truncate(size int64) : error +WriteString(s string) : ret int, err error +``` +In some applications it may make sense to define a new package that +simply exports the file system variable for easy access from anywhere. + +## Using Afero's utility functions + +Afero provides a set of functions to make it easier to use the underlying file systems. +These functions have been primarily ported from io & ioutil with some developed for Hugo. + +The afero utilities support all afero compatible backends. + +The list of utilities includes: + +```go +DirExists(path string) (bool, error) +Exists(path string) (bool, error) +FileContainsBytes(filename string, subslice []byte) (bool, error) +GetTempDir(subPath string) string +IsDir(path string) (bool, error) +IsEmpty(path string) (bool, error) +ReadDir(dirname string) ([]os.FileInfo, error) +ReadFile(filename string) ([]byte, error) +SafeWriteReader(path string, r io.Reader) (err error) +TempDir(dir, prefix string) (name string, err error) +TempFile(dir, prefix string) (f File, err error) +Walk(root string, walkFn filepath.WalkFunc) error +WriteFile(filename string, data []byte, perm os.FileMode) error +WriteReader(path string, r io.Reader) (err error) +``` +For a complete list see [Afero's GoDoc](https://godoc.org/github.com/spf13/afero) + +They are available under two different approaches to use. You can either call +them directly where the first parameter of each function will be the file +system, or you can declare a new `Afero`, a custom type used to bind these +functions as methods to a given filesystem. + +### Calling utilities directly + +```go +fs := new(afero.MemMapFs) +f, err := afero.TempFile(fs,"", "ioutil-test") + +``` + +### Calling via Afero + +```go +fs := afero.NewMemMapFs() +afs := &afero.Afero{Fs: fs} +f, err := afs.TempFile("", "ioutil-test") +``` + +## Using Afero for Testing + +There is a large benefit to using a mock filesystem for testing. It has a +completely blank state every time it is initialized and can be easily +reproducible regardless of OS. You could create files to your heart’s content +and the file access would be fast while also saving you from all the annoying +issues with deleting temporary files, Windows file locking, etc. The MemMapFs +backend is perfect for testing. + +* Much faster than performing I/O operations on disk +* Avoid security issues and permissions +* Far more control. 'rm -rf /' with confidence +* Test setup is far more easier to do +* No test cleanup needed + +One way to accomplish this is to define a variable as mentioned above. +In your application this will be set to afero.NewOsFs() during testing you +can set it to afero.NewMemMapFs(). + +It wouldn't be uncommon to have each test initialize a blank slate memory +backend. To do this I would define my `appFS = afero.NewOsFs()` somewhere +appropriate in my application code. This approach ensures that Tests are order +independent, with no test relying on the state left by an earlier test. + +Then in my tests I would initialize a new MemMapFs for each test: +```go +func TestExist(t *testing.T) { + appFS := afero.NewMemMapFs() + // create test files and directories + appFS.MkdirAll("src/a", 0755) + afero.WriteFile(appFS, "src/a/b", []byte("file b"), 0644) + afero.WriteFile(appFS, "src/c", []byte("file c"), 0644) + name := "src/c" + _, err := appFS.Stat(name) + if os.IsNotExist(err) { + t.Errorf("file \"%s\" does not exist.\n", name) + } +} +``` + +# Available Backends + +## Operating System Native + +### OsFs + +The first is simply a wrapper around the native OS calls. This makes it +very easy to use as all of the calls are the same as the existing OS +calls. It also makes it trivial to have your code use the OS during +operation and a mock filesystem during testing or as needed. + +```go +appfs := afero.NewOsFs() +appfs.MkdirAll("src/a", 0755) +``` + +## Memory Backed Storage + +### MemMapFs + +Afero also provides a fully atomic memory backed filesystem perfect for use in +mocking and to speed up unnecessary disk io when persistence isn’t +necessary. It is fully concurrent and will work within go routines +safely. + +```go +mm := afero.NewMemMapFs() +mm.MkdirAll("src/a", 0755) +``` + +#### InMemoryFile + +As part of MemMapFs, Afero also provides an atomic, fully concurrent memory +backed file implementation. This can be used in other memory backed file +systems with ease. Plans are to add a radix tree memory stored file +system using InMemoryFile. + +## Network Interfaces + +### SftpFs + +Afero has experimental support for secure file transfer protocol (sftp). Which can +be used to perform file operations over a encrypted channel. + +### GCSFs + +Afero has experimental support for Google Cloud Storage (GCS). You can either set the +`GOOGLE_APPLICATION_CREDENTIALS_JSON` env variable to your JSON credentials or use `opts` in +`NewGcsFS` to configure access to your GCS bucket. + +Some known limitations of the existing implementation: +* No Chmod support - The GCS ACL could probably be mapped to *nix style permissions but that would add another level of complexity and is ignored in this version. +* No Chtimes support - Could be simulated with attributes (gcs a/m-times are set implicitly) but that's is left for another version. +* Not thread safe - Also assumes all file operations are done through the same instance of the GcsFs. File operations between different GcsFs instances are not guaranteed to be consistent. + + +## Filtering Backends + +### BasePathFs + +The BasePathFs restricts all operations to a given path within an Fs. +The given file name to the operations on this Fs will be prepended with +the base path before calling the source Fs. + +```go +bp := afero.NewBasePathFs(afero.NewOsFs(), "/base/path") +``` + +### ReadOnlyFs + +A thin wrapper around the source Fs providing a read only view. + +```go +fs := afero.NewReadOnlyFs(afero.NewOsFs()) +_, err := fs.Create("/file.txt") +// err = syscall.EPERM +``` + +# RegexpFs + +A filtered view on file names, any file NOT matching +the passed regexp will be treated as non-existing. +Files not matching the regexp provided will not be created. +Directories are not filtered. + +```go +fs := afero.NewRegexpFs(afero.NewMemMapFs(), regexp.MustCompile(`\.txt$`)) +_, err := fs.Create("/file.html") +// err = syscall.ENOENT +``` + +### HttpFs + +Afero provides an http compatible backend which can wrap any of the existing +backends. + +The Http package requires a slightly specific version of Open which +returns an http.File type. + +Afero provides an httpFs file system which satisfies this requirement. +Any Afero FileSystem can be used as an httpFs. + +```go +httpFs := afero.NewHttpFs() +fileserver := http.FileServer(httpFs.Dir()) +http.Handle("/", fileserver) +``` + +## Composite Backends + +Afero provides the ability have two filesystems (or more) act as a single +file system. + +### CacheOnReadFs + +The CacheOnReadFs will lazily make copies of any accessed files from the base +layer into the overlay. Subsequent reads will be pulled from the overlay +directly permitting the request is within the cache duration of when it was +created in the overlay. + +If the base filesystem is writeable, any changes to files will be +done first to the base, then to the overlay layer. Write calls to open file +handles like `Write()` or `Truncate()` to the overlay first. + +To writing files to the overlay only, you can use the overlay Fs directly (not +via the union Fs). + +Cache files in the layer for the given time.Duration, a cache duration of 0 +means "forever" meaning the file will not be re-requested from the base ever. + +A read-only base will make the overlay also read-only but still copy files +from the base to the overlay when they're not present (or outdated) in the +caching layer. + +```go +base := afero.NewOsFs() +layer := afero.NewMemMapFs() +ufs := afero.NewCacheOnReadFs(base, layer, 100 * time.Second) +``` + +### CopyOnWriteFs() + +The CopyOnWriteFs is a read only base file system with a potentially +writeable layer on top. + +Read operations will first look in the overlay and if not found there, will +serve the file from the base. + +Changes to the file system will only be made in the overlay. + +Any attempt to modify a file found only in the base will copy the file to the +overlay layer before modification (including opening a file with a writable +handle). + +Removing and Renaming files present only in the base layer is not currently +permitted. If a file is present in the base layer and the overlay, only the +overlay will be removed/renamed. + +```go + base := afero.NewOsFs() + roBase := afero.NewReadOnlyFs(base) + ufs := afero.NewCopyOnWriteFs(roBase, afero.NewMemMapFs()) + + fh, _ = ufs.Create("/home/test/file2.txt") + fh.WriteString("This is a test") + fh.Close() +``` + +In this example all write operations will only occur in memory (MemMapFs) +leaving the base filesystem (OsFs) untouched. + + +## Desired/possible backends + +The following is a short list of possible backends we hope someone will +implement: + +* SSH +* S3 + +# About the project + +## What's in the name + +Afero comes from the latin roots Ad-Facere. + +**"Ad"** is a prefix meaning "to". + +**"Facere"** is a form of the root "faciō" making "make or do". + +The literal meaning of afero is "to make" or "to do" which seems very fitting +for a library that allows one to make files and directories and do things with them. + +The English word that shares the same roots as Afero is "affair". Affair shares +the same concept but as a noun it means "something that is made or done" or "an +object of a particular type". + +It's also nice that unlike some of my other libraries (hugo, cobra, viper) it +Googles very well. + +## Release Notes + +See the [Releases Page](https://github.com/spf13/afero/releases). + +## Contributing + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Add some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create new Pull Request + +## Contributors + +Names in no particular order: + +* [spf13](https://github.com/spf13) +* [jaqx0r](https://github.com/jaqx0r) +* [mbertschler](https://github.com/mbertschler) +* [xor-gate](https://github.com/xor-gate) + +## License + +Afero is released under the Apache 2.0 license. See +[LICENSE.txt](https://github.com/spf13/afero/blob/master/LICENSE.txt) diff --git a/vendor/github.com/spf13/afero/afero.go b/vendor/github.com/spf13/afero/afero.go new file mode 100644 index 00000000000..39f65852099 --- /dev/null +++ b/vendor/github.com/spf13/afero/afero.go @@ -0,0 +1,111 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package afero provides types and methods for interacting with the filesystem, +// as an abstraction layer. + +// Afero also provides a few implementations that are mostly interoperable. One that +// uses the operating system filesystem, one that uses memory to store files +// (cross platform) and an interface that should be implemented if you want to +// provide your own filesystem. + +package afero + +import ( + "errors" + "io" + "os" + "time" +) + +type Afero struct { + Fs +} + +// File represents a file in the filesystem. +type File interface { + io.Closer + io.Reader + io.ReaderAt + io.Seeker + io.Writer + io.WriterAt + + Name() string + Readdir(count int) ([]os.FileInfo, error) + Readdirnames(n int) ([]string, error) + Stat() (os.FileInfo, error) + Sync() error + Truncate(size int64) error + WriteString(s string) (ret int, err error) +} + +// Fs is the filesystem interface. +// +// Any simulated or real filesystem should implement this interface. +type Fs interface { + // Create creates a file in the filesystem, returning the file and an + // error, if any happens. + Create(name string) (File, error) + + // Mkdir creates a directory in the filesystem, return an error if any + // happens. + Mkdir(name string, perm os.FileMode) error + + // MkdirAll creates a directory path and all parents that does not exist + // yet. + MkdirAll(path string, perm os.FileMode) error + + // Open opens a file, returning it or an error, if any happens. + Open(name string) (File, error) + + // OpenFile opens a file using the given flags and the given mode. + OpenFile(name string, flag int, perm os.FileMode) (File, error) + + // Remove removes a file identified by name, returning an error, if any + // happens. + Remove(name string) error + + // RemoveAll removes a directory path and any children it contains. It + // does not fail if the path does not exist (return nil). + RemoveAll(path string) error + + // Rename renames a file. + Rename(oldname, newname string) error + + // Stat returns a FileInfo describing the named file, or an error, if any + // happens. + Stat(name string) (os.FileInfo, error) + + // The name of this FileSystem + Name() string + + // Chmod changes the mode of the named file to mode. + Chmod(name string, mode os.FileMode) error + + // Chown changes the uid and gid of the named file. + Chown(name string, uid, gid int) error + + // Chtimes changes the access and modification times of the named file + Chtimes(name string, atime time.Time, mtime time.Time) error +} + +var ( + ErrFileClosed = errors.New("File is closed") + ErrOutOfRange = errors.New("out of range") + ErrTooLarge = errors.New("too large") + ErrFileNotFound = os.ErrNotExist + ErrFileExists = os.ErrExist + ErrDestinationExists = os.ErrExist +) diff --git a/vendor/github.com/spf13/afero/appveyor.yml b/vendor/github.com/spf13/afero/appveyor.yml new file mode 100644 index 00000000000..65e20e8ca3f --- /dev/null +++ b/vendor/github.com/spf13/afero/appveyor.yml @@ -0,0 +1,10 @@ +# This currently does nothing. We have moved to GitHub action, but this is kept +# until spf13 has disabled this project in AppVeyor. +version: '{build}' +clone_folder: C:\gopath\src\github.com\spf13\afero +environment: + GOPATH: C:\gopath +build_script: +- cmd: >- + go version + diff --git a/vendor/github.com/spf13/afero/basepath.go b/vendor/github.com/spf13/afero/basepath.go new file mode 100644 index 00000000000..2e72793a3e1 --- /dev/null +++ b/vendor/github.com/spf13/afero/basepath.go @@ -0,0 +1,222 @@ +package afero + +import ( + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "time" +) + +var ( + _ Lstater = (*BasePathFs)(nil) + _ fs.ReadDirFile = (*BasePathFile)(nil) +) + +// The BasePathFs restricts all operations to a given path within an Fs. +// The given file name to the operations on this Fs will be prepended with +// the base path before calling the base Fs. +// Any file name (after filepath.Clean()) outside this base path will be +// treated as non existing file. +// +// Note that it does not clean the error messages on return, so you may +// reveal the real path on errors. +type BasePathFs struct { + source Fs + path string +} + +type BasePathFile struct { + File + path string +} + +func (f *BasePathFile) Name() string { + sourcename := f.File.Name() + return strings.TrimPrefix(sourcename, filepath.Clean(f.path)) +} + +func (f *BasePathFile) ReadDir(n int) ([]fs.DirEntry, error) { + if rdf, ok := f.File.(fs.ReadDirFile); ok { + return rdf.ReadDir(n) + } + return readDirFile{f.File}.ReadDir(n) +} + +func NewBasePathFs(source Fs, path string) Fs { + return &BasePathFs{source: source, path: path} +} + +// on a file outside the base path it returns the given file name and an error, +// else the given file with the base path prepended +func (b *BasePathFs) RealPath(name string) (path string, err error) { + if err := validateBasePathName(name); err != nil { + return name, err + } + + bpath := filepath.Clean(b.path) + path = filepath.Clean(filepath.Join(bpath, name)) + if !strings.HasPrefix(path, bpath) { + return name, os.ErrNotExist + } + + return path, nil +} + +func validateBasePathName(name string) error { + if runtime.GOOS != "windows" { + // Not much to do here; + // the virtual file paths all look absolute on *nix. + return nil + } + + // On Windows a common mistake would be to provide an absolute OS path + // We could strip out the base part, but that would not be very portable. + if filepath.IsAbs(name) { + return os.ErrNotExist + } + + return nil +} + +func (b *BasePathFs) Chtimes(name string, atime, mtime time.Time) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "chtimes", Path: name, Err: err} + } + return b.source.Chtimes(name, atime, mtime) +} + +func (b *BasePathFs) Chmod(name string, mode os.FileMode) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "chmod", Path: name, Err: err} + } + return b.source.Chmod(name, mode) +} + +func (b *BasePathFs) Chown(name string, uid, gid int) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "chown", Path: name, Err: err} + } + return b.source.Chown(name, uid, gid) +} + +func (b *BasePathFs) Name() string { + return "BasePathFs" +} + +func (b *BasePathFs) Stat(name string) (fi os.FileInfo, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "stat", Path: name, Err: err} + } + return b.source.Stat(name) +} + +func (b *BasePathFs) Rename(oldname, newname string) (err error) { + if oldname, err = b.RealPath(oldname); err != nil { + return &os.PathError{Op: "rename", Path: oldname, Err: err} + } + if newname, err = b.RealPath(newname); err != nil { + return &os.PathError{Op: "rename", Path: newname, Err: err} + } + return b.source.Rename(oldname, newname) +} + +func (b *BasePathFs) RemoveAll(name string) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "remove_all", Path: name, Err: err} + } + return b.source.RemoveAll(name) +} + +func (b *BasePathFs) Remove(name string) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "remove", Path: name, Err: err} + } + return b.source.Remove(name) +} + +func (b *BasePathFs) OpenFile(name string, flag int, mode os.FileMode) (f File, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "openfile", Path: name, Err: err} + } + sourcef, err := b.source.OpenFile(name, flag, mode) + if err != nil { + return nil, err + } + return &BasePathFile{sourcef, b.path}, nil +} + +func (b *BasePathFs) Open(name string) (f File, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "open", Path: name, Err: err} + } + sourcef, err := b.source.Open(name) + if err != nil { + return nil, err + } + return &BasePathFile{File: sourcef, path: b.path}, nil +} + +func (b *BasePathFs) Mkdir(name string, mode os.FileMode) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "mkdir", Path: name, Err: err} + } + return b.source.Mkdir(name, mode) +} + +func (b *BasePathFs) MkdirAll(name string, mode os.FileMode) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "mkdir", Path: name, Err: err} + } + return b.source.MkdirAll(name, mode) +} + +func (b *BasePathFs) Create(name string) (f File, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "create", Path: name, Err: err} + } + sourcef, err := b.source.Create(name) + if err != nil { + return nil, err + } + return &BasePathFile{File: sourcef, path: b.path}, nil +} + +func (b *BasePathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + name, err := b.RealPath(name) + if err != nil { + return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} + } + if lstater, ok := b.source.(Lstater); ok { + return lstater.LstatIfPossible(name) + } + fi, err := b.source.Stat(name) + return fi, false, err +} + +func (b *BasePathFs) SymlinkIfPossible(oldname, newname string) error { + oldname, err := b.RealPath(oldname) + if err != nil { + return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: err} + } + newname, err = b.RealPath(newname) + if err != nil { + return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: err} + } + if linker, ok := b.source.(Linker); ok { + return linker.SymlinkIfPossible(oldname, newname) + } + return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: ErrNoSymlink} +} + +func (b *BasePathFs) ReadlinkIfPossible(name string) (string, error) { + name, err := b.RealPath(name) + if err != nil { + return "", &os.PathError{Op: "readlink", Path: name, Err: err} + } + if reader, ok := b.source.(LinkReader); ok { + return reader.ReadlinkIfPossible(name) + } + return "", &os.PathError{Op: "readlink", Path: name, Err: ErrNoReadlink} +} diff --git a/vendor/github.com/spf13/afero/cacheOnReadFs.go b/vendor/github.com/spf13/afero/cacheOnReadFs.go new file mode 100644 index 00000000000..017d344fd53 --- /dev/null +++ b/vendor/github.com/spf13/afero/cacheOnReadFs.go @@ -0,0 +1,315 @@ +package afero + +import ( + "os" + "syscall" + "time" +) + +// If the cache duration is 0, cache time will be unlimited, i.e. once +// a file is in the layer, the base will never be read again for this file. +// +// For cache times greater than 0, the modification time of a file is +// checked. Note that a lot of file system implementations only allow a +// resolution of a second for timestamps... or as the godoc for os.Chtimes() +// states: "The underlying filesystem may truncate or round the values to a +// less precise time unit." +// +// This caching union will forward all write calls also to the base file +// system first. To prevent writing to the base Fs, wrap it in a read-only +// filter - Note: this will also make the overlay read-only, for writing files +// in the overlay, use the overlay Fs directly, not via the union Fs. +type CacheOnReadFs struct { + base Fs + layer Fs + cacheTime time.Duration +} + +func NewCacheOnReadFs(base Fs, layer Fs, cacheTime time.Duration) Fs { + return &CacheOnReadFs{base: base, layer: layer, cacheTime: cacheTime} +} + +type cacheState int + +const ( + // not present in the overlay, unknown if it exists in the base: + cacheMiss cacheState = iota + // present in the overlay and in base, base file is newer: + cacheStale + // present in the overlay - with cache time == 0 it may exist in the base, + // with cacheTime > 0 it exists in the base and is same age or newer in the + // overlay + cacheHit + // happens if someone writes directly to the overlay without + // going through this union + cacheLocal +) + +func (u *CacheOnReadFs) cacheStatus(name string) (state cacheState, fi os.FileInfo, err error) { + var lfi, bfi os.FileInfo + lfi, err = u.layer.Stat(name) + if err == nil { + if u.cacheTime == 0 { + return cacheHit, lfi, nil + } + if lfi.ModTime().Add(u.cacheTime).Before(time.Now()) { + bfi, err = u.base.Stat(name) + if err != nil { + return cacheLocal, lfi, nil + } + if bfi.ModTime().After(lfi.ModTime()) { + return cacheStale, bfi, nil + } + } + return cacheHit, lfi, nil + } + + if err == syscall.ENOENT || os.IsNotExist(err) { + return cacheMiss, nil, nil + } + + return cacheMiss, nil, err +} + +func (u *CacheOnReadFs) copyToLayer(name string) error { + return copyToLayer(u.base, u.layer, name) +} + +func (u *CacheOnReadFs) copyFileToLayer(name string, flag int, perm os.FileMode) error { + return copyFileToLayer(u.base, u.layer, name, flag, perm) +} + +func (u *CacheOnReadFs) Chtimes(name string, atime, mtime time.Time) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit: + err = u.base.Chtimes(name, atime, mtime) + case cacheStale, cacheMiss: + if err := u.copyToLayer(name); err != nil { + return err + } + err = u.base.Chtimes(name, atime, mtime) + } + if err != nil { + return err + } + return u.layer.Chtimes(name, atime, mtime) +} + +func (u *CacheOnReadFs) Chmod(name string, mode os.FileMode) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit: + err = u.base.Chmod(name, mode) + case cacheStale, cacheMiss: + if err := u.copyToLayer(name); err != nil { + return err + } + err = u.base.Chmod(name, mode) + } + if err != nil { + return err + } + return u.layer.Chmod(name, mode) +} + +func (u *CacheOnReadFs) Chown(name string, uid, gid int) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit: + err = u.base.Chown(name, uid, gid) + case cacheStale, cacheMiss: + if err := u.copyToLayer(name); err != nil { + return err + } + err = u.base.Chown(name, uid, gid) + } + if err != nil { + return err + } + return u.layer.Chown(name, uid, gid) +} + +func (u *CacheOnReadFs) Stat(name string) (os.FileInfo, error) { + st, fi, err := u.cacheStatus(name) + if err != nil { + return nil, err + } + switch st { + case cacheMiss: + return u.base.Stat(name) + default: // cacheStale has base, cacheHit and cacheLocal the layer os.FileInfo + return fi, nil + } +} + +func (u *CacheOnReadFs) Rename(oldname, newname string) error { + st, _, err := u.cacheStatus(oldname) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit: + err = u.base.Rename(oldname, newname) + case cacheStale, cacheMiss: + if err := u.copyToLayer(oldname); err != nil { + return err + } + err = u.base.Rename(oldname, newname) + } + if err != nil { + return err + } + return u.layer.Rename(oldname, newname) +} + +func (u *CacheOnReadFs) Remove(name string) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit, cacheStale, cacheMiss: + err = u.base.Remove(name) + } + if err != nil { + return err + } + return u.layer.Remove(name) +} + +func (u *CacheOnReadFs) RemoveAll(name string) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit, cacheStale, cacheMiss: + err = u.base.RemoveAll(name) + } + if err != nil { + return err + } + return u.layer.RemoveAll(name) +} + +func (u *CacheOnReadFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + st, _, err := u.cacheStatus(name) + if err != nil { + return nil, err + } + switch st { + case cacheLocal, cacheHit: + default: + if err := u.copyFileToLayer(name, flag, perm); err != nil { + return nil, err + } + } + if flag&(os.O_WRONLY|syscall.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC) != 0 { + bfi, err := u.base.OpenFile(name, flag, perm) + if err != nil { + return nil, err + } + lfi, err := u.layer.OpenFile(name, flag, perm) + if err != nil { + bfi.Close() // oops, what if O_TRUNC was set and file opening in the layer failed...? + return nil, err + } + return &UnionFile{Base: bfi, Layer: lfi}, nil + } + return u.layer.OpenFile(name, flag, perm) +} + +func (u *CacheOnReadFs) Open(name string) (File, error) { + st, fi, err := u.cacheStatus(name) + if err != nil { + return nil, err + } + + switch st { + case cacheLocal: + return u.layer.Open(name) + + case cacheMiss: + bfi, err := u.base.Stat(name) + if err != nil { + return nil, err + } + if bfi.IsDir() { + return u.base.Open(name) + } + if err := u.copyToLayer(name); err != nil { + return nil, err + } + return u.layer.Open(name) + + case cacheStale: + if !fi.IsDir() { + if err := u.copyToLayer(name); err != nil { + return nil, err + } + return u.layer.Open(name) + } + case cacheHit: + if !fi.IsDir() { + return u.layer.Open(name) + } + } + // the dirs from cacheHit, cacheStale fall down here: + bfile, _ := u.base.Open(name) + lfile, err := u.layer.Open(name) + if err != nil && bfile == nil { + return nil, err + } + return &UnionFile{Base: bfile, Layer: lfile}, nil +} + +func (u *CacheOnReadFs) Mkdir(name string, perm os.FileMode) error { + err := u.base.Mkdir(name, perm) + if err != nil { + return err + } + return u.layer.MkdirAll(name, perm) // yes, MkdirAll... we cannot assume it exists in the cache +} + +func (u *CacheOnReadFs) Name() string { + return "CacheOnReadFs" +} + +func (u *CacheOnReadFs) MkdirAll(name string, perm os.FileMode) error { + err := u.base.MkdirAll(name, perm) + if err != nil { + return err + } + return u.layer.MkdirAll(name, perm) +} + +func (u *CacheOnReadFs) Create(name string) (File, error) { + bfh, err := u.base.Create(name) + if err != nil { + return nil, err + } + lfh, err := u.layer.Create(name) + if err != nil { + // oops, see comment about OS_TRUNC above, should we remove? then we have to + // remember if the file did not exist before + bfh.Close() + return nil, err + } + return &UnionFile{Base: bfh, Layer: lfh}, nil +} diff --git a/vendor/github.com/spf13/afero/const_bsds.go b/vendor/github.com/spf13/afero/const_bsds.go new file mode 100644 index 00000000000..30855de572a --- /dev/null +++ b/vendor/github.com/spf13/afero/const_bsds.go @@ -0,0 +1,23 @@ +// Copyright © 2016 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build aix || darwin || openbsd || freebsd || netbsd || dragonfly || zos +// +build aix darwin openbsd freebsd netbsd dragonfly zos + +package afero + +import ( + "syscall" +) + +const BADFD = syscall.EBADF diff --git a/vendor/github.com/spf13/afero/const_win_unix.go b/vendor/github.com/spf13/afero/const_win_unix.go new file mode 100644 index 00000000000..12792d21e27 --- /dev/null +++ b/vendor/github.com/spf13/afero/const_win_unix.go @@ -0,0 +1,22 @@ +// Copyright © 2016 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//go:build !darwin && !openbsd && !freebsd && !dragonfly && !netbsd && !aix && !zos +// +build !darwin,!openbsd,!freebsd,!dragonfly,!netbsd,!aix,!zos + +package afero + +import ( + "syscall" +) + +const BADFD = syscall.EBADFD diff --git a/vendor/github.com/spf13/afero/copyOnWriteFs.go b/vendor/github.com/spf13/afero/copyOnWriteFs.go new file mode 100644 index 00000000000..184d6dd702a --- /dev/null +++ b/vendor/github.com/spf13/afero/copyOnWriteFs.go @@ -0,0 +1,327 @@ +package afero + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + "time" +) + +var _ Lstater = (*CopyOnWriteFs)(nil) + +// The CopyOnWriteFs is a union filesystem: a read only base file system with +// a possibly writeable layer on top. Changes to the file system will only +// be made in the overlay: Changing an existing file in the base layer which +// is not present in the overlay will copy the file to the overlay ("changing" +// includes also calls to e.g. Chtimes(), Chmod() and Chown()). +// +// Reading directories is currently only supported via Open(), not OpenFile(). +type CopyOnWriteFs struct { + base Fs + layer Fs +} + +func NewCopyOnWriteFs(base Fs, layer Fs) Fs { + return &CopyOnWriteFs{base: base, layer: layer} +} + +// Returns true if the file is not in the overlay +func (u *CopyOnWriteFs) isBaseFile(name string) (bool, error) { + if _, err := u.layer.Stat(name); err == nil { + return false, nil + } + _, err := u.base.Stat(name) + if err != nil { + if oerr, ok := err.(*os.PathError); ok { + if oerr.Err == os.ErrNotExist || oerr.Err == syscall.ENOENT || oerr.Err == syscall.ENOTDIR { + return false, nil + } + } + if err == syscall.ENOENT { + return false, nil + } + } + return true, err +} + +func (u *CopyOnWriteFs) copyToLayer(name string) error { + return copyToLayer(u.base, u.layer, name) +} + +func (u *CopyOnWriteFs) Chtimes(name string, atime, mtime time.Time) error { + b, err := u.isBaseFile(name) + if err != nil { + return err + } + if b { + if err := u.copyToLayer(name); err != nil { + return err + } + } + return u.layer.Chtimes(name, atime, mtime) +} + +func (u *CopyOnWriteFs) Chmod(name string, mode os.FileMode) error { + b, err := u.isBaseFile(name) + if err != nil { + return err + } + if b { + if err := u.copyToLayer(name); err != nil { + return err + } + } + return u.layer.Chmod(name, mode) +} + +func (u *CopyOnWriteFs) Chown(name string, uid, gid int) error { + b, err := u.isBaseFile(name) + if err != nil { + return err + } + if b { + if err := u.copyToLayer(name); err != nil { + return err + } + } + return u.layer.Chown(name, uid, gid) +} + +func (u *CopyOnWriteFs) Stat(name string) (os.FileInfo, error) { + fi, err := u.layer.Stat(name) + if err != nil { + isNotExist := u.isNotExist(err) + if isNotExist { + return u.base.Stat(name) + } + return nil, err + } + return fi, nil +} + +func (u *CopyOnWriteFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + llayer, ok1 := u.layer.(Lstater) + lbase, ok2 := u.base.(Lstater) + + if ok1 { + fi, b, err := llayer.LstatIfPossible(name) + if err == nil { + return fi, b, nil + } + + if !u.isNotExist(err) { + return nil, b, err + } + } + + if ok2 { + fi, b, err := lbase.LstatIfPossible(name) + if err == nil { + return fi, b, nil + } + if !u.isNotExist(err) { + return nil, b, err + } + } + + fi, err := u.Stat(name) + + return fi, false, err +} + +func (u *CopyOnWriteFs) SymlinkIfPossible(oldname, newname string) error { + if slayer, ok := u.layer.(Linker); ok { + return slayer.SymlinkIfPossible(oldname, newname) + } + + return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: ErrNoSymlink} +} + +func (u *CopyOnWriteFs) ReadlinkIfPossible(name string) (string, error) { + if rlayer, ok := u.layer.(LinkReader); ok { + return rlayer.ReadlinkIfPossible(name) + } + + if rbase, ok := u.base.(LinkReader); ok { + return rbase.ReadlinkIfPossible(name) + } + + return "", &os.PathError{Op: "readlink", Path: name, Err: ErrNoReadlink} +} + +func (u *CopyOnWriteFs) isNotExist(err error) bool { + if e, ok := err.(*os.PathError); ok { + err = e.Err + } + if err == os.ErrNotExist || err == syscall.ENOENT || err == syscall.ENOTDIR { + return true + } + return false +} + +// Renaming files present only in the base layer is not permitted +func (u *CopyOnWriteFs) Rename(oldname, newname string) error { + b, err := u.isBaseFile(oldname) + if err != nil { + return err + } + if b { + return syscall.EPERM + } + return u.layer.Rename(oldname, newname) +} + +// Removing files present only in the base layer is not permitted. If +// a file is present in the base layer and the overlay, only the overlay +// will be removed. +func (u *CopyOnWriteFs) Remove(name string) error { + err := u.layer.Remove(name) + switch err { + case syscall.ENOENT: + _, err = u.base.Stat(name) + if err == nil { + return syscall.EPERM + } + return syscall.ENOENT + default: + return err + } +} + +func (u *CopyOnWriteFs) RemoveAll(name string) error { + err := u.layer.RemoveAll(name) + switch err { + case syscall.ENOENT: + _, err = u.base.Stat(name) + if err == nil { + return syscall.EPERM + } + return syscall.ENOENT + default: + return err + } +} + +func (u *CopyOnWriteFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + b, err := u.isBaseFile(name) + if err != nil { + return nil, err + } + + if flag&(os.O_WRONLY|os.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC) != 0 { + if b { + if err = u.copyToLayer(name); err != nil { + return nil, err + } + return u.layer.OpenFile(name, flag, perm) + } + + dir := filepath.Dir(name) + isaDir, err := IsDir(u.base, dir) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + if isaDir { + if err = u.layer.MkdirAll(dir, 0o777); err != nil { + return nil, err + } + return u.layer.OpenFile(name, flag, perm) + } + + isaDir, err = IsDir(u.layer, dir) + if err != nil { + return nil, err + } + if isaDir { + return u.layer.OpenFile(name, flag, perm) + } + + return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOTDIR} // ...or os.ErrNotExist? + } + if b { + return u.base.OpenFile(name, flag, perm) + } + return u.layer.OpenFile(name, flag, perm) +} + +// This function handles the 9 different possibilities caused +// by the union which are the intersection of the following... +// +// layer: doesn't exist, exists as a file, and exists as a directory +// base: doesn't exist, exists as a file, and exists as a directory +func (u *CopyOnWriteFs) Open(name string) (File, error) { + // Since the overlay overrides the base we check that first + b, err := u.isBaseFile(name) + if err != nil { + return nil, err + } + + // If overlay doesn't exist, return the base (base state irrelevant) + if b { + return u.base.Open(name) + } + + // If overlay is a file, return it (base state irrelevant) + dir, err := IsDir(u.layer, name) + if err != nil { + return nil, err + } + if !dir { + return u.layer.Open(name) + } + + // Overlay is a directory, base state now matters. + // Base state has 3 states to check but 2 outcomes: + // A. It's a file or non-readable in the base (return just the overlay) + // B. It's an accessible directory in the base (return a UnionFile) + + // If base is file or nonreadable, return overlay + dir, err = IsDir(u.base, name) + if !dir || err != nil { + return u.layer.Open(name) + } + + // Both base & layer are directories + // Return union file (if opens are without error) + bfile, bErr := u.base.Open(name) + lfile, lErr := u.layer.Open(name) + + // If either have errors at this point something is very wrong. Return nil and the errors + if bErr != nil || lErr != nil { + return nil, fmt.Errorf("BaseErr: %v\nOverlayErr: %v", bErr, lErr) + } + + return &UnionFile{Base: bfile, Layer: lfile}, nil +} + +func (u *CopyOnWriteFs) Mkdir(name string, perm os.FileMode) error { + dir, err := IsDir(u.base, name) + if err != nil { + return u.layer.MkdirAll(name, perm) + } + if dir { + return ErrFileExists + } + return u.layer.MkdirAll(name, perm) +} + +func (u *CopyOnWriteFs) Name() string { + return "CopyOnWriteFs" +} + +func (u *CopyOnWriteFs) MkdirAll(name string, perm os.FileMode) error { + dir, err := IsDir(u.base, name) + if err != nil { + return u.layer.MkdirAll(name, perm) + } + if dir { + // This is in line with how os.MkdirAll behaves. + return nil + } + return u.layer.MkdirAll(name, perm) +} + +func (u *CopyOnWriteFs) Create(name string) (File, error) { + return u.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o666) +} diff --git a/vendor/github.com/spf13/afero/httpFs.go b/vendor/github.com/spf13/afero/httpFs.go new file mode 100644 index 00000000000..ac0de6d51f7 --- /dev/null +++ b/vendor/github.com/spf13/afero/httpFs.go @@ -0,0 +1,114 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "errors" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +type httpDir struct { + basePath string + fs HttpFs +} + +func (d httpDir) Open(name string) (http.File, error) { + if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) || + strings.Contains(name, "\x00") { + return nil, errors.New("http: invalid character in file path") + } + dir := string(d.basePath) + if dir == "" { + dir = "." + } + + f, err := d.fs.Open(filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name)))) + if err != nil { + return nil, err + } + return f, nil +} + +type HttpFs struct { + source Fs +} + +func NewHttpFs(source Fs) *HttpFs { + return &HttpFs{source: source} +} + +func (h HttpFs) Dir(s string) *httpDir { + return &httpDir{basePath: s, fs: h} +} + +func (h HttpFs) Name() string { return "h HttpFs" } + +func (h HttpFs) Create(name string) (File, error) { + return h.source.Create(name) +} + +func (h HttpFs) Chmod(name string, mode os.FileMode) error { + return h.source.Chmod(name, mode) +} + +func (h HttpFs) Chown(name string, uid, gid int) error { + return h.source.Chown(name, uid, gid) +} + +func (h HttpFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return h.source.Chtimes(name, atime, mtime) +} + +func (h HttpFs) Mkdir(name string, perm os.FileMode) error { + return h.source.Mkdir(name, perm) +} + +func (h HttpFs) MkdirAll(path string, perm os.FileMode) error { + return h.source.MkdirAll(path, perm) +} + +func (h HttpFs) Open(name string) (http.File, error) { + f, err := h.source.Open(name) + if err == nil { + if httpfile, ok := f.(http.File); ok { + return httpfile, nil + } + } + return nil, err +} + +func (h HttpFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + return h.source.OpenFile(name, flag, perm) +} + +func (h HttpFs) Remove(name string) error { + return h.source.Remove(name) +} + +func (h HttpFs) RemoveAll(path string) error { + return h.source.RemoveAll(path) +} + +func (h HttpFs) Rename(oldname, newname string) error { + return h.source.Rename(oldname, newname) +} + +func (h HttpFs) Stat(name string) (os.FileInfo, error) { + return h.source.Stat(name) +} diff --git a/vendor/github.com/spf13/afero/internal/common/adapters.go b/vendor/github.com/spf13/afero/internal/common/adapters.go new file mode 100644 index 00000000000..60685caa54d --- /dev/null +++ b/vendor/github.com/spf13/afero/internal/common/adapters.go @@ -0,0 +1,27 @@ +// Copyright © 2022 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import "io/fs" + +// FileInfoDirEntry provides an adapter from os.FileInfo to fs.DirEntry +type FileInfoDirEntry struct { + fs.FileInfo +} + +var _ fs.DirEntry = FileInfoDirEntry{} + +func (d FileInfoDirEntry) Type() fs.FileMode { return d.FileInfo.Mode().Type() } + +func (d FileInfoDirEntry) Info() (fs.FileInfo, error) { return d.FileInfo, nil } diff --git a/vendor/github.com/spf13/afero/iofs.go b/vendor/github.com/spf13/afero/iofs.go new file mode 100644 index 00000000000..938b9316e6b --- /dev/null +++ b/vendor/github.com/spf13/afero/iofs.go @@ -0,0 +1,298 @@ +//go:build go1.16 +// +build go1.16 + +package afero + +import ( + "io" + "io/fs" + "os" + "path" + "sort" + "time" + + "github.com/spf13/afero/internal/common" +) + +// IOFS adopts afero.Fs to stdlib io/fs.FS +type IOFS struct { + Fs +} + +func NewIOFS(fs Fs) IOFS { + return IOFS{Fs: fs} +} + +var ( + _ fs.FS = IOFS{} + _ fs.GlobFS = IOFS{} + _ fs.ReadDirFS = IOFS{} + _ fs.ReadFileFS = IOFS{} + _ fs.StatFS = IOFS{} + _ fs.SubFS = IOFS{} +) + +func (iofs IOFS) Open(name string) (fs.File, error) { + const op = "open" + + // by convention for fs.FS implementations we should perform this check + if !fs.ValidPath(name) { + return nil, iofs.wrapError(op, name, fs.ErrInvalid) + } + + file, err := iofs.Fs.Open(name) + if err != nil { + return nil, iofs.wrapError(op, name, err) + } + + // file should implement fs.ReadDirFile + if _, ok := file.(fs.ReadDirFile); !ok { + file = readDirFile{file} + } + + return file, nil +} + +func (iofs IOFS) Glob(pattern string) ([]string, error) { + const op = "glob" + + // afero.Glob does not perform this check but it's required for implementations + if _, err := path.Match(pattern, ""); err != nil { + return nil, iofs.wrapError(op, pattern, err) + } + + items, err := Glob(iofs.Fs, pattern) + if err != nil { + return nil, iofs.wrapError(op, pattern, err) + } + + return items, nil +} + +func (iofs IOFS) ReadDir(name string) ([]fs.DirEntry, error) { + f, err := iofs.Fs.Open(name) + if err != nil { + return nil, iofs.wrapError("readdir", name, err) + } + + defer f.Close() + + if rdf, ok := f.(fs.ReadDirFile); ok { + items, err := rdf.ReadDir(-1) + if err != nil { + return nil, iofs.wrapError("readdir", name, err) + } + sort.Slice(items, func(i, j int) bool { return items[i].Name() < items[j].Name() }) + return items, nil + } + + items, err := f.Readdir(-1) + if err != nil { + return nil, iofs.wrapError("readdir", name, err) + } + sort.Sort(byName(items)) + + ret := make([]fs.DirEntry, len(items)) + for i := range items { + ret[i] = common.FileInfoDirEntry{FileInfo: items[i]} + } + + return ret, nil +} + +func (iofs IOFS) ReadFile(name string) ([]byte, error) { + const op = "readfile" + + if !fs.ValidPath(name) { + return nil, iofs.wrapError(op, name, fs.ErrInvalid) + } + + bytes, err := ReadFile(iofs.Fs, name) + if err != nil { + return nil, iofs.wrapError(op, name, err) + } + + return bytes, nil +} + +func (iofs IOFS) Sub(dir string) (fs.FS, error) { return IOFS{NewBasePathFs(iofs.Fs, dir)}, nil } + +func (IOFS) wrapError(op, path string, err error) error { + if _, ok := err.(*fs.PathError); ok { + return err // don't need to wrap again + } + + return &fs.PathError{ + Op: op, + Path: path, + Err: err, + } +} + +// readDirFile provides adapter from afero.File to fs.ReadDirFile needed for correct Open +type readDirFile struct { + File +} + +var _ fs.ReadDirFile = readDirFile{} + +func (r readDirFile) ReadDir(n int) ([]fs.DirEntry, error) { + items, err := r.File.Readdir(n) + if err != nil { + return nil, err + } + + ret := make([]fs.DirEntry, len(items)) + for i := range items { + ret[i] = common.FileInfoDirEntry{FileInfo: items[i]} + } + + return ret, nil +} + +// FromIOFS adopts io/fs.FS to use it as afero.Fs +// Note that io/fs.FS is read-only so all mutating methods will return fs.PathError with fs.ErrPermission +// To store modifications you may use afero.CopyOnWriteFs +type FromIOFS struct { + fs.FS +} + +var _ Fs = FromIOFS{} + +func (f FromIOFS) Create(name string) (File, error) { return nil, notImplemented("create", name) } + +func (f FromIOFS) Mkdir(name string, perm os.FileMode) error { return notImplemented("mkdir", name) } + +func (f FromIOFS) MkdirAll(path string, perm os.FileMode) error { + return notImplemented("mkdirall", path) +} + +func (f FromIOFS) Open(name string) (File, error) { + file, err := f.FS.Open(name) + if err != nil { + return nil, err + } + + return fromIOFSFile{File: file, name: name}, nil +} + +func (f FromIOFS) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + return f.Open(name) +} + +func (f FromIOFS) Remove(name string) error { + return notImplemented("remove", name) +} + +func (f FromIOFS) RemoveAll(path string) error { + return notImplemented("removeall", path) +} + +func (f FromIOFS) Rename(oldname, newname string) error { + return notImplemented("rename", oldname) +} + +func (f FromIOFS) Stat(name string) (os.FileInfo, error) { return fs.Stat(f.FS, name) } + +func (f FromIOFS) Name() string { return "fromiofs" } + +func (f FromIOFS) Chmod(name string, mode os.FileMode) error { + return notImplemented("chmod", name) +} + +func (f FromIOFS) Chown(name string, uid, gid int) error { + return notImplemented("chown", name) +} + +func (f FromIOFS) Chtimes(name string, atime time.Time, mtime time.Time) error { + return notImplemented("chtimes", name) +} + +type fromIOFSFile struct { + fs.File + name string +} + +func (f fromIOFSFile) ReadAt(p []byte, off int64) (n int, err error) { + readerAt, ok := f.File.(io.ReaderAt) + if !ok { + return -1, notImplemented("readat", f.name) + } + + return readerAt.ReadAt(p, off) +} + +func (f fromIOFSFile) Seek(offset int64, whence int) (int64, error) { + seeker, ok := f.File.(io.Seeker) + if !ok { + return -1, notImplemented("seek", f.name) + } + + return seeker.Seek(offset, whence) +} + +func (f fromIOFSFile) Write(p []byte) (n int, err error) { + return -1, notImplemented("write", f.name) +} + +func (f fromIOFSFile) WriteAt(p []byte, off int64) (n int, err error) { + return -1, notImplemented("writeat", f.name) +} + +func (f fromIOFSFile) Name() string { return f.name } + +func (f fromIOFSFile) Readdir(count int) ([]os.FileInfo, error) { + rdfile, ok := f.File.(fs.ReadDirFile) + if !ok { + return nil, notImplemented("readdir", f.name) + } + + entries, err := rdfile.ReadDir(count) + if err != nil { + return nil, err + } + + ret := make([]os.FileInfo, len(entries)) + for i := range entries { + ret[i], err = entries[i].Info() + + if err != nil { + return nil, err + } + } + + return ret, nil +} + +func (f fromIOFSFile) Readdirnames(n int) ([]string, error) { + rdfile, ok := f.File.(fs.ReadDirFile) + if !ok { + return nil, notImplemented("readdir", f.name) + } + + entries, err := rdfile.ReadDir(n) + if err != nil { + return nil, err + } + + ret := make([]string, len(entries)) + for i := range entries { + ret[i] = entries[i].Name() + } + + return ret, nil +} + +func (f fromIOFSFile) Sync() error { return nil } + +func (f fromIOFSFile) Truncate(size int64) error { + return notImplemented("truncate", f.name) +} + +func (f fromIOFSFile) WriteString(s string) (ret int, err error) { + return -1, notImplemented("writestring", f.name) +} + +func notImplemented(op, path string) error { + return &fs.PathError{Op: op, Path: path, Err: fs.ErrPermission} +} diff --git a/vendor/github.com/spf13/afero/ioutil.go b/vendor/github.com/spf13/afero/ioutil.go new file mode 100644 index 00000000000..fa6abe1eeec --- /dev/null +++ b/vendor/github.com/spf13/afero/ioutil.go @@ -0,0 +1,243 @@ +// Copyright ©2015 The Go Authors +// Copyright ©2015 Steve Francia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "bytes" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// byName implements sort.Interface. +type byName []os.FileInfo + +func (f byName) Len() int { return len(f) } +func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() } +func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] } + +// ReadDir reads the directory named by dirname and returns +// a list of sorted directory entries. +func (a Afero) ReadDir(dirname string) ([]os.FileInfo, error) { + return ReadDir(a.Fs, dirname) +} + +func ReadDir(fs Fs, dirname string) ([]os.FileInfo, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + list, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + sort.Sort(byName(list)) + return list, nil +} + +// ReadFile reads the file named by filename and returns the contents. +// A successful call returns err == nil, not err == EOF. Because ReadFile +// reads the whole file, it does not treat an EOF from Read as an error +// to be reported. +func (a Afero) ReadFile(filename string) ([]byte, error) { + return ReadFile(a.Fs, filename) +} + +func ReadFile(fs Fs, filename string) ([]byte, error) { + f, err := fs.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + // It's a good but not certain bet that FileInfo will tell us exactly how much to + // read, so let's try it but be prepared for the answer to be wrong. + var n int64 + + if fi, err := f.Stat(); err == nil { + // Don't preallocate a huge buffer, just in case. + if size := fi.Size(); size < 1e9 { + n = size + } + } + // As initial capacity for readAll, use n + a little extra in case Size is zero, + // and to avoid another allocation after Read has filled the buffer. The readAll + // call will read into its allocated internal buffer cheaply. If the size was + // wrong, we'll either waste some space off the end or reallocate as needed, but + // in the overwhelmingly common case we'll get it just right. + return readAll(f, n+bytes.MinRead) +} + +// readAll reads from r until an error or EOF and returns the data it read +// from the internal buffer allocated with a specified capacity. +func readAll(r io.Reader, capacity int64) (b []byte, err error) { + buf := bytes.NewBuffer(make([]byte, 0, capacity)) + // If the buffer overflows, we will get bytes.ErrTooLarge. + // Return that as an error. Any other panic remains. + defer func() { + e := recover() + if e == nil { + return + } + if panicErr, ok := e.(error); ok && panicErr == bytes.ErrTooLarge { + err = panicErr + } else { + panic(e) + } + }() + _, err = buf.ReadFrom(r) + return buf.Bytes(), err +} + +// ReadAll reads from r until an error or EOF and returns the data it read. +// A successful call returns err == nil, not err == EOF. Because ReadAll is +// defined to read from src until EOF, it does not treat an EOF from Read +// as an error to be reported. +func ReadAll(r io.Reader) ([]byte, error) { + return readAll(r, bytes.MinRead) +} + +// WriteFile writes data to a file named by filename. +// If the file does not exist, WriteFile creates it with permissions perm; +// otherwise WriteFile truncates it before writing. +func (a Afero) WriteFile(filename string, data []byte, perm os.FileMode) error { + return WriteFile(a.Fs, filename, data, perm) +} + +func WriteFile(fs Fs, filename string, data []byte, perm os.FileMode) error { + f, err := fs.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + n, err := f.Write(data) + if err == nil && n < len(data) { + err = io.ErrShortWrite + } + if err1 := f.Close(); err == nil { + err = err1 + } + return err +} + +// Random number state. +// We generate random temporary file names so that there's a good +// chance the file doesn't exist yet - keeps the number of tries in +// TempFile to a minimum. +var ( + randNum uint32 + randmu sync.Mutex +) + +func reseed() uint32 { + return uint32(time.Now().UnixNano() + int64(os.Getpid())) +} + +func nextRandom() string { + randmu.Lock() + r := randNum + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + randNum = r + randmu.Unlock() + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} + +// TempFile creates a new temporary file in the directory dir, +// opens the file for reading and writing, and returns the resulting *os.File. +// The filename is generated by taking pattern and adding a random +// string to the end. If pattern includes a "*", the random string +// replaces the last "*". +// If dir is the empty string, TempFile uses the default directory +// for temporary files (see os.TempDir). +// Multiple programs calling TempFile simultaneously +// will not choose the same file. The caller can use f.Name() +// to find the pathname of the file. It is the caller's responsibility +// to remove the file when no longer needed. +func (a Afero) TempFile(dir, pattern string) (f File, err error) { + return TempFile(a.Fs, dir, pattern) +} + +func TempFile(fs Fs, dir, pattern string) (f File, err error) { + if dir == "" { + dir = os.TempDir() + } + + var prefix, suffix string + if pos := strings.LastIndex(pattern, "*"); pos != -1 { + prefix, suffix = pattern[:pos], pattern[pos+1:] + } else { + prefix = pattern + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextRandom()+suffix) + f, err = fs.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + randNum = reseed() + randmu.Unlock() + } + continue + } + break + } + return +} + +// TempDir creates a new temporary directory in the directory dir +// with a name beginning with prefix and returns the path of the +// new directory. If dir is the empty string, TempDir uses the +// default directory for temporary files (see os.TempDir). +// Multiple programs calling TempDir simultaneously +// will not choose the same directory. It is the caller's responsibility +// to remove the directory when no longer needed. +func (a Afero) TempDir(dir, prefix string) (name string, err error) { + return TempDir(a.Fs, dir, prefix) +} + +func TempDir(fs Fs, dir, prefix string) (name string, err error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + try := filepath.Join(dir, prefix+nextRandom()) + err = fs.Mkdir(try, 0o700) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + randNum = reseed() + randmu.Unlock() + } + continue + } + if err == nil { + name = try + } + break + } + return +} diff --git a/vendor/github.com/spf13/afero/lstater.go b/vendor/github.com/spf13/afero/lstater.go new file mode 100644 index 00000000000..89c1bfc0a7d --- /dev/null +++ b/vendor/github.com/spf13/afero/lstater.go @@ -0,0 +1,27 @@ +// Copyright © 2018 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" +) + +// Lstater is an optional interface in Afero. It is only implemented by the +// filesystems saying so. +// It will call Lstat if the filesystem iself is, or it delegates to, the os filesystem. +// Else it will call Stat. +// In addtion to the FileInfo, it will return a boolean telling whether Lstat was called or not. +type Lstater interface { + LstatIfPossible(name string) (os.FileInfo, bool, error) +} diff --git a/vendor/github.com/spf13/afero/match.go b/vendor/github.com/spf13/afero/match.go new file mode 100644 index 00000000000..7db4b7de6ed --- /dev/null +++ b/vendor/github.com/spf13/afero/match.go @@ -0,0 +1,110 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2009 The Go Authors. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "path/filepath" + "sort" + "strings" +) + +// Glob returns the names of all files matching pattern or nil +// if there is no matching file. The syntax of patterns is the same +// as in Match. The pattern may describe hierarchical names such as +// /usr/*/bin/ed (assuming the Separator is '/'). +// +// Glob ignores file system errors such as I/O errors reading directories. +// The only possible returned error is ErrBadPattern, when pattern +// is malformed. +// +// This was adapted from (http://golang.org/pkg/path/filepath) and uses several +// built-ins from that package. +func Glob(fs Fs, pattern string) (matches []string, err error) { + if !hasMeta(pattern) { + // Lstat not supported by a ll filesystems. + if _, err = lstatIfPossible(fs, pattern); err != nil { + return nil, nil + } + return []string{pattern}, nil + } + + dir, file := filepath.Split(pattern) + switch dir { + case "": + dir = "." + case string(filepath.Separator): + // nothing + default: + dir = dir[0 : len(dir)-1] // chop off trailing separator + } + + if !hasMeta(dir) { + return glob(fs, dir, file, nil) + } + + var m []string + m, err = Glob(fs, dir) + if err != nil { + return + } + for _, d := range m { + matches, err = glob(fs, d, file, matches) + if err != nil { + return + } + } + return +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. If the directory cannot be +// opened, it returns the existing matches. New matches are +// added in lexicographical order. +func glob(fs Fs, dir, pattern string, matches []string) (m []string, e error) { + m = matches + fi, err := fs.Stat(dir) + if err != nil { + return + } + if !fi.IsDir() { + return + } + d, err := fs.Open(dir) + if err != nil { + return + } + defer d.Close() + + names, _ := d.Readdirnames(-1) + sort.Strings(names) + + for _, n := range names { + matched, err := filepath.Match(pattern, n) + if err != nil { + return m, err + } + if matched { + m = append(m, filepath.Join(dir, n)) + } + } + return +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by Match. +func hasMeta(path string) bool { + // TODO(niemeyer): Should other magic characters be added here? + return strings.ContainsAny(path, "*?[") +} diff --git a/vendor/github.com/spf13/afero/mem/dir.go b/vendor/github.com/spf13/afero/mem/dir.go new file mode 100644 index 00000000000..e104013f457 --- /dev/null +++ b/vendor/github.com/spf13/afero/mem/dir.go @@ -0,0 +1,37 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mem + +type Dir interface { + Len() int + Names() []string + Files() []*FileData + Add(*FileData) + Remove(*FileData) +} + +func RemoveFromMemDir(dir *FileData, f *FileData) { + dir.memDir.Remove(f) +} + +func AddToMemDir(dir *FileData, f *FileData) { + dir.memDir.Add(f) +} + +func InitializeDir(d *FileData) { + if d.memDir == nil { + d.dir = true + d.memDir = &DirMap{} + } +} diff --git a/vendor/github.com/spf13/afero/mem/dirmap.go b/vendor/github.com/spf13/afero/mem/dirmap.go new file mode 100644 index 00000000000..03a57ee5b52 --- /dev/null +++ b/vendor/github.com/spf13/afero/mem/dirmap.go @@ -0,0 +1,43 @@ +// Copyright © 2015 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mem + +import "sort" + +type DirMap map[string]*FileData + +func (m DirMap) Len() int { return len(m) } +func (m DirMap) Add(f *FileData) { m[f.name] = f } +func (m DirMap) Remove(f *FileData) { delete(m, f.name) } +func (m DirMap) Files() (files []*FileData) { + for _, f := range m { + files = append(files, f) + } + sort.Sort(filesSorter(files)) + return files +} + +// implement sort.Interface for []*FileData +type filesSorter []*FileData + +func (s filesSorter) Len() int { return len(s) } +func (s filesSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s filesSorter) Less(i, j int) bool { return s[i].name < s[j].name } + +func (m DirMap) Names() (names []string) { + for x := range m { + names = append(names, x) + } + return names +} diff --git a/vendor/github.com/spf13/afero/mem/file.go b/vendor/github.com/spf13/afero/mem/file.go new file mode 100644 index 00000000000..62fe4498e19 --- /dev/null +++ b/vendor/github.com/spf13/afero/mem/file.go @@ -0,0 +1,359 @@ +// Copyright © 2015 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mem + +import ( + "bytes" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/spf13/afero/internal/common" +) + +const FilePathSeparator = string(filepath.Separator) + +var _ fs.ReadDirFile = &File{} + +type File struct { + // atomic requires 64-bit alignment for struct field access + at int64 + readDirCount int64 + closed bool + readOnly bool + fileData *FileData +} + +func NewFileHandle(data *FileData) *File { + return &File{fileData: data} +} + +func NewReadOnlyFileHandle(data *FileData) *File { + return &File{fileData: data, readOnly: true} +} + +func (f File) Data() *FileData { + return f.fileData +} + +type FileData struct { + sync.Mutex + name string + data []byte + memDir Dir + dir bool + mode os.FileMode + modtime time.Time + uid int + gid int +} + +func (d *FileData) Name() string { + d.Lock() + defer d.Unlock() + return d.name +} + +func CreateFile(name string) *FileData { + return &FileData{name: name, mode: os.ModeTemporary, modtime: time.Now()} +} + +func CreateDir(name string) *FileData { + return &FileData{name: name, memDir: &DirMap{}, dir: true, modtime: time.Now()} +} + +func ChangeFileName(f *FileData, newname string) { + f.Lock() + f.name = newname + f.Unlock() +} + +func SetMode(f *FileData, mode os.FileMode) { + f.Lock() + f.mode = mode + f.Unlock() +} + +func SetModTime(f *FileData, mtime time.Time) { + f.Lock() + setModTime(f, mtime) + f.Unlock() +} + +func setModTime(f *FileData, mtime time.Time) { + f.modtime = mtime +} + +func SetUID(f *FileData, uid int) { + f.Lock() + f.uid = uid + f.Unlock() +} + +func SetGID(f *FileData, gid int) { + f.Lock() + f.gid = gid + f.Unlock() +} + +func GetFileInfo(f *FileData) *FileInfo { + return &FileInfo{f} +} + +func (f *File) Open() error { + atomic.StoreInt64(&f.at, 0) + atomic.StoreInt64(&f.readDirCount, 0) + f.fileData.Lock() + f.closed = false + f.fileData.Unlock() + return nil +} + +func (f *File) Close() error { + f.fileData.Lock() + f.closed = true + if !f.readOnly { + setModTime(f.fileData, time.Now()) + } + f.fileData.Unlock() + return nil +} + +func (f *File) Name() string { + return f.fileData.Name() +} + +func (f *File) Stat() (os.FileInfo, error) { + return &FileInfo{f.fileData}, nil +} + +func (f *File) Sync() error { + return nil +} + +func (f *File) Readdir(count int) (res []os.FileInfo, err error) { + if !f.fileData.dir { + return nil, &os.PathError{Op: "readdir", Path: f.fileData.name, Err: errors.New("not a dir")} + } + var outLength int64 + + f.fileData.Lock() + files := f.fileData.memDir.Files()[f.readDirCount:] + if count > 0 { + if len(files) < count { + outLength = int64(len(files)) + } else { + outLength = int64(count) + } + if len(files) == 0 { + err = io.EOF + } + } else { + outLength = int64(len(files)) + } + f.readDirCount += outLength + f.fileData.Unlock() + + res = make([]os.FileInfo, outLength) + for i := range res { + res[i] = &FileInfo{files[i]} + } + + return res, err +} + +func (f *File) Readdirnames(n int) (names []string, err error) { + fi, err := f.Readdir(n) + names = make([]string, len(fi)) + for i, f := range fi { + _, names[i] = filepath.Split(f.Name()) + } + return names, err +} + +// Implements fs.ReadDirFile +func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { + fi, err := f.Readdir(n) + if err != nil { + return nil, err + } + di := make([]fs.DirEntry, len(fi)) + for i, f := range fi { + di[i] = common.FileInfoDirEntry{FileInfo: f} + } + return di, nil +} + +func (f *File) Read(b []byte) (n int, err error) { + f.fileData.Lock() + defer f.fileData.Unlock() + if f.closed { + return 0, ErrFileClosed + } + if len(b) > 0 && int(f.at) == len(f.fileData.data) { + return 0, io.EOF + } + if int(f.at) > len(f.fileData.data) { + return 0, io.ErrUnexpectedEOF + } + if len(f.fileData.data)-int(f.at) >= len(b) { + n = len(b) + } else { + n = len(f.fileData.data) - int(f.at) + } + copy(b, f.fileData.data[f.at:f.at+int64(n)]) + atomic.AddInt64(&f.at, int64(n)) + return +} + +func (f *File) ReadAt(b []byte, off int64) (n int, err error) { + prev := atomic.LoadInt64(&f.at) + atomic.StoreInt64(&f.at, off) + n, err = f.Read(b) + atomic.StoreInt64(&f.at, prev) + return +} + +func (f *File) Truncate(size int64) error { + if f.closed { + return ErrFileClosed + } + if f.readOnly { + return &os.PathError{Op: "truncate", Path: f.fileData.name, Err: errors.New("file handle is read only")} + } + if size < 0 { + return ErrOutOfRange + } + f.fileData.Lock() + defer f.fileData.Unlock() + if size > int64(len(f.fileData.data)) { + diff := size - int64(len(f.fileData.data)) + f.fileData.data = append(f.fileData.data, bytes.Repeat([]byte{0o0}, int(diff))...) + } else { + f.fileData.data = f.fileData.data[0:size] + } + setModTime(f.fileData, time.Now()) + return nil +} + +func (f *File) Seek(offset int64, whence int) (int64, error) { + if f.closed { + return 0, ErrFileClosed + } + switch whence { + case io.SeekStart: + atomic.StoreInt64(&f.at, offset) + case io.SeekCurrent: + atomic.AddInt64(&f.at, offset) + case io.SeekEnd: + atomic.StoreInt64(&f.at, int64(len(f.fileData.data))+offset) + } + return f.at, nil +} + +func (f *File) Write(b []byte) (n int, err error) { + if f.closed { + return 0, ErrFileClosed + } + if f.readOnly { + return 0, &os.PathError{Op: "write", Path: f.fileData.name, Err: errors.New("file handle is read only")} + } + n = len(b) + cur := atomic.LoadInt64(&f.at) + f.fileData.Lock() + defer f.fileData.Unlock() + diff := cur - int64(len(f.fileData.data)) + var tail []byte + if n+int(cur) < len(f.fileData.data) { + tail = f.fileData.data[n+int(cur):] + } + if diff > 0 { + f.fileData.data = append(f.fileData.data, append(bytes.Repeat([]byte{0o0}, int(diff)), b...)...) + f.fileData.data = append(f.fileData.data, tail...) + } else { + f.fileData.data = append(f.fileData.data[:cur], b...) + f.fileData.data = append(f.fileData.data, tail...) + } + setModTime(f.fileData, time.Now()) + + atomic.AddInt64(&f.at, int64(n)) + return +} + +func (f *File) WriteAt(b []byte, off int64) (n int, err error) { + atomic.StoreInt64(&f.at, off) + return f.Write(b) +} + +func (f *File) WriteString(s string) (ret int, err error) { + return f.Write([]byte(s)) +} + +func (f *File) Info() *FileInfo { + return &FileInfo{f.fileData} +} + +type FileInfo struct { + *FileData +} + +// Implements os.FileInfo +func (s *FileInfo) Name() string { + s.Lock() + _, name := filepath.Split(s.name) + s.Unlock() + return name +} + +func (s *FileInfo) Mode() os.FileMode { + s.Lock() + defer s.Unlock() + return s.mode +} + +func (s *FileInfo) ModTime() time.Time { + s.Lock() + defer s.Unlock() + return s.modtime +} + +func (s *FileInfo) IsDir() bool { + s.Lock() + defer s.Unlock() + return s.dir +} +func (s *FileInfo) Sys() interface{} { return nil } +func (s *FileInfo) Size() int64 { + if s.IsDir() { + return int64(42) + } + s.Lock() + defer s.Unlock() + return int64(len(s.data)) +} + +var ( + ErrFileClosed = errors.New("File is closed") + ErrOutOfRange = errors.New("out of range") + ErrTooLarge = errors.New("too large") + ErrFileNotFound = os.ErrNotExist + ErrFileExists = os.ErrExist + ErrDestinationExists = os.ErrExist +) diff --git a/vendor/github.com/spf13/afero/memmap.go b/vendor/github.com/spf13/afero/memmap.go new file mode 100644 index 00000000000..d6c744e8d56 --- /dev/null +++ b/vendor/github.com/spf13/afero/memmap.go @@ -0,0 +1,465 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "fmt" + "io" + + "log" + "os" + "path/filepath" + + "sort" + "strings" + "sync" + "time" + + "github.com/spf13/afero/mem" +) + +const chmodBits = os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky // Only a subset of bits are allowed to be changed. Documented under os.Chmod() + +type MemMapFs struct { + mu sync.RWMutex + data map[string]*mem.FileData + init sync.Once +} + +func NewMemMapFs() Fs { + return &MemMapFs{} +} + +func (m *MemMapFs) getData() map[string]*mem.FileData { + m.init.Do(func() { + m.data = make(map[string]*mem.FileData) + // Root should always exist, right? + // TODO: what about windows? + root := mem.CreateDir(FilePathSeparator) + mem.SetMode(root, os.ModeDir|0o755) + m.data[FilePathSeparator] = root + }) + return m.data +} + +func (*MemMapFs) Name() string { return "MemMapFS" } + +func (m *MemMapFs) Create(name string) (File, error) { + name = normalizePath(name) + m.mu.Lock() + file := mem.CreateFile(name) + m.getData()[name] = file + m.registerWithParent(file, 0) + m.mu.Unlock() + return mem.NewFileHandle(file), nil +} + +func (m *MemMapFs) unRegisterWithParent(fileName string) error { + f, err := m.lockfreeOpen(fileName) + if err != nil { + return err + } + parent := m.findParent(f) + if parent == nil { + log.Panic("parent of ", f.Name(), " is nil") + } + + parent.Lock() + mem.RemoveFromMemDir(parent, f) + parent.Unlock() + return nil +} + +func (m *MemMapFs) findParent(f *mem.FileData) *mem.FileData { + pdir, _ := filepath.Split(f.Name()) + pdir = filepath.Clean(pdir) + pfile, err := m.lockfreeOpen(pdir) + if err != nil { + return nil + } + return pfile +} + +func (m *MemMapFs) findDescendants(name string) []*mem.FileData { + fData := m.getData() + descendants := make([]*mem.FileData, 0, len(fData)) + for p, dFile := range fData { + if strings.HasPrefix(p, name+FilePathSeparator) { + descendants = append(descendants, dFile) + } + } + + sort.Slice(descendants, func(i, j int) bool { + cur := len(strings.Split(descendants[i].Name(), FilePathSeparator)) + next := len(strings.Split(descendants[j].Name(), FilePathSeparator)) + return cur < next + }) + + return descendants +} + +func (m *MemMapFs) registerWithParent(f *mem.FileData, perm os.FileMode) { + if f == nil { + return + } + parent := m.findParent(f) + if parent == nil { + pdir := filepath.Dir(filepath.Clean(f.Name())) + err := m.lockfreeMkdir(pdir, perm) + if err != nil { + // log.Println("Mkdir error:", err) + return + } + parent, err = m.lockfreeOpen(pdir) + if err != nil { + // log.Println("Open after Mkdir error:", err) + return + } + } + + parent.Lock() + mem.InitializeDir(parent) + mem.AddToMemDir(parent, f) + parent.Unlock() +} + +func (m *MemMapFs) lockfreeMkdir(name string, perm os.FileMode) error { + name = normalizePath(name) + x, ok := m.getData()[name] + if ok { + // Only return ErrFileExists if it's a file, not a directory. + i := mem.FileInfo{FileData: x} + if !i.IsDir() { + return ErrFileExists + } + } else { + item := mem.CreateDir(name) + mem.SetMode(item, os.ModeDir|perm) + m.getData()[name] = item + m.registerWithParent(item, perm) + } + return nil +} + +func (m *MemMapFs) Mkdir(name string, perm os.FileMode) error { + perm &= chmodBits + name = normalizePath(name) + + m.mu.RLock() + _, ok := m.getData()[name] + m.mu.RUnlock() + if ok { + return &os.PathError{Op: "mkdir", Path: name, Err: ErrFileExists} + } + + m.mu.Lock() + // Dobule check that it doesn't exist. + if _, ok := m.getData()[name]; ok { + m.mu.Unlock() + return &os.PathError{Op: "mkdir", Path: name, Err: ErrFileExists} + } + item := mem.CreateDir(name) + mem.SetMode(item, os.ModeDir|perm) + m.getData()[name] = item + m.registerWithParent(item, perm) + m.mu.Unlock() + + return m.setFileMode(name, perm|os.ModeDir) +} + +func (m *MemMapFs) MkdirAll(path string, perm os.FileMode) error { + err := m.Mkdir(path, perm) + if err != nil { + if err.(*os.PathError).Err == ErrFileExists { + return nil + } + return err + } + return nil +} + +// Handle some relative paths +func normalizePath(path string) string { + path = filepath.Clean(path) + + switch path { + case ".": + return FilePathSeparator + case "..": + return FilePathSeparator + default: + return path + } +} + +func (m *MemMapFs) Open(name string) (File, error) { + f, err := m.open(name) + if f != nil { + return mem.NewReadOnlyFileHandle(f), err + } + return nil, err +} + +func (m *MemMapFs) openWrite(name string) (File, error) { + f, err := m.open(name) + if f != nil { + return mem.NewFileHandle(f), err + } + return nil, err +} + +func (m *MemMapFs) open(name string) (*mem.FileData, error) { + name = normalizePath(name) + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return nil, &os.PathError{Op: "open", Path: name, Err: ErrFileNotFound} + } + return f, nil +} + +func (m *MemMapFs) lockfreeOpen(name string) (*mem.FileData, error) { + name = normalizePath(name) + f, ok := m.getData()[name] + if ok { + return f, nil + } else { + return nil, ErrFileNotFound + } +} + +func (m *MemMapFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + perm &= chmodBits + chmod := false + file, err := m.openWrite(name) + if err == nil && (flag&os.O_EXCL > 0) { + return nil, &os.PathError{Op: "open", Path: name, Err: ErrFileExists} + } + if os.IsNotExist(err) && (flag&os.O_CREATE > 0) { + file, err = m.Create(name) + chmod = true + } + if err != nil { + return nil, err + } + if flag == os.O_RDONLY { + file = mem.NewReadOnlyFileHandle(file.(*mem.File).Data()) + } + if flag&os.O_APPEND > 0 { + _, err = file.Seek(0, io.SeekEnd) + if err != nil { + file.Close() + return nil, err + } + } + if flag&os.O_TRUNC > 0 && flag&(os.O_RDWR|os.O_WRONLY) > 0 { + err = file.Truncate(0) + if err != nil { + file.Close() + return nil, err + } + } + if chmod { + return file, m.setFileMode(name, perm) + } + return file, nil +} + +func (m *MemMapFs) Remove(name string) error { + name = normalizePath(name) + + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.getData()[name]; ok { + err := m.unRegisterWithParent(name) + if err != nil { + return &os.PathError{Op: "remove", Path: name, Err: err} + } + delete(m.getData(), name) + } else { + return &os.PathError{Op: "remove", Path: name, Err: os.ErrNotExist} + } + return nil +} + +func (m *MemMapFs) RemoveAll(path string) error { + path = normalizePath(path) + m.mu.Lock() + m.unRegisterWithParent(path) + m.mu.Unlock() + + m.mu.RLock() + defer m.mu.RUnlock() + + for p := range m.getData() { + if p == path || strings.HasPrefix(p, path+FilePathSeparator) { + m.mu.RUnlock() + m.mu.Lock() + delete(m.getData(), p) + m.mu.Unlock() + m.mu.RLock() + } + } + return nil +} + +func (m *MemMapFs) Rename(oldname, newname string) error { + oldname = normalizePath(oldname) + newname = normalizePath(newname) + + if oldname == newname { + return nil + } + + m.mu.RLock() + defer m.mu.RUnlock() + if _, ok := m.getData()[oldname]; ok { + m.mu.RUnlock() + m.mu.Lock() + err := m.unRegisterWithParent(oldname) + if err != nil { + return err + } + + fileData := m.getData()[oldname] + mem.ChangeFileName(fileData, newname) + m.getData()[newname] = fileData + + err = m.renameDescendants(oldname, newname) + if err != nil { + return err + } + + delete(m.getData(), oldname) + + m.registerWithParent(fileData, 0) + m.mu.Unlock() + m.mu.RLock() + } else { + return &os.PathError{Op: "rename", Path: oldname, Err: ErrFileNotFound} + } + return nil +} + +func (m *MemMapFs) renameDescendants(oldname, newname string) error { + descendants := m.findDescendants(oldname) + removes := make([]string, 0, len(descendants)) + for _, desc := range descendants { + descNewName := strings.Replace(desc.Name(), oldname, newname, 1) + err := m.unRegisterWithParent(desc.Name()) + if err != nil { + return err + } + + removes = append(removes, desc.Name()) + mem.ChangeFileName(desc, descNewName) + m.getData()[descNewName] = desc + + m.registerWithParent(desc, 0) + } + for _, r := range removes { + delete(m.getData(), r) + } + + return nil +} + +func (m *MemMapFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fileInfo, err := m.Stat(name) + return fileInfo, false, err +} + +func (m *MemMapFs) Stat(name string) (os.FileInfo, error) { + f, err := m.Open(name) + if err != nil { + return nil, err + } + fi := mem.GetFileInfo(f.(*mem.File).Data()) + return fi, nil +} + +func (m *MemMapFs) Chmod(name string, mode os.FileMode) error { + mode &= chmodBits + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return &os.PathError{Op: "chmod", Path: name, Err: ErrFileNotFound} + } + prevOtherBits := mem.GetFileInfo(f).Mode() & ^chmodBits + + mode = prevOtherBits | mode + return m.setFileMode(name, mode) +} + +func (m *MemMapFs) setFileMode(name string, mode os.FileMode) error { + name = normalizePath(name) + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return &os.PathError{Op: "chmod", Path: name, Err: ErrFileNotFound} + } + + m.mu.Lock() + mem.SetMode(f, mode) + m.mu.Unlock() + + return nil +} + +func (m *MemMapFs) Chown(name string, uid, gid int) error { + name = normalizePath(name) + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return &os.PathError{Op: "chown", Path: name, Err: ErrFileNotFound} + } + + mem.SetUID(f, uid) + mem.SetGID(f, gid) + + return nil +} + +func (m *MemMapFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + name = normalizePath(name) + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return &os.PathError{Op: "chtimes", Path: name, Err: ErrFileNotFound} + } + + m.mu.Lock() + mem.SetModTime(f, mtime) + m.mu.Unlock() + + return nil +} + +func (m *MemMapFs) List() { + for _, x := range m.data { + y := mem.FileInfo{FileData: x} + fmt.Println(x.Name(), y.Size()) + } +} diff --git a/vendor/github.com/spf13/afero/os.go b/vendor/github.com/spf13/afero/os.go new file mode 100644 index 00000000000..f1366321ec5 --- /dev/null +++ b/vendor/github.com/spf13/afero/os.go @@ -0,0 +1,113 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" + "time" +) + +var _ Lstater = (*OsFs)(nil) + +// OsFs is a Fs implementation that uses functions provided by the os package. +// +// For details in any method, check the documentation of the os package +// (http://golang.org/pkg/os/). +type OsFs struct{} + +func NewOsFs() Fs { + return &OsFs{} +} + +func (OsFs) Name() string { return "OsFs" } + +func (OsFs) Create(name string) (File, error) { + f, e := os.Create(name) + if f == nil { + // while this looks strange, we need to return a bare nil (of type nil) not + // a nil value of type *os.File or nil won't be nil + return nil, e + } + return f, e +} + +func (OsFs) Mkdir(name string, perm os.FileMode) error { + return os.Mkdir(name, perm) +} + +func (OsFs) MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + +func (OsFs) Open(name string) (File, error) { + f, e := os.Open(name) + if f == nil { + // while this looks strange, we need to return a bare nil (of type nil) not + // a nil value of type *os.File or nil won't be nil + return nil, e + } + return f, e +} + +func (OsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + f, e := os.OpenFile(name, flag, perm) + if f == nil { + // while this looks strange, we need to return a bare nil (of type nil) not + // a nil value of type *os.File or nil won't be nil + return nil, e + } + return f, e +} + +func (OsFs) Remove(name string) error { + return os.Remove(name) +} + +func (OsFs) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +func (OsFs) Rename(oldname, newname string) error { + return os.Rename(oldname, newname) +} + +func (OsFs) Stat(name string) (os.FileInfo, error) { + return os.Stat(name) +} + +func (OsFs) Chmod(name string, mode os.FileMode) error { + return os.Chmod(name, mode) +} + +func (OsFs) Chown(name string, uid, gid int) error { + return os.Chown(name, uid, gid) +} + +func (OsFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return os.Chtimes(name, atime, mtime) +} + +func (OsFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fi, err := os.Lstat(name) + return fi, true, err +} + +func (OsFs) SymlinkIfPossible(oldname, newname string) error { + return os.Symlink(oldname, newname) +} + +func (OsFs) ReadlinkIfPossible(name string) (string, error) { + return os.Readlink(name) +} diff --git a/vendor/github.com/spf13/afero/path.go b/vendor/github.com/spf13/afero/path.go new file mode 100644 index 00000000000..18f60a0f6b6 --- /dev/null +++ b/vendor/github.com/spf13/afero/path.go @@ -0,0 +1,106 @@ +// Copyright ©2015 The Go Authors +// Copyright ©2015 Steve Francia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" + "path/filepath" + "sort" +) + +// readDirNames reads the directory named by dirname and returns +// a sorted list of directory entries. +// adapted from https://golang.org/src/path/filepath/path.go +func readDirNames(fs Fs, dirname string) ([]string, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + names, err := f.Readdirnames(-1) + f.Close() + if err != nil { + return nil, err + } + sort.Strings(names) + return names, nil +} + +// walk recursively descends path, calling walkFn +// adapted from https://golang.org/src/path/filepath/path.go +func walk(fs Fs, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { + err := walkFn(path, info, nil) + if err != nil { + if info.IsDir() && err == filepath.SkipDir { + return nil + } + return err + } + + if !info.IsDir() { + return nil + } + + names, err := readDirNames(fs, path) + if err != nil { + return walkFn(path, info, err) + } + + for _, name := range names { + filename := filepath.Join(path, name) + fileInfo, err := lstatIfPossible(fs, filename) + if err != nil { + if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { + return err + } + } else { + err = walk(fs, filename, fileInfo, walkFn) + if err != nil { + if !fileInfo.IsDir() || err != filepath.SkipDir { + return err + } + } + } + } + return nil +} + +// if the filesystem supports it, use Lstat, else use fs.Stat +func lstatIfPossible(fs Fs, path string) (os.FileInfo, error) { + if lfs, ok := fs.(Lstater); ok { + fi, _, err := lfs.LstatIfPossible(path) + return fi, err + } + return fs.Stat(path) +} + +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. All errors that arise visiting files +// and directories are filtered by walkFn. The files are walked in lexical +// order, which makes the output deterministic but means that for very +// large directories Walk can be inefficient. +// Walk does not follow symbolic links. + +func (a Afero) Walk(root string, walkFn filepath.WalkFunc) error { + return Walk(a.Fs, root, walkFn) +} + +func Walk(fs Fs, root string, walkFn filepath.WalkFunc) error { + info, err := lstatIfPossible(fs, root) + if err != nil { + return walkFn(root, nil, err) + } + return walk(fs, root, info, walkFn) +} diff --git a/vendor/github.com/spf13/afero/readonlyfs.go b/vendor/github.com/spf13/afero/readonlyfs.go new file mode 100644 index 00000000000..bd8f9264ddc --- /dev/null +++ b/vendor/github.com/spf13/afero/readonlyfs.go @@ -0,0 +1,96 @@ +package afero + +import ( + "os" + "syscall" + "time" +) + +var _ Lstater = (*ReadOnlyFs)(nil) + +type ReadOnlyFs struct { + source Fs +} + +func NewReadOnlyFs(source Fs) Fs { + return &ReadOnlyFs{source: source} +} + +func (r *ReadOnlyFs) ReadDir(name string) ([]os.FileInfo, error) { + return ReadDir(r.source, name) +} + +func (r *ReadOnlyFs) Chtimes(n string, a, m time.Time) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Chmod(n string, m os.FileMode) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Chown(n string, uid, gid int) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Name() string { + return "ReadOnlyFilter" +} + +func (r *ReadOnlyFs) Stat(name string) (os.FileInfo, error) { + return r.source.Stat(name) +} + +func (r *ReadOnlyFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + if lsf, ok := r.source.(Lstater); ok { + return lsf.LstatIfPossible(name) + } + fi, err := r.Stat(name) + return fi, false, err +} + +func (r *ReadOnlyFs) SymlinkIfPossible(oldname, newname string) error { + return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: ErrNoSymlink} +} + +func (r *ReadOnlyFs) ReadlinkIfPossible(name string) (string, error) { + if srdr, ok := r.source.(LinkReader); ok { + return srdr.ReadlinkIfPossible(name) + } + + return "", &os.PathError{Op: "readlink", Path: name, Err: ErrNoReadlink} +} + +func (r *ReadOnlyFs) Rename(o, n string) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) RemoveAll(p string) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Remove(n string) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + if flag&(os.O_WRONLY|syscall.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC) != 0 { + return nil, syscall.EPERM + } + return r.source.OpenFile(name, flag, perm) +} + +func (r *ReadOnlyFs) Open(n string) (File, error) { + return r.source.Open(n) +} + +func (r *ReadOnlyFs) Mkdir(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) MkdirAll(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Create(n string) (File, error) { + return nil, syscall.EPERM +} diff --git a/vendor/github.com/spf13/afero/regexpfs.go b/vendor/github.com/spf13/afero/regexpfs.go new file mode 100644 index 00000000000..218f3b235bd --- /dev/null +++ b/vendor/github.com/spf13/afero/regexpfs.go @@ -0,0 +1,223 @@ +package afero + +import ( + "os" + "regexp" + "syscall" + "time" +) + +// The RegexpFs filters files (not directories) by regular expression. Only +// files matching the given regexp will be allowed, all others get a ENOENT error ( +// "No such file or directory"). +type RegexpFs struct { + re *regexp.Regexp + source Fs +} + +func NewRegexpFs(source Fs, re *regexp.Regexp) Fs { + return &RegexpFs{source: source, re: re} +} + +type RegexpFile struct { + f File + re *regexp.Regexp +} + +func (r *RegexpFs) matchesName(name string) error { + if r.re == nil { + return nil + } + if r.re.MatchString(name) { + return nil + } + return syscall.ENOENT +} + +func (r *RegexpFs) dirOrMatches(name string) error { + dir, err := IsDir(r.source, name) + if err != nil { + return err + } + if dir { + return nil + } + return r.matchesName(name) +} + +func (r *RegexpFs) Chtimes(name string, a, m time.Time) error { + if err := r.dirOrMatches(name); err != nil { + return err + } + return r.source.Chtimes(name, a, m) +} + +func (r *RegexpFs) Chmod(name string, mode os.FileMode) error { + if err := r.dirOrMatches(name); err != nil { + return err + } + return r.source.Chmod(name, mode) +} + +func (r *RegexpFs) Chown(name string, uid, gid int) error { + if err := r.dirOrMatches(name); err != nil { + return err + } + return r.source.Chown(name, uid, gid) +} + +func (r *RegexpFs) Name() string { + return "RegexpFs" +} + +func (r *RegexpFs) Stat(name string) (os.FileInfo, error) { + if err := r.dirOrMatches(name); err != nil { + return nil, err + } + return r.source.Stat(name) +} + +func (r *RegexpFs) Rename(oldname, newname string) error { + dir, err := IsDir(r.source, oldname) + if err != nil { + return err + } + if dir { + return nil + } + if err := r.matchesName(oldname); err != nil { + return err + } + if err := r.matchesName(newname); err != nil { + return err + } + return r.source.Rename(oldname, newname) +} + +func (r *RegexpFs) RemoveAll(p string) error { + dir, err := IsDir(r.source, p) + if err != nil { + return err + } + if !dir { + if err := r.matchesName(p); err != nil { + return err + } + } + return r.source.RemoveAll(p) +} + +func (r *RegexpFs) Remove(name string) error { + if err := r.dirOrMatches(name); err != nil { + return err + } + return r.source.Remove(name) +} + +func (r *RegexpFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + if err := r.dirOrMatches(name); err != nil { + return nil, err + } + return r.source.OpenFile(name, flag, perm) +} + +func (r *RegexpFs) Open(name string) (File, error) { + dir, err := IsDir(r.source, name) + if err != nil { + return nil, err + } + if !dir { + if err := r.matchesName(name); err != nil { + return nil, err + } + } + f, err := r.source.Open(name) + if err != nil { + return nil, err + } + return &RegexpFile{f: f, re: r.re}, nil +} + +func (r *RegexpFs) Mkdir(n string, p os.FileMode) error { + return r.source.Mkdir(n, p) +} + +func (r *RegexpFs) MkdirAll(n string, p os.FileMode) error { + return r.source.MkdirAll(n, p) +} + +func (r *RegexpFs) Create(name string) (File, error) { + if err := r.matchesName(name); err != nil { + return nil, err + } + return r.source.Create(name) +} + +func (f *RegexpFile) Close() error { + return f.f.Close() +} + +func (f *RegexpFile) Read(s []byte) (int, error) { + return f.f.Read(s) +} + +func (f *RegexpFile) ReadAt(s []byte, o int64) (int, error) { + return f.f.ReadAt(s, o) +} + +func (f *RegexpFile) Seek(o int64, w int) (int64, error) { + return f.f.Seek(o, w) +} + +func (f *RegexpFile) Write(s []byte) (int, error) { + return f.f.Write(s) +} + +func (f *RegexpFile) WriteAt(s []byte, o int64) (int, error) { + return f.f.WriteAt(s, o) +} + +func (f *RegexpFile) Name() string { + return f.f.Name() +} + +func (f *RegexpFile) Readdir(c int) (fi []os.FileInfo, err error) { + var rfi []os.FileInfo + rfi, err = f.f.Readdir(c) + if err != nil { + return nil, err + } + for _, i := range rfi { + if i.IsDir() || f.re.MatchString(i.Name()) { + fi = append(fi, i) + } + } + return fi, nil +} + +func (f *RegexpFile) Readdirnames(c int) (n []string, err error) { + fi, err := f.Readdir(c) + if err != nil { + return nil, err + } + for _, s := range fi { + n = append(n, s.Name()) + } + return n, nil +} + +func (f *RegexpFile) Stat() (os.FileInfo, error) { + return f.f.Stat() +} + +func (f *RegexpFile) Sync() error { + return f.f.Sync() +} + +func (f *RegexpFile) Truncate(s int64) error { + return f.f.Truncate(s) +} + +func (f *RegexpFile) WriteString(s string) (int, error) { + return f.f.WriteString(s) +} diff --git a/vendor/github.com/spf13/afero/symlink.go b/vendor/github.com/spf13/afero/symlink.go new file mode 100644 index 00000000000..aa6ae125b65 --- /dev/null +++ b/vendor/github.com/spf13/afero/symlink.go @@ -0,0 +1,55 @@ +// Copyright © 2018 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "errors" +) + +// Symlinker is an optional interface in Afero. It is only implemented by the +// filesystems saying so. +// It indicates support for 3 symlink related interfaces that implement the +// behaviors of the os methods: +// - Lstat +// - Symlink, and +// - Readlink +type Symlinker interface { + Lstater + Linker + LinkReader +} + +// Linker is an optional interface in Afero. It is only implemented by the +// filesystems saying so. +// It will call Symlink if the filesystem itself is, or it delegates to, the os filesystem, +// or the filesystem otherwise supports Symlink's. +type Linker interface { + SymlinkIfPossible(oldname, newname string) error +} + +// ErrNoSymlink is the error that will be wrapped in an os.LinkError if a file system +// does not support Symlink's either directly or through its delegated filesystem. +// As expressed by support for the Linker interface. +var ErrNoSymlink = errors.New("symlink not supported") + +// LinkReader is an optional interface in Afero. It is only implemented by the +// filesystems saying so. +type LinkReader interface { + ReadlinkIfPossible(name string) (string, error) +} + +// ErrNoReadlink is the error that will be wrapped in an os.Path if a file system +// does not support the readlink operation either directly or through its delegated filesystem. +// As expressed by support for the LinkReader interface. +var ErrNoReadlink = errors.New("readlink not supported") diff --git a/vendor/github.com/spf13/afero/unionFile.go b/vendor/github.com/spf13/afero/unionFile.go new file mode 100644 index 00000000000..62dd6c93c83 --- /dev/null +++ b/vendor/github.com/spf13/afero/unionFile.go @@ -0,0 +1,330 @@ +package afero + +import ( + "io" + "os" + "path/filepath" + "syscall" +) + +// The UnionFile implements the afero.File interface and will be returned +// when reading a directory present at least in the overlay or opening a file +// for writing. +// +// The calls to +// Readdir() and Readdirnames() merge the file os.FileInfo / names from the +// base and the overlay - for files present in both layers, only those +// from the overlay will be used. +// +// When opening files for writing (Create() / OpenFile() with the right flags) +// the operations will be done in both layers, starting with the overlay. A +// successful read in the overlay will move the cursor position in the base layer +// by the number of bytes read. +type UnionFile struct { + Base File + Layer File + Merger DirsMerger + off int + files []os.FileInfo +} + +func (f *UnionFile) Close() error { + // first close base, so we have a newer timestamp in the overlay. If we'd close + // the overlay first, we'd get a cacheStale the next time we access this file + // -> cache would be useless ;-) + if f.Base != nil { + f.Base.Close() + } + if f.Layer != nil { + return f.Layer.Close() + } + return BADFD +} + +func (f *UnionFile) Read(s []byte) (int, error) { + if f.Layer != nil { + n, err := f.Layer.Read(s) + if (err == nil || err == io.EOF) && f.Base != nil { + // advance the file position also in the base file, the next + // call may be a write at this position (or a seek with SEEK_CUR) + if _, seekErr := f.Base.Seek(int64(n), io.SeekCurrent); seekErr != nil { + // only overwrite err in case the seek fails: we need to + // report an eventual io.EOF to the caller + err = seekErr + } + } + return n, err + } + if f.Base != nil { + return f.Base.Read(s) + } + return 0, BADFD +} + +func (f *UnionFile) ReadAt(s []byte, o int64) (int, error) { + if f.Layer != nil { + n, err := f.Layer.ReadAt(s, o) + if (err == nil || err == io.EOF) && f.Base != nil { + _, err = f.Base.Seek(o+int64(n), io.SeekStart) + } + return n, err + } + if f.Base != nil { + return f.Base.ReadAt(s, o) + } + return 0, BADFD +} + +func (f *UnionFile) Seek(o int64, w int) (pos int64, err error) { + if f.Layer != nil { + pos, err = f.Layer.Seek(o, w) + if (err == nil || err == io.EOF) && f.Base != nil { + _, err = f.Base.Seek(o, w) + } + return pos, err + } + if f.Base != nil { + return f.Base.Seek(o, w) + } + return 0, BADFD +} + +func (f *UnionFile) Write(s []byte) (n int, err error) { + if f.Layer != nil { + n, err = f.Layer.Write(s) + if err == nil && f.Base != nil { // hmm, do we have fixed size files where a write may hit the EOF mark? + _, err = f.Base.Write(s) + } + return n, err + } + if f.Base != nil { + return f.Base.Write(s) + } + return 0, BADFD +} + +func (f *UnionFile) WriteAt(s []byte, o int64) (n int, err error) { + if f.Layer != nil { + n, err = f.Layer.WriteAt(s, o) + if err == nil && f.Base != nil { + _, err = f.Base.WriteAt(s, o) + } + return n, err + } + if f.Base != nil { + return f.Base.WriteAt(s, o) + } + return 0, BADFD +} + +func (f *UnionFile) Name() string { + if f.Layer != nil { + return f.Layer.Name() + } + return f.Base.Name() +} + +// DirsMerger is how UnionFile weaves two directories together. +// It takes the FileInfo slices from the layer and the base and returns a +// single view. +type DirsMerger func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) + +var defaultUnionMergeDirsFn = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { + files := make(map[string]os.FileInfo) + + for _, fi := range lofi { + files[fi.Name()] = fi + } + + for _, fi := range bofi { + if _, exists := files[fi.Name()]; !exists { + files[fi.Name()] = fi + } + } + + rfi := make([]os.FileInfo, len(files)) + + i := 0 + for _, fi := range files { + rfi[i] = fi + i++ + } + + return rfi, nil +} + +// Readdir will weave the two directories together and +// return a single view of the overlayed directories. +// At the end of the directory view, the error is io.EOF if c > 0. +func (f *UnionFile) Readdir(c int) (ofi []os.FileInfo, err error) { + var merge DirsMerger = f.Merger + if merge == nil { + merge = defaultUnionMergeDirsFn + } + + if f.off == 0 { + var lfi []os.FileInfo + if f.Layer != nil { + lfi, err = f.Layer.Readdir(-1) + if err != nil { + return nil, err + } + } + + var bfi []os.FileInfo + if f.Base != nil { + bfi, err = f.Base.Readdir(-1) + if err != nil { + return nil, err + } + + } + merged, err := merge(lfi, bfi) + if err != nil { + return nil, err + } + f.files = append(f.files, merged...) + } + files := f.files[f.off:] + + if c <= 0 { + return files, nil + } + + if len(files) == 0 { + return nil, io.EOF + } + + if c > len(files) { + c = len(files) + } + + defer func() { f.off += c }() + return files[:c], nil +} + +func (f *UnionFile) Readdirnames(c int) ([]string, error) { + rfi, err := f.Readdir(c) + if err != nil { + return nil, err + } + var names []string + for _, fi := range rfi { + names = append(names, fi.Name()) + } + return names, nil +} + +func (f *UnionFile) Stat() (os.FileInfo, error) { + if f.Layer != nil { + return f.Layer.Stat() + } + if f.Base != nil { + return f.Base.Stat() + } + return nil, BADFD +} + +func (f *UnionFile) Sync() (err error) { + if f.Layer != nil { + err = f.Layer.Sync() + if err == nil && f.Base != nil { + err = f.Base.Sync() + } + return err + } + if f.Base != nil { + return f.Base.Sync() + } + return BADFD +} + +func (f *UnionFile) Truncate(s int64) (err error) { + if f.Layer != nil { + err = f.Layer.Truncate(s) + if err == nil && f.Base != nil { + err = f.Base.Truncate(s) + } + return err + } + if f.Base != nil { + return f.Base.Truncate(s) + } + return BADFD +} + +func (f *UnionFile) WriteString(s string) (n int, err error) { + if f.Layer != nil { + n, err = f.Layer.WriteString(s) + if err == nil && f.Base != nil { + _, err = f.Base.WriteString(s) + } + return n, err + } + if f.Base != nil { + return f.Base.WriteString(s) + } + return 0, BADFD +} + +func copyFile(base Fs, layer Fs, name string, bfh File) error { + // First make sure the directory exists + exists, err := Exists(layer, filepath.Dir(name)) + if err != nil { + return err + } + if !exists { + err = layer.MkdirAll(filepath.Dir(name), 0o777) // FIXME? + if err != nil { + return err + } + } + + // Create the file on the overlay + lfh, err := layer.Create(name) + if err != nil { + return err + } + n, err := io.Copy(lfh, bfh) + if err != nil { + // If anything fails, clean up the file + layer.Remove(name) + lfh.Close() + return err + } + + bfi, err := bfh.Stat() + if err != nil || bfi.Size() != n { + layer.Remove(name) + lfh.Close() + return syscall.EIO + } + + err = lfh.Close() + if err != nil { + layer.Remove(name) + lfh.Close() + return err + } + return layer.Chtimes(name, bfi.ModTime(), bfi.ModTime()) +} + +func copyToLayer(base Fs, layer Fs, name string) error { + bfh, err := base.Open(name) + if err != nil { + return err + } + defer bfh.Close() + + return copyFile(base, layer, name, bfh) +} + +func copyFileToLayer(base Fs, layer Fs, name string, flag int, perm os.FileMode) error { + bfh, err := base.OpenFile(name, flag, perm) + if err != nil { + return err + } + defer bfh.Close() + + return copyFile(base, layer, name, bfh) +} diff --git a/vendor/github.com/spf13/afero/util.go b/vendor/github.com/spf13/afero/util.go new file mode 100644 index 00000000000..9e4cba2746a --- /dev/null +++ b/vendor/github.com/spf13/afero/util.go @@ -0,0 +1,329 @@ +// Copyright ©2015 Steve Francia +// Portions Copyright ©2015 The Hugo Authors +// Portions Copyright 2016-present Bjørn Erik Pedersen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "unicode" + + "golang.org/x/text/runes" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" +) + +// Filepath separator defined by os.Separator. +const FilePathSeparator = string(filepath.Separator) + +// Takes a reader and a path and writes the content +func (a Afero) WriteReader(path string, r io.Reader) (err error) { + return WriteReader(a.Fs, path, r) +} + +func WriteReader(fs Fs, path string, r io.Reader) (err error) { + dir, _ := filepath.Split(path) + ospath := filepath.FromSlash(dir) + + if ospath != "" { + err = fs.MkdirAll(ospath, 0o777) // rwx, rw, r + if err != nil { + if err != os.ErrExist { + return err + } + } + } + + file, err := fs.Create(path) + if err != nil { + return + } + defer file.Close() + + _, err = io.Copy(file, r) + return +} + +// Same as WriteReader but checks to see if file/directory already exists. +func (a Afero) SafeWriteReader(path string, r io.Reader) (err error) { + return SafeWriteReader(a.Fs, path, r) +} + +func SafeWriteReader(fs Fs, path string, r io.Reader) (err error) { + dir, _ := filepath.Split(path) + ospath := filepath.FromSlash(dir) + + if ospath != "" { + err = fs.MkdirAll(ospath, 0o777) // rwx, rw, r + if err != nil { + return + } + } + + exists, err := Exists(fs, path) + if err != nil { + return + } + if exists { + return fmt.Errorf("%v already exists", path) + } + + file, err := fs.Create(path) + if err != nil { + return + } + defer file.Close() + + _, err = io.Copy(file, r) + return +} + +func (a Afero) GetTempDir(subPath string) string { + return GetTempDir(a.Fs, subPath) +} + +// GetTempDir returns the default temp directory with trailing slash +// if subPath is not empty then it will be created recursively with mode 777 rwx rwx rwx +func GetTempDir(fs Fs, subPath string) string { + addSlash := func(p string) string { + if FilePathSeparator != p[len(p)-1:] { + p = p + FilePathSeparator + } + return p + } + dir := addSlash(os.TempDir()) + + if subPath != "" { + // preserve windows backslash :-( + if FilePathSeparator == "\\" { + subPath = strings.Replace(subPath, "\\", "____", -1) + } + dir = dir + UnicodeSanitize((subPath)) + if FilePathSeparator == "\\" { + dir = strings.Replace(dir, "____", "\\", -1) + } + + if exists, _ := Exists(fs, dir); exists { + return addSlash(dir) + } + + err := fs.MkdirAll(dir, 0o777) + if err != nil { + panic(err) + } + dir = addSlash(dir) + } + return dir +} + +// Rewrite string to remove non-standard path characters +func UnicodeSanitize(s string) string { + source := []rune(s) + target := make([]rune, 0, len(source)) + + for _, r := range source { + if unicode.IsLetter(r) || + unicode.IsDigit(r) || + unicode.IsMark(r) || + r == '.' || + r == '/' || + r == '\\' || + r == '_' || + r == '-' || + r == '%' || + r == ' ' || + r == '#' { + target = append(target, r) + } + } + + return string(target) +} + +// Transform characters with accents into plain forms. +func NeuterAccents(s string) string { + t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) + result, _, _ := transform.String(t, string(s)) + + return result +} + +func (a Afero) FileContainsBytes(filename string, subslice []byte) (bool, error) { + return FileContainsBytes(a.Fs, filename, subslice) +} + +// Check if a file contains a specified byte slice. +func FileContainsBytes(fs Fs, filename string, subslice []byte) (bool, error) { + f, err := fs.Open(filename) + if err != nil { + return false, err + } + defer f.Close() + + return readerContainsAny(f, subslice), nil +} + +func (a Afero) FileContainsAnyBytes(filename string, subslices [][]byte) (bool, error) { + return FileContainsAnyBytes(a.Fs, filename, subslices) +} + +// Check if a file contains any of the specified byte slices. +func FileContainsAnyBytes(fs Fs, filename string, subslices [][]byte) (bool, error) { + f, err := fs.Open(filename) + if err != nil { + return false, err + } + defer f.Close() + + return readerContainsAny(f, subslices...), nil +} + +// readerContains reports whether any of the subslices is within r. +func readerContainsAny(r io.Reader, subslices ...[]byte) bool { + if r == nil || len(subslices) == 0 { + return false + } + + largestSlice := 0 + + for _, sl := range subslices { + if len(sl) > largestSlice { + largestSlice = len(sl) + } + } + + if largestSlice == 0 { + return false + } + + bufflen := largestSlice * 4 + halflen := bufflen / 2 + buff := make([]byte, bufflen) + var err error + var n, i int + + for { + i++ + if i == 1 { + n, err = io.ReadAtLeast(r, buff[:halflen], halflen) + } else { + if i != 2 { + // shift left to catch overlapping matches + copy(buff[:], buff[halflen:]) + } + n, err = io.ReadAtLeast(r, buff[halflen:], halflen) + } + + if n > 0 { + for _, sl := range subslices { + if bytes.Contains(buff, sl) { + return true + } + } + } + + if err != nil { + break + } + } + return false +} + +func (a Afero) DirExists(path string) (bool, error) { + return DirExists(a.Fs, path) +} + +// DirExists checks if a path exists and is a directory. +func DirExists(fs Fs, path string) (bool, error) { + fi, err := fs.Stat(path) + if err == nil && fi.IsDir() { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func (a Afero) IsDir(path string) (bool, error) { + return IsDir(a.Fs, path) +} + +// IsDir checks if a given path is a directory. +func IsDir(fs Fs, path string) (bool, error) { + fi, err := fs.Stat(path) + if err != nil { + return false, err + } + return fi.IsDir(), nil +} + +func (a Afero) IsEmpty(path string) (bool, error) { + return IsEmpty(a.Fs, path) +} + +// IsEmpty checks if a given file or directory is empty. +func IsEmpty(fs Fs, path string) (bool, error) { + if b, _ := Exists(fs, path); !b { + return false, fmt.Errorf("%q path does not exist", path) + } + fi, err := fs.Stat(path) + if err != nil { + return false, err + } + if fi.IsDir() { + f, err := fs.Open(path) + if err != nil { + return false, err + } + defer f.Close() + list, err := f.Readdir(-1) + if err != nil { + return false, err + } + return len(list) == 0, nil + } + return fi.Size() == 0, nil +} + +func (a Afero) Exists(path string) (bool, error) { + return Exists(a.Fs, path) +} + +// Check if a file or directory exists. +func Exists(fs Fs, path string) (bool, error) { + _, err := fs.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func FullBaseFsPath(basePathFs *BasePathFs, relativePath string) string { + combinedPath := filepath.Join(basePathFs.path, relativePath) + if parent, ok := basePathFs.source.(*BasePathFs); ok { + return FullBaseFsPath(parent, combinedPath) + } + + return combinedPath +} diff --git a/vendor/modules.txt b/vendor/modules.txt index be632f18aa5..7048a233580 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1732,6 +1732,11 @@ github.com/sony/gobreaker # github.com/spacewander/go-suffix-tree v0.0.0-20191010040751-0865e368c784 ## explicit github.com/spacewander/go-suffix-tree +# github.com/spf13/afero v1.11.0 +## explicit; go 1.19 +github.com/spf13/afero +github.com/spf13/afero/internal/common +github.com/spf13/afero/mem # github.com/spf13/cobra v1.8.0 ## explicit; go 1.15 github.com/spf13/cobra