From c82266264c00f3b41c9b77d5269eb63e6fb7f3c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 13 Mar 2020 12:04:57 +0100 Subject: [PATCH 01/18] initial version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/loader/loader.go | 2 + pkg/storage/fs/ocis/ocis.go | 479 +++++++++++++++++++++++++++++ pkg/storage/fs/ocis/path.go | 168 ++++++++++ pkg/storage/fs/ocis/persistence.go | 29 ++ pkg/storage/fs/ocis/tree.go | 194 ++++++++++++ 5 files changed, 872 insertions(+) create mode 100644 pkg/storage/fs/ocis/ocis.go create mode 100644 pkg/storage/fs/ocis/path.go create mode 100644 pkg/storage/fs/ocis/persistence.go create mode 100644 pkg/storage/fs/ocis/tree.go diff --git a/pkg/storage/fs/loader/loader.go b/pkg/storage/fs/loader/loader.go index 76604757c0..914d537cce 100644 --- a/pkg/storage/fs/loader/loader.go +++ b/pkg/storage/fs/loader/loader.go @@ -25,7 +25,9 @@ import ( _ "github.com/cs3org/reva/pkg/storage/fs/eoshome" _ "github.com/cs3org/reva/pkg/storage/fs/local" _ "github.com/cs3org/reva/pkg/storage/fs/localhome" + _ "github.com/cs3org/reva/pkg/storage/fs/ocis" _ "github.com/cs3org/reva/pkg/storage/fs/owncloud" _ "github.com/cs3org/reva/pkg/storage/fs/s3" + _ "github.com/cs3org/reva/pkg/storage/fs/stub" // Add your own here ) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go new file mode 100644 index 0000000000..d2a9d925bb --- /dev/null +++ b/pkg/storage/fs/ocis/ocis.go @@ -0,0 +1,479 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "path" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/mime" + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/templates" + "github.com/cs3org/reva/pkg/user" + "github.com/gofrs/uuid" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "github.com/pkg/xattr" +) + +func init() { + registry.Register("ocis", New) +} + +type config struct { + // ocis fs works on top of a dir of uuid nodes + DataDirectory string `mapstructure:"data_directory"` + + // UserLayout wraps the internal path with user information. + // Example: if conf.Namespace is /ocis/user and received path is /docs + // and the UserLayout is {{.Username}} the internal path will be: + // /ocis/user//docs + UserLayout string `mapstructure:"user_layout"` + + // EnableHome enables the creation of home directories. + EnableHome bool `mapstructure:"enable_home"` +} + +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "error decoding conf") + return nil, err + } + return c, nil +} + +func (c *config) init(m map[string]interface{}) { + if c.UserLayout == "" { + c.UserLayout = "{{.Username}}" + } + // c.DataDirectory should never end in / unless it is the root + c.DataDirectory = path.Clean(c.DataDirectory) + + // TODO we need a lot more mimetypes + mime.RegisterMime(".txt", "text/plain") +} + +// New returns an implementation to of the storage.FS interface that talk to +// a local filesystem. +func New(m map[string]interface{}) (storage.FS, error) { + c, err := parseConfig(m) + if err != nil { + return nil, err + } + c.init(m) + + dataPaths := []string{ + path.Join(c.DataDirectory, "users"), + path.Join(c.DataDirectory, "nodes"), + path.Join(c.DataDirectory, "trash/files"), + path.Join(c.DataDirectory, "trash/info"), + } + for _, v := range dataPaths { + if err := os.MkdirAll(v, 0700); err != nil { + logger.New().Error().Err(err). + Str("path", v). + Msg("could not create data dir") + } + } + + pw := &Path{ + DataDirectory: c.DataDirectory, + EnableHome: c.EnableHome, + UserLayout: c.UserLayout, + } + + tp, err := NewTree(pw, c.DataDirectory) + if err != nil { + return nil, err + } + + return &ocisfs{ + conf: c, + tp: tp, + pw: pw, + }, nil +} + +type ocisfs struct { + conf *config + tp TreePersistence + pw PathWrapper +} + +func (fs *ocisfs) Shutdown(ctx context.Context) error { + return nil +} + +func (fs *ocisfs) GetQuota(ctx context.Context) (int, int, error) { + return 0, 0, nil +} + +// Home discovery + +func (fs *ocisfs) CreateHome(ctx context.Context) error { + if !fs.conf.EnableHome || fs.conf.UserLayout == "" { + return errtypes.NotSupported("ocisfs: create home not supported") + } + + u, err := getUser(ctx) + if err != nil { + err = errors.Wrap(err, "ocisfs: wrap: no user in ctx and home is enabled") + return err + } + layout := templates.WithUser(u, fs.conf.UserLayout) + home := path.Join(fs.conf.DataDirectory, "users", layout) + + _, err = os.Stat(home) + if err == nil { // home already exists + return nil + } + + // create the users dir + parent := path.Dir(home) + err = os.MkdirAll(parent, 0700) + if err != nil { + // MkdirAll will return success on mkdir over an existing directory. + return errors.Wrap(err, "ocisfs: error creating dir") + } + + // create a directory node (with children subfolder) + nodeID := uuid.Must(uuid.NewV4()).String() + err = os.MkdirAll(path.Join(fs.conf.DataDirectory, "nodes", nodeID, "children"), 0700) + if err != nil { + return errors.Wrap(err, "ocisfs: error node dir") + } + + // link users home to node + return os.Symlink("../nodes/"+nodeID, home) +} + +// GetHome is called to look up the home path for a user +// It is NOT supposed to return the internal path but the external path +func (fs *ocisfs) GetHome(ctx context.Context) (string, error) { + if !fs.conf.EnableHome || fs.conf.UserLayout == "" { + return "", errtypes.NotSupported("ocisfs: get home not supported") + } + u, err := getUser(ctx) + if err != nil { + err = errors.Wrap(err, "ocisfs: wrap: no user in ctx and home is enabled") + return "", err + } + layout := templates.WithUser(u, fs.conf.UserLayout) + return path.Join(fs.conf.DataDirectory, layout), nil // TODO use a namespace? +} + +// Tree persistence + +// GetPathByID returns the fn pointed by the file id, without the internal namespace +func (fs *ocisfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { + return fs.tp.GetPathByID(ctx, id) +} + +func (fs *ocisfs) CreateDir(ctx context.Context, fn string) (err error) { + parent := path.Dir(fn) + var in string + if in, err = fs.pw.Wrap(ctx, parent); err != nil { + return + } + return fs.tp.CreateDir(ctx, in, path.Base(fn)) +} + +func (fs *ocisfs) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { + return fs.tp.CreateReference(ctx, path, targetURI) +} + +func (fs *ocisfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { + var oldInternal, newInternal string + if oldInternal, err = fs.pw.Resolve(ctx, oldRef); err != nil { + return + } + + if newInternal, err = fs.pw.Resolve(ctx, newRef); err != nil { + // TODO might not exist ... + return + } + return fs.tp.Move(ctx, oldInternal, newInternal) +} + +func (fs *ocisfs) GetMD(ctx context.Context, ref *provider.Reference) (ri *provider.ResourceInfo, err error) { + var in string + if in, err = fs.pw.Resolve(ctx, ref); err != nil { + return + } + var md os.FileInfo + md, err = fs.tp.GetMD(ctx, in) + if err != nil { + return nil, err + } + return fs.normalize(ctx, md, in) +} + +func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference) (finfos []*provider.ResourceInfo, err error) { + var in string + if in, err = fs.pw.Resolve(ctx, ref); err != nil { + return + } + var mds []os.FileInfo + mds, err = fs.tp.ListFolder(ctx, in) + if err != nil { + return + } + + for _, md := range mds { + var ri *provider.ResourceInfo + ri, err = fs.normalize(ctx, md, path.Join(in, "children", md.Name())) + if err != nil { + return + } + finfos = append(finfos, ri) + } + return +} + +func (fs *ocisfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { + var in string + if in, err = fs.pw.Resolve(ctx, ref); err != nil { + return + } + return fs.tp.Delete(ctx, in) +} + +// arbitrary metadata persistence + +func (fs *ocisfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { + return errtypes.NotSupported("operation not supported: SetArbitraryMetadata") +} + +func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { + return errtypes.NotSupported("operation not supported: UnsetArbitraryMetadata") +} + +// Data persistence + +func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + var in string // the internal path of the file node + + p := ref.GetPath() + if p != "" { + if p == "/" { + return fmt.Errorf("cannot upload into folder node /") + } + parent := path.Dir(p) + name := path.Base(p) + + inp, err := fs.pw.Wrap(ctx, parent) + + if _, err := os.Stat(path.Join(inp, "children")); os.IsNotExist(err) { + // TODO double check if node is a file + return fmt.Errorf("cannot upload into folder node " + path.Join(inp, "children")) + } + childEntry := path.Join(inp, "children", name) + + // try to determine nodeID by reading link + link, err := os.Readlink(childEntry) + if os.IsNotExist(err) { + // create a new file node + nodeID := uuid.Must(uuid.NewV4()).String() + + in = path.Join(fs.conf.DataDirectory, "nodes", nodeID) + + err = os.MkdirAll(in, 0700) + if err != nil { + return errors.Wrap(err, "ocisfs: could not create node dir") + } + // create back link + // we are not only linking back to the parent, but also to the filename + link = "../" + path.Base(inp) + "/children/" + name + err = os.Symlink(link, path.Join(in, "parentname")) + if err != nil { + return errors.Wrap(err, "ocisfs: could not symlink parent node") + } + + // link child name to node + err = os.Symlink("../../"+nodeID, path.Join(inp, "children", name)) + if err != nil { + return errors.Wrap(err, "ocisfs: could not symlink child entry") + } + } else { + // the nodeID is in the link + // TODO check if link has correct beginning? + nodeID := path.Base(link) + in = path.Join(fs.conf.DataDirectory, "nodes", nodeID) + } + } else if ref.GetId() != nil { + var err error + if in, err = fs.pw.WrapID(ctx, ref.GetId()); err != nil { + return err + } + } else { + return fmt.Errorf("invalid reference %+v", ref) + } + + tmp, err := ioutil.TempFile(in, "._reva_atomic_upload") + if err != nil { + return errors.Wrap(err, "ocisfs: error creating tmp fn at "+in) + } + + _, err = io.Copy(tmp, r) + if err != nil { + return errors.Wrap(err, "ocisfs: error writing to tmp file "+tmp.Name()) + } + + // TODO move old content to version + _ = os.RemoveAll(path.Join(in, "content")) + + err = os.Rename(tmp.Name(), path.Join(in, "content")) + if err != nil { + return err + } + return fs.tp.Propagate(ctx, in) +} + +func (fs *ocisfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { + in, err := fs.pw.Resolve(ctx, ref) + if err != nil { + return nil, errors.Wrap(err, "ocisfs: error resolving ref") + } + + contentPath := path.Join(in, "content") + + r, err := os.Open(contentPath) + if err != nil { + if os.IsNotExist(err) { + return nil, errtypes.NotFound(contentPath) + } + return nil, errors.Wrap(err, "ocisfs: error reading "+contentPath) + } + return r, nil +} + +// Version persistence + +func (fs *ocisfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { + return nil, errtypes.NotSupported("operation not supported: ListRevisions") +} +func (fs *ocisfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { + return nil, errtypes.NotSupported("operation not supported: DownloadRevision") +} + +func (fs *ocisfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { + return errtypes.NotSupported("operation not supported: RestoreRevision") +} + +// Trash persistence + +func (fs *ocisfs) PurgeRecycleItem(ctx context.Context, key string) error { + return errtypes.NotSupported("operation not supported: PurgeRecycleItem") +} + +func (fs *ocisfs) EmptyRecycle(ctx context.Context) error { + return errtypes.NotSupported("operation not supported: EmptyRecycle") +} + +func (fs *ocisfs) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) { + return nil, errtypes.NotSupported("operation not supported: ListRecycle") +} + +func (fs *ocisfs) RestoreRecycleItem(ctx context.Context, key string) error { + return errtypes.NotSupported("operation not supported: RestoreRecycleItem") +} + +// share persistence + +func (fs *ocisfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + return errtypes.NotSupported("operation not supported: AddGrant") +} + +func (fs *ocisfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { + return nil, errtypes.NotSupported("operation not supported: ListGrants") +} + +func (fs *ocisfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + return errtypes.NotSupported("operation not supported: RemoveGrant") +} + +func (fs *ocisfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + return errtypes.NotSupported("operation not supported: UpdateGrant") +} + +// supporting functions + +func getUser(ctx context.Context) (*userpb.User, error) { + u, ok := user.ContextGetUser(ctx) + if !ok { + err := errors.Wrap(errtypes.UserRequired(""), "ocisfs: error getting user from ctx") + return nil, err + } + return u, nil +} + +func (fs *ocisfs) normalize(ctx context.Context, fi os.FileInfo, internal string) (ri *provider.ResourceInfo, err error) { + var fn string + + fn, err = fs.pw.Unwrap(ctx, path.Join("/", internal)) + if err != nil { + return nil, err + } + // TODO GetMD should return the correct fileinfo + nodeType := provider.ResourceType_RESOURCE_TYPE_INVALID + if fi, err = os.Stat(path.Join(internal, "content")); err == nil { + nodeType = provider.ResourceType_RESOURCE_TYPE_FILE + } else if fi, err = os.Stat(path.Join(internal, "children")); err == nil { + nodeType = provider.ResourceType_RESOURCE_TYPE_CONTAINER + } else if fi, err = os.Stat(path.Join(internal, "reference")); err == nil { + // TODO handle references + nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE + } + + var etag []byte + if etag, err = xattr.Get(internal, "user.ocis.etag"); err != nil { + logger.New().Error().Err(err).Msg("could not read etag") + } + ri = &provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: path.Base(internal)}, + Path: fn, + Type: nodeType, + Etag: string(etag), + MimeType: mime.Detect(nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER, fn), + Size: uint64(fi.Size()), + // TODO fix permissions + PermissionSet: &provider.ResourcePermissions{ListContainer: true, CreateContainer: true}, + Mtime: &types.Timestamp{ + Seconds: uint64(fi.ModTime().Unix()), + }, + } + + logger.New().Debug(). + Interface("ri", ri). + Msg("normalized") + + return ri, nil +} diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go new file mode 100644 index 0000000000..0dba13d5d7 --- /dev/null +++ b/pkg/storage/fs/ocis/path.go @@ -0,0 +1,168 @@ +package ocis + +import ( + "context" + "fmt" + "os" + "path" + "strings" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/storage/templates" + "github.com/pkg/errors" +) + +type Path struct { + // ocis fs works on top of a dir of uuid nodes + DataDirectory string `mapstructure:"data_directory"` + + // UserLayout wraps the internal path with user information. + // Example: if conf.Namespace is /ocis/user and received path is /docs + // and the UserLayout is {{.Username}} the internal path will be: + // /ocis/user//docs + UserLayout string `mapstructure:"user_layout"` + + // EnableHome enables the creation of home directories. + EnableHome bool `mapstructure:"enable_home"` +} + +// Resolve takes in a request path or request id and converts it to an internal path. +func (pw *Path) Resolve(ctx context.Context, ref *provider.Reference) (string, error) { + if ref.GetPath() != "" { + return pw.Wrap(ctx, ref.GetPath()) + } + + if ref.GetId() != nil { + return pw.WrapID(ctx, ref.GetId()) + } + + // reference is invalid + return "", fmt.Errorf("invalid reference %+v", ref) +} + +func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error) { + var link, nodeID, root string + if fn == "" { + fn = "/" + } + if pw.EnableHome && pw.UserLayout != "" { + // start at the users root node + var u *userpb.User + + u, err = getUser(ctx) + if err != nil { + err = errors.Wrap(err, "ocisfs: Wrap: no user in ctx and home is enabled") + return + } + + layout := templates.WithUser(u, pw.UserLayout) + root = path.Join(pw.DataDirectory, "users", layout) + + } else { + // start at the storage root node + root = path.Join(pw.DataDirectory, "nodes", "root") + } + + // The symlink contains the nodeID + link, err = os.Readlink(root) + if os.IsNotExist(err) { + err = errtypes.NotFound(fn) + return + } + if err != nil { + err = errors.Wrap(err, "ocisfs: Wrap: readlink error") + return + } + + // extract the nodeID + if strings.HasPrefix(link, "../nodes/") { // TODO does not take into account the template + nodeID = link[9:] + if strings.Contains(nodeID, "/") { + err = fmt.Errorf("ocisfs: node id must not contain / %+v", nodeID) // TODO allow this to distribute nodeids over multiple folders + return + } + } else { + err = fmt.Errorf("ocisfs: expected '../nodes/ prefix, got' %+v", link) + return + } + + if fn != "/" { + // we need to walk the path + segments := strings.Split(strings.TrimLeft(fn, "/"), "/") + for i := range segments { + link, err = os.Readlink(path.Join(pw.DataDirectory, "nodes", nodeID, "children", segments[i])) + if os.IsNotExist(err) { + err = errtypes.NotFound(path.Join(pw.DataDirectory, "nodes", nodeID, "children", segments[i])) + return + } + if err != nil { + err = errors.Wrap(err, "ocisfs: Wrap: readlink error") + return + } + if strings.HasPrefix(link, "../../") { + nodeID = link[6:] + if strings.Contains(nodeID, "/") { + err = fmt.Errorf("ocisfs: node id must not contain / %+v", nodeID) + return + } + } else { + err = fmt.Errorf("ocisfs: expected '../../ prefix, got' %+v", link) + return + } + } + } + + internal = path.Join(pw.DataDirectory, "nodes", nodeID) + return +} + +// WrapID returns the internal path for the id +func (pw *Path) WrapID(ctx context.Context, id *provider.ResourceId) (string, error) { + if id == nil || id.GetOpaqueId() == "" { + return "", fmt.Errorf("invalid resource id %+v", id) + } + return path.Join(pw.DataDirectory, "nodes", id.GetOpaqueId()), nil +} + +func (pw *Path) Unwrap(ctx context.Context, internal string) (external string, err error) { + var link string + for err == nil { + link, err = os.Readlink(path.Join(internal, "parentname")) + if os.IsNotExist(err) { + err = nil + return + } + if err != nil { + err = errors.Wrap(err, "ocisfs: getNode: readlink error") + return + } + parentID := path.Base(path.Dir(path.Dir(link))) + internal = path.Join(pw.DataDirectory, "nodes", parentID) + external = path.Join(path.Base(link), external) + } + return +} + +// ReadParentName reads the symbolic link and extracts the parnetNodeID and the name of the child +func (pw *Path) ReadParentName(ctx context.Context, internal string) (parentNodeID string, name string, err error) { + + // The parentname symlink looks like `../76455834-769e-412a-8a01-68f265365b79/children/myname.txt` + link, err := os.Readlink(path.Join(internal, "parentname")) + if os.IsNotExist(err) { + err = errtypes.NotFound(internal) + return + } + + // check the link follows the correct schema + // TODO count slashes + if strings.HasPrefix(link, "../") { + name = path.Base(link) + parentNodeID = path.Base(path.Dir(path.Dir(link))) + } else { + err = fmt.Errorf("ocisfs: expected '../' prefix, got '%+v'", link) + return + } + return +} diff --git a/pkg/storage/fs/ocis/persistence.go b/pkg/storage/fs/ocis/persistence.go new file mode 100644 index 0000000000..9e2fae05f3 --- /dev/null +++ b/pkg/storage/fs/ocis/persistence.go @@ -0,0 +1,29 @@ +package ocis + +import ( + "context" + "net/url" + "os" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +type TreePersistence interface { + GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) + GetMD(ctx context.Context, internal string) (os.FileInfo, error) + ListFolder(ctx context.Context, internal string) ([]os.FileInfo, error) + CreateDir(ctx context.Context, internal string, newName string) (err error) + CreateReference(ctx context.Context, path string, targetURI *url.URL) error + Move(ctx context.Context, oldInternal string, newInternal string) (err error) + Delete(ctx context.Context, internal string) (err error) + + Propagate(ctx context.Context, internal string) (err error) +} + +type PathWrapper interface { + Resolve(ctx context.Context, ref *provider.Reference) (internal string, err error) + WrapID(ctx context.Context, id *provider.ResourceId) (internal string, err error) + Wrap(ctx context.Context, fn string) (internal string, err error) + Unwrap(ctx context.Context, np string) (external string, err error) + ReadParentName(ctx context.Context, internal string) (parentNodeID string, name string, err error) // Tree persistence? +} diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go new file mode 100644 index 0000000000..ceb1d98b08 --- /dev/null +++ b/pkg/storage/fs/ocis/tree.go @@ -0,0 +1,194 @@ +package ocis + +import ( + "context" + "encoding/hex" + "fmt" + "io/ioutil" + "math/rand" + "net/url" + "os" + "path" + "strings" + "time" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/gofrs/uuid" + "github.com/pkg/errors" + "github.com/pkg/xattr" + "github.com/rs/zerolog/log" +) + +type Tree struct { + pw PathWrapper + DataDirectory string +} + +func NewTree(pw PathWrapper, dataDirectory string) (TreePersistence, error) { + return &Tree{ + pw: pw, + DataDirectory: dataDirectory, + }, nil +} + +func (fs *Tree) GetMD(ctx context.Context, internal string) (os.FileInfo, error) { + md, err := os.Stat(internal) + if err != nil { + if os.IsNotExist(err) { + return nil, errtypes.NotFound(internal) + } + return nil, errors.Wrap(err, "tree: error stating "+internal) + } + + return md, nil +} + +// GetPathByID returns the fn pointed by the file id, without the internal namespace +func (fs *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relativeExternalPath string, err error) { + var internal string + internal, err = fs.pw.WrapID(ctx, id) + if err != nil { + return + } + + relativeExternalPath, err = fs.pw.Unwrap(ctx, path.Join("/", internal)) + if !strings.HasPrefix(relativeExternalPath, fs.DataDirectory) { + return "", fmt.Errorf("ocisfs: GetPathByID wrong prefix") + } + + relativeExternalPath = strings.TrimPrefix(relativeExternalPath, fs.DataDirectory) + return +} + +func (fs *Tree) CreateDir(ctx context.Context, internal string, newName string) (err error) { + + internalChild := path.Join(internal, "children", newName) + _, err = os.Stat(internalChild) + if err == nil { // child already exists + return nil + } + + // create a directory node (with children subfolder) + nodeID := uuid.Must(uuid.NewV4()).String() + + nodePath := path.Join(fs.DataDirectory, "nodes", nodeID) + + err = os.MkdirAll(path.Join(nodePath, "children"), 0700) + if err != nil { + return errors.Wrap(err, "ocisfs: could not create node dir") + } + + // create back link + // we are not only linking back to the parent, but also to the filename + err = os.Symlink("../"+path.Base(internal)+"/children/"+newName, path.Join(nodePath, "parentname")) + if err != nil { + return errors.Wrap(err, "ocisfs: could not symlink parent node") + } + + // link child name to node + err = os.Symlink("../../"+nodeID, internalChild) + if err != nil { + return + } + return fs.Propagate(ctx, nodePath) +} + +func (fs *Tree) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { + return errtypes.NotSupported("operation not supported: CreateReference") +} + +func (fs *Tree) Move(ctx context.Context, oldInternal string, newInternal string) (err error) { + oldParentID, oldName, err = fs.pw.ReadParentName(ctx, oldInternal) + if err != nil { + return err + } + newParentID, newName, err = fs.pw.ReadParentName(ctx, oldInternal) + if err != nil { + return err + } + if err := os.Rename(oldInternal, newInternal); err != nil { + return errors.Wrap(err, "localfs: error moving "+oldInternal+" to "+newInternal) + } + return errtypes.NotSupported("operation not supported: Move") +} + +func (fs *Tree) ListFolder(ctx context.Context, internal string) ([]os.FileInfo, error) { + + children := path.Join(internal, "children") + + mds, err := ioutil.ReadDir(children) + if err != nil { + if os.IsNotExist(err) { + return nil, errtypes.NotFound(children) + } + return nil, errors.Wrap(err, "tree: error listing "+children) + } + + return mds, nil +} + +func (fs *Tree) Delete(ctx context.Context, internal string) (err error) { + // resolve the parent + + // The nodes parentname symlink contains the nodeID and the file name + link, err := os.Readlink(path.Join(internal, "parentname")) + if os.IsNotExist(err) { + err = errtypes.NotFound(internal) + return + } + + // remove child entry from dir + + childName := path.Base(link) + parentNodeID := path.Base(path.Dir(path.Dir(link))) + os.Remove(path.Join(fs.DataDirectory, "nodes", parentNodeID, "children", childName)) + + nodeID := path.Base(internal) + + src := path.Join(fs.DataDirectory, "nodes", nodeID) + trashpath := path.Join(fs.DataDirectory, "trash/files", nodeID) + err = os.Rename(src, trashpath) + if err != nil { + return + } + + // write a trash info ... slightly violating the freedesktop trash spec + t := time.Now() + // TODO store the original Path + info := []byte("[Trash Info]\nParentID=" + parentNodeID + "\nDeletionDate=" + t.Format(time.RFC3339)) + infoPath := path.Join(fs.DataDirectory, "trash/info", nodeID+".trashinfo") + err = ioutil.WriteFile(infoPath, info, 0700) + if err != nil { + return + } + return fs.Propagate(ctx, path.Join(fs.DataDirectory, "nodes", parentNodeID)) +} + +func (fs *Tree) Propagate(ctx context.Context, internal string) (err error) { + // generate an etag + bytes := make([]byte, 16) + if _, err := rand.Read(bytes); err != nil { + return err + } + // store in extended attribute + etag := hex.EncodeToString(bytes) + var link string + for err == nil { + if err := xattr.Set(internal, "user.ocis.etag", []byte(etag)); err != nil { + log.Error().Err(err).Msg("error storing file id") + } + link, err = os.Readlink(path.Join(internal, "parentname")) + if os.IsNotExist(err) { + err = nil + return + } + if err != nil { + err = errors.Wrap(err, "ocisfs: getNode: readlink error") + return + } + parentID := path.Base(path.Dir(path.Dir(link))) + internal = path.Join(fs.DataDirectory, "nodes", parentID) + } + return +} From 4ebe56838d8fa8e2271ae7efc6e3eb5b24f5e463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 13 Mar 2020 21:45:20 +0100 Subject: [PATCH 02/18] introduce NodeInfo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../publicstorageprovider.go | 9 +- pkg/storage/fs/loader/loader.go | 1 - pkg/storage/fs/ocis/node.go | 58 +++ pkg/storage/fs/ocis/ocis.go | 212 +++------ pkg/storage/fs/ocis/path.go | 129 +++--- pkg/storage/fs/ocis/persistence.go | 29 +- pkg/storage/fs/ocis/tree.go | 212 ++++++--- pkg/storage/fs/ocis/upload.go | 438 ++++++++++++++++++ 8 files changed, 819 insertions(+), 269 deletions(-) create mode 100644 pkg/storage/fs/ocis/node.go create mode 100644 pkg/storage/fs/ocis/upload.go diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index afec9427c3..a9984a952b 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -46,14 +46,13 @@ func init() { type config struct { MountPath string `mapstructure:"mount_path"` - MountID string `mapstructure:"mount_id"` GatewayAddr string `mapstructure:"gateway_addr"` } type service struct { - conf *config - mountPath, mountID string - gateway gateway.GatewayAPIClient + conf *config + mountPath string + gateway gateway.GatewayAPIClient } func (s *service) Close() error { @@ -85,7 +84,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { } mountPath := c.MountPath - mountID := c.MountID gateway, err := pool.GetGatewayServiceClient(c.GatewayAddr) if err != nil { @@ -95,7 +93,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { service := &service{ conf: c, mountPath: mountPath, - mountID: mountID, gateway: gateway, } diff --git a/pkg/storage/fs/loader/loader.go b/pkg/storage/fs/loader/loader.go index 914d537cce..744265bc8f 100644 --- a/pkg/storage/fs/loader/loader.go +++ b/pkg/storage/fs/loader/loader.go @@ -28,6 +28,5 @@ import ( _ "github.com/cs3org/reva/pkg/storage/fs/ocis" _ "github.com/cs3org/reva/pkg/storage/fs/owncloud" _ "github.com/cs3org/reva/pkg/storage/fs/s3" - _ "github.com/cs3org/reva/pkg/storage/fs/stub" // Add your own here ) diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go new file mode 100644 index 0000000000..53569ae966 --- /dev/null +++ b/pkg/storage/fs/ocis/node.go @@ -0,0 +1,58 @@ +package ocis + +import ( + "os" + "path/filepath" + + "github.com/google/uuid" + "github.com/pkg/errors" +) + +// NodeInfo allows referencing a node by id and optionally a relative path +type NodeInfo struct { + ParentID string + ID string + Name string + Exists bool +} + +// BecomeParent rewrites the internal state to point to the parent id +func (n *NodeInfo) BecomeParent() { + n.ID = n.ParentID + n.ParentID = "" + n.Name = "" + n.Exists = false +} + +// Create creates a new node in the given root and add symlinks to parent node +// TODO use a reference to the tree to access tho root? +func (n *NodeInfo) Create(root string) (err error) { + + if n.ID != "" { + return errors.Wrap(err, "ocisfs: node already his an id") + } + // create a new file node + n.ID = uuid.New().String() + + nodePath := filepath.Join(root, "nodes", n.ID) + + err = os.MkdirAll(nodePath, 0700) + if err != nil { + return errors.Wrap(err, "ocisfs: could not create node dir") + } + // create back link + // we are not only linking back to the parent, but also to the filename + link := "../" + n.ParentID + "/children/" + n.Name + err = os.Symlink(link, filepath.Join(nodePath, "parentname")) + if err != nil { + return errors.Wrap(err, "ocisfs: could not symlink parent node") + } + + // link child name to node + err = os.Symlink("../../"+n.ID, filepath.Join(root, "nodes", n.ParentID, "children", n.Name)) + if err != nil { + return errors.Wrap(err, "ocisfs: could not symlink child entry") + } + + return nil +} diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index d2a9d925bb..fef47b9d72 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -20,12 +20,10 @@ package ocis import ( "context" - "fmt" "io" - "io/ioutil" "net/url" "os" - "path" + "path/filepath" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -35,7 +33,7 @@ import ( "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/storage/templates" + "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" "github.com/gofrs/uuid" "github.com/mitchellh/mapstructure" @@ -49,7 +47,7 @@ func init() { type config struct { // ocis fs works on top of a dir of uuid nodes - DataDirectory string `mapstructure:"data_directory"` + Root string `mapstructure:"root"` // UserLayout wraps the internal path with user information. // Example: if conf.Namespace is /ocis/user and received path is /docs @@ -72,10 +70,10 @@ func parseConfig(m map[string]interface{}) (*config, error) { func (c *config) init(m map[string]interface{}) { if c.UserLayout == "" { - c.UserLayout = "{{.Username}}" + c.UserLayout = "{{.Id.OpaqueId}}" } // c.DataDirectory should never end in / unless it is the root - c.DataDirectory = path.Clean(c.DataDirectory) + c.Root = filepath.Clean(c.Root) // TODO we need a lot more mimetypes mime.RegisterMime(".txt", "text/plain") @@ -91,10 +89,13 @@ func New(m map[string]interface{}) (storage.FS, error) { c.init(m) dataPaths := []string{ - path.Join(c.DataDirectory, "users"), - path.Join(c.DataDirectory, "nodes"), - path.Join(c.DataDirectory, "trash/files"), - path.Join(c.DataDirectory, "trash/info"), + filepath.Join(c.Root, "users"), + filepath.Join(c.Root, "nodes"), + // notes contain symlinks from nodes//uploads/ to ../../uploads/ + // better to keep uploads on a fast / volatile storage before a workflow finally moves them to the nodes dir + filepath.Join(c.Root, "uploads"), + filepath.Join(c.Root, "trash/files"), + filepath.Join(c.Root, "trash/info"), } for _, v := range dataPaths { if err := os.MkdirAll(v, 0700); err != nil { @@ -105,12 +106,12 @@ func New(m map[string]interface{}) (storage.FS, error) { } pw := &Path{ - DataDirectory: c.DataDirectory, - EnableHome: c.EnableHome, - UserLayout: c.UserLayout, + Root: c.Root, + EnableHome: c.EnableHome, + UserLayout: c.UserLayout, } - tp, err := NewTree(pw, c.DataDirectory) + tp, err := NewTree(pw, c.Root) if err != nil { return nil, err } @@ -136,8 +137,7 @@ func (fs *ocisfs) GetQuota(ctx context.Context) (int, int, error) { return 0, 0, nil } -// Home discovery - +// CreateHome creates a new root node that has no parent id func (fs *ocisfs) CreateHome(ctx context.Context) error { if !fs.conf.EnableHome || fs.conf.UserLayout == "" { return errtypes.NotSupported("ocisfs: create home not supported") @@ -149,7 +149,7 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { return err } layout := templates.WithUser(u, fs.conf.UserLayout) - home := path.Join(fs.conf.DataDirectory, "users", layout) + home := filepath.Join(fs.conf.Root, "users", layout) _, err = os.Stat(home) if err == nil { // home already exists @@ -157,7 +157,7 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { } // create the users dir - parent := path.Dir(home) + parent := filepath.Dir(home) err = os.MkdirAll(parent, 0700) if err != nil { // MkdirAll will return success on mkdir over an existing directory. @@ -166,7 +166,7 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { // create a directory node (with children subfolder) nodeID := uuid.Must(uuid.NewV4()).String() - err = os.MkdirAll(path.Join(fs.conf.DataDirectory, "nodes", nodeID, "children"), 0700) + err = os.MkdirAll(filepath.Join(fs.conf.Root, "nodes", nodeID, "children"), 0700) if err != nil { return errors.Wrap(err, "ocisfs: error node dir") } @@ -187,7 +187,7 @@ func (fs *ocisfs) GetHome(ctx context.Context) (string, error) { return "", err } layout := templates.WithUser(u, fs.conf.UserLayout) - return path.Join(fs.conf.DataDirectory, layout), nil // TODO use a namespace? + return filepath.Join(fs.conf.Root, layout), nil // TODO use a namespace? } // Tree persistence @@ -198,12 +198,11 @@ func (fs *ocisfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (str } func (fs *ocisfs) CreateDir(ctx context.Context, fn string) (err error) { - parent := path.Dir(fn) - var in string - if in, err = fs.pw.Wrap(ctx, parent); err != nil { + var node *NodeInfo + if node, err = fs.pw.Wrap(ctx, fn); err != nil { return } - return fs.tp.CreateDir(ctx, in, path.Base(fn)) + return fs.tp.CreateDir(ctx, node) } func (fs *ocisfs) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { @@ -211,45 +210,51 @@ func (fs *ocisfs) CreateReference(ctx context.Context, path string, targetURI *u } func (fs *ocisfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { - var oldInternal, newInternal string - if oldInternal, err = fs.pw.Resolve(ctx, oldRef); err != nil { + var oldNode, newNode *NodeInfo + if oldNode, err = fs.pw.Resolve(ctx, oldRef); err != nil { + return + } + if !oldNode.Exists { + err = errtypes.NotFound(filepath.Join(oldNode.ParentID, oldNode.Name)) return } - if newInternal, err = fs.pw.Resolve(ctx, newRef); err != nil { - // TODO might not exist ... + if newNode, err = fs.pw.Resolve(ctx, newRef); err != nil { return } - return fs.tp.Move(ctx, oldInternal, newInternal) + return fs.tp.Move(ctx, oldNode, newNode) } -func (fs *ocisfs) GetMD(ctx context.Context, ref *provider.Reference) (ri *provider.ResourceInfo, err error) { - var in string - if in, err = fs.pw.Resolve(ctx, ref); err != nil { +func (fs *ocisfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (ri *provider.ResourceInfo, err error) { + var node *NodeInfo + if node, err = fs.pw.Resolve(ctx, ref); err != nil { return } - var md os.FileInfo - md, err = fs.tp.GetMD(ctx, in) - if err != nil { - return nil, err + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return } - return fs.normalize(ctx, md, in) + return fs.normalize(ctx, node) } -func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference) (finfos []*provider.ResourceInfo, err error) { - var in string - if in, err = fs.pw.Resolve(ctx, ref); err != nil { +func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) (finfos []*provider.ResourceInfo, err error) { + var node *NodeInfo + if node, err = fs.pw.Resolve(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) return } - var mds []os.FileInfo - mds, err = fs.tp.ListFolder(ctx, in) + var children []*NodeInfo + children, err = fs.tp.ListFolder(ctx, node) if err != nil { return } - for _, md := range mds { + for _, child := range children { var ri *provider.ResourceInfo - ri, err = fs.normalize(ctx, md, path.Join(in, "children", md.Name())) + ri, err = fs.normalize(ctx, child) if err != nil { return } @@ -259,11 +264,15 @@ func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference) (finf } func (fs *ocisfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { - var in string - if in, err = fs.pw.Resolve(ctx, ref); err != nil { + var node *NodeInfo + if node, err = fs.pw.Resolve(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) return } - return fs.tp.Delete(ctx, in) + return fs.tp.Delete(ctx, node) } // arbitrary metadata persistence @@ -278,92 +287,22 @@ func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refe // Data persistence -func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { - var in string // the internal path of the file node - - p := ref.GetPath() - if p != "" { - if p == "/" { - return fmt.Errorf("cannot upload into folder node /") - } - parent := path.Dir(p) - name := path.Base(p) - - inp, err := fs.pw.Wrap(ctx, parent) - - if _, err := os.Stat(path.Join(inp, "children")); os.IsNotExist(err) { - // TODO double check if node is a file - return fmt.Errorf("cannot upload into folder node " + path.Join(inp, "children")) - } - childEntry := path.Join(inp, "children", name) - - // try to determine nodeID by reading link - link, err := os.Readlink(childEntry) - if os.IsNotExist(err) { - // create a new file node - nodeID := uuid.Must(uuid.NewV4()).String() - - in = path.Join(fs.conf.DataDirectory, "nodes", nodeID) - - err = os.MkdirAll(in, 0700) - if err != nil { - return errors.Wrap(err, "ocisfs: could not create node dir") - } - // create back link - // we are not only linking back to the parent, but also to the filename - link = "../" + path.Base(inp) + "/children/" + name - err = os.Symlink(link, path.Join(in, "parentname")) - if err != nil { - return errors.Wrap(err, "ocisfs: could not symlink parent node") - } - - // link child name to node - err = os.Symlink("../../"+nodeID, path.Join(inp, "children", name)) - if err != nil { - return errors.Wrap(err, "ocisfs: could not symlink child entry") - } - } else { - // the nodeID is in the link - // TODO check if link has correct beginning? - nodeID := path.Base(link) - in = path.Join(fs.conf.DataDirectory, "nodes", nodeID) - } - } else if ref.GetId() != nil { - var err error - if in, err = fs.pw.WrapID(ctx, ref.GetId()); err != nil { - return err - } - } else { - return fmt.Errorf("invalid reference %+v", ref) - } - - tmp, err := ioutil.TempFile(in, "._reva_atomic_upload") - if err != nil { - return errors.Wrap(err, "ocisfs: error creating tmp fn at "+in) - } - - _, err = io.Copy(tmp, r) - if err != nil { - return errors.Wrap(err, "ocisfs: error writing to tmp file "+tmp.Name()) - } - - // TODO move old content to version - _ = os.RemoveAll(path.Join(in, "content")) - - err = os.Rename(tmp.Name(), path.Join(in, "content")) - if err != nil { - return err - } - return fs.tp.Propagate(ctx, in) +func (fs *ocisfs) ContentPath(node *NodeInfo) string { + return filepath.Join(fs.conf.Root, "nodes", node.ID, "content") } func (fs *ocisfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { - in, err := fs.pw.Resolve(ctx, ref) + node, err := fs.pw.Resolve(ctx, ref) if err != nil { return nil, errors.Wrap(err, "ocisfs: error resolving ref") } - contentPath := path.Join(in, "content") + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return nil, err + } + + contentPath := fs.ContentPath(node) r, err := os.Open(contentPath) if err != nil { @@ -435,30 +374,33 @@ func getUser(ctx context.Context) (*userpb.User, error) { return u, nil } -func (fs *ocisfs) normalize(ctx context.Context, fi os.FileInfo, internal string) (ri *provider.ResourceInfo, err error) { +func (fs *ocisfs) normalize(ctx context.Context, node *NodeInfo) (ri *provider.ResourceInfo, err error) { var fn string - fn, err = fs.pw.Unwrap(ctx, path.Join("/", internal)) + fn, err = fs.pw.Unwrap(ctx, node) if err != nil { return nil, err } - // TODO GetMD should return the correct fileinfo + nodePath := filepath.Join(fs.conf.Root, "nodes", node.ID) + + var fi os.FileInfo nodeType := provider.ResourceType_RESOURCE_TYPE_INVALID - if fi, err = os.Stat(path.Join(internal, "content")); err == nil { + if fi, err = os.Stat(filepath.Join(nodePath, "content")); err == nil { nodeType = provider.ResourceType_RESOURCE_TYPE_FILE - } else if fi, err = os.Stat(path.Join(internal, "children")); err == nil { + } else if fi, err = os.Stat(filepath.Join(nodePath, "children")); err == nil { nodeType = provider.ResourceType_RESOURCE_TYPE_CONTAINER - } else if fi, err = os.Stat(path.Join(internal, "reference")); err == nil { + } else if fi, err = os.Stat(filepath.Join(nodePath, "reference")); err == nil { // TODO handle references nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE } var etag []byte - if etag, err = xattr.Get(internal, "user.ocis.etag"); err != nil { + // TODO store etag in node folder or on content and child nodes? + if etag, err = xattr.Get(nodePath, "user.ocis.etag"); err != nil { logger.New().Error().Err(err).Msg("could not read etag") } ri = &provider.ResourceInfo{ - Id: &provider.ResourceId{OpaqueId: path.Base(internal)}, + Id: &provider.ResourceId{OpaqueId: node.ID}, Path: fn, Type: nodeType, Etag: string(etag), diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go index 0dba13d5d7..63ed78568b 100644 --- a/pkg/storage/fs/ocis/path.go +++ b/pkg/storage/fs/ocis/path.go @@ -4,19 +4,20 @@ import ( "context" "fmt" "os" - "path" + "path/filepath" "strings" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage/templates" + "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/pkg/errors" ) +// Path implements transformations from filepath to node and back type Path struct { // ocis fs works on top of a dir of uuid nodes - DataDirectory string `mapstructure:"data_directory"` + Root string `mapstructure:"root"` // UserLayout wraps the internal path with user information. // Example: if conf.Namespace is /ocis/user and received path is /docs @@ -28,8 +29,8 @@ type Path struct { EnableHome bool `mapstructure:"enable_home"` } -// Resolve takes in a request path or request id and converts it to an internal path. -func (pw *Path) Resolve(ctx context.Context, ref *provider.Reference) (string, error) { +// Resolve takes in a request path or request id and converts it to a NodeInfo +func (pw *Path) Resolve(ctx context.Context, ref *provider.Reference) (*NodeInfo, error) { if ref.GetPath() != "" { return pw.Wrap(ctx, ref.GetPath()) } @@ -39,11 +40,12 @@ func (pw *Path) Resolve(ctx context.Context, ref *provider.Reference) (string, e } // reference is invalid - return "", fmt.Errorf("invalid reference %+v", ref) + return nil, fmt.Errorf("invalid reference %+v", ref) } -func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error) { - var link, nodeID, root string +// Wrap converts a filename into a NodeInfo +func (pw *Path) Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) { + var link, root string if fn == "" { fn = "/" } @@ -58,33 +60,16 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error } layout := templates.WithUser(u, pw.UserLayout) - root = path.Join(pw.DataDirectory, "users", layout) + root = filepath.Join(pw.Root, "users", layout) } else { // start at the storage root node - root = path.Join(pw.DataDirectory, "nodes", "root") + root = filepath.Join(pw.Root, "nodes/root") } + node, err = pw.ReadRootLink(root) // The symlink contains the nodeID - link, err = os.Readlink(root) - if os.IsNotExist(err) { - err = errtypes.NotFound(fn) - return - } if err != nil { - err = errors.Wrap(err, "ocisfs: Wrap: readlink error") - return - } - - // extract the nodeID - if strings.HasPrefix(link, "../nodes/") { // TODO does not take into account the template - nodeID = link[9:] - if strings.Contains(nodeID, "/") { - err = fmt.Errorf("ocisfs: node id must not contain / %+v", nodeID) // TODO allow this to distribute nodeids over multiple folders - return - } - } else { - err = fmt.Errorf("ocisfs: expected '../nodes/ prefix, got' %+v", link) return } @@ -92,9 +77,20 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error // we need to walk the path segments := strings.Split(strings.TrimLeft(fn, "/"), "/") for i := range segments { - link, err = os.Readlink(path.Join(pw.DataDirectory, "nodes", nodeID, "children", segments[i])) + node.ParentID = node.ID + node.ID = "" + node.Name = segments[i] + + link, err = os.Readlink(filepath.Join(pw.Root, "nodes", node.ParentID, "children", node.Name)) if os.IsNotExist(err) { - err = errtypes.NotFound(path.Join(pw.DataDirectory, "nodes", nodeID, "children", segments[i])) + node.Exists = false + // if this is the last segment we can use it as the node name + if i == len(segments)-1 { + err = nil + return + } + + err = errtypes.NotFound(filepath.Join(pw.Root, "nodes", node.ParentID, "children", node.Name)) return } if err != nil { @@ -102,11 +98,7 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error return } if strings.HasPrefix(link, "../../") { - nodeID = link[6:] - if strings.Contains(nodeID, "/") { - err = fmt.Errorf("ocisfs: node id must not contain / %+v", nodeID) - return - } + node.ID = filepath.Base(link) } else { err = fmt.Errorf("ocisfs: expected '../../ prefix, got' %+v", link) return @@ -114,55 +106,84 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error } } - internal = path.Join(pw.DataDirectory, "nodes", nodeID) return } // WrapID returns the internal path for the id -func (pw *Path) WrapID(ctx context.Context, id *provider.ResourceId) (string, error) { - if id == nil || id.GetOpaqueId() == "" { - return "", fmt.Errorf("invalid resource id %+v", id) +func (pw *Path) WrapID(ctx context.Context, id *provider.ResourceId) (*NodeInfo, error) { + if id == nil || id.OpaqueId == "" { + return nil, fmt.Errorf("invalid resource id %+v", id) } - return path.Join(pw.DataDirectory, "nodes", id.GetOpaqueId()), nil + return &NodeInfo{ID: id.OpaqueId}, nil } -func (pw *Path) Unwrap(ctx context.Context, internal string) (external string, err error) { - var link string +func (pw *Path) Unwrap(ctx context.Context, ni *NodeInfo) (external string, err error) { for err == nil { - link, err = os.Readlink(path.Join(internal, "parentname")) + err = pw.FillParentAndName(ni) if os.IsNotExist(err) { err = nil return } if err != nil { - err = errors.Wrap(err, "ocisfs: getNode: readlink error") + err = errors.Wrap(err, "ocisfs: Unwrap: could not fill node") return } - parentID := path.Base(path.Dir(path.Dir(link))) - internal = path.Join(pw.DataDirectory, "nodes", parentID) - external = path.Join(path.Base(link), external) + external = filepath.Join(ni.Name, external) + ni.BecomeParent() } return } -// ReadParentName reads the symbolic link and extracts the parnetNodeID and the name of the child -func (pw *Path) ReadParentName(ctx context.Context, internal string) (parentNodeID string, name string, err error) { +// FillParentAndName reads the symbolic link and extracts the parent ID and the name of the node if necessary +func (pw *Path) FillParentAndName(node *NodeInfo) (err error) { + + if node == nil || node.ID == "" { + err = fmt.Errorf("ocisfs: invalid node info '%+v'", node) + } + // check if node is already filled + if node.ParentID != "" && node.Name != "" { + return + } + + var link string // The parentname symlink looks like `../76455834-769e-412a-8a01-68f265365b79/children/myname.txt` - link, err := os.Readlink(path.Join(internal, "parentname")) - if os.IsNotExist(err) { - err = errtypes.NotFound(internal) + link, err = os.Readlink(filepath.Join(pw.Root, "nodes", node.ID, "parentname")) + if err != nil { return } // check the link follows the correct schema // TODO count slashes if strings.HasPrefix(link, "../") { - name = path.Base(link) - parentNodeID = path.Base(path.Dir(path.Dir(link))) + node.Name = filepath.Base(link) + node.ParentID = filepath.Base(filepath.Dir(filepath.Dir(link))) + node.Exists = true } else { err = fmt.Errorf("ocisfs: expected '../' prefix, got '%+v'", link) return } return } + +// ReadRootLink reads the symbolic link and extracts the node id +func (pw *Path) ReadRootLink(root string) (node *NodeInfo, err error) { + + // A root symlink looks like `../nodes/76455834-769e-412a-8a01-68f265365b79` + link, err := os.Readlink(root) + if os.IsNotExist(err) { + err = errtypes.NotFound(root) + return + } + + // extract the nodeID + if strings.HasPrefix(link, "../nodes/") { + node = &NodeInfo{ + ID: filepath.Base(link), + Exists: true, + } + } else { + err = fmt.Errorf("ocisfs: expected '../nodes/ prefix, got' %+v", link) + } + return +} diff --git a/pkg/storage/fs/ocis/persistence.go b/pkg/storage/fs/ocis/persistence.go index 9e2fae05f3..0698585a3a 100644 --- a/pkg/storage/fs/ocis/persistence.go +++ b/pkg/storage/fs/ocis/persistence.go @@ -8,22 +8,29 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ) +// TreePersistence is used to manage a tree hierarchy type TreePersistence interface { GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) - GetMD(ctx context.Context, internal string) (os.FileInfo, error) - ListFolder(ctx context.Context, internal string) ([]os.FileInfo, error) - CreateDir(ctx context.Context, internal string, newName string) (err error) + GetMD(ctx context.Context, node *NodeInfo) (os.FileInfo, error) + ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, error) + CreateDir(ctx context.Context, node *NodeInfo) (err error) CreateReference(ctx context.Context, path string, targetURI *url.URL) error - Move(ctx context.Context, oldInternal string, newInternal string) (err error) - Delete(ctx context.Context, internal string) (err error) + Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) (err error) + Delete(ctx context.Context, node *NodeInfo) (err error) - Propagate(ctx context.Context, internal string) (err error) + Propagate(ctx context.Context, node *NodeInfo) (err error) } +// PathWrapper is used to encapsulate path transformations type PathWrapper interface { - Resolve(ctx context.Context, ref *provider.Reference) (internal string, err error) - WrapID(ctx context.Context, id *provider.ResourceId) (internal string, err error) - Wrap(ctx context.Context, fn string) (internal string, err error) - Unwrap(ctx context.Context, np string) (external string, err error) - ReadParentName(ctx context.Context, internal string) (parentNodeID string, name string, err error) // Tree persistence? + Resolve(ctx context.Context, ref *provider.Reference) (node *NodeInfo, err error) + WrapID(ctx context.Context, id *provider.ResourceId) (node *NodeInfo, err error) + + // Wrap returns a NodeInfo object: + // - if the node exists with the node id, name and parent + // - if only the parent exists, the node id is empty + Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) + Unwrap(ctx context.Context, node *NodeInfo) (external string, err error) + FillParentAndName(node *NodeInfo) (err error) // Tree persistence? + ReadRootLink(root string) (node *NodeInfo, err error) } diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index ceb1d98b08..39520c410c 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -3,13 +3,11 @@ package ocis import ( "context" "encoding/hex" - "fmt" "io/ioutil" "math/rand" "net/url" "os" - "path" - "strings" + "path/filepath" "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -20,11 +18,13 @@ import ( "github.com/rs/zerolog/log" ) +// Tree manages a hierarchical tree type Tree struct { pw PathWrapper DataDirectory string } +// NewTree creates a new Tree instance func NewTree(pw PathWrapper, dataDirectory string) (TreePersistence, error) { return &Tree{ pw: pw, @@ -32,13 +32,14 @@ func NewTree(pw PathWrapper, dataDirectory string) (TreePersistence, error) { }, nil } -func (fs *Tree) GetMD(ctx context.Context, internal string) (os.FileInfo, error) { - md, err := os.Stat(internal) +// GetMD returns the metadata of a node in the tree +func (fs *Tree) GetMD(ctx context.Context, node *NodeInfo) (os.FileInfo, error) { + md, err := os.Stat(filepath.Join(fs.DataDirectory, "nodes", node.ID)) if err != nil { if os.IsNotExist(err) { - return nil, errtypes.NotFound(internal) + return nil, errtypes.NotFound(node.ID) } - return nil, errors.Wrap(err, "tree: error stating "+internal) + return nil, errors.Wrap(err, "tree: error stating "+node.ID) } return md, nil @@ -46,78 +47,149 @@ func (fs *Tree) GetMD(ctx context.Context, internal string) (os.FileInfo, error) // GetPathByID returns the fn pointed by the file id, without the internal namespace func (fs *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relativeExternalPath string, err error) { - var internal string - internal, err = fs.pw.WrapID(ctx, id) + var node *NodeInfo + node, err = fs.pw.WrapID(ctx, id) if err != nil { return } - relativeExternalPath, err = fs.pw.Unwrap(ctx, path.Join("/", internal)) - if !strings.HasPrefix(relativeExternalPath, fs.DataDirectory) { - return "", fmt.Errorf("ocisfs: GetPathByID wrong prefix") - } - - relativeExternalPath = strings.TrimPrefix(relativeExternalPath, fs.DataDirectory) + relativeExternalPath, err = fs.pw.Unwrap(ctx, node) return } -func (fs *Tree) CreateDir(ctx context.Context, internal string, newName string) (err error) { +// CreateDir creates a new directory entry in the tree +func (fs *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { - internalChild := path.Join(internal, "children", newName) - _, err = os.Stat(internalChild) - if err == nil { // child already exists - return nil + // TODO always try to fill node? + if node.Exists || node.ID != "" { // child already exists + return } // create a directory node (with children subfolder) - nodeID := uuid.Must(uuid.NewV4()).String() + node.ID = uuid.Must(uuid.NewV4()).String() - nodePath := path.Join(fs.DataDirectory, "nodes", nodeID) + newPath := filepath.Join(fs.DataDirectory, "nodes", node.ID) - err = os.MkdirAll(path.Join(nodePath, "children"), 0700) + err = os.MkdirAll(filepath.Join(newPath, "children"), 0700) if err != nil { return errors.Wrap(err, "ocisfs: could not create node dir") } // create back link // we are not only linking back to the parent, but also to the filename - err = os.Symlink("../"+path.Base(internal)+"/children/"+newName, path.Join(nodePath, "parentname")) + err = os.Symlink("../"+node.ParentID+"/children/"+node.Name, filepath.Join(newPath, "parentname")) if err != nil { return errors.Wrap(err, "ocisfs: could not symlink parent node") } - // link child name to node - err = os.Symlink("../../"+nodeID, internalChild) + // make child appear in listings + err = os.Symlink("../../"+node.ID, filepath.Join(fs.DataDirectory, "nodes", node.ParentID, "children", node.Name)) if err != nil { return } - return fs.Propagate(ctx, nodePath) + return fs.Propagate(ctx, node) } +// CreateReference creates a new reference entry in the tree func (fs *Tree) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { return errtypes.NotSupported("operation not supported: CreateReference") } -func (fs *Tree) Move(ctx context.Context, oldInternal string, newInternal string) (err error) { - oldParentID, oldName, err = fs.pw.ReadParentName(ctx, oldInternal) +// Move replaces the target with the source +func (fs *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) (err error) { + err = fs.pw.FillParentAndName(newNode) + if os.IsNotExist(err) { + err = nil + return + } + + // if target exists delete it without trashing it + if newNode.Exists { + err = fs.pw.FillParentAndName(newNode) + if os.IsNotExist(err) { + err = nil + return + } + if err := os.RemoveAll(filepath.Join(fs.DataDirectory, "nodes", newNode.ID)); err != nil { + return errors.Wrap(err, "ocisfs: Move: error deleting target node "+newNode.ID) + } + } + // are we renaming? + if oldNode.ParentID == newNode.ParentID { + + nodePath := filepath.Join(fs.DataDirectory, "nodes", oldNode.ID) + + // update back link + // we are not only linking back to the parent, but also to the filename + err = os.Remove(filepath.Join(nodePath, "parentname")) + if err != nil { + return errors.Wrap(err, "ocisfs: could not remove parent link") + } + err = os.Symlink("../"+oldNode.ParentID+"/children/"+newNode.Name, filepath.Join(nodePath, "parentname")) + if err != nil { + return errors.Wrap(err, "ocisfs: could not symlink parent") + } + + // rename child + err = os.Rename( + filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, "children", oldNode.Name), + filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, "children", newNode.Name), + ) + if err != nil { + return errors.Wrap(err, "ocisfs: could not rename symlink") + } + return fs.Propagate(ctx, oldNode) + } + // we are moving the node to a new parent, any target has been removed + // bring old node to the new parent + + nodePath := filepath.Join(fs.DataDirectory, "nodes", oldNode.ID) + + // update back link + // we are not only linking back to the parent, but also to the filename + err = os.Remove(filepath.Join(nodePath, "parentname")) if err != nil { - return err + return errors.Wrap(err, "ocisfs: could not remove parent link") } - newParentID, newName, err = fs.pw.ReadParentName(ctx, oldInternal) + err = os.Symlink("../"+newNode.ParentID+"/children/"+newNode.Name, filepath.Join(nodePath, "parentname")) if err != nil { - return err + return errors.Wrap(err, "ocisfs: could not symlink parent") + } + + // rename child + err = os.Rename( + filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, "children", oldNode.Name), + filepath.Join(fs.DataDirectory, "nodes", newNode.ParentID, "children", newNode.Name), + ) + if err != nil { + return errors.Wrap(err, "ocisfs: could not rename symlink") + } + + // TODO inefficient because we might update several nodes twice, only propagate unchanged nodes? + // collect in a list, then only stat each node once + // also do this in a go routine ... webdav should check the etag async + err = fs.Propagate(ctx, oldNode) + if err != nil { + return errors.Wrap(err, "ocisfs: Move: could not propagate old node") } - if err := os.Rename(oldInternal, newInternal); err != nil { - return errors.Wrap(err, "localfs: error moving "+oldInternal+" to "+newInternal) + err = fs.Propagate(ctx, newNode) + if err != nil { + return errors.Wrap(err, "ocisfs: Move: could not propagate old node") } - return errtypes.NotSupported("operation not supported: Move") + return nil } -func (fs *Tree) ListFolder(ctx context.Context, internal string) ([]os.FileInfo, error) { +// ChildrenPath returns the absolute path to childrens in a node +// TODO move to node? +func (fs *Tree) ChildrenPath(node *NodeInfo) string { + return filepath.Join(fs.DataDirectory, "nodes", node.ID, "children") +} - children := path.Join(internal, "children") +// ListFolder lists the children inside a folder +func (fs *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, error) { - mds, err := ioutil.ReadDir(children) + children := fs.ChildrenPath(node) + f, err := os.Open(children) if err != nil { if os.IsNotExist(err) { return nil, errtypes.NotFound(children) @@ -125,29 +197,44 @@ func (fs *Tree) ListFolder(ctx context.Context, internal string) ([]os.FileInfo, return nil, errors.Wrap(err, "tree: error listing "+children) } - return mds, nil -} + names, err := f.Readdirnames(0) + nodes := []*NodeInfo{} + for i := range names { + link, err := os.Readlink(filepath.Join(children, names[i])) + if err != nil { + // TODO log + continue + } + n := &NodeInfo{ + ParentID: node.ID, + ID: filepath.Base(link), + Name: names[i], + Exists: true, // TODO + } -func (fs *Tree) Delete(ctx context.Context, internal string) (err error) { - // resolve the parent + nodes = append(nodes, n) + } + return nodes, nil +} - // The nodes parentname symlink contains the nodeID and the file name - link, err := os.Readlink(path.Join(internal, "parentname")) +// Delete deletes a node in the tree +func (fs *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { + err = fs.pw.FillParentAndName(node) if os.IsNotExist(err) { - err = errtypes.NotFound(internal) + err = nil + return + } + if err != nil { + err = errors.Wrap(err, "ocisfs: Delete: FillParentAndName error") return } // remove child entry from dir - childName := path.Base(link) - parentNodeID := path.Base(path.Dir(path.Dir(link))) - os.Remove(path.Join(fs.DataDirectory, "nodes", parentNodeID, "children", childName)) + os.Remove(filepath.Join(fs.DataDirectory, "nodes", node.ParentID, "children", node.Name)) - nodeID := path.Base(internal) - - src := path.Join(fs.DataDirectory, "nodes", nodeID) - trashpath := path.Join(fs.DataDirectory, "trash/files", nodeID) + src := filepath.Join(fs.DataDirectory, "nodes", node.ID) + trashpath := filepath.Join(fs.DataDirectory, "trash/files", node.ID) err = os.Rename(src, trashpath) if err != nil { return @@ -156,16 +243,18 @@ func (fs *Tree) Delete(ctx context.Context, internal string) (err error) { // write a trash info ... slightly violating the freedesktop trash spec t := time.Now() // TODO store the original Path - info := []byte("[Trash Info]\nParentID=" + parentNodeID + "\nDeletionDate=" + t.Format(time.RFC3339)) - infoPath := path.Join(fs.DataDirectory, "trash/info", nodeID+".trashinfo") + info := []byte("[Trash Info]\nParentID=" + node.ParentID + "\nDeletionDate=" + t.Format(time.RFC3339)) + infoPath := filepath.Join(fs.DataDirectory, "trash/info", node.ID+".trashinfo") err = ioutil.WriteFile(infoPath, info, 0700) if err != nil { return } - return fs.Propagate(ctx, path.Join(fs.DataDirectory, "nodes", parentNodeID)) + + return fs.Propagate(ctx, &NodeInfo{ID: node.ParentID}) } -func (fs *Tree) Propagate(ctx context.Context, internal string) (err error) { +// Propagate propagates changes to the root of the tree +func (fs *Tree) Propagate(ctx context.Context, node *NodeInfo) (err error) { // generate an etag bytes := make([]byte, 16) if _, err := rand.Read(bytes); err != nil { @@ -173,22 +262,21 @@ func (fs *Tree) Propagate(ctx context.Context, internal string) (err error) { } // store in extended attribute etag := hex.EncodeToString(bytes) - var link string for err == nil { - if err := xattr.Set(internal, "user.ocis.etag", []byte(etag)); err != nil { + if err := xattr.Set(filepath.Join(fs.DataDirectory, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { log.Error().Err(err).Msg("error storing file id") } - link, err = os.Readlink(path.Join(internal, "parentname")) + err = fs.pw.FillParentAndName(node) if os.IsNotExist(err) { err = nil return } if err != nil { - err = errors.Wrap(err, "ocisfs: getNode: readlink error") + err = errors.Wrap(err, "ocisfs: Propagate: readlink error") return } - parentID := path.Base(path.Dir(path.Dir(link))) - internal = path.Join(fs.DataDirectory, "nodes", parentID) + + node.BecomeParent() } return } diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go new file mode 100644 index 0000000000..53e8a4f410 --- /dev/null +++ b/pkg/storage/fs/ocis/upload.go @@ -0,0 +1,438 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/storage/utils/templates" + "github.com/cs3org/reva/pkg/user" + "github.com/google/uuid" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" +) + +var defaultFilePerm = os.FileMode(0664) + +// TODO deprecated ... use tus + +func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + + node, err := fs.pw.Resolve(ctx, ref) + if err != nil { + return err + } + + if !node.Exists { + err = node.Create(fs.conf.Root) + if err != nil { + return errors.Wrap(err, "ocisfs: could not create node") + } + } + nodePath := filepath.Join(fs.conf.Root, "nodes", node.ID) + + tmp, err := ioutil.TempFile(nodePath, "._reva_atomic_upload") + if err != nil { + return errors.Wrap(err, "ocisfs: error creating tmp fn at "+nodePath) + } + + _, err = io.Copy(tmp, r) + if err != nil { + return errors.Wrap(err, "ocisfs: error writing to tmp file "+tmp.Name()) + } + + // TODO move old content to version + //_ = os.RemoveAll(path.Join(nodePath, "content")) + + err = os.Rename(tmp.Name(), filepath.Join(nodePath, "content")) + if err != nil { + return err + } + return fs.tp.Propagate(ctx, node) + +} + +// InitiateUpload returns an upload id that can be used for uploads with tus +// TODO read optional content for small files in this request +func (fs *ocisfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (uploadID string, err error) { + var relative string // the internal path of the file node + + node, err := fs.pw.Resolve(ctx, ref) + if err != nil { + return "", err + } + relative, err = fs.pw.Unwrap(ctx, node) + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": node.Name, + "dir": filepath.Dir(relative), + }, + Size: uploadLength, + } + + if metadata != nil && metadata["mtime"] != "" { + info.MetaData["mtime"] = metadata["mtime"] + } + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return "", err + } + + info, _ = upload.GetInfo(ctx) + + return info.ID, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *ocisfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) + composer.UseConcater(fs) + composer.UseLengthDeferrer(fs) +} + +// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol +// - the storage needs to implement NewUpload and GetUpload +// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload + +func (fs *ocisfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("ocisfs: NewUpload") + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("ocisfs: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("ocisfs: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + node, err := fs.pw.Wrap(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) + + log.Debug().Interface("info", info).Interface("node", node).Msg("ocisfs: resolved filename") + + info.ID = uuid.New().String() + + binPath, err := fs.getUploadPath(ctx, info.ID) + if err != nil { + return nil, errors.Wrap(err, "ocisfs: error resolving upload path") + } + usr := user.ContextMustGetUser(ctx) + info.Storage = map[string]string{ + "Type": "OCISStore", + "BinPath": binPath, + + "NodeId": node.ID, + "NodeParentId": node.ParentID, + "NodeName": node.Name, + + "Idp": usr.Id.Idp, + "UserId": usr.Id.OpaqueId, + "UserName": usr.Username, + + "LogLevel": log.GetLevel().String(), + } + // Create binary file in the upload folder with no content + log.Debug().Interface("info", info).Msg("ocisfs: built storage info") + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + u := &fileUpload{ + info: info, + binPath: binPath, + infoPath: filepath.Join(fs.conf.Root, "uploads", info.ID+".info"), + fs: fs, + ctx: ctx, + } + + if !info.SizeIsDeferred && info.Size == 0 { + log.Debug().Interface("info", info).Msg("ocisfs: finishing upload for empty file") + // no need to create info file and finish directly + err := u.FinishUpload(ctx) + if err != nil { + return nil, err + } + return u, nil + } + + // writeInfo creates the file by itself if necessary + err = u.writeInfo() + if err != nil { + return nil, err + } + + return u, nil +} + +func (fs *ocisfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { + u, ok := user.ContextGetUser(ctx) + if !ok { + err := errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") + return "", err + } + layout := templates.WithUser(u, fs.conf.UserLayout) + return filepath.Join(fs.conf.Root, layout, "uploads", uploadID), nil +} + +// GetUpload returns the Upload for the given upload id +func (fs *ocisfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + infoPath := filepath.Join(fs.conf.Root, "uploads", id+".info") + + info := tusd.FileInfo{} + data, err := ioutil.ReadFile(infoPath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + stat, err := os.Stat(info.Storage["BinPath"]) + if err != nil { + return nil, err + } + + info.Offset = stat.Size() + + u := &userpb.User{ + Id: &userpb.UserId{ + Idp: info.Storage["Idp"], + OpaqueId: info.Storage["UserId"], + }, + Username: info.Storage["UserName"], + } + + ctx = user.ContextSetUser(ctx, u) + // TODO configure the logger the same way ... store and add traceid in file info + + var opts []logger.Option + opts = append(opts, logger.WithLevel(info.Storage["LogLevel"])) + opts = append(opts, logger.WithWriter(os.Stderr, logger.ConsoleMode)) + l := logger.New(opts...) + + sub := l.With().Int("pid", os.Getpid()).Logger() + + ctx = appctx.WithLogger(ctx, &sub) + + return &fileUpload{ + info: info, + binPath: info.Storage["BinPath"], + infoPath: infoPath, + fs: fs, + ctx: ctx, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *ocisfs + // a context with a user + // TODO add logger as well? + ctx context.Context +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for OwnCloudStore it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() // TODO info is written here ... we need to truncate in DiscardChunk + + return n, err +} + +// GetReader returns an io.Reader for the upload +func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(upload.binPath) +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + if err != nil { + return err + } + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) error { + + node := &NodeInfo{ + ID: upload.info.Storage["NodeId"], + ParentID: upload.info.Storage["NodeParentId"], + Name: upload.info.Storage["NodeName"], + } + + if node.ID == "" { + err := node.Create(upload.fs.conf.Root) + if err != nil { + return errors.Wrap(err, "ocisfs: could not create node") + } + } + contentPath := filepath.Join(upload.fs.conf.Root, "nodes", node.ID, "content") + + log := appctx.GetLogger(upload.ctx) + err := os.Rename(upload.binPath, contentPath) + if err != nil { + log.Err(err).Interface("info", upload.info). + Str("binPath", upload.binPath). + Str("contentPath", contentPath). + Msg("ocisfs: could not rename") + return err + } + + // only delete the upload if it was successfully written to the storage + if err := os.Remove(upload.infoPath); err != nil { + if !os.IsNotExist(err) { + log.Err(err).Interface("info", upload.info).Msg("ocisfs: could not delete upload info") + return err + } + } + // use set arbitrary metadata? + /*if upload.info.MetaData["mtime"] != "" { + err := upload.fs.SetMtime(ctx, np, upload.info.MetaData["mtime"]) + if err != nil { + log.Err(err).Interface("info", upload.info).Msg("ocisfs: could not set mtime metadata") + return err + } + }*/ + + return upload.fs.tp.Propagate(upload.ctx, node) +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returns a TerminatableUpload +func (fs *ocisfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) error { + if err := os.Remove(upload.infoPath); err != nil { + if !os.IsNotExist(err) { + return err + } + } + if err := os.Remove(upload.binPath); err != nil { + if !os.IsNotExist(err) { + return err + } + } + return nil +} + +// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation +// - the storage needs to implement AsLengthDeclarableUpload +// - the upload needs to implement DeclareLength + +// AsLengthDeclarableUpload returns a LengthDeclarableUpload +func (fs *ocisfs) AsLengthDeclarableUpload(upload tusd.Upload) tusd.LengthDeclarableUpload { + return upload.(*fileUpload) +} + +// DeclareLength updates the upload length information +func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error { + upload.info.Size = length + upload.info.SizeIsDeferred = false + return upload.writeInfo() +} + +// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation +// - the storage needs to implement AsConcatableUpload +// - the upload needs to implement ConcatUploads + +// AsConcatableUpload returns a ConcatableUpload +func (fs *ocisfs) AsConcatableUpload(upload tusd.Upload) tusd.ConcatableUpload { + return upload.(*fileUpload) +} + +// ConcatUploads concatenates multiple uploads +func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Upload) (err error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return err + } + defer file.Close() + + for _, partialUpload := range uploads { + fileUpload := partialUpload.(*fileUpload) + + src, err := os.Open(fileUpload.binPath) + if err != nil { + return err + } + + if _, err := io.Copy(file, src); err != nil { + return err + } + } + + return +} From 60335b0c2d2e286585598437f129ff5f0cd8dcdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 1 Sep 2020 15:52:47 +0200 Subject: [PATCH 03/18] fix metadata normalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/ocis.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index fef47b9d72..8795979cbe 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -252,13 +252,10 @@ func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKey return } - for _, child := range children { - var ri *provider.ResourceInfo - ri, err = fs.normalize(ctx, child) - if err != nil { - return + for i := range children { + if ri, err := fs.normalize(ctx, children[i]); err == nil { + finfos = append(finfos, ri) } - finfos = append(finfos, ri) } return } @@ -377,10 +374,6 @@ func getUser(ctx context.Context) (*userpb.User, error) { func (fs *ocisfs) normalize(ctx context.Context, node *NodeInfo) (ri *provider.ResourceInfo, err error) { var fn string - fn, err = fs.pw.Unwrap(ctx, node) - if err != nil { - return nil, err - } nodePath := filepath.Join(fs.conf.Root, "nodes", node.ID) var fi os.FileInfo @@ -399,8 +392,15 @@ func (fs *ocisfs) normalize(ctx context.Context, node *NodeInfo) (ri *provider.R if etag, err = xattr.Get(nodePath, "user.ocis.etag"); err != nil { logger.New().Error().Err(err).Msg("could not read etag") } + + id := &provider.ResourceId{OpaqueId: node.ID} + // Unwrap changes the node because it traverses the tree + fn, err = fs.pw.Unwrap(ctx, node) + if err != nil { + return nil, err + } ri = &provider.ResourceInfo{ - Id: &provider.ResourceId{OpaqueId: node.ID}, + Id: id, Path: fn, Type: nodeType, Etag: string(etag), From 4942d3a91984abcf910d0a23f44493a65ad55ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 3 Sep 2020 10:51:13 +0200 Subject: [PATCH 04/18] stick to github.com/google/uuid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/ocis.go | 4 ++-- pkg/storage/fs/ocis/tree.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 8795979cbe..79f03ced25 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -35,7 +35,7 @@ import ( "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" - "github.com/gofrs/uuid" + "github.com/google/uuid" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/pkg/xattr" @@ -165,7 +165,7 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { } // create a directory node (with children subfolder) - nodeID := uuid.Must(uuid.NewV4()).String() + nodeID := uuid.New().String() err = os.MkdirAll(filepath.Join(fs.conf.Root, "nodes", nodeID, "children"), 0700) if err != nil { return errors.Wrap(err, "ocisfs: error node dir") diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index 39520c410c..9f9e8996fc 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -12,7 +12,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" - "github.com/gofrs/uuid" + "github.com/google/uuid" "github.com/pkg/errors" "github.com/pkg/xattr" "github.com/rs/zerolog/log" @@ -66,7 +66,7 @@ func (fs *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { } // create a directory node (with children subfolder) - node.ID = uuid.Must(uuid.NewV4()).String() + node.ID = uuid.New().String() newPath := filepath.Join(fs.DataDirectory, "nodes", node.ID) From 4ff3ecf6a7f4da977ec1a1b617c6d5db4f9cf575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 3 Sep 2020 15:26:32 +0200 Subject: [PATCH 05/18] simpler layout, parentid and name as ext attr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/node.go | 41 ------------ pkg/storage/fs/ocis/ocis.go | 70 ++++++++++---------- pkg/storage/fs/ocis/path.go | 55 ++++++++-------- pkg/storage/fs/ocis/tree.go | 121 ++++++++++++++-------------------- pkg/storage/fs/ocis/upload.go | 73 +++++++++++--------- 5 files changed, 155 insertions(+), 205 deletions(-) diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go index 53569ae966..8db46f3130 100644 --- a/pkg/storage/fs/ocis/node.go +++ b/pkg/storage/fs/ocis/node.go @@ -1,13 +1,5 @@ package ocis -import ( - "os" - "path/filepath" - - "github.com/google/uuid" - "github.com/pkg/errors" -) - // NodeInfo allows referencing a node by id and optionally a relative path type NodeInfo struct { ParentID string @@ -23,36 +15,3 @@ func (n *NodeInfo) BecomeParent() { n.Name = "" n.Exists = false } - -// Create creates a new node in the given root and add symlinks to parent node -// TODO use a reference to the tree to access tho root? -func (n *NodeInfo) Create(root string) (err error) { - - if n.ID != "" { - return errors.Wrap(err, "ocisfs: node already his an id") - } - // create a new file node - n.ID = uuid.New().String() - - nodePath := filepath.Join(root, "nodes", n.ID) - - err = os.MkdirAll(nodePath, 0700) - if err != nil { - return errors.Wrap(err, "ocisfs: could not create node dir") - } - // create back link - // we are not only linking back to the parent, but also to the filename - link := "../" + n.ParentID + "/children/" + n.Name - err = os.Symlink(link, filepath.Join(nodePath, "parentname")) - if err != nil { - return errors.Wrap(err, "ocisfs: could not symlink parent node") - } - - // link child name to node - err = os.Symlink("../../"+n.ID, filepath.Join(root, "nodes", n.ParentID, "children", n.Name)) - if err != nil { - return errors.Wrap(err, "ocisfs: could not symlink child entry") - } - - return nil -} diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 79f03ced25..7086a58ca8 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -25,9 +25,9 @@ import ( "os" "path/filepath" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/logger" "github.com/cs3org/reva/pkg/mime" @@ -94,8 +94,8 @@ func New(m map[string]interface{}) (storage.FS, error) { // notes contain symlinks from nodes//uploads/ to ../../uploads/ // better to keep uploads on a fast / volatile storage before a workflow finally moves them to the nodes dir filepath.Join(c.Root, "uploads"), - filepath.Join(c.Root, "trash/files"), - filepath.Join(c.Root, "trash/info"), + filepath.Join(c.Root, "trash"), + filepath.Join(c.Root, "versions"), } for _, v := range dataPaths { if err := os.MkdirAll(v, 0700); err != nil { @@ -143,15 +143,11 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { return errtypes.NotSupported("ocisfs: create home not supported") } - u, err := getUser(ctx) - if err != nil { - err = errors.Wrap(err, "ocisfs: wrap: no user in ctx and home is enabled") - return err - } + u := user.ContextMustGetUser(ctx) layout := templates.WithUser(u, fs.conf.UserLayout) home := filepath.Join(fs.conf.Root, "users", layout) - _, err = os.Stat(home) + _, err := os.Stat(home) if err == nil { // home already exists return nil } @@ -164,11 +160,19 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { return errors.Wrap(err, "ocisfs: error creating dir") } - // create a directory node (with children subfolder) + // create a directory node nodeID := uuid.New().String() - err = os.MkdirAll(filepath.Join(fs.conf.Root, "nodes", nodeID, "children"), 0700) + nodePath := filepath.Join(fs.conf.Root, "nodes", nodeID) + err = os.MkdirAll(nodePath, 0700) if err != nil { - return errors.Wrap(err, "ocisfs: error node dir") + return errors.Wrap(err, "ocisfs: error creating node dir") + } + + if err := xattr.Set(nodePath, "user.ocis.parentid", []byte("root")); err != nil { + return errors.Wrap(err, "ocisfs: could not set parentid attribute") + } + if err := xattr.Set(nodePath, "user.ocis.name", []byte("")); err != nil { + return errors.Wrap(err, "ocisfs: could not set name attribute") } // link users home to node @@ -181,11 +185,7 @@ func (fs *ocisfs) GetHome(ctx context.Context) (string, error) { if !fs.conf.EnableHome || fs.conf.UserLayout == "" { return "", errtypes.NotSupported("ocisfs: get home not supported") } - u, err := getUser(ctx) - if err != nil { - err = errors.Wrap(err, "ocisfs: wrap: no user in ctx and home is enabled") - return "", err - } + u := user.ContextMustGetUser(ctx) layout := templates.WithUser(u, fs.conf.UserLayout) return filepath.Join(fs.conf.Root, layout), nil // TODO use a namespace? } @@ -285,7 +285,7 @@ func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refe // Data persistence func (fs *ocisfs) ContentPath(node *NodeInfo) string { - return filepath.Join(fs.conf.Root, "nodes", node.ID, "content") + return filepath.Join(fs.conf.Root, "nodes", node.ID) } func (fs *ocisfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { @@ -362,35 +362,32 @@ func (fs *ocisfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *p // supporting functions -func getUser(ctx context.Context) (*userpb.User, error) { - u, ok := user.ContextGetUser(ctx) - if !ok { - err := errors.Wrap(errtypes.UserRequired(""), "ocisfs: error getting user from ctx") - return nil, err - } - return u, nil -} - func (fs *ocisfs) normalize(ctx context.Context, node *NodeInfo) (ri *provider.ResourceInfo, err error) { var fn string nodePath := filepath.Join(fs.conf.Root, "nodes", node.ID) var fi os.FileInfo + nodeType := provider.ResourceType_RESOURCE_TYPE_INVALID - if fi, err = os.Stat(filepath.Join(nodePath, "content")); err == nil { - nodeType = provider.ResourceType_RESOURCE_TYPE_FILE - } else if fi, err = os.Stat(filepath.Join(nodePath, "children")); err == nil { + if fi, err = os.Lstat(nodePath); err != nil { + return + } + if fi.IsDir() { nodeType = provider.ResourceType_RESOURCE_TYPE_CONTAINER - } else if fi, err = os.Stat(filepath.Join(nodePath, "reference")); err == nil { - // TODO handle references - nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE + } else if fi.Mode().IsRegular() { + nodeType = provider.ResourceType_RESOURCE_TYPE_FILE + } else if fi.Mode()&os.ModeSymlink != 0 { + nodeType = provider.ResourceType_RESOURCE_TYPE_SYMLINK + // TODO reference using ext attr on a symlink + // nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE } var etag []byte - // TODO store etag in node folder or on content and child nodes? + // TODO optionally store etag in new `root/attributes/` file if etag, err = xattr.Get(nodePath, "user.ocis.etag"); err != nil { - logger.New().Error().Err(err).Msg("could not read etag") + log := appctx.GetLogger(ctx) + log.Debug().Err(err).Msg("could not read etag") } id := &provider.ResourceId{OpaqueId: node.ID} @@ -413,7 +410,8 @@ func (fs *ocisfs) normalize(ctx context.Context, node *NodeInfo) (ri *provider.R }, } - logger.New().Debug(). + log := appctx.GetLogger(ctx) + log.Debug(). Interface("ri", ri). Msg("normalized") diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go index 63ed78568b..33f9eaf227 100644 --- a/pkg/storage/fs/ocis/path.go +++ b/pkg/storage/fs/ocis/path.go @@ -7,11 +7,12 @@ import ( "path/filepath" "strings" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage/utils/templates" + "github.com/cs3org/reva/pkg/user" "github.com/pkg/errors" + "github.com/pkg/xattr" ) // Path implements transformations from filepath to node and back @@ -51,14 +52,7 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) } if pw.EnableHome && pw.UserLayout != "" { // start at the users root node - var u *userpb.User - - u, err = getUser(ctx) - if err != nil { - err = errors.Wrap(err, "ocisfs: Wrap: no user in ctx and home is enabled") - return - } - + u := user.ContextMustGetUser(ctx) layout := templates.WithUser(u, pw.UserLayout) root = filepath.Join(pw.Root, "users", layout) @@ -81,7 +75,7 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) node.ID = "" node.Name = segments[i] - link, err = os.Readlink(filepath.Join(pw.Root, "nodes", node.ParentID, "children", node.Name)) + link, err = os.Readlink(filepath.Join(pw.Root, "nodes", node.ParentID, node.Name)) if os.IsNotExist(err) { node.Exists = false // if this is the last segment we can use it as the node name @@ -90,17 +84,17 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) return } - err = errtypes.NotFound(filepath.Join(pw.Root, "nodes", node.ParentID, "children", node.Name)) + err = errtypes.NotFound(filepath.Join(pw.Root, "nodes", node.ParentID, node.Name)) return } if err != nil { err = errors.Wrap(err, "ocisfs: Wrap: readlink error") return } - if strings.HasPrefix(link, "../../") { + if strings.HasPrefix(link, "../") { node.ID = filepath.Base(link) } else { - err = fmt.Errorf("ocisfs: expected '../../ prefix, got' %+v", link) + err = fmt.Errorf("ocisfs: expected '../ prefix, got' %+v", link) return } } @@ -118,9 +112,14 @@ func (pw *Path) WrapID(ctx context.Context, id *provider.ResourceId) (*NodeInfo, } func (pw *Path) Unwrap(ctx context.Context, ni *NodeInfo) (external string, err error) { + // check if this a not yet existing node + if ni.ID == "" && ni.Name != "" && ni.ParentID != "" { + external = ni.Name + ni.BecomeParent() + } for err == nil { err = pw.FillParentAndName(ni) - if os.IsNotExist(err) { + if os.IsNotExist(err) || ni.ParentID == "root" { err = nil return } @@ -146,23 +145,23 @@ func (pw *Path) FillParentAndName(node *NodeInfo) (err error) { return } - var link string - // The parentname symlink looks like `../76455834-769e-412a-8a01-68f265365b79/children/myname.txt` - link, err = os.Readlink(filepath.Join(pw.Root, "nodes", node.ID, "parentname")) - if err != nil { - return + nodePath := filepath.Join(pw.Root, "nodes", node.ID) + + var attrBytes []byte + if attrBytes, err = xattr.Get(nodePath, "user.ocis.parentid"); err == nil { + node.ParentID = string(attrBytes) } + // no error if it is empty -> root node + // TODO better identify a root node, eg by using `root` as the parentid? or `root:userid`? - // check the link follows the correct schema - // TODO count slashes - if strings.HasPrefix(link, "../") { - node.Name = filepath.Base(link) - node.ParentID = filepath.Base(filepath.Dir(filepath.Dir(link))) - node.Exists = true - } else { - err = fmt.Errorf("ocisfs: expected '../' prefix, got '%+v'", link) - return + if attrBytes, err = xattr.Get(nodePath, "user.ocis.name"); err == nil { + node.Name = string(attrBytes) } + + // no error if it is empty -> root node + // TODO better identify a root node? + + node.Exists = true return } diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index 9f9e8996fc..81de0ed7a7 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -3,12 +3,10 @@ package ocis import ( "context" "encoding/hex" - "io/ioutil" "math/rand" "net/url" "os" "path/filepath" - "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" @@ -60,30 +58,30 @@ func (fs *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relat // CreateDir creates a new directory entry in the tree func (fs *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { - // TODO always try to fill node? + // TODO always try to fill node? if node.Exists || node.ID != "" { // child already exists return } - // create a directory node (with children subfolder) + // create a directory node node.ID = uuid.New().String() newPath := filepath.Join(fs.DataDirectory, "nodes", node.ID) - err = os.MkdirAll(filepath.Join(newPath, "children"), 0700) + err = os.MkdirAll(newPath, 0700) if err != nil { return errors.Wrap(err, "ocisfs: could not create node dir") } - // create back link - // we are not only linking back to the parent, but also to the filename - err = os.Symlink("../"+node.ParentID+"/children/"+node.Name, filepath.Join(newPath, "parentname")) - if err != nil { - return errors.Wrap(err, "ocisfs: could not symlink parent node") + if err := xattr.Set(newPath, "user.ocis.parentid", []byte(node.ParentID)); err != nil { + return errors.Wrap(err, "ocisfs: could not set parentid attribute") + } + if err := xattr.Set(newPath, "user.ocis.name", []byte(node.Name)); err != nil { + return errors.Wrap(err, "ocisfs: could not set name attribute") } // make child appear in listings - err = os.Symlink("../../"+node.ID, filepath.Join(fs.DataDirectory, "nodes", node.ParentID, "children", node.Name)) + err = os.Symlink("../"+node.ID, filepath.Join(fs.DataDirectory, "nodes", node.ParentID, node.Name)) if err != nil { return } @@ -110,6 +108,7 @@ func (fs *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) err = nil return } + // TODO make sure all children are deleted if err := os.RemoveAll(filepath.Join(fs.DataDirectory, "nodes", newNode.ID)); err != nil { return errors.Wrap(err, "ocisfs: Move: error deleting target node "+newNode.ID) } @@ -117,90 +116,79 @@ func (fs *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) // are we renaming? if oldNode.ParentID == newNode.ParentID { - nodePath := filepath.Join(fs.DataDirectory, "nodes", oldNode.ID) - - // update back link - // we are not only linking back to the parent, but also to the filename - err = os.Remove(filepath.Join(nodePath, "parentname")) - if err != nil { - return errors.Wrap(err, "ocisfs: could not remove parent link") - } - err = os.Symlink("../"+oldNode.ParentID+"/children/"+newNode.Name, filepath.Join(nodePath, "parentname")) - if err != nil { - return errors.Wrap(err, "ocisfs: could not symlink parent") - } + parentPath := filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID) // rename child err = os.Rename( - filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, "children", oldNode.Name), - filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, "children", newNode.Name), + filepath.Join(parentPath, oldNode.Name), + filepath.Join(parentPath, newNode.Name), ) if err != nil { - return errors.Wrap(err, "ocisfs: could not rename symlink") + return errors.Wrap(err, "ocisfs: could not rename child") } - return fs.Propagate(ctx, oldNode) - } - // we are moving the node to a new parent, any target has been removed - // bring old node to the new parent - nodePath := filepath.Join(fs.DataDirectory, "nodes", oldNode.ID) + tgtPath := filepath.Join(fs.DataDirectory, "nodes", newNode.ID) - // update back link - // we are not only linking back to the parent, but also to the filename - err = os.Remove(filepath.Join(nodePath, "parentname")) - if err != nil { - return errors.Wrap(err, "ocisfs: could not remove parent link") - } - err = os.Symlink("../"+newNode.ParentID+"/children/"+newNode.Name, filepath.Join(nodePath, "parentname")) - if err != nil { - return errors.Wrap(err, "ocisfs: could not symlink parent") + // update name attribute + if err := xattr.Set(tgtPath, "user.ocis.name", []byte(newNode.Name)); err != nil { + return errors.Wrap(err, "ocisfs: could not set name attribute") + } + + return fs.Propagate(ctx, newNode) } + // we are moving the node to a new parent, any target has been removed + // bring old node to the new parent + // rename child err = os.Rename( - filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, "children", oldNode.Name), - filepath.Join(fs.DataDirectory, "nodes", newNode.ParentID, "children", newNode.Name), + filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, oldNode.Name), + filepath.Join(fs.DataDirectory, "nodes", newNode.ParentID, newNode.Name), ) if err != nil { - return errors.Wrap(err, "ocisfs: could not rename symlink") + return errors.Wrap(err, "ocisfs: could not move child") + } + + tgtPath := filepath.Join(fs.DataDirectory, "nodes", newNode.ID) + + if err := xattr.Set(tgtPath, "user.ocis.parentid", []byte(newNode.ParentID)); err != nil { + return errors.Wrap(err, "ocisfs: could not set parentid attribute") + } + if err := xattr.Set(tgtPath, "user.ocis.name", []byte(newNode.Name)); err != nil { + return errors.Wrap(err, "ocisfs: could not set name attribute") } // TODO inefficient because we might update several nodes twice, only propagate unchanged nodes? // collect in a list, then only stat each node once // also do this in a go routine ... webdav should check the etag async + err = fs.Propagate(ctx, oldNode) if err != nil { return errors.Wrap(err, "ocisfs: Move: could not propagate old node") } err = fs.Propagate(ctx, newNode) if err != nil { - return errors.Wrap(err, "ocisfs: Move: could not propagate old node") + return errors.Wrap(err, "ocisfs: Move: could not propagate new node") } return nil } -// ChildrenPath returns the absolute path to childrens in a node -// TODO move to node? -func (fs *Tree) ChildrenPath(node *NodeInfo) string { - return filepath.Join(fs.DataDirectory, "nodes", node.ID, "children") -} - -// ListFolder lists the children inside a folder +// ListFolder lists the content of a folder node func (fs *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, error) { - children := fs.ChildrenPath(node) - f, err := os.Open(children) + dir := filepath.Join(fs.DataDirectory, "nodes", node.ID) + f, err := os.Open(dir) if err != nil { if os.IsNotExist(err) { - return nil, errtypes.NotFound(children) + return nil, errtypes.NotFound(dir) } - return nil, errors.Wrap(err, "tree: error listing "+children) + return nil, errors.Wrap(err, "tree: error listing "+dir) } names, err := f.Readdirnames(0) nodes := []*NodeInfo{} for i := range names { - link, err := os.Readlink(filepath.Join(children, names[i])) + link, err := os.Readlink(filepath.Join(dir, names[i])) if err != nil { // TODO log continue @@ -229,23 +217,16 @@ func (fs *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { return } - // remove child entry from dir - - os.Remove(filepath.Join(fs.DataDirectory, "nodes", node.ParentID, "children", node.Name)) - - src := filepath.Join(fs.DataDirectory, "nodes", node.ID) - trashpath := filepath.Join(fs.DataDirectory, "trash/files", node.ID) - err = os.Rename(src, trashpath) + src := filepath.Join(fs.DataDirectory, "nodes", node.ParentID, node.Name) + err = os.Remove(src) if err != nil { return } - // write a trash info ... slightly violating the freedesktop trash spec - t := time.Now() - // TODO store the original Path - info := []byte("[Trash Info]\nParentID=" + node.ParentID + "\nDeletionDate=" + t.Format(time.RFC3339)) - infoPath := filepath.Join(fs.DataDirectory, "trash/info", node.ID+".trashinfo") - err = ioutil.WriteFile(infoPath, info, 0700) + // make node appear in trash + // parent id and name are stored as extended attributes in the node itself + trashpath := filepath.Join(fs.DataDirectory, "trash", node.ID) + err = os.Symlink("../nodes/"+node.ID, trashpath) if err != nil { return } @@ -267,7 +248,7 @@ func (fs *Tree) Propagate(ctx context.Context, node *NodeInfo) (err error) { log.Error().Err(err).Msg("error storing file id") } err = fs.pw.FillParentAndName(node) - if os.IsNotExist(err) { + if os.IsNotExist(err) || node.ParentID == "root" { err = nil return } diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go index 53e8a4f410..08967590bd 100644 --- a/pkg/storage/fs/ocis/upload.go +++ b/pkg/storage/fs/ocis/upload.go @@ -29,12 +29,11 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/logger" - "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" "github.com/google/uuid" "github.com/pkg/errors" + "github.com/pkg/xattr" tusd "github.com/tus/tusd/pkg/handler" ) @@ -49,12 +48,10 @@ func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.Read return err } - if !node.Exists { - err = node.Create(fs.conf.Root) - if err != nil { - return errors.Wrap(err, "ocisfs: could not create node") - } + if node.ID == "" { + node.ID = uuid.New().String() } + nodePath := filepath.Join(fs.conf.Root, "nodes", node.ID) tmp, err := ioutil.TempFile(nodePath, "._reva_atomic_upload") @@ -70,7 +67,7 @@ func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.Read // TODO move old content to version //_ = os.RemoveAll(path.Join(nodePath, "content")) - err = os.Rename(tmp.Name(), filepath.Join(nodePath, "content")) + err = os.Rename(tmp.Name(), nodePath) if err != nil { return err } @@ -81,17 +78,24 @@ func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.Read // InitiateUpload returns an upload id that can be used for uploads with tus // TODO read optional content for small files in this request func (fs *ocisfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (uploadID string, err error) { + + log := appctx.GetLogger(ctx) + var relative string // the internal path of the file node node, err := fs.pw.Resolve(ctx, ref) if err != nil { return "", err } + relative, err = fs.pw.Unwrap(ctx, node) + if err != nil { + return "", err + } info := tusd.FileInfo{ MetaData: tusd.MetaData{ - "filename": node.Name, + "filename": filepath.Base(relative), "dir": filepath.Dir(relative), }, Size: uploadLength, @@ -101,6 +105,8 @@ func (fs *ocisfs) InitiateUpload(ctx context.Context, ref *provider.Reference, u info.MetaData["mtime"] = metadata["mtime"] } + log.Debug().Interface("info", info).Interface("node", node).Interface("metadata", metadata).Msg("ocisfs: resolved filename") + upload, err := fs.NewUpload(ctx, info) if err != nil { return "", err @@ -141,6 +147,9 @@ func (fs *ocisfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tus info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) node, err := fs.pw.Wrap(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) + if err != nil { + return nil, errors.Wrap(err, "ocisfs: error wrapping filename") + } log.Debug().Interface("info", info).Interface("node", node).Msg("ocisfs: resolved filename") @@ -201,13 +210,7 @@ func (fs *ocisfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tus } func (fs *ocisfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { - u, ok := user.ContextGetUser(ctx) - if !ok { - err := errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") - return "", err - } - layout := templates.WithUser(u, fs.conf.UserLayout) - return filepath.Join(fs.conf.Root, layout, "uploads", uploadID), nil + return filepath.Join(fs.conf.Root, "uploads", uploadID), nil } // GetUpload returns the Upload for the given upload id @@ -290,7 +293,7 @@ func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.R // If the HTTP PATCH request gets interrupted in the middle (e.g. because // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. - // However, for OwnCloudStore it's not important whether the stream has ended + // However, for the ocis driver it's not important whether the stream has ended // on purpose or accidentally. if err != nil { if err != io.ErrUnexpectedEOF { @@ -321,37 +324,47 @@ func (upload *fileUpload) writeInfo() error { // FinishUpload finishes an upload and moves the file to the internal destination func (upload *fileUpload) FinishUpload(ctx context.Context) error { - node := &NodeInfo{ + n := &NodeInfo{ ID: upload.info.Storage["NodeId"], ParentID: upload.info.Storage["NodeParentId"], Name: upload.info.Storage["NodeName"], } - if node.ID == "" { - err := node.Create(upload.fs.conf.Root) - if err != nil { - return errors.Wrap(err, "ocisfs: could not create node") - } + if n.ID == "" { + n.ID = uuid.New().String() } - contentPath := filepath.Join(upload.fs.conf.Root, "nodes", node.ID, "content") + targetPath := filepath.Join(upload.fs.conf.Root, "nodes", n.ID) - log := appctx.GetLogger(upload.ctx) - err := os.Rename(upload.binPath, contentPath) + err := os.Rename(upload.binPath, targetPath) if err != nil { + log := appctx.GetLogger(upload.ctx) log.Err(err).Interface("info", upload.info). Str("binPath", upload.binPath). - Str("contentPath", contentPath). + Str("targetPath", targetPath). Msg("ocisfs: could not rename") return err } + if err := xattr.Set(targetPath, "user.ocis.parentid", []byte(n.ParentID)); err != nil { + return errors.Wrap(err, "ocisfs: could not set parentid attribute") + } + if err := xattr.Set(targetPath, "user.ocis.name", []byte(n.Name)); err != nil { + return errors.Wrap(err, "ocisfs: could not set name attribute") + } + + // link child name to parent + err = os.Symlink("../"+n.ID, filepath.Join(upload.fs.conf.Root, "nodes", n.ParentID, n.Name)) + if err != nil { + return errors.Wrap(err, "ocisfs: could not symlink child entry") + } + // only delete the upload if it was successfully written to the storage - if err := os.Remove(upload.infoPath); err != nil { + /*if err := os.Remove(upload.infoPath); err != nil { if !os.IsNotExist(err) { log.Err(err).Interface("info", upload.info).Msg("ocisfs: could not delete upload info") return err } - } + }*/ // use set arbitrary metadata? /*if upload.info.MetaData["mtime"] != "" { err := upload.fs.SetMtime(ctx, np, upload.info.MetaData["mtime"]) @@ -361,7 +374,7 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error { } }*/ - return upload.fs.tp.Propagate(upload.ctx, node) + return upload.fs.tp.Propagate(upload.ctx, n) } // To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination From ef23de5e2cb88873b646628a76615958d1a1486f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 3 Sep 2020 15:46:48 +0200 Subject: [PATCH 06/18] manually adding mimetypes is no longer needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/ocis.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 7086a58ca8..56378fd220 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -74,9 +74,6 @@ func (c *config) init(m map[string]interface{}) { } // c.DataDirectory should never end in / unless it is the root c.Root = filepath.Clean(c.Root) - - // TODO we need a lot more mimetypes - mime.RegisterMime(".txt", "text/plain") } // New returns an implementation to of the storage.FS interface that talk to From c188f79b9185b2d7d9975952547b903b064b1295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 3 Sep 2020 17:01:04 +0200 Subject: [PATCH 07/18] rename Wrap to NodeFrom* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/ocis.go | 18 ++++---- pkg/storage/fs/ocis/path.go | 48 ++++++++++++++++------ pkg/storage/fs/ocis/persistence.go | 11 ++--- pkg/storage/fs/ocis/tree.go | 66 +++++++++++++++--------------- pkg/storage/fs/ocis/upload.go | 13 +++--- 5 files changed, 91 insertions(+), 65 deletions(-) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 56378fd220..1803e3be29 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -196,7 +196,7 @@ func (fs *ocisfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (str func (fs *ocisfs) CreateDir(ctx context.Context, fn string) (err error) { var node *NodeInfo - if node, err = fs.pw.Wrap(ctx, fn); err != nil { + if node, err = fs.pw.NodeFromPath(ctx, fn); err != nil { return } return fs.tp.CreateDir(ctx, node) @@ -208,7 +208,7 @@ func (fs *ocisfs) CreateReference(ctx context.Context, path string, targetURI *u func (fs *ocisfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { var oldNode, newNode *NodeInfo - if oldNode, err = fs.pw.Resolve(ctx, oldRef); err != nil { + if oldNode, err = fs.pw.NodeFromResource(ctx, oldRef); err != nil { return } if !oldNode.Exists { @@ -216,7 +216,7 @@ func (fs *ocisfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) return } - if newNode, err = fs.pw.Resolve(ctx, newRef); err != nil { + if newNode, err = fs.pw.NodeFromResource(ctx, newRef); err != nil { return } return fs.tp.Move(ctx, oldNode, newNode) @@ -224,7 +224,7 @@ func (fs *ocisfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) func (fs *ocisfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (ri *provider.ResourceInfo, err error) { var node *NodeInfo - if node, err = fs.pw.Resolve(ctx, ref); err != nil { + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { return } if !node.Exists { @@ -236,7 +236,7 @@ func (fs *ocisfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []s func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) (finfos []*provider.ResourceInfo, err error) { var node *NodeInfo - if node, err = fs.pw.Resolve(ctx, ref); err != nil { + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { return } if !node.Exists { @@ -259,7 +259,7 @@ func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKey func (fs *ocisfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { var node *NodeInfo - if node, err = fs.pw.Resolve(ctx, ref); err != nil { + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { return } if !node.Exists { @@ -286,7 +286,7 @@ func (fs *ocisfs) ContentPath(node *NodeInfo) string { } func (fs *ocisfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { - node, err := fs.pw.Resolve(ctx, ref) + node, err := fs.pw.NodeFromResource(ctx, ref) if err != nil { return nil, errors.Wrap(err, "ocisfs: error resolving ref") } @@ -388,8 +388,8 @@ func (fs *ocisfs) normalize(ctx context.Context, node *NodeInfo) (ri *provider.R } id := &provider.ResourceId{OpaqueId: node.ID} - // Unwrap changes the node because it traverses the tree - fn, err = fs.pw.Unwrap(ctx, node) + // Path changes the node because it traverses the tree + fn, err = fs.pw.Path(ctx, node) if err != nil { return nil, err } diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go index 33f9eaf227..057f647545 100644 --- a/pkg/storage/fs/ocis/path.go +++ b/pkg/storage/fs/ocis/path.go @@ -30,22 +30,22 @@ type Path struct { EnableHome bool `mapstructure:"enable_home"` } -// Resolve takes in a request path or request id and converts it to a NodeInfo -func (pw *Path) Resolve(ctx context.Context, ref *provider.Reference) (*NodeInfo, error) { +// NodeFromResource takes in a request path or request id and converts it to a NodeInfo +func (pw *Path) NodeFromResource(ctx context.Context, ref *provider.Reference) (*NodeInfo, error) { if ref.GetPath() != "" { - return pw.Wrap(ctx, ref.GetPath()) + return pw.NodeFromPath(ctx, ref.GetPath()) } if ref.GetId() != nil { - return pw.WrapID(ctx, ref.GetId()) + return pw.NodeFromID(ctx, ref.GetId()) } // reference is invalid return nil, fmt.Errorf("invalid reference %+v", ref) } -// Wrap converts a filename into a NodeInfo -func (pw *Path) Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) { +// NodeFromPath converts a filename into a NodeInfo +func (pw *Path) NodeFromPath(ctx context.Context, fn string) (node *NodeInfo, err error) { var link, root string if fn == "" { fn = "/" @@ -61,7 +61,7 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) root = filepath.Join(pw.Root, "nodes/root") } - node, err = pw.ReadRootLink(root) + node, err = pw.readRootLink(root) // The symlink contains the nodeID if err != nil { return @@ -103,15 +103,15 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) return } -// WrapID returns the internal path for the id -func (pw *Path) WrapID(ctx context.Context, id *provider.ResourceId) (*NodeInfo, error) { +// NodeFromID returns the internal path for the id +func (pw *Path) NodeFromID(ctx context.Context, id *provider.ResourceId) (*NodeInfo, error) { if id == nil || id.OpaqueId == "" { return nil, fmt.Errorf("invalid resource id %+v", id) } return &NodeInfo{ID: id.OpaqueId}, nil } -func (pw *Path) Unwrap(ctx context.Context, ni *NodeInfo) (external string, err error) { +func (pw *Path) Path(ctx context.Context, ni *NodeInfo) (external string, err error) { // check if this a not yet existing node if ni.ID == "" && ni.Name != "" && ni.ParentID != "" { external = ni.Name @@ -165,9 +165,8 @@ func (pw *Path) FillParentAndName(node *NodeInfo) (err error) { return } -// ReadRootLink reads the symbolic link and extracts the node id -func (pw *Path) ReadRootLink(root string) (node *NodeInfo, err error) { - +// readRootLink reads the symbolic link and extracts the node id +func (pw *Path) readRootLink(root string) (node *NodeInfo, err error) { // A root symlink looks like `../nodes/76455834-769e-412a-8a01-68f265365b79` link, err := os.Readlink(root) if os.IsNotExist(err) { @@ -186,3 +185,26 @@ func (pw *Path) ReadRootLink(root string) (node *NodeInfo, err error) { } return } + +// RootNode returns the root node of a tree, +// taking into account the user layout if EnableHome is true +func (pw *Path) RootNode(ctx context.Context) (node *NodeInfo, err error) { + var root string + if pw.EnableHome && pw.UserLayout != "" { + // start at the users root node + u := user.ContextMustGetUser(ctx) + layout := templates.WithUser(u, pw.UserLayout) + root = filepath.Join(pw.Root, "users", layout) + + } else { + // start at the storage root node + root = filepath.Join(pw.Root, "nodes/root") + } + + // The symlink contains the nodeID + node, err = pw.readRootLink(root) + if err != nil { + return + } + return +} diff --git a/pkg/storage/fs/ocis/persistence.go b/pkg/storage/fs/ocis/persistence.go index 0698585a3a..2407b3fc1a 100644 --- a/pkg/storage/fs/ocis/persistence.go +++ b/pkg/storage/fs/ocis/persistence.go @@ -23,14 +23,15 @@ type TreePersistence interface { // PathWrapper is used to encapsulate path transformations type PathWrapper interface { - Resolve(ctx context.Context, ref *provider.Reference) (node *NodeInfo, err error) - WrapID(ctx context.Context, id *provider.ResourceId) (node *NodeInfo, err error) + NodeFromResource(ctx context.Context, ref *provider.Reference) (node *NodeInfo, err error) + NodeFromID(ctx context.Context, id *provider.ResourceId) (node *NodeInfo, err error) // Wrap returns a NodeInfo object: // - if the node exists with the node id, name and parent // - if only the parent exists, the node id is empty - Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) - Unwrap(ctx context.Context, node *NodeInfo) (external string, err error) + NodeFromPath(ctx context.Context, fn string) (node *NodeInfo, err error) + Path(ctx context.Context, node *NodeInfo) (path string, err error) + FillParentAndName(node *NodeInfo) (err error) // Tree persistence? - ReadRootLink(root string) (node *NodeInfo, err error) + RootNode(ctx context.Context) (node *NodeInfo, err error) } diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index 81de0ed7a7..ddcd190ae6 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -31,8 +31,8 @@ func NewTree(pw PathWrapper, dataDirectory string) (TreePersistence, error) { } // GetMD returns the metadata of a node in the tree -func (fs *Tree) GetMD(ctx context.Context, node *NodeInfo) (os.FileInfo, error) { - md, err := os.Stat(filepath.Join(fs.DataDirectory, "nodes", node.ID)) +func (t *Tree) GetMD(ctx context.Context, node *NodeInfo) (os.FileInfo, error) { + md, err := os.Stat(filepath.Join(t.DataDirectory, "nodes", node.ID)) if err != nil { if os.IsNotExist(err) { return nil, errtypes.NotFound(node.ID) @@ -44,19 +44,19 @@ func (fs *Tree) GetMD(ctx context.Context, node *NodeInfo) (os.FileInfo, error) } // GetPathByID returns the fn pointed by the file id, without the internal namespace -func (fs *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relativeExternalPath string, err error) { +func (t *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relativeExternalPath string, err error) { var node *NodeInfo - node, err = fs.pw.WrapID(ctx, id) + node, err = t.pw.NodeFromID(ctx, id) if err != nil { return } - relativeExternalPath, err = fs.pw.Unwrap(ctx, node) + relativeExternalPath, err = t.pw.Path(ctx, node) return } // CreateDir creates a new directory entry in the tree -func (fs *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { +func (t *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { // TODO always try to fill node? if node.Exists || node.ID != "" { // child already exists @@ -66,7 +66,7 @@ func (fs *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { // create a directory node node.ID = uuid.New().String() - newPath := filepath.Join(fs.DataDirectory, "nodes", node.ID) + newPath := filepath.Join(t.DataDirectory, "nodes", node.ID) err = os.MkdirAll(newPath, 0700) if err != nil { @@ -81,21 +81,21 @@ func (fs *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { } // make child appear in listings - err = os.Symlink("../"+node.ID, filepath.Join(fs.DataDirectory, "nodes", node.ParentID, node.Name)) + err = os.Symlink("../"+node.ID, filepath.Join(t.DataDirectory, "nodes", node.ParentID, node.Name)) if err != nil { return } - return fs.Propagate(ctx, node) + return t.Propagate(ctx, node) } // CreateReference creates a new reference entry in the tree -func (fs *Tree) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { +func (t *Tree) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { return errtypes.NotSupported("operation not supported: CreateReference") } // Move replaces the target with the source -func (fs *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) (err error) { - err = fs.pw.FillParentAndName(newNode) +func (t *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) (err error) { + err = t.pw.FillParentAndName(newNode) if os.IsNotExist(err) { err = nil return @@ -103,20 +103,20 @@ func (fs *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) // if target exists delete it without trashing it if newNode.Exists { - err = fs.pw.FillParentAndName(newNode) + err = t.pw.FillParentAndName(newNode) if os.IsNotExist(err) { err = nil return } // TODO make sure all children are deleted - if err := os.RemoveAll(filepath.Join(fs.DataDirectory, "nodes", newNode.ID)); err != nil { + if err := os.RemoveAll(filepath.Join(t.DataDirectory, "nodes", newNode.ID)); err != nil { return errors.Wrap(err, "ocisfs: Move: error deleting target node "+newNode.ID) } } // are we renaming? if oldNode.ParentID == newNode.ParentID { - parentPath := filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID) + parentPath := filepath.Join(t.DataDirectory, "nodes", oldNode.ParentID) // rename child err = os.Rename( @@ -127,14 +127,14 @@ func (fs *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) return errors.Wrap(err, "ocisfs: could not rename child") } - tgtPath := filepath.Join(fs.DataDirectory, "nodes", newNode.ID) + tgtPath := filepath.Join(t.DataDirectory, "nodes", newNode.ID) // update name attribute if err := xattr.Set(tgtPath, "user.ocis.name", []byte(newNode.Name)); err != nil { return errors.Wrap(err, "ocisfs: could not set name attribute") } - return fs.Propagate(ctx, newNode) + return t.Propagate(ctx, newNode) } // we are moving the node to a new parent, any target has been removed @@ -142,14 +142,14 @@ func (fs *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) // rename child err = os.Rename( - filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, oldNode.Name), - filepath.Join(fs.DataDirectory, "nodes", newNode.ParentID, newNode.Name), + filepath.Join(t.DataDirectory, "nodes", oldNode.ParentID, oldNode.Name), + filepath.Join(t.DataDirectory, "nodes", newNode.ParentID, newNode.Name), ) if err != nil { return errors.Wrap(err, "ocisfs: could not move child") } - tgtPath := filepath.Join(fs.DataDirectory, "nodes", newNode.ID) + tgtPath := filepath.Join(t.DataDirectory, "nodes", newNode.ID) if err := xattr.Set(tgtPath, "user.ocis.parentid", []byte(newNode.ParentID)); err != nil { return errors.Wrap(err, "ocisfs: could not set parentid attribute") @@ -162,11 +162,11 @@ func (fs *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) // collect in a list, then only stat each node once // also do this in a go routine ... webdav should check the etag async - err = fs.Propagate(ctx, oldNode) + err = t.Propagate(ctx, oldNode) if err != nil { return errors.Wrap(err, "ocisfs: Move: could not propagate old node") } - err = fs.Propagate(ctx, newNode) + err = t.Propagate(ctx, newNode) if err != nil { return errors.Wrap(err, "ocisfs: Move: could not propagate new node") } @@ -174,9 +174,9 @@ func (fs *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) } // ListFolder lists the content of a folder node -func (fs *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, error) { +func (t *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, error) { - dir := filepath.Join(fs.DataDirectory, "nodes", node.ID) + dir := filepath.Join(t.DataDirectory, "nodes", node.ID) f, err := os.Open(dir) if err != nil { if os.IsNotExist(err) { @@ -206,8 +206,8 @@ func (fs *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, er } // Delete deletes a node in the tree -func (fs *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { - err = fs.pw.FillParentAndName(node) +func (t *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { + err = t.pw.FillParentAndName(node) if os.IsNotExist(err) { err = nil return @@ -217,7 +217,7 @@ func (fs *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { return } - src := filepath.Join(fs.DataDirectory, "nodes", node.ParentID, node.Name) + src := filepath.Join(t.DataDirectory, "nodes", node.ParentID, node.Name) err = os.Remove(src) if err != nil { return @@ -225,17 +225,17 @@ func (fs *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { // make node appear in trash // parent id and name are stored as extended attributes in the node itself - trashpath := filepath.Join(fs.DataDirectory, "trash", node.ID) + trashpath := filepath.Join(t.DataDirectory, "trash", node.ID) err = os.Symlink("../nodes/"+node.ID, trashpath) if err != nil { return } - return fs.Propagate(ctx, &NodeInfo{ID: node.ParentID}) + return t.Propagate(ctx, &NodeInfo{ID: node.ParentID}) } // Propagate propagates changes to the root of the tree -func (fs *Tree) Propagate(ctx context.Context, node *NodeInfo) (err error) { +func (t *Tree) Propagate(ctx context.Context, node *NodeInfo) (err error) { // generate an etag bytes := make([]byte, 16) if _, err := rand.Read(bytes); err != nil { @@ -244,10 +244,12 @@ func (fs *Tree) Propagate(ctx context.Context, node *NodeInfo) (err error) { // store in extended attribute etag := hex.EncodeToString(bytes) for err == nil { - if err := xattr.Set(filepath.Join(fs.DataDirectory, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { + if err := xattr.Set(filepath.Join(t.DataDirectory, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { log.Error().Err(err).Msg("error storing file id") } - err = fs.pw.FillParentAndName(node) + // TODO propagate mtime + // TODO size accounting + err = t.pw.FillParentAndName(node) if os.IsNotExist(err) || node.ParentID == "root" { err = nil return diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go index 08967590bd..7609a67616 100644 --- a/pkg/storage/fs/ocis/upload.go +++ b/pkg/storage/fs/ocis/upload.go @@ -34,6 +34,7 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" "github.com/pkg/xattr" + "github.com/rs/zerolog/log" tusd "github.com/tus/tusd/pkg/handler" ) @@ -43,7 +44,7 @@ var defaultFilePerm = os.FileMode(0664) func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { - node, err := fs.pw.Resolve(ctx, ref) + node, err := fs.pw.NodeFromResource(ctx, ref) if err != nil { return err } @@ -83,12 +84,12 @@ func (fs *ocisfs) InitiateUpload(ctx context.Context, ref *provider.Reference, u var relative string // the internal path of the file node - node, err := fs.pw.Resolve(ctx, ref) + node, err := fs.pw.NodeFromResource(ctx, ref) if err != nil { return "", err } - relative, err = fs.pw.Unwrap(ctx, node) + relative, err = fs.pw.Path(ctx, node) if err != nil { return "", err } @@ -146,7 +147,7 @@ func (fs *ocisfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tus } info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) - node, err := fs.pw.Wrap(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) + node, err := fs.pw.NodeFromPath(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) if err != nil { return nil, errors.Wrap(err, "ocisfs: error wrapping filename") } @@ -359,12 +360,12 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error { } // only delete the upload if it was successfully written to the storage - /*if err := os.Remove(upload.infoPath); err != nil { + if err := os.Remove(upload.infoPath); err != nil { if !os.IsNotExist(err) { log.Err(err).Interface("info", upload.info).Msg("ocisfs: could not delete upload info") return err } - }*/ + } // use set arbitrary metadata? /*if upload.info.MetaData["mtime"] != "" { err := upload.fs.SetMtime(ctx, np, upload.info.MetaData["mtime"]) From c3954402fb489a61f86958ab6c85aa40ba5854ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 4 Sep 2020 13:45:06 +0200 Subject: [PATCH 08/18] refactor node implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/node.go | 126 ++++++++++++++++++++++++-- pkg/storage/fs/ocis/ocis.go | 2 +- pkg/storage/fs/ocis/path.go | 138 ++++++++--------------------- pkg/storage/fs/ocis/persistence.go | 3 +- pkg/storage/fs/ocis/tree.go | 34 ++----- pkg/storage/fs/ocis/upload.go | 1 + 6 files changed, 167 insertions(+), 137 deletions(-) diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go index 8db46f3130..9e92c1cd59 100644 --- a/pkg/storage/fs/ocis/node.go +++ b/pkg/storage/fs/ocis/node.go @@ -1,17 +1,131 @@ package ocis +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "github.com/pkg/xattr" +) + // NodeInfo allows referencing a node by id and optionally a relative path type NodeInfo struct { + pw PathWrapper ParentID string ID string Name string Exists bool } -// BecomeParent rewrites the internal state to point to the parent id -func (n *NodeInfo) BecomeParent() { - n.ID = n.ParentID - n.ParentID = "" - n.Name = "" - n.Exists = false +// NewNode creates a new instance and checks if it exists +func NewNode(pw PathWrapper, id string) (n *NodeInfo, err error) { + n = &NodeInfo{ + pw: n.pw, + ID: id, + } + + nodePath := filepath.Join(n.pw.Root(), "nodes", n.ID) + + // lookup parent id in extended attributes + var attrBytes []byte + if attrBytes, err = xattr.Get(nodePath, "user.ocis.parentid"); err == nil { + n.ParentID = string(attrBytes) + } else { + // TODO log error + return + } + // lookup name in extended attributes + if attrBytes, err = xattr.Get(nodePath, "user.ocis.name"); err == nil { + n.Name = string(attrBytes) + } else { + // TODO log error + return + } + + parentID := n.ParentID + + for parentID != "root" { + // walk to root to check node is not part of a deleted subtree + parentPath := filepath.Join(n.pw.Root(), "nodes", parentID) + + if attrBytes, err = xattr.Get(parentPath, "user.ocis.parentid"); err == nil { + parentID = string(attrBytes) + } else { + if os.IsNotExist(err) { + return + } else { + // TODO log error + return + } + } + } + + n.Exists = true + + return +} + +// Child returns the child node with the given name +func (n *NodeInfo) Child(name string) (c *NodeInfo, err error) { + c = &NodeInfo{ + pw: n.pw, + ParentID: n.ID, + Name: name, + } + var link string + link, err = os.Readlink(filepath.Join(n.pw.Root(), "nodes", n.ID, name)) + if os.IsNotExist(err) { + return + } + if err != nil { + err = errors.Wrap(err, "ocisfs: Wrap: readlink error") + return + } + if strings.HasPrefix(link, "../") { + c.Exists = true + c.ID = filepath.Base(link) + } else { + err = fmt.Errorf("ocisfs: expected '../ prefix, got' %+v", link) + } + return +} + +// IsRoot returns true when the node is the root of a tree +func (n *NodeInfo) IsRoot() bool { + return n.ParentID == "root" +} + +// Parent returns the parent node +func (n *NodeInfo) Parent() (p *NodeInfo, err error) { + if n.ParentID == "root" { + return nil, fmt.Errorf("ocisfs: root has no parent") + } + p = &NodeInfo{ + pw: n.pw, + ID: n.ParentID, + } + + parentPath := filepath.Join(n.pw.Root(), "nodes", n.ParentID) + + // lookup parent id in extended attributes + var attrBytes []byte + if attrBytes, err = xattr.Get(parentPath, "user.ocis.parentid"); err == nil { + p.ParentID = string(attrBytes) + } else { + return + } + // lookup name in extended attributes + if attrBytes, err = xattr.Get(parentPath, "user.ocis.name"); err == nil { + p.Name = string(attrBytes) + } else { + return + } + + // check node exists + if _, err := os.Stat(parentPath); err == nil { + p.Exists = true + } + return } diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 1803e3be29..97c3e58135 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -103,7 +103,7 @@ func New(m map[string]interface{}) (storage.FS, error) { } pw := &Path{ - Root: c.Root, + root: c.Root, EnableHome: c.EnableHome, UserLayout: c.UserLayout, } diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go index 057f647545..1a0bd64038 100644 --- a/pkg/storage/fs/ocis/path.go +++ b/pkg/storage/fs/ocis/path.go @@ -8,17 +8,16 @@ import ( "strings" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" - "github.com/pkg/errors" - "github.com/pkg/xattr" ) // Path implements transformations from filepath to node and back type Path struct { // ocis fs works on top of a dir of uuid nodes - Root string `mapstructure:"root"` + root string `mapstructure:"root"` // UserLayout wraps the internal path with user information. // Example: if conf.Namespace is /ocis/user and received path is /docs @@ -46,125 +45,58 @@ func (pw *Path) NodeFromResource(ctx context.Context, ref *provider.Reference) ( // NodeFromPath converts a filename into a NodeInfo func (pw *Path) NodeFromPath(ctx context.Context, fn string) (node *NodeInfo, err error) { - var link, root string - if fn == "" { - fn = "/" - } - if pw.EnableHome && pw.UserLayout != "" { - // start at the users root node - u := user.ContextMustGetUser(ctx) - layout := templates.WithUser(u, pw.UserLayout) - root = filepath.Join(pw.Root, "users", layout) - - } else { - // start at the storage root node - root = filepath.Join(pw.Root, "nodes/root") - } - - node, err = pw.readRootLink(root) - // The symlink contains the nodeID - if err != nil { - return - } + log := appctx.GetLogger(ctx) + log.Debug().Interface("fn", fn).Msg("NodeFromPath()") + node, err = pw.RootNode(ctx) if fn != "/" { - // we need to walk the path + // walk the path segments := strings.Split(strings.TrimLeft(fn, "/"), "/") for i := range segments { - node.ParentID = node.ID - node.ID = "" - node.Name = segments[i] - - link, err = os.Readlink(filepath.Join(pw.Root, "nodes", node.ParentID, node.Name)) - if os.IsNotExist(err) { - node.Exists = false - // if this is the last segment we can use it as the node name - if i == len(segments)-1 { - err = nil - return - } - - err = errtypes.NotFound(filepath.Join(pw.Root, "nodes", node.ParentID, node.Name)) - return - } - if err != nil { - err = errors.Wrap(err, "ocisfs: Wrap: readlink error") - return - } - if strings.HasPrefix(link, "../") { - node.ID = filepath.Base(link) - } else { - err = fmt.Errorf("ocisfs: expected '../ prefix, got' %+v", link) - return + if node, err = node.Child(segments[i]); err != nil { + break } + log.Debug().Interface("node", node).Str("segment", segments[i]).Msg("NodeFromPath()") } } + // if a node does not exist that is fine + if os.IsNotExist(err) { + err = nil + } + return } // NodeFromID returns the internal path for the id -func (pw *Path) NodeFromID(ctx context.Context, id *provider.ResourceId) (*NodeInfo, error) { +func (pw *Path) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *NodeInfo, err error) { if id == nil || id.OpaqueId == "" { return nil, fmt.Errorf("invalid resource id %+v", id) } - return &NodeInfo{ID: id.OpaqueId}, nil + return NewNode(pw, id.OpaqueId) } -func (pw *Path) Path(ctx context.Context, ni *NodeInfo) (external string, err error) { - // check if this a not yet existing node - if ni.ID == "" && ni.Name != "" && ni.ParentID != "" { - external = ni.Name - ni.BecomeParent() - } - for err == nil { - err = pw.FillParentAndName(ni) - if os.IsNotExist(err) || ni.ParentID == "root" { - err = nil - return +// Path returns the path for node +func (pw *Path) Path(ctx context.Context, n *NodeInfo) (p string, err error) { + log := appctx.GetLogger(ctx) + log.Debug().Interface("node", n).Msg("Path()") + /* + // check if this a not yet existing node + if n.ID == "" && n.Name != "" && n.ParentID != "" { + path = n.Name + n, err = n.Parent() } - if err != nil { - err = errors.Wrap(err, "ocisfs: Unwrap: could not fill node") + */ + for !n.IsRoot() { + p = filepath.Join(n.Name, p) + if n, err = n.Parent(); err != nil { + log.Error().Err(err).Str("path", p).Interface("node", n).Msg("Path()") return } - external = filepath.Join(ni.Name, external) - ni.BecomeParent() } return } -// FillParentAndName reads the symbolic link and extracts the parent ID and the name of the node if necessary -func (pw *Path) FillParentAndName(node *NodeInfo) (err error) { - - if node == nil || node.ID == "" { - err = fmt.Errorf("ocisfs: invalid node info '%+v'", node) - } - - // check if node is already filled - if node.ParentID != "" && node.Name != "" { - return - } - - nodePath := filepath.Join(pw.Root, "nodes", node.ID) - - var attrBytes []byte - if attrBytes, err = xattr.Get(nodePath, "user.ocis.parentid"); err == nil { - node.ParentID = string(attrBytes) - } - // no error if it is empty -> root node - // TODO better identify a root node, eg by using `root` as the parentid? or `root:userid`? - - if attrBytes, err = xattr.Get(nodePath, "user.ocis.name"); err == nil { - node.Name = string(attrBytes) - } - - // no error if it is empty -> root node - // TODO better identify a root node? - - node.Exists = true - return -} - // readRootLink reads the symbolic link and extracts the node id func (pw *Path) readRootLink(root string) (node *NodeInfo, err error) { // A root symlink looks like `../nodes/76455834-769e-412a-8a01-68f265365b79` @@ -177,6 +109,7 @@ func (pw *Path) readRootLink(root string) (node *NodeInfo, err error) { // extract the nodeID if strings.HasPrefix(link, "../nodes/") { node = &NodeInfo{ + pw: pw, ID: filepath.Base(link), Exists: true, } @@ -194,11 +127,11 @@ func (pw *Path) RootNode(ctx context.Context) (node *NodeInfo, err error) { // start at the users root node u := user.ContextMustGetUser(ctx) layout := templates.WithUser(u, pw.UserLayout) - root = filepath.Join(pw.Root, "users", layout) + root = filepath.Join(pw.Root(), "users", layout) } else { // start at the storage root node - root = filepath.Join(pw.Root, "nodes/root") + root = filepath.Join(pw.Root(), "nodes/root") } // The symlink contains the nodeID @@ -208,3 +141,8 @@ func (pw *Path) RootNode(ctx context.Context) (node *NodeInfo, err error) { } return } + +// Root returns the root of the storags +func (pw *Path) Root() string { + return pw.root +} diff --git a/pkg/storage/fs/ocis/persistence.go b/pkg/storage/fs/ocis/persistence.go index 2407b3fc1a..c73f736407 100644 --- a/pkg/storage/fs/ocis/persistence.go +++ b/pkg/storage/fs/ocis/persistence.go @@ -32,6 +32,7 @@ type PathWrapper interface { NodeFromPath(ctx context.Context, fn string) (node *NodeInfo, err error) Path(ctx context.Context, node *NodeInfo) (path string, err error) - FillParentAndName(node *NodeInfo) (err error) // Tree persistence? RootNode(ctx context.Context) (node *NodeInfo, err error) + // Root returns the internal root of the storage + Root() string } diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index ddcd190ae6..cefe8059ee 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -95,19 +95,8 @@ func (t *Tree) CreateReference(ctx context.Context, path string, targetURI *url. // Move replaces the target with the source func (t *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) (err error) { - err = t.pw.FillParentAndName(newNode) - if os.IsNotExist(err) { - err = nil - return - } - // if target exists delete it without trashing it if newNode.Exists { - err = t.pw.FillParentAndName(newNode) - if os.IsNotExist(err) { - err = nil - return - } // TODO make sure all children are deleted if err := os.RemoveAll(filepath.Join(t.DataDirectory, "nodes", newNode.ID)); err != nil { return errors.Wrap(err, "ocisfs: Move: error deleting target node "+newNode.ID) @@ -194,6 +183,7 @@ func (t *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, err continue } n := &NodeInfo{ + pw: t.pw, ParentID: node.ID, ID: filepath.Base(link), Name: names[i], @@ -207,16 +197,6 @@ func (t *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, err // Delete deletes a node in the tree func (t *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { - err = t.pw.FillParentAndName(node) - if os.IsNotExist(err) { - err = nil - return - } - if err != nil { - err = errors.Wrap(err, "ocisfs: Delete: FillParentAndName error") - return - } - src := filepath.Join(t.DataDirectory, "nodes", node.ParentID, node.Name) err = os.Remove(src) if err != nil { @@ -231,7 +211,7 @@ func (t *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { return } - return t.Propagate(ctx, &NodeInfo{ID: node.ParentID}) + return t.Propagate(ctx, &NodeInfo{pw: t.pw, ID: node.ParentID}) } // Propagate propagates changes to the root of the tree @@ -243,23 +223,19 @@ func (t *Tree) Propagate(ctx context.Context, node *NodeInfo) (err error) { } // store in extended attribute etag := hex.EncodeToString(bytes) - for err == nil { + for err == nil && !node.IsRoot() { if err := xattr.Set(filepath.Join(t.DataDirectory, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { log.Error().Err(err).Msg("error storing file id") } // TODO propagate mtime // TODO size accounting - err = t.pw.FillParentAndName(node) - if os.IsNotExist(err) || node.ParentID == "root" { - err = nil - return - } + if err != nil { err = errors.Wrap(err, "ocisfs: Propagate: readlink error") return } - node.BecomeParent() + node, err = node.Parent() } return } diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go index 7609a67616..82fa9013f6 100644 --- a/pkg/storage/fs/ocis/upload.go +++ b/pkg/storage/fs/ocis/upload.go @@ -326,6 +326,7 @@ func (upload *fileUpload) writeInfo() error { func (upload *fileUpload) FinishUpload(ctx context.Context) error { n := &NodeInfo{ + pw: upload.fs.pw, ID: upload.info.Storage["NodeId"], ParentID: upload.info.Storage["NodeParentId"], Name: upload.info.Storage["NodeName"], From b5e9dcfc43ba767a89852f9903ea1dfcb2bfa9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 4 Sep 2020 15:13:49 +0200 Subject: [PATCH 09/18] fix delete and overwrite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/ocis.go | 1 - pkg/storage/fs/ocis/path.go | 7 ++--- pkg/storage/fs/ocis/tree.go | 19 +++++++++++--- pkg/storage/fs/ocis/upload.go | 48 ++++++++++++++++++++++++++++++----- 4 files changed, 61 insertions(+), 14 deletions(-) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 97c3e58135..9483e4d259 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -92,7 +92,6 @@ func New(m map[string]interface{}) (storage.FS, error) { // better to keep uploads on a fast / volatile storage before a workflow finally moves them to the nodes dir filepath.Join(c.Root, "uploads"), filepath.Join(c.Root, "trash"), - filepath.Join(c.Root, "versions"), } for _, v := range dataPaths { if err := os.MkdirAll(v, 0700); err != nil { diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go index 1a0bd64038..c23b0ba09d 100644 --- a/pkg/storage/fs/ocis/path.go +++ b/pkg/storage/fs/ocis/path.go @@ -109,9 +109,10 @@ func (pw *Path) readRootLink(root string) (node *NodeInfo, err error) { // extract the nodeID if strings.HasPrefix(link, "../nodes/") { node = &NodeInfo{ - pw: pw, - ID: filepath.Base(link), - Exists: true, + pw: pw, + ID: filepath.Base(link), + ParentID: "root", + Exists: true, } } else { err = fmt.Errorf("ocisfs: expected '../nodes/ prefix, got' %+v", link) diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index cefe8059ee..d308ec76c7 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "path/filepath" + "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" @@ -203,15 +204,25 @@ func (t *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { return } + nodePath := filepath.Join(t.DataDirectory, "nodes", node.ID) + trashPath := nodePath + ".T." + time.Now().UTC().Format(time.RFC3339Nano) + err = os.Rename(nodePath, trashPath) + if err != nil { + return + } + // make node appear in trash // parent id and name are stored as extended attributes in the node itself - trashpath := filepath.Join(t.DataDirectory, "trash", node.ID) - err = os.Symlink("../nodes/"+node.ID, trashpath) + trashLink := filepath.Join(t.DataDirectory, "trash", node.ID) + err = os.Symlink("../nodes/"+node.ID+".T."+time.Now().UTC().Format(time.RFC3339Nano), trashLink) if err != nil { return } - - return t.Propagate(ctx, &NodeInfo{pw: t.pw, ID: node.ParentID}) + p, err := node.Parent() + if err != nil { + return + } + return t.Propagate(ctx, p) } // Propagate propagates changes to the root of the tree diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go index 82fa9013f6..bfe5bfdbd7 100644 --- a/pkg/storage/fs/ocis/upload.go +++ b/pkg/storage/fs/ocis/upload.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "os" "path/filepath" + "time" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -337,8 +338,25 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error { } targetPath := filepath.Join(upload.fs.conf.Root, "nodes", n.ID) - err := os.Rename(upload.binPath, targetPath) - if err != nil { + // if target exists create new version + if fi, err := os.Stat(targetPath); err == nil { + // versions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries + versionsPath := filepath.Join(upload.fs.conf.Root, "nodes", n.ID+"."+fi.ModTime().UTC().Format(time.RFC3339Nano)) + + if err := os.Rename(targetPath, versionsPath); err != nil { + log := appctx.GetLogger(upload.ctx) + log.Err(err).Interface("info", upload.info). + Str("binPath", upload.binPath). + Str("targetPath", targetPath). + Msg("ocisfs: could not create version") + return err + } + } + + // now rename the upload to the target path + // TODO put uploads on the same underlying storage as the destination dir? + // TODO trigger a workflow as the final rename might eg involve antivirus scanning + if err := os.Rename(upload.binPath, targetPath); err != nil { log := appctx.GetLogger(upload.ctx) log.Err(err).Interface("info", upload.info). Str("binPath", upload.binPath). @@ -354,10 +372,26 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error { return errors.Wrap(err, "ocisfs: could not set name attribute") } - // link child name to parent - err = os.Symlink("../"+n.ID, filepath.Join(upload.fs.conf.Root, "nodes", n.ParentID, n.Name)) - if err != nil { - return errors.Wrap(err, "ocisfs: could not symlink child entry") + // link child name to parent if it is new + childNameLink := filepath.Join(upload.fs.conf.Root, "nodes", n.ParentID, n.Name) + link, err := os.Readlink(childNameLink) + if err == nil && link != "../"+n.ID { + log.Err(err). + Interface("info", upload.info). + Interface("node", n). + Str("targetPath", targetPath). + Str("childNameLink", childNameLink). + Str("link", link). + Msg("ocisfs: child name link has wrong target id, repairing") + + if err := os.Remove(childNameLink); err != nil { + return errors.Wrap(err, "ocisfs: could not remove symlink child entry") + } + } + if os.IsNotExist(err) || link != "../"+n.ID { + if err = os.Symlink("../"+n.ID, childNameLink); err != nil { + return errors.Wrap(err, "ocisfs: could not symlink child entry") + } } // only delete the upload if it was successfully written to the storage @@ -376,6 +410,8 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error { } }*/ + n.Exists = true + return upload.fs.tp.Propagate(upload.ctx, n) } From 745d4bc4301f5ca7ee15e743f5da26276bd2d1c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 4 Sep 2020 23:18:51 +0200 Subject: [PATCH 10/18] initial sharing, store owner as ext attr, fix NewNode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/grants.go | 380 ++++++++++++++++++++++++++++++++++ pkg/storage/fs/ocis/node.go | 28 ++- pkg/storage/fs/ocis/ocis.go | 125 +++++++++-- pkg/storage/fs/ocis/tree.go | 11 + pkg/storage/fs/ocis/upload.go | 10 + 5 files changed, 536 insertions(+), 18 deletions(-) create mode 100644 pkg/storage/fs/ocis/grants.go diff --git a/pkg/storage/fs/ocis/grants.go b/pkg/storage/fs/ocis/grants.go new file mode 100644 index 0000000000..5681f72a6d --- /dev/null +++ b/pkg/storage/fs/ocis/grants.go @@ -0,0 +1,380 @@ +package ocis + +import ( + "context" + "encoding/csv" + "fmt" + "path/filepath" + "strconv" + "strings" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/pkg/xattr" +) + +func (fs *ocisfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + log := appctx.GetLogger(ctx) + log.Debug().Interface("ref", ref).Interface("grant", g).Msg("AddGrant()") + var node *NodeInfo + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + + e, err := fs.getACE(g) + if err != nil { + return err + } + + var attr string + if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { + attr = sharePrefix + "g:" + e.Principal + } else { + attr = sharePrefix + "u:" + e.Principal + } + + np := filepath.Join(fs.pw.Root(), "nodes", node.ID) + if err := xattr.Set(np, attr, getValue(e)); err != nil { + return err + } + return fs.tp.Propagate(ctx, node) +} + +func getValue(e *ace) []byte { + // first byte will be replaced after converting to byte array + val := fmt.Sprintf("_t=%s:f=%s:p=%s", e.Type, e.Flags, e.Permissions) + b := []byte(val) + b[0] = 0 // indicate key value + return b +} + +func getACEPerm(set *provider.ResourcePermissions) (string, error) { + var b strings.Builder + + if set.Stat || set.InitiateFileDownload || set.ListContainer { + b.WriteString("r") + } + if set.InitiateFileUpload || set.Move { + b.WriteString("w") + } + if set.CreateContainer { + b.WriteString("a") + } + if set.Delete { + b.WriteString("d") + } + + // sharing + if set.AddGrant || set.RemoveGrant || set.UpdateGrant { + b.WriteString("C") + } + if set.ListGrants { + b.WriteString("c") + } + + // trash + if set.ListRecycle { + b.WriteString("u") + } + if set.RestoreRecycleItem { + b.WriteString("U") + } + if set.PurgeRecycle { + b.WriteString("P") + } + + // versions + if set.ListFileVersions { + b.WriteString("v") + } + if set.RestoreFileVersion { + b.WriteString("V") + } + + // quota + if set.GetQuota { + b.WriteString("q") + } + // TODO set quota permission? + // TODO GetPath + return b.String(), nil +} + +func (fs *ocisfs) getACE(g *provider.Grant) (*ace, error) { + permissions, err := getACEPerm(g.Permissions) + if err != nil { + return nil, err + } + e := &ace{ + Principal: g.Grantee.Id.OpaqueId, + Permissions: permissions, + // TODO creator ... + Type: "A", + } + if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { + e.Flags = "g" + } + return e, nil +} + +type ace struct { + //NFSv4 acls + Type string // t + Flags string // f + Principal string // im key + Permissions string // p + + // sharing specific + ShareTime int // s + Creator string // c + Expires int // e + Password string // w passWord TODO h = hash + Label string // l +} + +func unmarshalACE(v []byte) (*ace, error) { + // first byte indicates type of value + switch v[0] { + case 0: // = ':' separated key=value pairs + s := string(v[1:]) + return unmarshalKV(s) + default: + return nil, fmt.Errorf("unknown ace encoding") + } +} + +func unmarshalKV(s string) (*ace, error) { + e := &ace{} + r := csv.NewReader(strings.NewReader(s)) + r.Comma = ':' + r.Comment = 0 + r.FieldsPerRecord = -1 + r.LazyQuotes = false + r.TrimLeadingSpace = false + records, err := r.ReadAll() + if err != nil { + return nil, err + } + if len(records) != 1 { + return nil, fmt.Errorf("more than one row of ace kvs") + } + for i := range records[0] { + kv := strings.Split(records[0][i], "=") + switch kv[0] { + case "t": + e.Type = kv[1] + case "f": + e.Flags = kv[1] + case "p": + e.Permissions = kv[1] + case "s": + v, err := strconv.Atoi(kv[1]) + if err != nil { + return nil, err + } + e.ShareTime = v + case "c": + e.Creator = kv[1] + case "e": + v, err := strconv.Atoi(kv[1]) + if err != nil { + return nil, err + } + e.Expires = v + case "w": + e.Password = kv[1] + case "l": + e.Label = kv[1] + // TODO default ... log unknown keys? or add as opaque? hm we need that for tagged shares ... + } + } + return e, nil +} + +// Parse parses an acl string with the given delimiter (LongTextForm or ShortTextForm) +func getACEs(ctx context.Context, fsfn string, attrs []string) (entries []*ace, err error) { + log := appctx.GetLogger(ctx) + entries = []*ace{} + for i := range attrs { + if strings.HasPrefix(attrs[i], sharePrefix) { + principal := attrs[i][len(sharePrefix):] + var value []byte + if value, err = xattr.Get(fsfn, attrs[i]); err != nil { + log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute") + continue + } + var e *ace + if e, err = unmarshalACE(value); err != nil { + log.Error().Err(err).Str("attr", attrs[i]).Msg("could unmarshal ace") + continue + } + e.Principal = principal[2:] + // check consistency of Flags and principal type + if strings.Contains(e.Flags, "g") { + if principal[:1] != "g" { + log.Error().Str("attr", attrs[i]).Interface("ace", e).Msg("inconsistent ace: expected group") + continue + } + } else { + if principal[:1] != "u" { + log.Error().Str("attr", attrs[i]).Interface("ace", e).Msg("inconsistent ace: expected user") + continue + } + } + entries = append(entries, e) + } + } + return entries, nil +} + +func (fs *ocisfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { + var node *NodeInfo + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + log := appctx.GetLogger(ctx) + np := filepath.Join(fs.pw.Root(), "nodes", node.ID) + var attrs []string + if attrs, err = xattr.List(np); err != nil { + log.Error().Err(err).Msg("error listing attributes") + return nil, err + } + + log.Debug().Interface("attrs", attrs).Msg("read attributes") + // filter attributes + var aces []*ace + if aces, err = getACEs(ctx, np, attrs); err != nil { + log.Error().Err(err).Msg("error getting aces") + return nil, err + } + + grants = make([]*provider.Grant, 0, len(aces)) + for i := range aces { + grantee := &provider.Grantee{ + // TODO lookup uid from principal + Id: &userpb.UserId{OpaqueId: aces[i].Principal}, + Type: fs.getGranteeType(aces[i]), + } + grants = append(grants, &provider.Grant{ + Grantee: grantee, + Permissions: fs.getGrantPermissionSet(aces[i].Permissions), + }) + } + + return grants, nil +} + +func (fs *ocisfs) getGranteeType(e *ace) provider.GranteeType { + if strings.Contains(e.Flags, "g") { + return provider.GranteeType_GRANTEE_TYPE_GROUP + } + return provider.GranteeType_GRANTEE_TYPE_USER +} + +func (fs *ocisfs) getGrantPermissionSet(mode string) *provider.ResourcePermissions { + p := &provider.ResourcePermissions{} + // r + if strings.Contains(mode, "r") { + p.Stat = true + p.InitiateFileDownload = true + p.ListContainer = true + } + // w + if strings.Contains(mode, "w") { + p.InitiateFileUpload = true + if p.InitiateFileDownload { + p.Move = true + } + } + //a + if strings.Contains(mode, "a") { + // TODO append data to file permission? + p.CreateContainer = true + } + //x + //if strings.Contains(mode, "x") { + // TODO execute file permission? + // TODO change directory permission? + //} + //d + if strings.Contains(mode, "d") { + p.Delete = true + } + //D ? + + // sharing + if strings.Contains(mode, "C") { + p.AddGrant = true + p.RemoveGrant = true + p.UpdateGrant = true + } + if strings.Contains(mode, "c") { + p.ListGrants = true + } + + // trash + if strings.Contains(mode, "u") { // u = undelete + p.ListRecycle = true + } + if strings.Contains(mode, "U") { + p.RestoreRecycleItem = true + } + if strings.Contains(mode, "P") { + p.PurgeRecycle = true + } + + // versions + if strings.Contains(mode, "v") { + p.ListFileVersions = true + } + if strings.Contains(mode, "V") { + p.RestoreFileVersion = true + } + + // ? + // TODO GetPath + if strings.Contains(mode, "q") { + p.GetQuota = true + } + // TODO set quota permission? + return p +} + +func (fs *ocisfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + var node *NodeInfo + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + + var attr string + if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { + attr = sharePrefix + "g:" + g.Grantee.Id.OpaqueId + } else { + attr = sharePrefix + "u:" + g.Grantee.Id.OpaqueId + } + + np := filepath.Join(fs.pw.Root(), "nodes", node.ID) + if err = xattr.Remove(np, attr); err != nil { + return + } + + return fs.tp.Propagate(ctx, node) +} + +func (fs *ocisfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + return fs.AddGrant(ctx, ref, g) +} diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go index 9e92c1cd59..295c3b5d44 100644 --- a/pkg/storage/fs/ocis/node.go +++ b/pkg/storage/fs/ocis/node.go @@ -16,13 +16,15 @@ type NodeInfo struct { ParentID string ID string Name string + ownerID string + ownerIDP string Exists bool } // NewNode creates a new instance and checks if it exists func NewNode(pw PathWrapper, id string) (n *NodeInfo, err error) { n = &NodeInfo{ - pw: n.pw, + pw: pw, ID: id, } @@ -129,3 +131,27 @@ func (n *NodeInfo) Parent() (p *NodeInfo, err error) { } return } + +// Owner returns the cached owner id or reads it from the extended attributes +func (n *NodeInfo) Owner() (id string, idp string, err error) { + if n.ownerID != "" && n.ownerIDP != "" { + return n.ownerID, n.ownerIDP, nil + } + + nodePath := filepath.Join(n.pw.Root(), "nodes", n.ParentID) + // lookup parent id in extended attributes + var attrBytes []byte + // lookup name in extended attributes + if attrBytes, err = xattr.Get(nodePath, "user.ocis.owner.id"); err == nil { + n.ownerID = string(attrBytes) + } else { + // TODO log error + } + // lookup name in extended attributes + if attrBytes, err = xattr.Get(nodePath, "user.ocis.owner.idp"); err == nil { + n.ownerIDP = string(attrBytes) + } else { + // TODO log error + } + return n.ownerID, n.ownerIDP, err +} diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 9483e4d259..cd6c1bd330 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" @@ -41,6 +42,99 @@ import ( "github.com/pkg/xattr" ) +const ( + // TODO the below comment is currently copied from the owncloud driver, revisit + // Currently,extended file attributes have four separated + // namespaces (user, trusted, security and system) followed by a dot. + // A non root user can only manipulate the user. namespace, which is what + // we will use to store ownCloud specific metadata. To prevent name + // collisions with other apps We are going to introduce a sub namespace + // "user.ocis." + + // shares are persisted using extended attributes. We are going to mimic + // NFS4 ACLs, with one extended attribute per share, following Access + // Control Entries (ACEs). The following is taken from the nfs4_acl man page, + // see https://linux.die.net/man/5/nfs4_acl: + // the extended attributes will look like this + // "user.oc.acl.:::" + // - *type* will be limited to A for now + // A: Allow - allow *principal* to perform actions requiring *permissions* + // In the future we can use: + // U: aUdit - log any attempted access by principal which requires + // permissions. + // L: aLarm - generate a system alarm at any attempted access by + // principal which requires permissions + // D for deny is not recommended + // - *flags* for now empty or g for group, no inheritance yet + // - d directory-inherit - newly-created subdirectories will inherit the + // ACE. + // - f file-inherit - newly-created files will inherit the ACE, minus its + // inheritance flags. Newly-created subdirectories + // will inherit the ACE; if directory-inherit is not + // also specified in the parent ACE, inherit-only will + // be added to the inherited ACE. + // - n no-propagate-inherit - newly-created subdirectories will inherit + // the ACE, minus its inheritance flags. + // - i inherit-only - the ACE is not considered in permissions checks, + // but it is heritable; however, the inherit-only + // flag is stripped from inherited ACEs. + // - *principal* a named user, group or special principal + // - the oidc sub@iss maps nicely to this + // - 'OWNER@', 'GROUP@', and 'EVERYONE@', which are, respectively, analogous to the POSIX user/group/other + // - *permissions* + // - r read-data (files) / list-directory (directories) + // - w write-data (files) / create-file (directories) + // - a append-data (files) / create-subdirectory (directories) + // - x execute (files) / change-directory (directories) + // - d delete - delete the file/directory. Some servers will allow a delete to occur if either this permission is set in the file/directory or if the delete-child permission is set in its parent directory. + // - D delete-child - remove a file or subdirectory from within the given directory (directories only) + // - t read-attributes - read the attributes of the file/directory. + // - T write-attributes - write the attributes of the file/directory. + // - n read-named-attributes - read the named attributes of the file/directory. + // - N write-named-attributes - write the named attributes of the file/directory. + // - c read-ACL - read the file/directory NFSv4 ACL. + // - C write-ACL - write the file/directory NFSv4 ACL. + // - o write-owner - change ownership of the file/directory. + // - y synchronize - allow clients to use synchronous I/O with the server. + // TODO implement OWNER@ as "user.oc.acl.A::OWNER@:rwaDxtTnNcCy" + // attribute names are limited to 255 chars by the linux kernel vfs, values to 64 kb + // ext3 extended attributes must fit inside a single filesystem block ... 4096 bytes + // that leaves us with "user.oc.acl.A::someonewithaslightlylongersubject@whateverissuer:rwaDxtTnNcCy" ~80 chars + // 4096/80 = 51 shares ... with luck we might move the actual permissions to the value, saving ~15 chars + // 4096/64 = 64 shares ... still meh ... we can do better by using ints instead of strings for principals + // "user.ocis.acl.u:100000" is pretty neat, but we can still do better: base64 encode the int + // "user.ocis.acl.u:6Jqg" but base64 always has at least 4 chars, maybe hex is better for smaller numbers + // well use 4 chars in addition to the ace: "user.oc.acl.u:////" = 65535 -> 18 chars + // 4096/18 = 227 shares + // still ... ext attrs for this are not infinite scale ... + // so .. attach shares via fileid. + // /metadata//shares, similar to /files + // /metadata//shares/u///A:fdi:rwaDxtTnNcCy permissions as filename to keep them in the stat cache? + // + // whatever ... 50 shares is good enough. If more is needed we can delegate to the metadata + // if "user.oc.acl.M" is present look inside the metadata app. + // - if we cannot set an ace we might get an io error. + // in that case convert all shares to metadata and try to set "user.oc.acl.m" + // + // what about metadata like share creator, share time, expiry? + // - creator is same as owner, but can be set + // - share date, or abbreviated st is a unix timestamp + // - expiry is a unix timestamp + // - can be put inside the value + // - we need to reorder the fields: + // "user.oc.acl.:" -> "kv:t=:f=:p=:st=:c=:e=:pw=:n=" + // "user.oc.acl.:" -> "v1::::::::" + // or the first byte determines the format + // 0x00 = key value + // 0x01 = v1 ... + // + // SharePrefix is the prefix for sharing related extended attributes + sharePrefix string = "user.ocis.acl." + favPrefix string = "user.ocis.fav." // favorite flag, per user + etagPrefix string = "user.ocis.etag." // allow overriding a calculated etag with one from the extended attributes + //checksumPrefix string = "user.ocis.cs." // TODO add checksum support +) + func init() { registry.Register("ocis", New) } @@ -170,6 +264,12 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { if err := xattr.Set(nodePath, "user.ocis.name", []byte("")); err != nil { return errors.Wrap(err, "ocisfs: could not set name attribute") } + if err := xattr.Set(nodePath, "user.ocis.owner.id", []byte(u.Id.OpaqueId)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner id attribute") + } + if err := xattr.Set(nodePath, "user.ocis.owner.idp", []byte(u.Id.Idp)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner idp attribute") + } // link users home to node return os.Symlink("../nodes/"+nodeID, home) @@ -338,23 +438,7 @@ func (fs *ocisfs) RestoreRecycleItem(ctx context.Context, key string) error { return errtypes.NotSupported("operation not supported: RestoreRecycleItem") } -// share persistence - -func (fs *ocisfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - return errtypes.NotSupported("operation not supported: AddGrant") -} - -func (fs *ocisfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { - return nil, errtypes.NotSupported("operation not supported: ListGrants") -} - -func (fs *ocisfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { - return errtypes.NotSupported("operation not supported: RemoveGrant") -} - -func (fs *ocisfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - return errtypes.NotSupported("operation not supported: UpdateGrant") -} +// share persistence is grants.go // supporting functions @@ -403,9 +487,16 @@ func (fs *ocisfs) normalize(ctx context.Context, node *NodeInfo) (ri *provider.R PermissionSet: &provider.ResourcePermissions{ListContainer: true, CreateContainer: true}, Mtime: &types.Timestamp{ Seconds: uint64(fi.ModTime().Unix()), + // TODO read nanos from where? Nanos: fi.MTimeNanos, }, } + if owner, idp, err := node.Owner(); err == nil { + ri.Owner = &userpb.UserId{ + Idp: idp, + OpaqueId: owner, + } + } log := appctx.GetLogger(ctx) log.Debug(). Interface("ri", ri). diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index d308ec76c7..e007ad6138 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -11,6 +11,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/user" "github.com/google/uuid" "github.com/pkg/errors" "github.com/pkg/xattr" @@ -80,6 +81,16 @@ func (t *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { if err := xattr.Set(newPath, "user.ocis.name", []byte(node.Name)); err != nil { return errors.Wrap(err, "ocisfs: could not set name attribute") } + if u, ok := user.ContextGetUser(ctx); ok { + if err := xattr.Set(newPath, "user.ocis.owner.id", []byte(u.Id.OpaqueId)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner id attribute") + } + if err := xattr.Set(newPath, "user.ocis.owner.idp", []byte(u.Id.Idp)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner idp attribute") + } + } else { + // TODO no user in context, log as error when home enabled + } // make child appear in listings err = os.Symlink("../"+node.ID, filepath.Join(t.DataDirectory, "nodes", node.ParentID, node.Name)) diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go index bfe5bfdbd7..e0c60d44fd 100644 --- a/pkg/storage/fs/ocis/upload.go +++ b/pkg/storage/fs/ocis/upload.go @@ -371,6 +371,16 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error { if err := xattr.Set(targetPath, "user.ocis.name", []byte(n.Name)); err != nil { return errors.Wrap(err, "ocisfs: could not set name attribute") } + if u, ok := user.ContextGetUser(ctx); ok { + if err := xattr.Set(targetPath, "user.ocis.owner.id", []byte(u.Id.OpaqueId)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner id attribute") + } + if err := xattr.Set(targetPath, "user.ocis.owner.idp", []byte(u.Id.Idp)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner idp attribute") + } + } else { + // TODO no user in context, log as error when home enabled + } // link child name to parent if it is new childNameLink := filepath.Join(upload.fs.conf.Root, "nodes", n.ParentID, n.Name) From 571073890531672f52763ada793eeedca89c607e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 4 Sep 2020 23:29:08 +0200 Subject: [PATCH 11/18] separate files for metadada, revisions and recycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/metadata.go | 16 ++++++++++++ pkg/storage/fs/ocis/ocis.go | 43 +++----------------------------- pkg/storage/fs/ocis/recycle.go | 24 ++++++++++++++++++ pkg/storage/fs/ocis/revisions.go | 20 +++++++++++++++ 4 files changed, 64 insertions(+), 39 deletions(-) create mode 100644 pkg/storage/fs/ocis/metadata.go create mode 100644 pkg/storage/fs/ocis/recycle.go create mode 100644 pkg/storage/fs/ocis/revisions.go diff --git a/pkg/storage/fs/ocis/metadata.go b/pkg/storage/fs/ocis/metadata.go new file mode 100644 index 0000000000..463883b5bf --- /dev/null +++ b/pkg/storage/fs/ocis/metadata.go @@ -0,0 +1,16 @@ +package ocis + +import ( + "context" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" +) + +func (fs *ocisfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { + return errtypes.NotSupported("operation not supported: SetArbitraryMetadata") +} + +func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { + return errtypes.NotSupported("operation not supported: UnsetArbitraryMetadata") +} diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index cd6c1bd330..8a52b5c360 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -368,16 +368,6 @@ func (fs *ocisfs) Delete(ctx context.Context, ref *provider.Reference) (err erro return fs.tp.Delete(ctx, node) } -// arbitrary metadata persistence - -func (fs *ocisfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { - return errtypes.NotSupported("operation not supported: SetArbitraryMetadata") -} - -func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { - return errtypes.NotSupported("operation not supported: UnsetArbitraryMetadata") -} - // Data persistence func (fs *ocisfs) ContentPath(node *NodeInfo) string { @@ -407,38 +397,13 @@ func (fs *ocisfs) Download(ctx context.Context, ref *provider.Reference) (io.Rea return r, nil } -// Version persistence +// arbitrary metadata persistence in metadata.go -func (fs *ocisfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { - return nil, errtypes.NotSupported("operation not supported: ListRevisions") -} -func (fs *ocisfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { - return nil, errtypes.NotSupported("operation not supported: DownloadRevision") -} - -func (fs *ocisfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { - return errtypes.NotSupported("operation not supported: RestoreRevision") -} +// Version persistence in revisions.go -// Trash persistence - -func (fs *ocisfs) PurgeRecycleItem(ctx context.Context, key string) error { - return errtypes.NotSupported("operation not supported: PurgeRecycleItem") -} - -func (fs *ocisfs) EmptyRecycle(ctx context.Context) error { - return errtypes.NotSupported("operation not supported: EmptyRecycle") -} - -func (fs *ocisfs) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) { - return nil, errtypes.NotSupported("operation not supported: ListRecycle") -} - -func (fs *ocisfs) RestoreRecycleItem(ctx context.Context, key string) error { - return errtypes.NotSupported("operation not supported: RestoreRecycleItem") -} +// Trash persistence in recycle.go -// share persistence is grants.go +// share persistence in grants.go // supporting functions diff --git a/pkg/storage/fs/ocis/recycle.go b/pkg/storage/fs/ocis/recycle.go new file mode 100644 index 0000000000..c0cdbfbcd8 --- /dev/null +++ b/pkg/storage/fs/ocis/recycle.go @@ -0,0 +1,24 @@ +package ocis + +import ( + "context" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" +) + +func (fs *ocisfs) PurgeRecycleItem(ctx context.Context, key string) error { + return errtypes.NotSupported("operation not supported: PurgeRecycleItem") +} + +func (fs *ocisfs) EmptyRecycle(ctx context.Context) error { + return errtypes.NotSupported("operation not supported: EmptyRecycle") +} + +func (fs *ocisfs) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) { + return nil, errtypes.NotSupported("operation not supported: ListRecycle") +} + +func (fs *ocisfs) RestoreRecycleItem(ctx context.Context, key string) error { + return errtypes.NotSupported("operation not supported: RestoreRecycleItem") +} diff --git a/pkg/storage/fs/ocis/revisions.go b/pkg/storage/fs/ocis/revisions.go new file mode 100644 index 0000000000..e6a1159f64 --- /dev/null +++ b/pkg/storage/fs/ocis/revisions.go @@ -0,0 +1,20 @@ +package ocis + +import ( + "context" + "io" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" +) + +func (fs *ocisfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { + return nil, errtypes.NotSupported("operation not supported: ListRevisions") +} +func (fs *ocisfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { + return nil, errtypes.NotSupported("operation not supported: DownloadRevision") +} + +func (fs *ocisfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { + return errtypes.NotSupported("operation not supported: RestoreRevision") +} From 492312125771ab2df7d2dc2a97e7aa3d442e62c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 4 Sep 2020 23:53:51 +0200 Subject: [PATCH 12/18] list revisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/revisions.go | 29 +++++++++++++++++++++++++++-- pkg/storage/fs/ocis/upload.go | 2 +- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/pkg/storage/fs/ocis/revisions.go b/pkg/storage/fs/ocis/revisions.go index e6a1159f64..f93e77be56 100644 --- a/pkg/storage/fs/ocis/revisions.go +++ b/pkg/storage/fs/ocis/revisions.go @@ -3,13 +3,38 @@ package ocis import ( "context" "io" + "os" + "path/filepath" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" ) -func (fs *ocisfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { - return nil, errtypes.NotSupported("operation not supported: ListRevisions") +func (fs *ocisfs) ListRevisions(ctx context.Context, ref *provider.Reference) (revisions []*provider.FileVersion, err error) { + var node *NodeInfo + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + + revisions = []*provider.FileVersion{} + nodePath := filepath.Join(fs.pw.Root(), "nodes", node.ID) + if items, err := filepath.Glob(nodePath + ".REV.*"); err == nil { + for i := range items { + if fi, err := os.Stat(items[i]); err == nil { + rev := &provider.FileVersion{ + Key: filepath.Base(items[i]), + Size: uint64(fi.Size()), + Mtime: uint64(fi.ModTime().Unix()), + } + revisions = append(revisions, rev) + } + } + } + return } func (fs *ocisfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { return nil, errtypes.NotSupported("operation not supported: DownloadRevision") diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go index e0c60d44fd..e9857b7e44 100644 --- a/pkg/storage/fs/ocis/upload.go +++ b/pkg/storage/fs/ocis/upload.go @@ -341,7 +341,7 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error { // if target exists create new version if fi, err := os.Stat(targetPath); err == nil { // versions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries - versionsPath := filepath.Join(upload.fs.conf.Root, "nodes", n.ID+"."+fi.ModTime().UTC().Format(time.RFC3339Nano)) + versionsPath := filepath.Join(upload.fs.conf.Root, "nodes", n.ID+".REV."+fi.ModTime().UTC().Format(time.RFC3339Nano)) if err := os.Rename(targetPath, versionsPath); err != nil { log := appctx.GetLogger(upload.ctx) From aff7ef54877eebb73c8ea7049e3b5ac66cba4bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 7 Sep 2020 15:28:41 +0200 Subject: [PATCH 13/18] refactor and use ACE package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/grants.go | 332 +++---------------------- pkg/storage/fs/ocis/node.go | 100 ++++++-- pkg/storage/fs/ocis/ocis.go | 87 +------ pkg/storage/fs/ocis/path.go | 40 ++- pkg/storage/fs/ocis/persistence.go | 24 +- pkg/storage/fs/ocis/revisions.go | 2 +- pkg/storage/fs/ocis/tree.go | 19 +- pkg/storage/fs/ocis/upload.go | 2 +- pkg/storage/fs/owncloud/owncloud.go | 370 ++-------------------------- pkg/storage/utils/ace/ace.go | 346 ++++++++++++++++++++++++++ 10 files changed, 529 insertions(+), 793 deletions(-) create mode 100644 pkg/storage/utils/ace/ace.go diff --git a/pkg/storage/fs/ocis/grants.go b/pkg/storage/fs/ocis/grants.go index 5681f72a6d..b151ef99db 100644 --- a/pkg/storage/fs/ocis/grants.go +++ b/pkg/storage/fs/ocis/grants.go @@ -2,23 +2,20 @@ package ocis import ( "context" - "encoding/csv" - "fmt" "path/filepath" - "strconv" "strings" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/storage/utils/ace" "github.com/pkg/xattr" ) func (fs *ocisfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { log := appctx.GetLogger(ctx) log.Debug().Interface("ref", ref).Interface("grant", g).Msg("AddGrant()") - var node *NodeInfo + var node *Node if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { return } @@ -27,214 +24,17 @@ func (fs *ocisfs) AddGrant(ctx context.Context, ref *provider.Reference, g *prov return } - e, err := fs.getACE(g) - if err != nil { - return err - } - - var attr string - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - attr = sharePrefix + "g:" + e.Principal - } else { - attr = sharePrefix + "u:" + e.Principal - } - np := filepath.Join(fs.pw.Root(), "nodes", node.ID) - if err := xattr.Set(np, attr, getValue(e)); err != nil { + e := ace.FromGrant(g) + principal, value := e.Marshal() + if err := xattr.Set(np, sharePrefix+principal, value); err != nil { return err } return fs.tp.Propagate(ctx, node) } -func getValue(e *ace) []byte { - // first byte will be replaced after converting to byte array - val := fmt.Sprintf("_t=%s:f=%s:p=%s", e.Type, e.Flags, e.Permissions) - b := []byte(val) - b[0] = 0 // indicate key value - return b -} - -func getACEPerm(set *provider.ResourcePermissions) (string, error) { - var b strings.Builder - - if set.Stat || set.InitiateFileDownload || set.ListContainer { - b.WriteString("r") - } - if set.InitiateFileUpload || set.Move { - b.WriteString("w") - } - if set.CreateContainer { - b.WriteString("a") - } - if set.Delete { - b.WriteString("d") - } - - // sharing - if set.AddGrant || set.RemoveGrant || set.UpdateGrant { - b.WriteString("C") - } - if set.ListGrants { - b.WriteString("c") - } - - // trash - if set.ListRecycle { - b.WriteString("u") - } - if set.RestoreRecycleItem { - b.WriteString("U") - } - if set.PurgeRecycle { - b.WriteString("P") - } - - // versions - if set.ListFileVersions { - b.WriteString("v") - } - if set.RestoreFileVersion { - b.WriteString("V") - } - - // quota - if set.GetQuota { - b.WriteString("q") - } - // TODO set quota permission? - // TODO GetPath - return b.String(), nil -} - -func (fs *ocisfs) getACE(g *provider.Grant) (*ace, error) { - permissions, err := getACEPerm(g.Permissions) - if err != nil { - return nil, err - } - e := &ace{ - Principal: g.Grantee.Id.OpaqueId, - Permissions: permissions, - // TODO creator ... - Type: "A", - } - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - e.Flags = "g" - } - return e, nil -} - -type ace struct { - //NFSv4 acls - Type string // t - Flags string // f - Principal string // im key - Permissions string // p - - // sharing specific - ShareTime int // s - Creator string // c - Expires int // e - Password string // w passWord TODO h = hash - Label string // l -} - -func unmarshalACE(v []byte) (*ace, error) { - // first byte indicates type of value - switch v[0] { - case 0: // = ':' separated key=value pairs - s := string(v[1:]) - return unmarshalKV(s) - default: - return nil, fmt.Errorf("unknown ace encoding") - } -} - -func unmarshalKV(s string) (*ace, error) { - e := &ace{} - r := csv.NewReader(strings.NewReader(s)) - r.Comma = ':' - r.Comment = 0 - r.FieldsPerRecord = -1 - r.LazyQuotes = false - r.TrimLeadingSpace = false - records, err := r.ReadAll() - if err != nil { - return nil, err - } - if len(records) != 1 { - return nil, fmt.Errorf("more than one row of ace kvs") - } - for i := range records[0] { - kv := strings.Split(records[0][i], "=") - switch kv[0] { - case "t": - e.Type = kv[1] - case "f": - e.Flags = kv[1] - case "p": - e.Permissions = kv[1] - case "s": - v, err := strconv.Atoi(kv[1]) - if err != nil { - return nil, err - } - e.ShareTime = v - case "c": - e.Creator = kv[1] - case "e": - v, err := strconv.Atoi(kv[1]) - if err != nil { - return nil, err - } - e.Expires = v - case "w": - e.Password = kv[1] - case "l": - e.Label = kv[1] - // TODO default ... log unknown keys? or add as opaque? hm we need that for tagged shares ... - } - } - return e, nil -} - -// Parse parses an acl string with the given delimiter (LongTextForm or ShortTextForm) -func getACEs(ctx context.Context, fsfn string, attrs []string) (entries []*ace, err error) { - log := appctx.GetLogger(ctx) - entries = []*ace{} - for i := range attrs { - if strings.HasPrefix(attrs[i], sharePrefix) { - principal := attrs[i][len(sharePrefix):] - var value []byte - if value, err = xattr.Get(fsfn, attrs[i]); err != nil { - log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute") - continue - } - var e *ace - if e, err = unmarshalACE(value); err != nil { - log.Error().Err(err).Str("attr", attrs[i]).Msg("could unmarshal ace") - continue - } - e.Principal = principal[2:] - // check consistency of Flags and principal type - if strings.Contains(e.Flags, "g") { - if principal[:1] != "g" { - log.Error().Str("attr", attrs[i]).Interface("ace", e).Msg("inconsistent ace: expected group") - continue - } - } else { - if principal[:1] != "u" { - log.Error().Str("attr", attrs[i]).Interface("ace", e).Msg("inconsistent ace: expected user") - continue - } - } - entries = append(entries, e) - } - } - return entries, nil -} - func (fs *ocisfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { - var node *NodeInfo + var node *Node if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { return } @@ -251,107 +51,19 @@ func (fs *ocisfs) ListGrants(ctx context.Context, ref *provider.Reference) (gran } log.Debug().Interface("attrs", attrs).Msg("read attributes") - // filter attributes - var aces []*ace - if aces, err = getACEs(ctx, np, attrs); err != nil { - log.Error().Err(err).Msg("error getting aces") - return nil, err - } + + aces := extractACEsFromAttrs(ctx, np, attrs) grants = make([]*provider.Grant, 0, len(aces)) for i := range aces { - grantee := &provider.Grantee{ - // TODO lookup uid from principal - Id: &userpb.UserId{OpaqueId: aces[i].Principal}, - Type: fs.getGranteeType(aces[i]), - } - grants = append(grants, &provider.Grant{ - Grantee: grantee, - Permissions: fs.getGrantPermissionSet(aces[i].Permissions), - }) + grants = append(grants, aces[i].Grant()) } return grants, nil } -func (fs *ocisfs) getGranteeType(e *ace) provider.GranteeType { - if strings.Contains(e.Flags, "g") { - return provider.GranteeType_GRANTEE_TYPE_GROUP - } - return provider.GranteeType_GRANTEE_TYPE_USER -} - -func (fs *ocisfs) getGrantPermissionSet(mode string) *provider.ResourcePermissions { - p := &provider.ResourcePermissions{} - // r - if strings.Contains(mode, "r") { - p.Stat = true - p.InitiateFileDownload = true - p.ListContainer = true - } - // w - if strings.Contains(mode, "w") { - p.InitiateFileUpload = true - if p.InitiateFileDownload { - p.Move = true - } - } - //a - if strings.Contains(mode, "a") { - // TODO append data to file permission? - p.CreateContainer = true - } - //x - //if strings.Contains(mode, "x") { - // TODO execute file permission? - // TODO change directory permission? - //} - //d - if strings.Contains(mode, "d") { - p.Delete = true - } - //D ? - - // sharing - if strings.Contains(mode, "C") { - p.AddGrant = true - p.RemoveGrant = true - p.UpdateGrant = true - } - if strings.Contains(mode, "c") { - p.ListGrants = true - } - - // trash - if strings.Contains(mode, "u") { // u = undelete - p.ListRecycle = true - } - if strings.Contains(mode, "U") { - p.RestoreRecycleItem = true - } - if strings.Contains(mode, "P") { - p.PurgeRecycle = true - } - - // versions - if strings.Contains(mode, "v") { - p.ListFileVersions = true - } - if strings.Contains(mode, "V") { - p.RestoreFileVersion = true - } - - // ? - // TODO GetPath - if strings.Contains(mode, "q") { - p.GetQuota = true - } - // TODO set quota permission? - return p -} - func (fs *ocisfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { - var node *NodeInfo + var node *Node if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { return } @@ -378,3 +90,27 @@ func (fs *ocisfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *p func (fs *ocisfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { return fs.AddGrant(ctx, ref, g) } + +// extractACEsFromAttrs reads ACEs in the list of attrs from the node +func extractACEsFromAttrs(ctx context.Context, fsfn string, attrs []string) (entries []*ace.ACE) { + log := appctx.GetLogger(ctx) + entries = []*ace.ACE{} + for i := range attrs { + if strings.HasPrefix(attrs[i], sharePrefix) { + var value []byte + var err error + if value, err = xattr.Get(fsfn, attrs[i]); err != nil { + log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute") + continue + } + var e *ace.ACE + principal := attrs[i][len(sharePrefix):] + if e, err = ace.Unmarshal(principal, value); err != nil { + log.Error().Err(err).Str("principal", principal).Str("attr", attrs[i]).Msg("could unmarshal ace") + continue + } + entries = append(entries, e) + } + } + return +} diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go index 295c3b5d44..16ef1d00a3 100644 --- a/pkg/storage/fs/ocis/node.go +++ b/pkg/storage/fs/ocis/node.go @@ -1,17 +1,23 @@ package ocis import ( + "context" "fmt" "os" "path/filepath" "strings" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/mime" "github.com/pkg/errors" "github.com/pkg/xattr" ) -// NodeInfo allows referencing a node by id and optionally a relative path -type NodeInfo struct { +// Node represents a node in the tree and provides methods to get a Parent or Child instance +type Node struct { pw PathWrapper ParentID string ID string @@ -22,8 +28,8 @@ type NodeInfo struct { } // NewNode creates a new instance and checks if it exists -func NewNode(pw PathWrapper, id string) (n *NodeInfo, err error) { - n = &NodeInfo{ +func NewNode(pw PathWrapper, id string) (n *Node, err error) { + n = &Node{ pw: pw, ID: id, } @@ -35,14 +41,12 @@ func NewNode(pw PathWrapper, id string) (n *NodeInfo, err error) { if attrBytes, err = xattr.Get(nodePath, "user.ocis.parentid"); err == nil { n.ParentID = string(attrBytes) } else { - // TODO log error return } // lookup name in extended attributes if attrBytes, err = xattr.Get(nodePath, "user.ocis.name"); err == nil { n.Name = string(attrBytes) } else { - // TODO log error return } @@ -57,10 +61,8 @@ func NewNode(pw PathWrapper, id string) (n *NodeInfo, err error) { } else { if os.IsNotExist(err) { return - } else { - // TODO log error - return } + return } } @@ -70,8 +72,8 @@ func NewNode(pw PathWrapper, id string) (n *NodeInfo, err error) { } // Child returns the child node with the given name -func (n *NodeInfo) Child(name string) (c *NodeInfo, err error) { - c = &NodeInfo{ +func (n *Node) Child(name string) (c *Node, err error) { + c = &Node{ pw: n.pw, ParentID: n.ID, Name: name, @@ -95,16 +97,16 @@ func (n *NodeInfo) Child(name string) (c *NodeInfo, err error) { } // IsRoot returns true when the node is the root of a tree -func (n *NodeInfo) IsRoot() bool { +func (n *Node) IsRoot() bool { return n.ParentID == "root" } // Parent returns the parent node -func (n *NodeInfo) Parent() (p *NodeInfo, err error) { +func (n *Node) Parent() (p *Node, err error) { if n.ParentID == "root" { return nil, fmt.Errorf("ocisfs: root has no parent") } - p = &NodeInfo{ + p = &Node{ pw: n.pw, ID: n.ParentID, } @@ -133,7 +135,7 @@ func (n *NodeInfo) Parent() (p *NodeInfo, err error) { } // Owner returns the cached owner id or reads it from the extended attributes -func (n *NodeInfo) Owner() (id string, idp string, err error) { +func (n *Node) Owner() (id string, idp string, err error) { if n.ownerID != "" && n.ownerIDP != "" { return n.ownerID, n.ownerIDP, nil } @@ -145,13 +147,77 @@ func (n *NodeInfo) Owner() (id string, idp string, err error) { if attrBytes, err = xattr.Get(nodePath, "user.ocis.owner.id"); err == nil { n.ownerID = string(attrBytes) } else { - // TODO log error + return } // lookup name in extended attributes if attrBytes, err = xattr.Get(nodePath, "user.ocis.owner.idp"); err == nil { n.ownerIDP = string(attrBytes) } else { - // TODO log error + return } return n.ownerID, n.ownerIDP, err } + +// AsResourceInfo return the node as CS3 ResourceInfo +func (n *Node) AsResourceInfo(ctx context.Context) (ri *provider.ResourceInfo, err error) { + var fn string + + nodePath := filepath.Join(n.pw.Root(), "nodes", n.ID) + + var fi os.FileInfo + + nodeType := provider.ResourceType_RESOURCE_TYPE_INVALID + if fi, err = os.Lstat(nodePath); err != nil { + return + } + if fi.IsDir() { + nodeType = provider.ResourceType_RESOURCE_TYPE_CONTAINER + } else if fi.Mode().IsRegular() { + nodeType = provider.ResourceType_RESOURCE_TYPE_FILE + } else if fi.Mode()&os.ModeSymlink != 0 { + nodeType = provider.ResourceType_RESOURCE_TYPE_SYMLINK + // TODO reference using ext attr on a symlink + // nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE + } + + var etag []byte + // TODO optionally store etag in new `root/attributes/` file + if etag, err = xattr.Get(nodePath, "user.ocis.etag"); err != nil { + log := appctx.GetLogger(ctx) + log.Error().Err(err).Interface("node", n).Msg("could not read etag") + } + + id := &provider.ResourceId{OpaqueId: n.ID} + + fn, err = n.pw.Path(ctx, n) + if err != nil { + return nil, err + } + ri = &provider.ResourceInfo{ + Id: id, + Path: fn, + Type: nodeType, + Etag: string(etag), + MimeType: mime.Detect(nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER, fn), + Size: uint64(fi.Size()), + // TODO fix permissions + PermissionSet: &provider.ResourcePermissions{ListContainer: true, CreateContainer: true}, + Mtime: &types.Timestamp{ + Seconds: uint64(fi.ModTime().Unix()), + // TODO read nanos from where? Nanos: fi.MTimeNanos, + }, + } + + if owner, idp, err := n.Owner(); err == nil { + ri.Owner = &userpb.UserId{ + Idp: idp, + OpaqueId: owner, + } + } + log := appctx.GetLogger(ctx) + log.Debug(). + Interface("ri", ri). + Msg("AsResourceInfo") + + return ri, nil +} diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 8a52b5c360..caf5e981e6 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -25,13 +25,9 @@ import ( "os" "path/filepath" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/logger" - "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/templates" @@ -294,7 +290,7 @@ func (fs *ocisfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (str } func (fs *ocisfs) CreateDir(ctx context.Context, fn string) (err error) { - var node *NodeInfo + var node *Node if node, err = fs.pw.NodeFromPath(ctx, fn); err != nil { return } @@ -306,7 +302,7 @@ func (fs *ocisfs) CreateReference(ctx context.Context, path string, targetURI *u } func (fs *ocisfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { - var oldNode, newNode *NodeInfo + var oldNode, newNode *Node if oldNode, err = fs.pw.NodeFromResource(ctx, oldRef); err != nil { return } @@ -322,7 +318,7 @@ func (fs *ocisfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) } func (fs *ocisfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (ri *provider.ResourceInfo, err error) { - var node *NodeInfo + var node *Node if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { return } @@ -330,11 +326,11 @@ func (fs *ocisfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []s err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) return } - return fs.normalize(ctx, node) + return node.AsResourceInfo(ctx) } func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) (finfos []*provider.ResourceInfo, err error) { - var node *NodeInfo + var node *Node if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { return } @@ -342,14 +338,14 @@ func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKey err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) return } - var children []*NodeInfo + var children []*Node children, err = fs.tp.ListFolder(ctx, node) if err != nil { return } for i := range children { - if ri, err := fs.normalize(ctx, children[i]); err == nil { + if ri, err := children[i].AsResourceInfo(ctx); err == nil { finfos = append(finfos, ri) } } @@ -357,7 +353,7 @@ func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKey } func (fs *ocisfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { - var node *NodeInfo + var node *Node if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { return } @@ -370,7 +366,7 @@ func (fs *ocisfs) Delete(ctx context.Context, ref *provider.Reference) (err erro // Data persistence -func (fs *ocisfs) ContentPath(node *NodeInfo) string { +func (fs *ocisfs) ContentPath(node *Node) string { return filepath.Join(fs.conf.Root, "nodes", node.ID) } @@ -404,68 +400,3 @@ func (fs *ocisfs) Download(ctx context.Context, ref *provider.Reference) (io.Rea // Trash persistence in recycle.go // share persistence in grants.go - -// supporting functions - -func (fs *ocisfs) normalize(ctx context.Context, node *NodeInfo) (ri *provider.ResourceInfo, err error) { - var fn string - - nodePath := filepath.Join(fs.conf.Root, "nodes", node.ID) - - var fi os.FileInfo - - nodeType := provider.ResourceType_RESOURCE_TYPE_INVALID - if fi, err = os.Lstat(nodePath); err != nil { - return - } - if fi.IsDir() { - nodeType = provider.ResourceType_RESOURCE_TYPE_CONTAINER - } else if fi.Mode().IsRegular() { - nodeType = provider.ResourceType_RESOURCE_TYPE_FILE - } else if fi.Mode()&os.ModeSymlink != 0 { - nodeType = provider.ResourceType_RESOURCE_TYPE_SYMLINK - // TODO reference using ext attr on a symlink - // nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE - } - - var etag []byte - // TODO optionally store etag in new `root/attributes/` file - if etag, err = xattr.Get(nodePath, "user.ocis.etag"); err != nil { - log := appctx.GetLogger(ctx) - log.Debug().Err(err).Msg("could not read etag") - } - - id := &provider.ResourceId{OpaqueId: node.ID} - // Path changes the node because it traverses the tree - fn, err = fs.pw.Path(ctx, node) - if err != nil { - return nil, err - } - ri = &provider.ResourceInfo{ - Id: id, - Path: fn, - Type: nodeType, - Etag: string(etag), - MimeType: mime.Detect(nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER, fn), - Size: uint64(fi.Size()), - // TODO fix permissions - PermissionSet: &provider.ResourcePermissions{ListContainer: true, CreateContainer: true}, - Mtime: &types.Timestamp{ - Seconds: uint64(fi.ModTime().Unix()), - // TODO read nanos from where? Nanos: fi.MTimeNanos, - }, - } - - if owner, idp, err := node.Owner(); err == nil { - ri.Owner = &userpb.UserId{ - Idp: idp, - OpaqueId: owner, - } - } - log := appctx.GetLogger(ctx) - log.Debug(). - Interface("ri", ri). - Msg("normalized") - - return ri, nil -} diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go index c23b0ba09d..98dbc3a4c8 100644 --- a/pkg/storage/fs/ocis/path.go +++ b/pkg/storage/fs/ocis/path.go @@ -19,18 +19,17 @@ type Path struct { // ocis fs works on top of a dir of uuid nodes root string `mapstructure:"root"` - // UserLayout wraps the internal path with user information. - // Example: if conf.Namespace is /ocis/user and received path is /docs - // and the UserLayout is {{.Username}} the internal path will be: - // /ocis/user//docs + // UserLayout wraps the internal path in the users folder with user information. UserLayout string `mapstructure:"user_layout"` + // TODO NodeLayout option to save nodes as eg. nodes/1d/d8/1dd84abf-9466-4e14-bb86-02fc4ea3abcf + // EnableHome enables the creation of home directories. EnableHome bool `mapstructure:"enable_home"` } -// NodeFromResource takes in a request path or request id and converts it to a NodeInfo -func (pw *Path) NodeFromResource(ctx context.Context, ref *provider.Reference) (*NodeInfo, error) { +// NodeFromResource takes in a request path or request id and converts it to a Node +func (pw *Path) NodeFromResource(ctx context.Context, ref *provider.Reference) (*Node, error) { if ref.GetPath() != "" { return pw.NodeFromPath(ctx, ref.GetPath()) } @@ -43,8 +42,8 @@ func (pw *Path) NodeFromResource(ctx context.Context, ref *provider.Reference) ( return nil, fmt.Errorf("invalid reference %+v", ref) } -// NodeFromPath converts a filename into a NodeInfo -func (pw *Path) NodeFromPath(ctx context.Context, fn string) (node *NodeInfo, err error) { +// NodeFromPath converts a filename into a Node +func (pw *Path) NodeFromPath(ctx context.Context, fn string) (node *Node, err error) { log := appctx.GetLogger(ctx) log.Debug().Interface("fn", fn).Msg("NodeFromPath()") node, err = pw.RootNode(ctx) @@ -69,7 +68,7 @@ func (pw *Path) NodeFromPath(ctx context.Context, fn string) (node *NodeInfo, er } // NodeFromID returns the internal path for the id -func (pw *Path) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *NodeInfo, err error) { +func (pw *Path) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *Node, err error) { if id == nil || id.OpaqueId == "" { return nil, fmt.Errorf("invalid resource id %+v", id) } @@ -77,20 +76,15 @@ func (pw *Path) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *Nod } // Path returns the path for node -func (pw *Path) Path(ctx context.Context, n *NodeInfo) (p string, err error) { - log := appctx.GetLogger(ctx) - log.Debug().Interface("node", n).Msg("Path()") - /* - // check if this a not yet existing node - if n.ID == "" && n.Name != "" && n.ParentID != "" { - path = n.Name - n, err = n.Parent() - } - */ +func (pw *Path) Path(ctx context.Context, n *Node) (p string, err error) { for !n.IsRoot() { p = filepath.Join(n.Name, p) if n, err = n.Parent(); err != nil { - log.Error().Err(err).Str("path", p).Interface("node", n).Msg("Path()") + appctx.GetLogger(ctx). + Error().Err(err). + Str("path", p). + Interface("node", n). + Msg("Path()") return } } @@ -98,7 +92,7 @@ func (pw *Path) Path(ctx context.Context, n *NodeInfo) (p string, err error) { } // readRootLink reads the symbolic link and extracts the node id -func (pw *Path) readRootLink(root string) (node *NodeInfo, err error) { +func (pw *Path) readRootLink(root string) (node *Node, err error) { // A root symlink looks like `../nodes/76455834-769e-412a-8a01-68f265365b79` link, err := os.Readlink(root) if os.IsNotExist(err) { @@ -108,7 +102,7 @@ func (pw *Path) readRootLink(root string) (node *NodeInfo, err error) { // extract the nodeID if strings.HasPrefix(link, "../nodes/") { - node = &NodeInfo{ + node = &Node{ pw: pw, ID: filepath.Base(link), ParentID: "root", @@ -122,7 +116,7 @@ func (pw *Path) readRootLink(root string) (node *NodeInfo, err error) { // RootNode returns the root node of a tree, // taking into account the user layout if EnableHome is true -func (pw *Path) RootNode(ctx context.Context) (node *NodeInfo, err error) { +func (pw *Path) RootNode(ctx context.Context) (node *Node, err error) { var root string if pw.EnableHome && pw.UserLayout != "" { // start at the users root node diff --git a/pkg/storage/fs/ocis/persistence.go b/pkg/storage/fs/ocis/persistence.go index c73f736407..7b86d14fb4 100644 --- a/pkg/storage/fs/ocis/persistence.go +++ b/pkg/storage/fs/ocis/persistence.go @@ -11,28 +11,28 @@ import ( // TreePersistence is used to manage a tree hierarchy type TreePersistence interface { GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) - GetMD(ctx context.Context, node *NodeInfo) (os.FileInfo, error) - ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, error) - CreateDir(ctx context.Context, node *NodeInfo) (err error) + GetMD(ctx context.Context, node *Node) (os.FileInfo, error) + ListFolder(ctx context.Context, node *Node) ([]*Node, error) + CreateDir(ctx context.Context, node *Node) (err error) CreateReference(ctx context.Context, path string, targetURI *url.URL) error - Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) (err error) - Delete(ctx context.Context, node *NodeInfo) (err error) + Move(ctx context.Context, oldNode *Node, newNode *Node) (err error) + Delete(ctx context.Context, node *Node) (err error) - Propagate(ctx context.Context, node *NodeInfo) (err error) + Propagate(ctx context.Context, node *Node) (err error) } // PathWrapper is used to encapsulate path transformations type PathWrapper interface { - NodeFromResource(ctx context.Context, ref *provider.Reference) (node *NodeInfo, err error) - NodeFromID(ctx context.Context, id *provider.ResourceId) (node *NodeInfo, err error) + NodeFromResource(ctx context.Context, ref *provider.Reference) (node *Node, err error) + NodeFromID(ctx context.Context, id *provider.ResourceId) (node *Node, err error) - // Wrap returns a NodeInfo object: + // Wrap returns a Node object: // - if the node exists with the node id, name and parent // - if only the parent exists, the node id is empty - NodeFromPath(ctx context.Context, fn string) (node *NodeInfo, err error) - Path(ctx context.Context, node *NodeInfo) (path string, err error) + NodeFromPath(ctx context.Context, fn string) (node *Node, err error) + Path(ctx context.Context, node *Node) (path string, err error) - RootNode(ctx context.Context) (node *NodeInfo, err error) + RootNode(ctx context.Context) (node *Node, err error) // Root returns the internal root of the storage Root() string } diff --git a/pkg/storage/fs/ocis/revisions.go b/pkg/storage/fs/ocis/revisions.go index f93e77be56..f02d116495 100644 --- a/pkg/storage/fs/ocis/revisions.go +++ b/pkg/storage/fs/ocis/revisions.go @@ -11,7 +11,7 @@ import ( ) func (fs *ocisfs) ListRevisions(ctx context.Context, ref *provider.Reference) (revisions []*provider.FileVersion, err error) { - var node *NodeInfo + var node *Node if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { return } diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index e007ad6138..993422117f 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -33,7 +33,7 @@ func NewTree(pw PathWrapper, dataDirectory string) (TreePersistence, error) { } // GetMD returns the metadata of a node in the tree -func (t *Tree) GetMD(ctx context.Context, node *NodeInfo) (os.FileInfo, error) { +func (t *Tree) GetMD(ctx context.Context, node *Node) (os.FileInfo, error) { md, err := os.Stat(filepath.Join(t.DataDirectory, "nodes", node.ID)) if err != nil { if os.IsNotExist(err) { @@ -47,7 +47,7 @@ func (t *Tree) GetMD(ctx context.Context, node *NodeInfo) (os.FileInfo, error) { // GetPathByID returns the fn pointed by the file id, without the internal namespace func (t *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relativeExternalPath string, err error) { - var node *NodeInfo + var node *Node node, err = t.pw.NodeFromID(ctx, id) if err != nil { return @@ -58,7 +58,7 @@ func (t *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relati } // CreateDir creates a new directory entry in the tree -func (t *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { +func (t *Tree) CreateDir(ctx context.Context, node *Node) (err error) { // TODO always try to fill node? if node.Exists || node.ID != "" { // child already exists @@ -102,11 +102,12 @@ func (t *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { // CreateReference creates a new reference entry in the tree func (t *Tree) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { + // TODO use symlink, but set extended attribute on link (not on target) return errtypes.NotSupported("operation not supported: CreateReference") } // Move replaces the target with the source -func (t *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) (err error) { +func (t *Tree) Move(ctx context.Context, oldNode *Node, newNode *Node) (err error) { // if target exists delete it without trashing it if newNode.Exists { // TODO make sure all children are deleted @@ -175,7 +176,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) ( } // ListFolder lists the content of a folder node -func (t *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, error) { +func (t *Tree) ListFolder(ctx context.Context, node *Node) ([]*Node, error) { dir := filepath.Join(t.DataDirectory, "nodes", node.ID) f, err := os.Open(dir) @@ -187,14 +188,14 @@ func (t *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, err } names, err := f.Readdirnames(0) - nodes := []*NodeInfo{} + nodes := []*Node{} for i := range names { link, err := os.Readlink(filepath.Join(dir, names[i])) if err != nil { // TODO log continue } - n := &NodeInfo{ + n := &Node{ pw: t.pw, ParentID: node.ID, ID: filepath.Base(link), @@ -208,7 +209,7 @@ func (t *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, err } // Delete deletes a node in the tree -func (t *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { +func (t *Tree) Delete(ctx context.Context, node *Node) (err error) { src := filepath.Join(t.DataDirectory, "nodes", node.ParentID, node.Name) err = os.Remove(src) if err != nil { @@ -237,7 +238,7 @@ func (t *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { } // Propagate propagates changes to the root of the tree -func (t *Tree) Propagate(ctx context.Context, node *NodeInfo) (err error) { +func (t *Tree) Propagate(ctx context.Context, node *Node) (err error) { // generate an etag bytes := make([]byte, 16) if _, err := rand.Read(bytes); err != nil { diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go index e9857b7e44..e372b7f4c4 100644 --- a/pkg/storage/fs/ocis/upload.go +++ b/pkg/storage/fs/ocis/upload.go @@ -326,7 +326,7 @@ func (upload *fileUpload) writeInfo() error { // FinishUpload finishes an upload and moves the file to the internal destination func (upload *fileUpload) FinishUpload(ctx context.Context) error { - n := &NodeInfo{ + n := &Node{ pw: upload.fs.pw, ID: upload.info.Storage["NodeId"], ParentID: upload.info.Storage["NodeParentId"], diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index cb51617be4..c899caf932 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -20,7 +20,6 @@ package owncloud import ( "context" - "encoding/csv" "fmt" "io" "io/ioutil" @@ -44,6 +43,7 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/utils/ace" "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" "github.com/gofrs/uuid" @@ -64,83 +64,6 @@ const ( // idAttribute is the name of the filesystem extended attribute that is used to store the uuid in idAttribute string = "user.oc.id" - // shares are persisted using extended attributes. We are going to mimic - // NFS4 ACLs, with one extended attribute per share, following Access - // Control Entries (ACEs). The following is taken from the nfs4_acl man page, - // see https://linux.die.net/man/5/nfs4_acl: - // the extended attributes will look like this - // "user.oc.acl.:::" - // - *type* will be limited to A for now - // A: Allow - allow *principal* to perform actions requiring *permissions* - // In the future we can use: - // U: aUdit - log any attempted access by principal which requires - // permissions. - // L: aLarm - generate a system alarm at any attempted access by - // principal which requires permissions - // D for deny is not recommended - // - *flags* for now empty or g for group, no inheritance yet - // - d directory-inherit - newly-created subdirectories will inherit the - // ACE. - // - f file-inherit - newly-created files will inherit the ACE, minus its - // inheritance flags. Newly-created subdirectories - // will inherit the ACE; if directory-inherit is not - // also specified in the parent ACE, inherit-only will - // be added to the inherited ACE. - // - n no-propagate-inherit - newly-created subdirectories will inherit - // the ACE, minus its inheritance flags. - // - i inherit-only - the ACE is not considered in permissions checks, - // but it is heritable; however, the inherit-only - // flag is stripped from inherited ACEs. - // - *principal* a named user, group or special principal - // - the oidc sub@iss maps nicely to this - // - 'OWNER@', 'GROUP@', and 'EVERYONE@', which are, respectively, analogous to the POSIX user/group/other - // - *permissions* - // - r read-data (files) / list-directory (directories) - // - w write-data (files) / create-file (directories) - // - a append-data (files) / create-subdirectory (directories) - // - x execute (files) / change-directory (directories) - // - d delete - delete the file/directory. Some servers will allow a delete to occur if either this permission is set in the file/directory or if the delete-child permission is set in its parent directory. - // - D delete-child - remove a file or subdirectory from within the given directory (directories only) - // - t read-attributes - read the attributes of the file/directory. - // - T write-attributes - write the attributes of the file/directory. - // - n read-named-attributes - read the named attributes of the file/directory. - // - N write-named-attributes - write the named attributes of the file/directory. - // - c read-ACL - read the file/directory NFSv4 ACL. - // - C write-ACL - write the file/directory NFSv4 ACL. - // - o write-owner - change ownership of the file/directory. - // - y synchronize - allow clients to use synchronous I/O with the server. - // TODO implement OWNER@ as "user.oc.acl.A::OWNER@:rwaDxtTnNcCy" - // attribute names are limited to 255 chars by the linux kernel vfs, values to 64 kb - // ext3 extended attributes must fit inside a single filesystem block ... 4096 bytes - // that leaves us with "user.oc.acl.A::someonewithaslightlylongersubject@whateverissuer:rwaDxtTnNcCy" ~80 chars - // 4096/80 = 51 shares ... with luck we might move the actual permissions to the value, saving ~15 chars - // 4096/64 = 64 shares ... still meh ... we can do better by using ints instead of strings for principals - // "user.oc.acl.u:100000" is pretty neat, but we can still do better: base64 encode the int - // "user.oc.acl.u:6Jqg" but base64 always has at least 4 chars, maybe hex is better for smaller numbers - // well use 4 chars in addition to the ace: "user.oc.acl.u:////" = 65535 -> 18 chars - // 4096/18 = 227 shares - // still ... ext attrs for this are not infinite scale ... - // so .. attach shares via fileid. - // /metadata//shares, similar to /files - // /metadata//shares/u///A:fdi:rwaDxtTnNcCy permissions as filename to keep them in the stat cache? - // - // whatever ... 50 shares is good enough. If more is needed we can delegate to the metadata - // if "user.oc.acl.M" is present look inside the metadata app. - // - if we cannot set an ace we might get an io error. - // in that case convert all shares to metadata and try to set "user.oc.acl.m" - // - // what about metadata like share creator, share time, expiry? - // - creator is same as owner, but can be set - // - share date, or abbreviated st is a unix timestamp - // - expiry is a unix timestamp - // - can be put inside the value - // - we need to reorder the fields: - // "user.oc.acl.:" -> "kv:t=:f=:p=:st=:c=:e=:pw=:n=" - // "user.oc.acl.:" -> "v1::::::::" - // or the first byte determines the format - // 0x00 = key value - // 0x01 = v1 ... - // // SharePrefix is the prefix for sharing related extended attributes sharePrefix string = "user.oc.acl." trashOriginPrefix string = "user.oc.o" @@ -750,209 +673,36 @@ func (fs *ocfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provid return errors.Wrap(err, "ocfs: error resolving reference") } - e, err := fs.getACE(g) - if err != nil { - return err - } - - var attr string - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - attr = sharePrefix + "g:" + e.Principal - } else { - attr = sharePrefix + "u:" + e.Principal - } - - if err := xattr.Set(np, attr, getValue(e)); err != nil { + e := ace.FromGrant(g) + principal, value := e.Marshal() + if err := xattr.Set(np, sharePrefix+principal, value); err != nil { return err } return fs.propagate(ctx, np) } -func getValue(e *ace) []byte { - // first byte will be replaced after converting to byte array - val := fmt.Sprintf("_t=%s:f=%s:p=%s", e.Type, e.Flags, e.Permissions) - b := []byte(val) - b[0] = 0 // indicate key value - return b -} - -func getACEPerm(set *provider.ResourcePermissions) (string, error) { - var b strings.Builder - - if set.Stat || set.InitiateFileDownload || set.ListContainer { - b.WriteString("r") - } - if set.InitiateFileUpload || set.Move { - b.WriteString("w") - } - if set.CreateContainer { - b.WriteString("a") - } - if set.Delete { - b.WriteString("d") - } - - // sharing - if set.AddGrant || set.RemoveGrant || set.UpdateGrant { - b.WriteString("C") - } - if set.ListGrants { - b.WriteString("c") - } - - // trash - if set.ListRecycle { - b.WriteString("u") - } - if set.RestoreRecycleItem { - b.WriteString("U") - } - if set.PurgeRecycle { - b.WriteString("P") - } - - // versions - if set.ListFileVersions { - b.WriteString("v") - } - if set.RestoreFileVersion { - b.WriteString("V") - } - - // quota - if set.GetQuota { - b.WriteString("q") - } - // TODO set quota permission? - // TODO GetPath - return b.String(), nil -} - -func (fs *ocfs) getACE(g *provider.Grant) (*ace, error) { - permissions, err := getACEPerm(g.Permissions) - if err != nil { - return nil, err - } - e := &ace{ - Principal: g.Grantee.Id.OpaqueId, - Permissions: permissions, - // TODO creator ... - Type: "A", - } - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - e.Flags = "g" - } - return e, nil -} - -type ace struct { - //NFSv4 acls - Type string // t - Flags string // f - Principal string // im key - Permissions string // p - - // sharing specific - ShareTime int // s - Creator string // c - Expires int // e - Password string // w passWord TODO h = hash - Label string // l -} - -func unmarshalACE(v []byte) (*ace, error) { - // first byte indicates type of value - switch v[0] { - case 0: // = ':' separated key=value pairs - s := string(v[1:]) - return unmarshalKV(s) - default: - return nil, fmt.Errorf("unknown ace encoding") - } -} - -func unmarshalKV(s string) (*ace, error) { - e := &ace{} - r := csv.NewReader(strings.NewReader(s)) - r.Comma = ':' - r.Comment = 0 - r.FieldsPerRecord = -1 - r.LazyQuotes = false - r.TrimLeadingSpace = false - records, err := r.ReadAll() - if err != nil { - return nil, err - } - if len(records) != 1 { - return nil, fmt.Errorf("more than one row of ace kvs") - } - for i := range records[0] { - kv := strings.Split(records[0][i], "=") - switch kv[0] { - case "t": - e.Type = kv[1] - case "f": - e.Flags = kv[1] - case "p": - e.Permissions = kv[1] - case "s": - v, err := strconv.Atoi(kv[1]) - if err != nil { - return nil, err - } - e.ShareTime = v - case "c": - e.Creator = kv[1] - case "e": - v, err := strconv.Atoi(kv[1]) - if err != nil { - return nil, err - } - e.Expires = v - case "w": - e.Password = kv[1] - case "l": - e.Label = kv[1] - // TODO default ... log unknown keys? or add as opaque? hm we need that for tagged shares ... - } - } - return e, nil -} - -// Parse parses an acl string with the given delimiter (LongTextForm or ShortTextForm) -func getACEs(ctx context.Context, fsfn string, attrs []string) (entries []*ace, err error) { +// extractACEsFromAttrs reads ACEs in the list of attrs from the file +func extractACEsFromAttrs(ctx context.Context, fsfn string, attrs []string) (entries []*ace.ACE) { log := appctx.GetLogger(ctx) - entries = []*ace{} + entries = []*ace.ACE{} for i := range attrs { if strings.HasPrefix(attrs[i], sharePrefix) { - principal := attrs[i][len(sharePrefix):] var value []byte + var err error if value, err = xattr.Get(fsfn, attrs[i]); err != nil { log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute") continue } - var e *ace - if e, err = unmarshalACE(value); err != nil { - log.Error().Err(err).Str("attr", attrs[i]).Msg("could unmarshal ace") + var e *ace.ACE + principal := attrs[i][len(sharePrefix):] + if e, err = ace.Unmarshal(principal, value); err != nil { + log.Error().Err(err).Str("principal", principal).Str("attr", attrs[i]).Msg("could not unmarshal ace") continue } - e.Principal = principal[2:] - // check consistency of Flags and principal type - if strings.Contains(e.Flags, "g") { - if principal[:1] != "g" { - log.Error().Str("attr", attrs[i]).Interface("ace", e).Msg("inconsistent ace: expected group") - continue - } - } else { - if principal[:1] != "u" { - log.Error().Str("attr", attrs[i]).Interface("ace", e).Msg("inconsistent ace: expected user") - continue - } - } entries = append(entries, e) } } - return entries, nil + return } func (fs *ocfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { @@ -968,105 +718,17 @@ func (fs *ocfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants } log.Debug().Interface("attrs", attrs).Msg("read attributes") - // filter attributes - var aces []*ace - if aces, err = getACEs(ctx, np, attrs); err != nil { - log.Error().Err(err).Msg("error getting aces") - return nil, err - } + + aces := extractACEsFromAttrs(ctx, np, attrs) grants = make([]*provider.Grant, 0, len(aces)) for i := range aces { - grantee := &provider.Grantee{ - // TODO lookup uid from principal - Id: &userpb.UserId{OpaqueId: aces[i].Principal}, - Type: fs.getGranteeType(aces[i]), - } - grants = append(grants, &provider.Grant{ - Grantee: grantee, - Permissions: fs.getGrantPermissionSet(aces[i].Permissions), - }) + grants = append(grants, aces[i].Grant()) } return grants, nil } -func (fs *ocfs) getGranteeType(e *ace) provider.GranteeType { - if strings.Contains(e.Flags, "g") { - return provider.GranteeType_GRANTEE_TYPE_GROUP - } - return provider.GranteeType_GRANTEE_TYPE_USER -} - -func (fs *ocfs) getGrantPermissionSet(mode string) *provider.ResourcePermissions { - p := &provider.ResourcePermissions{} - // r - if strings.Contains(mode, "r") { - p.Stat = true - p.InitiateFileDownload = true - p.ListContainer = true - } - // w - if strings.Contains(mode, "w") { - p.InitiateFileUpload = true - if p.InitiateFileDownload { - p.Move = true - } - } - //a - if strings.Contains(mode, "a") { - // TODO append data to file permission? - p.CreateContainer = true - } - //x - //if strings.Contains(mode, "x") { - // TODO execute file permission? - // TODO change directory permission? - //} - //d - if strings.Contains(mode, "d") { - p.Delete = true - } - //D ? - - // sharing - if strings.Contains(mode, "C") { - p.AddGrant = true - p.RemoveGrant = true - p.UpdateGrant = true - } - if strings.Contains(mode, "c") { - p.ListGrants = true - } - - // trash - if strings.Contains(mode, "u") { // u = undelete - p.ListRecycle = true - } - if strings.Contains(mode, "U") { - p.RestoreRecycleItem = true - } - if strings.Contains(mode, "P") { - p.PurgeRecycle = true - } - - // versions - if strings.Contains(mode, "v") { - p.ListFileVersions = true - } - if strings.Contains(mode, "V") { - p.RestoreFileVersion = true - } - - // ? - // TODO GetPath - if strings.Contains(mode, "q") { - p.GetQuota = true - } - // TODO set quota permission? - return p -} - func (fs *ocfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { var np string diff --git a/pkg/storage/utils/ace/ace.go b/pkg/storage/utils/ace/ace.go new file mode 100644 index 0000000000..734fab1a26 --- /dev/null +++ b/pkg/storage/utils/ace/ace.go @@ -0,0 +1,346 @@ +package ace + +import ( + "encoding/csv" + "fmt" + "strconv" + "strings" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +// ACE represents an Access Control Entry, mimicing NFSv4 ACLs +// +// The following is taken from the nfs4_acl man page, +// see https://linux.die.net/man/5/nfs4_acl: +// the extended attributes will look like this +// "user.oc.acl.:::" +// - *type* will be limited to A for now +// A: Allow - allow *principal* to perform actions requiring *permissions* +// In the future we can use: +// U: aUdit - log any attempted access by principal which requires +// permissions. +// L: aLarm - generate a system alarm at any attempted access by +// principal which requires permissions +// D for deny is not recommended +// - *flags* for now empty or g for group, no inheritance yet +// - d directory-inherit - newly-created subdirectories will inherit the +// ACE. +// - f file-inherit - newly-created files will inherit the ACE, minus its +// inheritance flags. Newly-created subdirectories +// will inherit the ACE; if directory-inherit is not +// also specified in the parent ACE, inherit-only will +// be added to the inherited ACE. +// - n no-propagate-inherit - newly-created subdirectories will inherit +// the ACE, minus its inheritance flags. +// - i inherit-only - the ACE is not considered in permissions checks, +// but it is heritable; however, the inherit-only +// flag is stripped from inherited ACEs. +// - *principal* a named user, group or special principal +// - the oidc sub@iss maps nicely to this +// - 'OWNER@', 'GROUP@', and 'EVERYONE@', which are, respectively, analogous to the POSIX user/group/other +// - *permissions* +// - r read-data (files) / list-directory (directories) +// - w write-data (files) / create-file (directories) +// - a append-data (files) / create-subdirectory (directories) +// - x execute (files) / change-directory (directories) +// - d delete - delete the file/directory. Some servers will allow a delete to occur if either this permission is set in the file/directory or if the delete-child permission is set in its parent directory. +// - D delete-child - remove a file or subdirectory from within the given directory (directories only) +// - t read-attributes - read the attributes of the file/directory. +// - T write-attributes - write the attributes of the file/directory. +// - n read-named-attributes - read the named attributes of the file/directory. +// - N write-named-attributes - write the named attributes of the file/directory. +// - c read-ACL - read the file/directory NFSv4 ACL. +// - C write-ACL - write the file/directory NFSv4 ACL. +// - o write-owner - change ownership of the file/directory. +// - y synchronize - allow clients to use synchronous I/O with the server. +// TODO implement OWNER@ as "user.oc.acl.A::OWNER@:rwaDxtTnNcCy" +// attribute names are limited to 255 chars by the linux kernel vfs, values to 64 kb +// ext3 extended attributes must fit inside a single filesystem block ... 4096 bytes +// that leaves us with "user.oc.acl.A::someonewithaslightlylongersubject@whateverissuer:rwaDxtTnNcCy" ~80 chars +// 4096/80 = 51 shares ... with luck we might move the actual permissions to the value, saving ~15 chars +// 4096/64 = 64 shares ... still meh ... we can do better by using ints instead of strings for principals +// "user.oc.acl.u:100000" is pretty neat, but we can still do better: base64 encode the int +// "user.oc.acl.u:6Jqg" but base64 always has at least 4 chars, maybe hex is better for smaller numbers +// well use 4 chars in addition to the ace: "user.oc.acl.u:////" = 65535 -> 18 chars +// 4096/18 = 227 shares +// still ... ext attrs for this are not infinite scale ... +// so .. attach shares via fileid. +// /metadata//shares, similar to /files +// /metadata//shares/u///A:fdi:rwaDxtTnNcCy permissions as filename to keep them in the stat cache? +// +// whatever ... 50 shares is good enough. If more is needed we can delegate to the metadata +// if "user.oc.acl.M" is present look inside the metadata app. +// - if we cannot set an ace we might get an io error. +// in that case convert all shares to metadata and try to set "user.oc.acl.m" +// +// what about metadata like share creator, share time, expiry? +// - creator is same as owner, but can be set +// - share date, or abbreviated st is a unix timestamp +// - expiry is a unix timestamp +// - can be put inside the value +// - we need to reorder the fields: +// "user.oc.acl.:" -> "kv:t=:f=:p=:st=:c=:e=:pw=:n=" +// "user.oc.acl.:" -> "v1::::::::" +// or the first byte determines the format +// 0x00 = key value +// 0x01 = v1 ... +type ACE struct { + //NFSv4 acls + _type string // t + flags string // f + principal string // im key + permissions string // p + + // sharing specific + shareTime int // s + creator string // c + expires int // e + password string // w passWord TODO h = hash + label string // l +} + +// FromGrant creates an ACE from a CS3 grant +func FromGrant(g *provider.Grant) *ACE { + e := &ACE{ + _type: "A", + permissions: getACEPerm(g.Permissions), + // TODO creator ... + } + if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { + e.flags = "g" + e.principal = "g:" + g.Grantee.Id.OpaqueId + } else { + e.principal = "u:" + g.Grantee.Id.OpaqueId + } + return e +} + +// Principal returns the principal of the ACE, eg. `u:` or `g:` +func (e *ACE) Principal() string { + return e.principal +} + +// Marshal renders a principal and byte[] that can be used to persist the ACE as an extended attribute +func (e *ACE) Marshal() (string, []byte) { + // first byte will be replaced after converting to byte array + val := fmt.Sprintf("_t=%s:f=%s:p=%s", e._type, e.flags, e.permissions) + b := []byte(val) + b[0] = 0 // indicate key value + return e.principal, b +} + +// Unmarshal parses a principal string and byte[] into an ACE +func Unmarshal(principal string, v []byte) (e *ACE, err error) { + // first byte indicates type of value + switch v[0] { + case 0: // = ':' separated key=value pairs + s := string(v[1:]) + if e, err = unmarshalKV(s); err == nil { + e.principal = principal + } + // check consistency of Flags and principal type + if strings.Contains(e.flags, "g") { + if principal[:1] != "g" { + return nil, fmt.Errorf("inconsistent ace: expected group") + } + } else { + if principal[:1] != "u" { + return nil, fmt.Errorf("inconsistent ace: expected user") + } + } + default: + return nil, fmt.Errorf("unknown ace encoding") + } + return +} + +// Grant returns a CS3 grant +func (e *ACE) Grant() *provider.Grant { + return &provider.Grant{ + Grantee: &provider.Grantee{ + Id: &userpb.UserId{OpaqueId: e.principal}, + Type: e.granteeType(), + }, + Permissions: e.grantPermissionSet(), + } +} + +// granteeType returns the CS3 grantee type +func (e *ACE) granteeType() provider.GranteeType { + if strings.Contains(e.flags, "g") { + return provider.GranteeType_GRANTEE_TYPE_GROUP + } + return provider.GranteeType_GRANTEE_TYPE_USER +} + +// grantPermissionSet returns the set of CS3 resource permissions representing the ACE +func (e *ACE) grantPermissionSet() *provider.ResourcePermissions { + p := &provider.ResourcePermissions{} + // r + if strings.Contains(e.permissions, "r") { + p.Stat = true + p.InitiateFileDownload = true + p.ListContainer = true + } + // w + if strings.Contains(e.permissions, "w") { + p.InitiateFileUpload = true + if p.InitiateFileDownload { + p.Move = true + } + } + //a + if strings.Contains(e.permissions, "a") { + // TODO append data to file permission? + p.CreateContainer = true + } + //x + //if strings.Contains(e.Permissions, "x") { + // TODO execute file permission? + // TODO change directory permission? + //} + //d + if strings.Contains(e.permissions, "d") { + p.Delete = true + } + //D ? + + // sharing + if strings.Contains(e.permissions, "C") { + p.AddGrant = true + p.RemoveGrant = true + p.UpdateGrant = true + } + if strings.Contains(e.permissions, "c") { + p.ListGrants = true + } + + // trash + if strings.Contains(e.permissions, "u") { // u = undelete + p.ListRecycle = true + } + if strings.Contains(e.permissions, "U") { + p.RestoreRecycleItem = true + } + if strings.Contains(e.permissions, "P") { + p.PurgeRecycle = true + } + + // versions + if strings.Contains(e.permissions, "v") { + p.ListFileVersions = true + } + if strings.Contains(e.permissions, "V") { + p.RestoreFileVersion = true + } + + // ? + // TODO GetPath + if strings.Contains(e.permissions, "q") { + p.GetQuota = true + } + // TODO set quota permission? + return p +} + +func unmarshalKV(s string) (*ACE, error) { + e := &ACE{} + r := csv.NewReader(strings.NewReader(s)) + r.Comma = ':' + r.Comment = 0 + r.FieldsPerRecord = -1 + r.LazyQuotes = false + r.TrimLeadingSpace = false + records, err := r.ReadAll() + if err != nil { + return nil, err + } + if len(records) != 1 { + return nil, fmt.Errorf("more than one row of ace kvs") + } + for i := range records[0] { + kv := strings.Split(records[0][i], "=") + switch kv[0] { + case "t": + e._type = kv[1] + case "f": + e.flags = kv[1] + case "p": + e.permissions = kv[1] + case "s": + v, err := strconv.Atoi(kv[1]) + if err != nil { + return nil, err + } + e.shareTime = v + case "c": + e.creator = kv[1] + case "e": + v, err := strconv.Atoi(kv[1]) + if err != nil { + return nil, err + } + e.expires = v + case "w": + e.password = kv[1] + case "l": + e.label = kv[1] + // TODO default ... log unknown keys? or add as opaque? hm we need that for tagged shares ... + } + } + return e, nil +} + +func getACEPerm(set *provider.ResourcePermissions) string { + var b strings.Builder + + if set.Stat || set.InitiateFileDownload || set.ListContainer { + b.WriteString("r") + } + if set.InitiateFileUpload || set.Move { + b.WriteString("w") + } + if set.CreateContainer { + b.WriteString("a") + } + if set.Delete { + b.WriteString("d") + } + + // sharing + if set.AddGrant || set.RemoveGrant || set.UpdateGrant { + b.WriteString("C") + } + if set.ListGrants { + b.WriteString("c") + } + + // trash + if set.ListRecycle { + b.WriteString("u") + } + if set.RestoreRecycleItem { + b.WriteString("U") + } + if set.PurgeRecycle { + b.WriteString("P") + } + + // versions + if set.ListFileVersions { + b.WriteString("v") + } + if set.RestoreFileVersion { + b.WriteString("V") + } + + // quota + if set.GetQuota { + b.WriteString("q") + } + // TODO set quota permission? + // TODO GetPath + return b.String() +} From 705e7fdc4968dd125e59538a0e63da3c85cb9a97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 7 Sep 2020 17:05:37 +0200 Subject: [PATCH 14/18] refactor node api MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/node.go | 51 ++++++++++++++++++++-- pkg/storage/fs/ocis/ocis.go | 19 +-------- pkg/storage/fs/ocis/path.go | 2 +- pkg/storage/fs/ocis/persistence.go | 6 +-- pkg/storage/fs/ocis/recycle.go | 8 ++++ pkg/storage/fs/ocis/revisions.go | 8 ++++ pkg/storage/fs/ocis/tree.go | 68 ++++++++++++++++++++---------- 7 files changed, 112 insertions(+), 50 deletions(-) diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go index 16ef1d00a3..47aa0f43e6 100644 --- a/pkg/storage/fs/ocis/node.go +++ b/pkg/storage/fs/ocis/node.go @@ -12,6 +12,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/mime" + "github.com/google/uuid" "github.com/pkg/errors" "github.com/pkg/xattr" ) @@ -22,13 +23,54 @@ type Node struct { ParentID string ID string Name string - ownerID string - ownerIDP string + ownerID string // used to cache the owner id + ownerIDP string // used to cache the owner idp Exists bool } -// NewNode creates a new instance and checks if it exists -func NewNode(pw PathWrapper, id string) (n *Node, err error) { +// CreateDir creates a new child directory node with a new id and the given name +// owner is optional +// TODO use in tree CreateDir +func (n *Node) CreateDir(pw PathWrapper, name string, owner *userpb.UserId) (c *Node, err error) { + c = &Node{ + pw: pw, + ParentID: n.ID, + ID: uuid.New().String(), + Name: name, + } + + // create a directory node + nodePath := filepath.Join(pw.Root(), "nodes", c.ID) + if err = os.MkdirAll(nodePath, 0700); err != nil { + return nil, errors.Wrap(err, "ocisfs: error creating node child dir") + } + + c.writeMetadata(nodePath, owner) + + c.Exists = true + return +} + +func (n *Node) writeMetadata(nodePath string, owner *userpb.UserId) (err error) { + if err = xattr.Set(nodePath, "user.ocis.parentid", []byte(n.ParentID)); err != nil { + return errors.Wrap(err, "ocisfs: could not set parentid attribute") + } + if err = xattr.Set(nodePath, "user.ocis.name", []byte(n.Name)); err != nil { + return errors.Wrap(err, "ocisfs: could not set name attribute") + } + if owner != nil { + if err = xattr.Set(nodePath, "user.ocis.owner.id", []byte(owner.OpaqueId)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner id attribute") + } + if err = xattr.Set(nodePath, "user.ocis.owner.idp", []byte(owner.Idp)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner idp attribute") + } + } + return +} + +// ReadNode creates a new instance from an id and checks if it exists +func ReadNode(pw PathWrapper, id string) (n *Node, err error) { n = &Node{ pw: pw, ID: id, @@ -135,6 +177,7 @@ func (n *Node) Parent() (p *Node, err error) { } // Owner returns the cached owner id or reads it from the extended attributes +// TODO can be private as only the AsResourceInfo uses it func (n *Node) Owner() (id string, idp string, err error) { if n.ownerID != "" && n.ownerIDP != "" { return n.ownerID, n.ownerIDP, nil diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index caf5e981e6..4fc0156b4e 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -35,7 +35,6 @@ import ( "github.com/google/uuid" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - "github.com/pkg/xattr" ) const ( @@ -248,24 +247,8 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { // create a directory node nodeID := uuid.New().String() - nodePath := filepath.Join(fs.conf.Root, "nodes", nodeID) - err = os.MkdirAll(nodePath, 0700) - if err != nil { - return errors.Wrap(err, "ocisfs: error creating node dir") - } - if err := xattr.Set(nodePath, "user.ocis.parentid", []byte("root")); err != nil { - return errors.Wrap(err, "ocisfs: could not set parentid attribute") - } - if err := xattr.Set(nodePath, "user.ocis.name", []byte("")); err != nil { - return errors.Wrap(err, "ocisfs: could not set name attribute") - } - if err := xattr.Set(nodePath, "user.ocis.owner.id", []byte(u.Id.OpaqueId)); err != nil { - return errors.Wrap(err, "ocisfs: could not set owner id attribute") - } - if err := xattr.Set(nodePath, "user.ocis.owner.idp", []byte(u.Id.Idp)); err != nil { - return errors.Wrap(err, "ocisfs: could not set owner idp attribute") - } + fs.tp.CreateRoot(nodeID, u.Id) // link users home to node return os.Symlink("../nodes/"+nodeID, home) diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go index 98dbc3a4c8..4ffe731182 100644 --- a/pkg/storage/fs/ocis/path.go +++ b/pkg/storage/fs/ocis/path.go @@ -72,7 +72,7 @@ func (pw *Path) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *Nod if id == nil || id.OpaqueId == "" { return nil, fmt.Errorf("invalid resource id %+v", id) } - return NewNode(pw, id.OpaqueId) + return ReadNode(pw, id.OpaqueId) } // Path returns the path for node diff --git a/pkg/storage/fs/ocis/persistence.go b/pkg/storage/fs/ocis/persistence.go index 7b86d14fb4..e163230479 100644 --- a/pkg/storage/fs/ocis/persistence.go +++ b/pkg/storage/fs/ocis/persistence.go @@ -5,6 +5,7 @@ import ( "net/url" "os" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ) @@ -13,6 +14,7 @@ type TreePersistence interface { GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) GetMD(ctx context.Context, node *Node) (os.FileInfo, error) ListFolder(ctx context.Context, node *Node) ([]*Node, error) + CreateRoot(id string, owner *userpb.UserId) (n *Node, err error) CreateDir(ctx context.Context, node *Node) (err error) CreateReference(ctx context.Context, path string, targetURI *url.URL) error Move(ctx context.Context, oldNode *Node, newNode *Node) (err error) @@ -25,10 +27,6 @@ type TreePersistence interface { type PathWrapper interface { NodeFromResource(ctx context.Context, ref *provider.Reference) (node *Node, err error) NodeFromID(ctx context.Context, id *provider.ResourceId) (node *Node, err error) - - // Wrap returns a Node object: - // - if the node exists with the node id, name and parent - // - if only the parent exists, the node id is empty NodeFromPath(ctx context.Context, fn string) (node *Node, err error) Path(ctx context.Context, node *Node) (path string, err error) diff --git a/pkg/storage/fs/ocis/recycle.go b/pkg/storage/fs/ocis/recycle.go index c0cdbfbcd8..e1d68bdf55 100644 --- a/pkg/storage/fs/ocis/recycle.go +++ b/pkg/storage/fs/ocis/recycle.go @@ -7,6 +7,14 @@ import ( "github.com/cs3org/reva/pkg/errtypes" ) +// Recycle items are stored inside the node folder and start with the uuid of the deleted node. +// The `.T.` indicates it is a trash item and what follows is the timestamp of the deletion. +// The deleted file is kept in the same location/dir as the original node. This prevents deletes +// from triggering cross storage moves when the trash is accidentally stored on another partition, +// because the admin mounted a different partition there. +// TODO For an efficient listing of deleted nodes the ocis storages trash folder should have +// contain a directory with symlinks to trash files for every userid/"root" + func (fs *ocisfs) PurgeRecycleItem(ctx context.Context, key string) error { return errtypes.NotSupported("operation not supported: PurgeRecycleItem") } diff --git a/pkg/storage/fs/ocis/revisions.go b/pkg/storage/fs/ocis/revisions.go index f02d116495..59afb47a72 100644 --- a/pkg/storage/fs/ocis/revisions.go +++ b/pkg/storage/fs/ocis/revisions.go @@ -10,6 +10,14 @@ import ( "github.com/cs3org/reva/pkg/errtypes" ) +// Revision entries are stored inside the node folder and start with the same uuid as the current version. +// The `.REV.` indicates it is a revision and what follows is a timestamp, so multiple versions +// can be kept in the same location as the current file content. This prevents new fileuploads +// to trigger cross storage moves when revisions accidentally are stored on another partition, +// because the admin mounted a different partition there. +// We can add a background process to move old revisions to a slower storage +// and replace the revision file with a symbolic link in the future, if necessary. + func (fs *ocisfs) ListRevisions(ctx context.Context, ref *provider.Reference) (revisions []*provider.FileVersion, err error) { var node *Node if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index 993422117f..9c12a4aedd 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -9,6 +9,7 @@ import ( "path/filepath" "time" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/user" @@ -20,21 +21,21 @@ import ( // Tree manages a hierarchical tree type Tree struct { - pw PathWrapper - DataDirectory string + pw PathWrapper + Root string } // NewTree creates a new Tree instance -func NewTree(pw PathWrapper, dataDirectory string) (TreePersistence, error) { +func NewTree(pw PathWrapper, Root string) (TreePersistence, error) { return &Tree{ - pw: pw, - DataDirectory: dataDirectory, + pw: pw, + Root: Root, }, nil } // GetMD returns the metadata of a node in the tree func (t *Tree) GetMD(ctx context.Context, node *Node) (os.FileInfo, error) { - md, err := os.Stat(filepath.Join(t.DataDirectory, "nodes", node.ID)) + md, err := os.Stat(filepath.Join(t.Root, "nodes", node.ID)) if err != nil { if os.IsNotExist(err) { return nil, errtypes.NotFound(node.ID) @@ -57,18 +58,38 @@ func (t *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relati return } +// CreateRoot creates a new root node with parentid = "root" +func (t *Tree) CreateRoot(id string, owner *userpb.UserId) (n *Node, err error) { + n = &Node{ + ParentID: "root", + pw: t.pw, + ID: id, + } + + // create a directory node + nodePath := filepath.Join(t.Root, "nodes", n.ID) + if err = os.MkdirAll(nodePath, 0700); err != nil { + return nil, errors.Wrap(err, "ocisfs: error creating root node dir") + } + + n.writeMetadata(nodePath, owner) + + n.Exists = true + return +} + // CreateDir creates a new directory entry in the tree +// TODO use parentnode and name instead of node? would make the exists stuff clearer? maybe obsolete? func (t *Tree) CreateDir(ctx context.Context, node *Node) (err error) { - // TODO always try to fill node? - if node.Exists || node.ID != "" { // child already exists - return + if node.Exists || node.ID != "" { + return errtypes.AlreadyExists(node.ID) // path? } // create a directory node node.ID = uuid.New().String() - newPath := filepath.Join(t.DataDirectory, "nodes", node.ID) + newPath := filepath.Join(t.Root, "nodes", node.ID) err = os.MkdirAll(newPath, 0700) if err != nil { @@ -93,7 +114,7 @@ func (t *Tree) CreateDir(ctx context.Context, node *Node) (err error) { } // make child appear in listings - err = os.Symlink("../"+node.ID, filepath.Join(t.DataDirectory, "nodes", node.ParentID, node.Name)) + err = os.Symlink("../"+node.ID, filepath.Join(t.Root, "nodes", node.ParentID, node.Name)) if err != nil { return } @@ -111,14 +132,14 @@ func (t *Tree) Move(ctx context.Context, oldNode *Node, newNode *Node) (err erro // if target exists delete it without trashing it if newNode.Exists { // TODO make sure all children are deleted - if err := os.RemoveAll(filepath.Join(t.DataDirectory, "nodes", newNode.ID)); err != nil { + if err := os.RemoveAll(filepath.Join(t.Root, "nodes", newNode.ID)); err != nil { return errors.Wrap(err, "ocisfs: Move: error deleting target node "+newNode.ID) } } - // are we renaming? + // are we just renaming (parent stays the same)? if oldNode.ParentID == newNode.ParentID { - parentPath := filepath.Join(t.DataDirectory, "nodes", oldNode.ParentID) + parentPath := filepath.Join(t.Root, "nodes", oldNode.ParentID) // rename child err = os.Rename( @@ -129,7 +150,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *Node, newNode *Node) (err erro return errors.Wrap(err, "ocisfs: could not rename child") } - tgtPath := filepath.Join(t.DataDirectory, "nodes", newNode.ID) + tgtPath := filepath.Join(t.Root, "nodes", newNode.ID) // update name attribute if err := xattr.Set(tgtPath, "user.ocis.name", []byte(newNode.Name)); err != nil { @@ -144,14 +165,15 @@ func (t *Tree) Move(ctx context.Context, oldNode *Node, newNode *Node) (err erro // rename child err = os.Rename( - filepath.Join(t.DataDirectory, "nodes", oldNode.ParentID, oldNode.Name), - filepath.Join(t.DataDirectory, "nodes", newNode.ParentID, newNode.Name), + filepath.Join(t.Root, "nodes", oldNode.ParentID, oldNode.Name), + filepath.Join(t.Root, "nodes", newNode.ParentID, newNode.Name), ) if err != nil { return errors.Wrap(err, "ocisfs: could not move child") } - tgtPath := filepath.Join(t.DataDirectory, "nodes", newNode.ID) + // update parentid and name + tgtPath := filepath.Join(t.Root, "nodes", newNode.ID) if err := xattr.Set(tgtPath, "user.ocis.parentid", []byte(newNode.ParentID)); err != nil { return errors.Wrap(err, "ocisfs: could not set parentid attribute") @@ -178,7 +200,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *Node, newNode *Node) (err erro // ListFolder lists the content of a folder node func (t *Tree) ListFolder(ctx context.Context, node *Node) ([]*Node, error) { - dir := filepath.Join(t.DataDirectory, "nodes", node.ID) + dir := filepath.Join(t.Root, "nodes", node.ID) f, err := os.Open(dir) if err != nil { if os.IsNotExist(err) { @@ -210,13 +232,13 @@ func (t *Tree) ListFolder(ctx context.Context, node *Node) ([]*Node, error) { // Delete deletes a node in the tree func (t *Tree) Delete(ctx context.Context, node *Node) (err error) { - src := filepath.Join(t.DataDirectory, "nodes", node.ParentID, node.Name) + src := filepath.Join(t.Root, "nodes", node.ParentID, node.Name) err = os.Remove(src) if err != nil { return } - nodePath := filepath.Join(t.DataDirectory, "nodes", node.ID) + nodePath := filepath.Join(t.Root, "nodes", node.ID) trashPath := nodePath + ".T." + time.Now().UTC().Format(time.RFC3339Nano) err = os.Rename(nodePath, trashPath) if err != nil { @@ -225,7 +247,7 @@ func (t *Tree) Delete(ctx context.Context, node *Node) (err error) { // make node appear in trash // parent id and name are stored as extended attributes in the node itself - trashLink := filepath.Join(t.DataDirectory, "trash", node.ID) + trashLink := filepath.Join(t.Root, "trash", node.ID) err = os.Symlink("../nodes/"+node.ID+".T."+time.Now().UTC().Format(time.RFC3339Nano), trashLink) if err != nil { return @@ -247,7 +269,7 @@ func (t *Tree) Propagate(ctx context.Context, node *Node) (err error) { // store in extended attribute etag := hex.EncodeToString(bytes) for err == nil && !node.IsRoot() { - if err := xattr.Set(filepath.Join(t.DataDirectory, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { + if err := xattr.Set(filepath.Join(t.Root, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { log.Error().Err(err).Msg("error storing file id") } // TODO propagate mtime From 0412aa71e58a7e8094df4a98ccdc83470778b616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 8 Sep 2020 11:07:40 +0200 Subject: [PATCH 15/18] add license headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/grants.go | 18 ++++++++++++++++++ pkg/storage/fs/ocis/metadata.go | 18 ++++++++++++++++++ pkg/storage/fs/ocis/node.go | 18 ++++++++++++++++++ pkg/storage/fs/ocis/path.go | 18 ++++++++++++++++++ pkg/storage/fs/ocis/persistence.go | 18 ++++++++++++++++++ pkg/storage/fs/ocis/recycle.go | 18 ++++++++++++++++++ pkg/storage/fs/ocis/revisions.go | 18 ++++++++++++++++++ pkg/storage/fs/ocis/tree.go | 18 ++++++++++++++++++ pkg/storage/utils/ace/ace.go | 18 ++++++++++++++++++ 9 files changed, 162 insertions(+) diff --git a/pkg/storage/fs/ocis/grants.go b/pkg/storage/fs/ocis/grants.go index b151ef99db..4186ed81c6 100644 --- a/pkg/storage/fs/ocis/grants.go +++ b/pkg/storage/fs/ocis/grants.go @@ -1,3 +1,21 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package ocis import ( diff --git a/pkg/storage/fs/ocis/metadata.go b/pkg/storage/fs/ocis/metadata.go index 463883b5bf..3ff45babf8 100644 --- a/pkg/storage/fs/ocis/metadata.go +++ b/pkg/storage/fs/ocis/metadata.go @@ -1,3 +1,21 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package ocis import ( diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go index 47aa0f43e6..a68c164a73 100644 --- a/pkg/storage/fs/ocis/node.go +++ b/pkg/storage/fs/ocis/node.go @@ -1,3 +1,21 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package ocis import ( diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go index 4ffe731182..46ece88944 100644 --- a/pkg/storage/fs/ocis/path.go +++ b/pkg/storage/fs/ocis/path.go @@ -1,3 +1,21 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package ocis import ( diff --git a/pkg/storage/fs/ocis/persistence.go b/pkg/storage/fs/ocis/persistence.go index e163230479..b1cdc8104b 100644 --- a/pkg/storage/fs/ocis/persistence.go +++ b/pkg/storage/fs/ocis/persistence.go @@ -1,3 +1,21 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package ocis import ( diff --git a/pkg/storage/fs/ocis/recycle.go b/pkg/storage/fs/ocis/recycle.go index e1d68bdf55..ae4001db10 100644 --- a/pkg/storage/fs/ocis/recycle.go +++ b/pkg/storage/fs/ocis/recycle.go @@ -1,3 +1,21 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package ocis import ( diff --git a/pkg/storage/fs/ocis/revisions.go b/pkg/storage/fs/ocis/revisions.go index 59afb47a72..5eebe4583c 100644 --- a/pkg/storage/fs/ocis/revisions.go +++ b/pkg/storage/fs/ocis/revisions.go @@ -1,3 +1,21 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package ocis import ( diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index 9c12a4aedd..acabe0c638 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -1,3 +1,21 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package ocis import ( diff --git a/pkg/storage/utils/ace/ace.go b/pkg/storage/utils/ace/ace.go index 734fab1a26..14154bf468 100644 --- a/pkg/storage/utils/ace/ace.go +++ b/pkg/storage/utils/ace/ace.go @@ -1,3 +1,21 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package ace import ( From 50b66e144f8603fa557d790e44dba2ddd08ffdf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 8 Sep 2020 14:54:50 +0200 Subject: [PATCH 16/18] add changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/ocis-driver.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/ocis-driver.md diff --git a/changelog/unreleased/ocis-driver.md b/changelog/unreleased/ocis-driver.md new file mode 100644 index 0000000000..72f86fc8f4 --- /dev/null +++ b/changelog/unreleased/ocis-driver.md @@ -0,0 +1,5 @@ +Enhancement: Introduce ocis storage driver + +We introduced a now storage driver `ocis` that deconstructs a filesystem and uses a node first approach to implement an efficient lookup of files by path as well as by file id. + +https://github.com/cs3org/reva/pull/559 From e0587cca846b37157ed621a49d31e75e6cc58f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 8 Sep 2020 15:00:22 +0200 Subject: [PATCH 17/18] remove duplicate docs (moved to ace.go) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/ocis.go | 77 ------------------------------------- 1 file changed, 77 deletions(-) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 4fc0156b4e..5791f979b2 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -46,83 +46,6 @@ const ( // collisions with other apps We are going to introduce a sub namespace // "user.ocis." - // shares are persisted using extended attributes. We are going to mimic - // NFS4 ACLs, with one extended attribute per share, following Access - // Control Entries (ACEs). The following is taken from the nfs4_acl man page, - // see https://linux.die.net/man/5/nfs4_acl: - // the extended attributes will look like this - // "user.oc.acl.:::" - // - *type* will be limited to A for now - // A: Allow - allow *principal* to perform actions requiring *permissions* - // In the future we can use: - // U: aUdit - log any attempted access by principal which requires - // permissions. - // L: aLarm - generate a system alarm at any attempted access by - // principal which requires permissions - // D for deny is not recommended - // - *flags* for now empty or g for group, no inheritance yet - // - d directory-inherit - newly-created subdirectories will inherit the - // ACE. - // - f file-inherit - newly-created files will inherit the ACE, minus its - // inheritance flags. Newly-created subdirectories - // will inherit the ACE; if directory-inherit is not - // also specified in the parent ACE, inherit-only will - // be added to the inherited ACE. - // - n no-propagate-inherit - newly-created subdirectories will inherit - // the ACE, minus its inheritance flags. - // - i inherit-only - the ACE is not considered in permissions checks, - // but it is heritable; however, the inherit-only - // flag is stripped from inherited ACEs. - // - *principal* a named user, group or special principal - // - the oidc sub@iss maps nicely to this - // - 'OWNER@', 'GROUP@', and 'EVERYONE@', which are, respectively, analogous to the POSIX user/group/other - // - *permissions* - // - r read-data (files) / list-directory (directories) - // - w write-data (files) / create-file (directories) - // - a append-data (files) / create-subdirectory (directories) - // - x execute (files) / change-directory (directories) - // - d delete - delete the file/directory. Some servers will allow a delete to occur if either this permission is set in the file/directory or if the delete-child permission is set in its parent directory. - // - D delete-child - remove a file or subdirectory from within the given directory (directories only) - // - t read-attributes - read the attributes of the file/directory. - // - T write-attributes - write the attributes of the file/directory. - // - n read-named-attributes - read the named attributes of the file/directory. - // - N write-named-attributes - write the named attributes of the file/directory. - // - c read-ACL - read the file/directory NFSv4 ACL. - // - C write-ACL - write the file/directory NFSv4 ACL. - // - o write-owner - change ownership of the file/directory. - // - y synchronize - allow clients to use synchronous I/O with the server. - // TODO implement OWNER@ as "user.oc.acl.A::OWNER@:rwaDxtTnNcCy" - // attribute names are limited to 255 chars by the linux kernel vfs, values to 64 kb - // ext3 extended attributes must fit inside a single filesystem block ... 4096 bytes - // that leaves us with "user.oc.acl.A::someonewithaslightlylongersubject@whateverissuer:rwaDxtTnNcCy" ~80 chars - // 4096/80 = 51 shares ... with luck we might move the actual permissions to the value, saving ~15 chars - // 4096/64 = 64 shares ... still meh ... we can do better by using ints instead of strings for principals - // "user.ocis.acl.u:100000" is pretty neat, but we can still do better: base64 encode the int - // "user.ocis.acl.u:6Jqg" but base64 always has at least 4 chars, maybe hex is better for smaller numbers - // well use 4 chars in addition to the ace: "user.oc.acl.u:////" = 65535 -> 18 chars - // 4096/18 = 227 shares - // still ... ext attrs for this are not infinite scale ... - // so .. attach shares via fileid. - // /metadata//shares, similar to /files - // /metadata//shares/u///A:fdi:rwaDxtTnNcCy permissions as filename to keep them in the stat cache? - // - // whatever ... 50 shares is good enough. If more is needed we can delegate to the metadata - // if "user.oc.acl.M" is present look inside the metadata app. - // - if we cannot set an ace we might get an io error. - // in that case convert all shares to metadata and try to set "user.oc.acl.m" - // - // what about metadata like share creator, share time, expiry? - // - creator is same as owner, but can be set - // - share date, or abbreviated st is a unix timestamp - // - expiry is a unix timestamp - // - can be put inside the value - // - we need to reorder the fields: - // "user.oc.acl.:" -> "kv:t=:f=:p=:st=:c=:e=:pw=:n=" - // "user.oc.acl.:" -> "v1::::::::" - // or the first byte determines the format - // 0x00 = key value - // 0x01 = v1 ... - // // SharePrefix is the prefix for sharing related extended attributes sharePrefix string = "user.ocis.acl." favPrefix string = "user.ocis.fav." // favorite flag, per user From c3dd7a3924c056f4a8df0820f58089d9d32b8a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 8 Sep 2020 15:21:44 +0200 Subject: [PATCH 18/18] make linter happy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/node.go | 9 +++++---- pkg/storage/fs/ocis/ocis.go | 10 +++++++--- pkg/storage/fs/ocis/tree.go | 15 ++++++++++----- pkg/storage/fs/ocis/upload.go | 3 ++- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go index a68c164a73..8c958c83d9 100644 --- a/pkg/storage/fs/ocis/node.go +++ b/pkg/storage/fs/ocis/node.go @@ -63,7 +63,7 @@ func (n *Node) CreateDir(pw PathWrapper, name string, owner *userpb.UserId) (c * return nil, errors.Wrap(err, "ocisfs: error creating node child dir") } - c.writeMetadata(nodePath, owner) + err = c.writeMetadata(nodePath, owner) c.Exists = true return @@ -231,11 +231,12 @@ func (n *Node) AsResourceInfo(ctx context.Context) (ri *provider.ResourceInfo, e if fi, err = os.Lstat(nodePath); err != nil { return } - if fi.IsDir() { + switch { + case fi.IsDir(): nodeType = provider.ResourceType_RESOURCE_TYPE_CONTAINER - } else if fi.Mode().IsRegular() { + case fi.Mode().IsRegular(): nodeType = provider.ResourceType_RESOURCE_TYPE_FILE - } else if fi.Mode()&os.ModeSymlink != 0 { + case fi.Mode()&os.ModeSymlink != 0: nodeType = provider.ResourceType_RESOURCE_TYPE_SYMLINK // TODO reference using ext attr on a symlink // nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 5791f979b2..0d0d076d6c 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -48,8 +48,10 @@ const ( // SharePrefix is the prefix for sharing related extended attributes sharePrefix string = "user.ocis.acl." - favPrefix string = "user.ocis.fav." // favorite flag, per user - etagPrefix string = "user.ocis.etag." // allow overriding a calculated etag with one from the extended attributes + // TODO implement favorites metadata flag + //favPrefix string = "user.ocis.fav." // favorite flag, per user + // TODO use etag prefix instead of single etag property + //etagPrefix string = "user.ocis.etag." // allow overriding a calculated etag with one from the extended attributes //checksumPrefix string = "user.ocis.cs." // TODO add checksum support ) @@ -171,7 +173,9 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { // create a directory node nodeID := uuid.New().String() - fs.tp.CreateRoot(nodeID, u.Id) + if _, err = fs.tp.CreateRoot(nodeID, u.Id); err != nil { + return err + } // link users home to node return os.Symlink("../nodes/"+nodeID, home) diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index acabe0c638..76b4bb806e 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -29,12 +29,12 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/user" "github.com/google/uuid" "github.com/pkg/errors" "github.com/pkg/xattr" - "github.com/rs/zerolog/log" ) // Tree manages a hierarchical tree @@ -44,10 +44,10 @@ type Tree struct { } // NewTree creates a new Tree instance -func NewTree(pw PathWrapper, Root string) (TreePersistence, error) { +func NewTree(pw PathWrapper, root string) (TreePersistence, error) { return &Tree{ pw: pw, - Root: Root, + Root: root, }, nil } @@ -90,7 +90,7 @@ func (t *Tree) CreateRoot(id string, owner *userpb.UserId) (n *Node, err error) return nil, errors.Wrap(err, "ocisfs: error creating root node dir") } - n.writeMetadata(nodePath, owner) + err = n.writeMetadata(nodePath, owner) n.Exists = true return @@ -128,7 +128,8 @@ func (t *Tree) CreateDir(ctx context.Context, node *Node) (err error) { return errors.Wrap(err, "ocisfs: could not set owner idp attribute") } } else { - // TODO no user in context, log as error when home enabled + log := appctx.GetLogger(ctx) + log.Error().Msg("home enabled but no user in context") } // make child appear in listings @@ -228,6 +229,9 @@ func (t *Tree) ListFolder(ctx context.Context, node *Node) ([]*Node, error) { } names, err := f.Readdirnames(0) + if err != nil { + return nil, err + } nodes := []*Node{} for i := range names { link, err := os.Readlink(filepath.Join(dir, names[i])) @@ -288,6 +292,7 @@ func (t *Tree) Propagate(ctx context.Context, node *Node) (err error) { etag := hex.EncodeToString(bytes) for err == nil && !node.IsRoot() { if err := xattr.Set(filepath.Join(t.Root, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { + log := appctx.GetLogger(ctx) log.Error().Err(err).Msg("error storing file id") } // TODO propagate mtime diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go index e372b7f4c4..3d43aa7931 100644 --- a/pkg/storage/fs/ocis/upload.go +++ b/pkg/storage/fs/ocis/upload.go @@ -379,7 +379,8 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error { return errors.Wrap(err, "ocisfs: could not set owner idp attribute") } } else { - // TODO no user in context, log as error when home enabled + log := appctx.GetLogger(upload.ctx) + log.Error().Msg("home enabled but no user in context") } // link child name to parent if it is new