From 0c57d4bbdda5e5e61f406d8b1cb2433e683f90c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 7 Sep 2020 17:05:37 +0200 Subject: [PATCH] refactor node api MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/node.go | 51 ++++++++++++++++++++-- pkg/storage/fs/ocis/ocis.go | 19 +-------- pkg/storage/fs/ocis/path.go | 2 +- pkg/storage/fs/ocis/persistence.go | 6 +-- pkg/storage/fs/ocis/recycle.go | 8 ++++ pkg/storage/fs/ocis/revisions.go | 8 ++++ pkg/storage/fs/ocis/tree.go | 68 ++++++++++++++++++++---------- 7 files changed, 112 insertions(+), 50 deletions(-) diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go index 16ef1d00a3a..47aa0f43e66 100644 --- a/pkg/storage/fs/ocis/node.go +++ b/pkg/storage/fs/ocis/node.go @@ -12,6 +12,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/mime" + "github.com/google/uuid" "github.com/pkg/errors" "github.com/pkg/xattr" ) @@ -22,13 +23,54 @@ type Node struct { ParentID string ID string Name string - ownerID string - ownerIDP string + ownerID string // used to cache the owner id + ownerIDP string // used to cache the owner idp Exists bool } -// NewNode creates a new instance and checks if it exists -func NewNode(pw PathWrapper, id string) (n *Node, err error) { +// CreateDir creates a new child directory node with a new id and the given name +// owner is optional +// TODO use in tree CreateDir +func (n *Node) CreateDir(pw PathWrapper, name string, owner *userpb.UserId) (c *Node, err error) { + c = &Node{ + pw: pw, + ParentID: n.ID, + ID: uuid.New().String(), + Name: name, + } + + // create a directory node + nodePath := filepath.Join(pw.Root(), "nodes", c.ID) + if err = os.MkdirAll(nodePath, 0700); err != nil { + return nil, errors.Wrap(err, "ocisfs: error creating node child dir") + } + + c.writeMetadata(nodePath, owner) + + c.Exists = true + return +} + +func (n *Node) writeMetadata(nodePath string, owner *userpb.UserId) (err error) { + if err = xattr.Set(nodePath, "user.ocis.parentid", []byte(n.ParentID)); err != nil { + return errors.Wrap(err, "ocisfs: could not set parentid attribute") + } + if err = xattr.Set(nodePath, "user.ocis.name", []byte(n.Name)); err != nil { + return errors.Wrap(err, "ocisfs: could not set name attribute") + } + if owner != nil { + if err = xattr.Set(nodePath, "user.ocis.owner.id", []byte(owner.OpaqueId)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner id attribute") + } + if err = xattr.Set(nodePath, "user.ocis.owner.idp", []byte(owner.Idp)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner idp attribute") + } + } + return +} + +// ReadNode creates a new instance from an id and checks if it exists +func ReadNode(pw PathWrapper, id string) (n *Node, err error) { n = &Node{ pw: pw, ID: id, @@ -135,6 +177,7 @@ func (n *Node) Parent() (p *Node, err error) { } // Owner returns the cached owner id or reads it from the extended attributes +// TODO can be private as only the AsResourceInfo uses it func (n *Node) Owner() (id string, idp string, err error) { if n.ownerID != "" && n.ownerIDP != "" { return n.ownerID, n.ownerIDP, nil diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index caf5e981e65..4fc0156b4e0 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -35,7 +35,6 @@ import ( "github.com/google/uuid" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - "github.com/pkg/xattr" ) const ( @@ -248,24 +247,8 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { // create a directory node nodeID := uuid.New().String() - nodePath := filepath.Join(fs.conf.Root, "nodes", nodeID) - err = os.MkdirAll(nodePath, 0700) - if err != nil { - return errors.Wrap(err, "ocisfs: error creating node dir") - } - if err := xattr.Set(nodePath, "user.ocis.parentid", []byte("root")); err != nil { - return errors.Wrap(err, "ocisfs: could not set parentid attribute") - } - if err := xattr.Set(nodePath, "user.ocis.name", []byte("")); err != nil { - return errors.Wrap(err, "ocisfs: could not set name attribute") - } - if err := xattr.Set(nodePath, "user.ocis.owner.id", []byte(u.Id.OpaqueId)); err != nil { - return errors.Wrap(err, "ocisfs: could not set owner id attribute") - } - if err := xattr.Set(nodePath, "user.ocis.owner.idp", []byte(u.Id.Idp)); err != nil { - return errors.Wrap(err, "ocisfs: could not set owner idp attribute") - } + fs.tp.CreateRoot(nodeID, u.Id) // link users home to node return os.Symlink("../nodes/"+nodeID, home) diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go index 98dbc3a4c86..4ffe731182b 100644 --- a/pkg/storage/fs/ocis/path.go +++ b/pkg/storage/fs/ocis/path.go @@ -72,7 +72,7 @@ func (pw *Path) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *Nod if id == nil || id.OpaqueId == "" { return nil, fmt.Errorf("invalid resource id %+v", id) } - return NewNode(pw, id.OpaqueId) + return ReadNode(pw, id.OpaqueId) } // Path returns the path for node diff --git a/pkg/storage/fs/ocis/persistence.go b/pkg/storage/fs/ocis/persistence.go index 7b86d14fb4a..e1632304797 100644 --- a/pkg/storage/fs/ocis/persistence.go +++ b/pkg/storage/fs/ocis/persistence.go @@ -5,6 +5,7 @@ import ( "net/url" "os" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ) @@ -13,6 +14,7 @@ type TreePersistence interface { GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) GetMD(ctx context.Context, node *Node) (os.FileInfo, error) ListFolder(ctx context.Context, node *Node) ([]*Node, error) + CreateRoot(id string, owner *userpb.UserId) (n *Node, err error) CreateDir(ctx context.Context, node *Node) (err error) CreateReference(ctx context.Context, path string, targetURI *url.URL) error Move(ctx context.Context, oldNode *Node, newNode *Node) (err error) @@ -25,10 +27,6 @@ type TreePersistence interface { type PathWrapper interface { NodeFromResource(ctx context.Context, ref *provider.Reference) (node *Node, err error) NodeFromID(ctx context.Context, id *provider.ResourceId) (node *Node, err error) - - // Wrap returns a Node object: - // - if the node exists with the node id, name and parent - // - if only the parent exists, the node id is empty NodeFromPath(ctx context.Context, fn string) (node *Node, err error) Path(ctx context.Context, node *Node) (path string, err error) diff --git a/pkg/storage/fs/ocis/recycle.go b/pkg/storage/fs/ocis/recycle.go index c0cdbfbcd8f..e1d68bdf553 100644 --- a/pkg/storage/fs/ocis/recycle.go +++ b/pkg/storage/fs/ocis/recycle.go @@ -7,6 +7,14 @@ import ( "github.com/cs3org/reva/pkg/errtypes" ) +// Recycle items are stored inside the node folder and start with the uuid of the deleted node. +// The `.T.` indicates it is a trash item and what follows is the timestamp of the deletion. +// The deleted file is kept in the same location/dir as the original node. This prevents deletes +// from triggering cross storage moves when the trash is accidentally stored on another partition, +// because the admin mounted a different partition there. +// TODO For an efficient listing of deleted nodes the ocis storages trash folder should have +// contain a directory with symlinks to trash files for every userid/"root" + func (fs *ocisfs) PurgeRecycleItem(ctx context.Context, key string) error { return errtypes.NotSupported("operation not supported: PurgeRecycleItem") } diff --git a/pkg/storage/fs/ocis/revisions.go b/pkg/storage/fs/ocis/revisions.go index f02d1164950..59afb47a72e 100644 --- a/pkg/storage/fs/ocis/revisions.go +++ b/pkg/storage/fs/ocis/revisions.go @@ -10,6 +10,14 @@ import ( "github.com/cs3org/reva/pkg/errtypes" ) +// Revision entries are stored inside the node folder and start with the same uuid as the current version. +// The `.REV.` indicates it is a revision and what follows is a timestamp, so multiple versions +// can be kept in the same location as the current file content. This prevents new fileuploads +// to trigger cross storage moves when revisions accidentally are stored on another partition, +// because the admin mounted a different partition there. +// We can add a background process to move old revisions to a slower storage +// and replace the revision file with a symbolic link in the future, if necessary. + func (fs *ocisfs) ListRevisions(ctx context.Context, ref *provider.Reference) (revisions []*provider.FileVersion, err error) { var node *Node if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index 993422117f4..9c12a4aedd1 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -9,6 +9,7 @@ import ( "path/filepath" "time" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/user" @@ -20,21 +21,21 @@ import ( // Tree manages a hierarchical tree type Tree struct { - pw PathWrapper - DataDirectory string + pw PathWrapper + Root string } // NewTree creates a new Tree instance -func NewTree(pw PathWrapper, dataDirectory string) (TreePersistence, error) { +func NewTree(pw PathWrapper, Root string) (TreePersistence, error) { return &Tree{ - pw: pw, - DataDirectory: dataDirectory, + pw: pw, + Root: Root, }, nil } // GetMD returns the metadata of a node in the tree func (t *Tree) GetMD(ctx context.Context, node *Node) (os.FileInfo, error) { - md, err := os.Stat(filepath.Join(t.DataDirectory, "nodes", node.ID)) + md, err := os.Stat(filepath.Join(t.Root, "nodes", node.ID)) if err != nil { if os.IsNotExist(err) { return nil, errtypes.NotFound(node.ID) @@ -57,18 +58,38 @@ func (t *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relati return } +// CreateRoot creates a new root node with parentid = "root" +func (t *Tree) CreateRoot(id string, owner *userpb.UserId) (n *Node, err error) { + n = &Node{ + ParentID: "root", + pw: t.pw, + ID: id, + } + + // create a directory node + nodePath := filepath.Join(t.Root, "nodes", n.ID) + if err = os.MkdirAll(nodePath, 0700); err != nil { + return nil, errors.Wrap(err, "ocisfs: error creating root node dir") + } + + n.writeMetadata(nodePath, owner) + + n.Exists = true + return +} + // CreateDir creates a new directory entry in the tree +// TODO use parentnode and name instead of node? would make the exists stuff clearer? maybe obsolete? func (t *Tree) CreateDir(ctx context.Context, node *Node) (err error) { - // TODO always try to fill node? - if node.Exists || node.ID != "" { // child already exists - return + if node.Exists || node.ID != "" { + return errtypes.AlreadyExists(node.ID) // path? } // create a directory node node.ID = uuid.New().String() - newPath := filepath.Join(t.DataDirectory, "nodes", node.ID) + newPath := filepath.Join(t.Root, "nodes", node.ID) err = os.MkdirAll(newPath, 0700) if err != nil { @@ -93,7 +114,7 @@ func (t *Tree) CreateDir(ctx context.Context, node *Node) (err error) { } // make child appear in listings - err = os.Symlink("../"+node.ID, filepath.Join(t.DataDirectory, "nodes", node.ParentID, node.Name)) + err = os.Symlink("../"+node.ID, filepath.Join(t.Root, "nodes", node.ParentID, node.Name)) if err != nil { return } @@ -111,14 +132,14 @@ func (t *Tree) Move(ctx context.Context, oldNode *Node, newNode *Node) (err erro // if target exists delete it without trashing it if newNode.Exists { // TODO make sure all children are deleted - if err := os.RemoveAll(filepath.Join(t.DataDirectory, "nodes", newNode.ID)); err != nil { + if err := os.RemoveAll(filepath.Join(t.Root, "nodes", newNode.ID)); err != nil { return errors.Wrap(err, "ocisfs: Move: error deleting target node "+newNode.ID) } } - // are we renaming? + // are we just renaming (parent stays the same)? if oldNode.ParentID == newNode.ParentID { - parentPath := filepath.Join(t.DataDirectory, "nodes", oldNode.ParentID) + parentPath := filepath.Join(t.Root, "nodes", oldNode.ParentID) // rename child err = os.Rename( @@ -129,7 +150,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *Node, newNode *Node) (err erro return errors.Wrap(err, "ocisfs: could not rename child") } - tgtPath := filepath.Join(t.DataDirectory, "nodes", newNode.ID) + tgtPath := filepath.Join(t.Root, "nodes", newNode.ID) // update name attribute if err := xattr.Set(tgtPath, "user.ocis.name", []byte(newNode.Name)); err != nil { @@ -144,14 +165,15 @@ func (t *Tree) Move(ctx context.Context, oldNode *Node, newNode *Node) (err erro // rename child err = os.Rename( - filepath.Join(t.DataDirectory, "nodes", oldNode.ParentID, oldNode.Name), - filepath.Join(t.DataDirectory, "nodes", newNode.ParentID, newNode.Name), + filepath.Join(t.Root, "nodes", oldNode.ParentID, oldNode.Name), + filepath.Join(t.Root, "nodes", newNode.ParentID, newNode.Name), ) if err != nil { return errors.Wrap(err, "ocisfs: could not move child") } - tgtPath := filepath.Join(t.DataDirectory, "nodes", newNode.ID) + // update parentid and name + tgtPath := filepath.Join(t.Root, "nodes", newNode.ID) if err := xattr.Set(tgtPath, "user.ocis.parentid", []byte(newNode.ParentID)); err != nil { return errors.Wrap(err, "ocisfs: could not set parentid attribute") @@ -178,7 +200,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *Node, newNode *Node) (err erro // ListFolder lists the content of a folder node func (t *Tree) ListFolder(ctx context.Context, node *Node) ([]*Node, error) { - dir := filepath.Join(t.DataDirectory, "nodes", node.ID) + dir := filepath.Join(t.Root, "nodes", node.ID) f, err := os.Open(dir) if err != nil { if os.IsNotExist(err) { @@ -210,13 +232,13 @@ func (t *Tree) ListFolder(ctx context.Context, node *Node) ([]*Node, error) { // Delete deletes a node in the tree func (t *Tree) Delete(ctx context.Context, node *Node) (err error) { - src := filepath.Join(t.DataDirectory, "nodes", node.ParentID, node.Name) + src := filepath.Join(t.Root, "nodes", node.ParentID, node.Name) err = os.Remove(src) if err != nil { return } - nodePath := filepath.Join(t.DataDirectory, "nodes", node.ID) + nodePath := filepath.Join(t.Root, "nodes", node.ID) trashPath := nodePath + ".T." + time.Now().UTC().Format(time.RFC3339Nano) err = os.Rename(nodePath, trashPath) if err != nil { @@ -225,7 +247,7 @@ func (t *Tree) Delete(ctx context.Context, node *Node) (err error) { // make node appear in trash // parent id and name are stored as extended attributes in the node itself - trashLink := filepath.Join(t.DataDirectory, "trash", node.ID) + trashLink := filepath.Join(t.Root, "trash", node.ID) err = os.Symlink("../nodes/"+node.ID+".T."+time.Now().UTC().Format(time.RFC3339Nano), trashLink) if err != nil { return @@ -247,7 +269,7 @@ func (t *Tree) Propagate(ctx context.Context, node *Node) (err error) { // store in extended attribute etag := hex.EncodeToString(bytes) for err == nil && !node.IsRoot() { - if err := xattr.Set(filepath.Join(t.DataDirectory, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { + if err := xattr.Set(filepath.Join(t.Root, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { log.Error().Err(err).Msg("error storing file id") } // TODO propagate mtime