From 0843b5ba07dcc7719f1bb9d4488a2d00c36fd716 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Fri, 28 Aug 2015 02:25:27 -0600 Subject: [PATCH 1/2] findByPath: handle files clashing by path --- src/changes.go | 20 +++++-- src/copy.go | 90 +++++++++++++++++++++-------- src/list.go | 46 ++++++++------- src/move.go | 151 ++++++++++++++++++++++++++++++++++--------------- src/publish.go | 49 +++++++++------- src/pull.go | 21 ++++--- src/push.go | 42 +++++++++++--- src/remote.go | 106 +++++++++++++++++++++++++++------- src/share.go | 30 ++++++---- src/stat.go | 30 +++++----- src/touch.go | 66 ++++++++++++--------- src/trash.go | 93 +++++++++++++++++------------- src/url.go | 16 +++--- 13 files changed, 509 insertions(+), 251 deletions(-) diff --git a/src/changes.go b/src/changes.go index 225fa84a..a54581f6 100644 --- a/src/changes.go +++ b/src/changes.go @@ -130,17 +130,27 @@ func (g *Commands) byRemoteResolve(relToRoot, fsPath string, r *File, push bool) } func (g *Commands) changeListResolve(relToRoot, fsPath string, push bool) (cl, clashes []*Change, err error) { - var r *File - r, err = g.rem.FindByPath(relToRoot) + var rl []*File + rl, err = g.rem.FindByPath(relToRoot) if err != nil && err != ErrPathNotExists { return } - if r != nil && anyMatch(g.opts.IgnoreRegexp, r.Name) { - return + for _, r := range rl { + if r != nil && anyMatch(g.opts.IgnoreRegexp, r.Name) { + return + } + + ccl, cclashes, clErr := g.byRemoteResolve(relToRoot, fsPath, r, push) + if clErr != nil { + g.log.LogErrf("changeListResolve:: %s %v\n", relToRoot, clErr) + } + + cl = append(cl, ccl...) + clashes = append(clashes, cclashes...) } - return g.byRemoteResolve(relToRoot, fsPath, r, push) + return } func (g *Commands) doChangeListRecv(relToRoot, fsPath string, l, r *File, push bool) (cl, clashes []*Change, err error) { diff --git a/src/copy.go b/src/copy.go index 5545f9e6..582e8301 100644 --- a/src/copy.go +++ b/src/copy.go @@ -42,46 +42,65 @@ func (g *Commands) Copy(byId bool) error { end := argc - 1 sources, dest := g.opts.Sources[:end], g.opts.Sources[end] - destFile, err := g.rem.FindByPath(dest) + destFiles, err := g.rem.FindByPath(dest) if err != nil && err != ErrPathNotExists { return fmt.Errorf("destination: %s err: %v", dest, err) } + for _, destFile := range destFiles { + if destFile == nil { + continue + } + + errs := copy_(g, dest, destFile, sources, byId) + for _, err := range errs { + g.log.LogErrf("copy_: %s %v\n", destFile.Id, err) + } + } + + return nil +} + +func copy_(g *Commands, dest string, destFile *File, sources []string, byId bool) (errs []error) { multiPaths := len(sources) > 1 if multiPaths { if destFile != nil && !destFile.IsDir { - return fmt.Errorf("%s: %v", dest, ErrPathNotDir) + errs = append(errs, fmt.Errorf("%s: %v", dest, ErrPathNotDir)) + return } _, err := g.remoteMkdirAll(dest) if err != nil { - return err + errs = append(errs, err) + return } } srcResolver := g.rem.FindByPath if byId { - srcResolver = g.rem.FindById + srcResolver = g.rem.FindByIdMulti } done := make(chan bool) waitCount := uint64(0) for _, srcPath := range sources { - srcFile, srcErr := srcResolver(srcPath) + srcFiles, srcErr := srcResolver(srcPath) if srcErr != nil { g.log.LogErrf("%s: %v\n", srcPath, srcErr) continue } - waitCount += 1 + for _, srcFile := range srcFiles { + waitCount += 1 - go func(fromPath, toPath string, fromFile *File) { - _, copyErr := g.copy(fromFile, toPath) - if copyErr != nil { - g.log.LogErrf("%s: %v\n", fromPath, copyErr) - } - done <- true - }(srcPath, dest, srcFile) + go func(fromPath, toPath string, fromFile *File) { + _, copyErr := g.copy(fromFile, toPath) + if copyErr != nil { + g.log.LogErrf("%s: %v\n", fromPath, copyErr) + } + done <- true + }(srcPath, dest, srcFile) + } } for i := uint64(0); i < waitCount; i += 1 { @@ -89,40 +108,60 @@ func (g *Commands) Copy(byId bool) error { } return nil + } -func (g *Commands) copy(src *File, destPath string) (*File, error) { +func (g *Commands) copy(src *File, destPath string) (copies []*File, errs []error) { if src == nil { - return nil, fmt.Errorf("non existant src") + errs = append(errs, fmt.Errorf("non existant src")) + return } if !src.IsDir { if !src.Copyable { - return nil, fmt.Errorf("%s is non-copyable", src.Name) + errs = append(errs, fmt.Errorf("%s is non-copyable", src.Name)) + return } destDir, destBase := g.pathSplitter(destPath) destParent, destParErr := g.remoteMkdirAll(destDir) if destParErr != nil { - return nil, destParErr + errs = append(errs, destParErr) + return } parentId := destParent.Id - destFile, destErr := g.rem.FindByPath(destPath) + destFiles, destErr := g.rem.FindByPath(destPath) if destErr != nil && destErr != ErrPathNotExists { - return nil, destErr + errs = append(errs, destErr) + return } - if destFile != nil && destFile.IsDir { - parentId = destFile.Id - destBase = src.Name + + for _, destFile := range destFiles { + if destFile != nil && destFile.IsDir { + parentId = destFile.Id + destBase = src.Name + } + + copy, cpErr := g.rem.copy(destBase, parentId, src) + + if copy != nil { + copies = append(copies, copy) + } + + if cpErr != nil { + errs = append(errs, cpErr) + } } - return g.rem.copy(destBase, parentId, src) + + return } destFile, destErr := g.remoteMkdirAll(destPath) if destErr != nil { - return nil, destErr + errs = append(errs, destErr) + return } children := g.rem.findChildren(src.Id, false) @@ -138,5 +177,6 @@ func (g *Commands) copy(src *File, destPath string) (*File, error) { } } - return destFile, nil + copies = append(copies, destFile) + return } diff --git a/src/list.go b/src/list.go index a781c43c..7b7ca62d 100644 --- a/src/list.go +++ b/src/list.go @@ -179,7 +179,7 @@ func (g *Commands) List(byId bool) error { resolver := g.rem.FindByPath if byId { - resolver = g.rem.FindById + resolver = g.rem.FindByIdMulti } else if g.opts.InTrash { resolver = g.rem.FindByPathTrashed } @@ -187,34 +187,36 @@ func (g *Commands) List(byId bool) error { mq := g.createMatchQuery(true) for _, relPath := range g.opts.Sources { - r, rErr := resolver(relPath) + rl, rErr := resolver(relPath) if rErr != nil && rErr != ErrPathNotExists { return fmt.Errorf("%v: '%s'", rErr, relPath) } - if r == nil { - g.log.LogErrf("remote: %s is nil\n", strconv.Quote(relPath)) - continue - } + for _, r := range rl { + if r == nil { + g.log.LogErrf("remote: %s is nil\n", strconv.Quote(relPath)) + continue + } - parentPath := "" - if !byId { - parentPath = g.parentPather(relPath) - } else { - parentPath = r.Id - } + parentPath := "" + if !byId { + parentPath = g.parentPather(relPath) + } else { + parentPath = r.Id + } - if remoteRootLike(parentPath) { - parentPath = "" - } - if remoteRootLike(r.Name) { - r.Name = "" - } - if rootLike(parentPath) { - parentPath = "" - } + if remoteRootLike(parentPath) { + parentPath = "" + } + if remoteRootLike(r.Name) { + r.Name = "" + } + if rootLike(parentPath) { + parentPath = "" + } - kvList = append(kvList, &keyValue{key: parentPath, value: r}) + kvList = append(kvList, &keyValue{key: parentPath, value: r}) + } } spin := g.playabler() diff --git a/src/move.go b/src/move.go index ca87d43d..b8bdd639 100644 --- a/src/move.go +++ b/src/move.go @@ -20,9 +20,11 @@ import ( ) type moveOpt struct { - src string - dest string - byId bool + src string + dest string + byId bool + srcFile *File + destFile *File } func (g *Commands) Move(byId bool) (err error) { @@ -47,67 +49,100 @@ func (g *Commands) Move(byId bool) (err error) { byId: byId, } - err = g.move(&opt) - if err != nil { - // TODO: Actually throw the error? Impact on UX if thrown? - fmt.Printf("move: %s: %v\n", src, err) + errs := g.move(&opt) + for _, err := range errs { + if err != nil { + // TODO: Actually throw the error? Impact on UX if thrown? + fmt.Printf("move: %s: %v\n", src, err) + } } } return nil } -func (g *Commands) move(opt *moveOpt) (err error) { - var newParent, remSrc *File +func (g *Commands) move(opt *moveOpt) (errs []error) { srcResolver := g.rem.FindByPath if opt.byId { - srcResolver = g.rem.FindById + srcResolver = g.rem.FindByIdMulti } - if remSrc, err = srcResolver(opt.src); err != nil { - return fmt.Errorf("src('%s') %v", opt.src, err) + sources, err := srcResolver(opt.src) + if err != nil { + errs = append(errs, fmt.Errorf("src('%s') %v", opt.src, err)) + return } - if remSrc == nil { - return fmt.Errorf("src: '%s' could not be found", opt.src) + newParents, parErr := g.rem.FindByPath(opt.dest) + if parErr != nil { + errs = append(errs, fmt.Errorf("dest: '%s' %v", opt.dest, parErr)) + return } - if newParent, err = g.rem.FindByPath(opt.dest); err != nil { - return fmt.Errorf("dest: '%s' %v", opt.dest, err) + for _, src := range sources { + + for _, newParent := range newParents { + if newParent == nil || !newParent.IsDir { + errs = append(errs, fmt.Errorf("dest: '%s' must be an existant folder", opt.dest)) + continue + } + + opt.srcFile = src + opt.destFile = newParent + if err := move_(g, opt); err != nil { + errs = append(errs, err) + } + } } - if newParent == nil || !newParent.IsDir { - return fmt.Errorf("dest: '%s' must be an existant folder", opt.dest) + return +} + +func move_(g *Commands, opt *moveOpt) error { + var err error + + remSrc := opt.srcFile + newParent := opt.destFile + + if remSrc == nil { + return fmt.Errorf("src: '%s' could not be found", opt.src) } if !opt.byId { parentPath := g.parentPather(opt.src) - oldParent, parErr := g.rem.FindByPath(parentPath) + oldParents, parErr := g.rem.FindByPath(parentPath) if parErr != nil && parErr != ErrPathNotExists { return parErr } - // TODO: If oldParent is not found, retry since it may have been moved temporarily at least - if oldParent != nil && oldParent.Id == newParent.Id { - return fmt.Errorf("src and dest are the same srcParentId %s destParentId %s", - customQuote(oldParent.Id), customQuote(newParent.Id)) + for _, oldParent := range oldParents { + // TODO: If oldParent is not found, retry since it may have been moved temporarily at least + if oldParent != nil && oldParent.Id == newParent.Id { + return fmt.Errorf("src and dest are the same srcParentId %s destParentId %s", + customQuote(oldParent.Id), customQuote(newParent.Id)) + } } } newFullPath := filepath.Join(opt.dest, remSrc.Name) // Check for a duplicate - var dupCheck *File - dupCheck, err = g.rem.FindByPath(newFullPath) + var dupChecks []*File + dupChecks, err = g.rem.FindByPath(newFullPath) if err != nil && err != ErrPathNotExists { return err } - if dupCheck != nil { - if dupCheck.Id == remSrc.Id { // Trying to move to self - return fmt.Errorf("move: trying to move fileId:%s to self fileId:%s", customQuote(dupCheck.Id), customQuote(remSrc.Id)) + for _, dup := range dupChecks { + if dup == nil { + continue + } + + if dup.Id == remSrc.Id { // Trying to move to self + return fmt.Errorf("move: trying to move fileId:%s to self fileId:%s", customQuote(dup.Id), customQuote(remSrc.Id)) } + if !g.opts.Force { return fmt.Errorf("%s already exists. Use `%s` flag to override this behaviour", newFullPath, ForceKey) } @@ -130,14 +165,22 @@ func (g *Commands) move(opt *moveOpt) (err error) { func (g *Commands) removeParent(fileId, relToRootPath string) error { parentPath := g.parentPather(relToRootPath) - parent, pErr := g.rem.FindByPath(parentPath) + parents, pErr := g.rem.FindByPath(parentPath) if pErr != nil { return pErr } - if parent == nil { - return fmt.Errorf("non existant parent '%s' for src", parentPath) + + for _, parent := range parents { + if parent == nil { + return fmt.Errorf("non existant parent '%s' for src", parentPath) + } + + if err := g.rem.removeParent(fileId, parent.Id); err != nil { + g.log.LogErrf("removeParent:: %s %s %v\n", fileId, relToRootPath, err) + } } - return g.rem.removeParent(fileId, parent.Id) + + return nil } func (g *Commands) Rename(byId bool) error { @@ -148,16 +191,28 @@ func (g *Commands) Rename(byId bool) error { src := g.opts.Sources[0] resolver := g.rem.FindByPath if byId { - resolver = g.rem.FindById + resolver = g.rem.FindByIdMulti } - remSrc, err := resolver(src) + + remoteSources, err := resolver(src) if err != nil { return fmt.Errorf("%s: %v", src, err) } - if remSrc == nil { - return fmt.Errorf("%s does not exist", src) + + for _, remSrc := range remoteSources { + if remSrc == nil { + g.log.LogErrf("%s does not exist", src) + } + + if err = rename_(g, src, remSrc, byId); err != nil { + g.log.LogErrf("%s %v\n", src, err) + } } + return nil +} + +func rename_(g *Commands, src string, remSrc *File, byId bool) error { var parentPath string if !byId { parentPath = g.parentPather(src) @@ -169,18 +224,24 @@ func (g *Commands) Rename(byId bool) error { urlBoundName := urlToPath(newName, true) newFullPath := filepath.Join(parentPath, urlBoundName) - var dupCheck *File - dupCheck, err = g.rem.FindByPath(newFullPath) + dupChecks, err := g.rem.FindByPath(newFullPath) - if err == nil && dupCheck != nil { - if dupCheck.Id == remSrc.Id { // Trying to rename self - return nil - } - if !g.opts.Force { - return fmt.Errorf("%s already exists. Use `%s` flag to override this behaviour", newFullPath, ForceKey) + if err == nil && len(dupChecks) >= 1 { + for _, dup := range dupChecks { + if dup.Id == remSrc.Id { // Trying to rename self + continue + } + + if !g.opts.Force { + g.log.LogErrf("%s already exists. Use `%s` flag to override this behaviour", newFullPath, ForceKey) + continue + } + + if _, err = g.rem.rename(remSrc.Id, newName); err != nil { + g.log.LogErrf("rename: %s %s %v\n", remSrc.Id, newName, err) + } } } - _, err = g.rem.rename(remSrc.Id, newName) return err } diff --git a/src/publish.go b/src/publish.go index 9e180d01..750a90fa 100644 --- a/src/publish.go +++ b/src/publish.go @@ -27,34 +27,38 @@ func (c *Commands) Publish(byId bool) (err error) { return } -func (c *Commands) remFileResolve(relToRoot string, byId bool) (*File, error) { +func (c *Commands) remFileResolve(relToRoot string, byId bool) ([]*File, error) { resolver := c.rem.FindByPath if byId { - resolver = c.rem.FindById + resolver = c.rem.FindByIdMulti } return resolver(relToRoot) } -func (c *Commands) pub(relToRoot string, byId bool) (err error) { - file, err := c.remFileResolve(relToRoot, byId) - if err != nil || file == nil { - return err - } - - var link string - link, err = c.rem.Publish(file.Id) - if err != nil { +func (c *Commands) pub(relToRoot string, byId bool) (errs []error) { + files, err := c.remFileResolve(relToRoot, byId) + if err != nil || len(files) < 1 { + errs = append(errs, err) return } - link = file.Url() + for _, file := range files { + link, err := c.rem.Publish(file.Id) + if err != nil { + errs = append(errs, err) + continue + } - if byId { - relToRoot = fmt.Sprintf("%s aka %s", relToRoot, file.Name) + link = file.Url() + + if byId { + relToRoot = fmt.Sprintf("%s aka %s", relToRoot, file.Name) + } + + c.log.Logf("%s published on %s\n", relToRoot, link) } - c.log.Logf("%s published on %s\n", relToRoot, link) return } @@ -67,11 +71,18 @@ func (c *Commands) Unpublish(byId bool) error { return nil } -func (c *Commands) unpub(relToRoot string, byId bool) error { - file, err := c.remFileResolve(relToRoot, byId) +func (c *Commands) unpub(relToRoot string, byId bool) (errs []error) { + files, err := c.remFileResolve(relToRoot, byId) if err != nil { - return err + errs = append(errs, err) + return + } + + for _, file := range files { + if upErr := c.rem.Unpublish(file.Id); upErr != nil { + errs = append(errs, upErr) + } } - return c.rem.Unpublish(file.Id) + return errs } diff --git a/src/pull.go b/src/pull.go index aca764aa..922a5445 100644 --- a/src/pull.go +++ b/src/pull.go @@ -186,23 +186,28 @@ func (g *Commands) PullMatches() (err error) { func (g *Commands) PullPiped(byId bool) (err error) { resolver := g.rem.FindByPath if byId { - resolver = g.rem.FindById + resolver = g.rem.FindByIdMulti } for _, relToRootPath := range g.opts.Sources { - rem, err := resolver(relToRootPath) + remotes, err := resolver(relToRootPath) + if err != nil { return fmt.Errorf("%s: %v", relToRootPath, err) } - if rem == nil { - continue - } - err = g.pullAndDownload(relToRootPath, os.Stdout, rem, true) - if err != nil { - return err + for _, rem := range remotes { + if rem == nil { + continue + } + + err = g.pullAndDownload(relToRootPath, os.Stdout, rem, true) + if err != nil { + g.log.LogErrf("pullPiped:: %s %s: %v\n", relToRootPath, rem.Id, err) + } } } + return nil } diff --git a/src/push.go b/src/push.go index 9aad1c0c..19077090 100644 --- a/src/push.go +++ b/src/push.go @@ -168,10 +168,18 @@ func (g *Commands) resolveConflicts(cl []*Change, push bool) (*[]*Change, *[]*Ch func (g *Commands) PushPiped() (err error) { // Cannot push asynchronously because the push order must be maintained for _, relToRootPath := range g.opts.Sources { - rem, resErr := g.rem.FindByPath(relToRootPath) + remotes, resErr := g.rem.FindByPath(relToRootPath) if resErr != nil && resErr != ErrPathNotExists { return resErr } + + if len(remotes) < 1 { + continue + } + + // TODO: Decide if pushing to all those remotes should proceed + rem := remotes[0] + if rem != nil && !g.opts.Force { return fmt.Errorf("%s already exists remotely, use `%s` to override this behaviour.\n", relToRootPath, ForceKey) } @@ -187,8 +195,12 @@ func (g *Commands) PushPiped() (err error) { } parentPath := g.parentPather(relToRootPath) - parent, pErr := g.rem.FindByPath(parentPath) - if pErr != nil { + parents, pErr := g.rem.FindByPath(parentPath) + var parent *File + + if pErr == nil && len(parents) >= 1 { + parent = parents[0] + } else { spin := g.playabler() spin.play() parent, pErr = g.remoteMkdirAll(parentPath) @@ -355,12 +367,17 @@ func (g *Commands) playPushChanges(cl []*Change, opMap *map[Operation]sizeCounte } func lonePush(g *Commands, parent, absPath, path string) (cl, clashes []*Change, err error) { - r, err := g.rem.FindByPath(absPath) + remotes, err := g.rem.FindByPath(absPath) if err != nil && err != ErrPathNotExists { return } - var l *File + var r, l *File + + if len(remotes) >= 1 { + r = remotes[0] + } + localinfo, _ := os.Stat(path) if localinfo != nil { l = NewLocalFile(path, localinfo) @@ -537,7 +554,13 @@ func (g *Commands) remoteMkdirAll(d string) (file *File, err error) { } // Try the lookup one last time in case a coroutine raced us to it. - retrFile, retryErr := g.rem.FindByPath(d) + retrFiles, retryErr := g.rem.FindByPath(d) + + var retrFile *File + + if len(retrFiles) >= 1 { + retrFile = retrFiles[0] + } if retryErr != nil && retryErr != ErrPathNotExists { mkdirAllMu.Unlock() @@ -551,7 +574,12 @@ func (g *Commands) remoteMkdirAll(d string) (file *File, err error) { parDirPath, last := remotePathSplit(d) - parent, parentErr := g.rem.FindByPath(parDirPath) + parents, parentErr := g.rem.FindByPath(parDirPath) + + var parent *File + if len(parents) >= 1 { + parent = parents[0] + } if parentErr != nil && parentErr != ErrPathNotExists { mkdirAllMu.Unlock() diff --git a/src/remote.go b/src/remote.go index a7d0e489..f53180e5 100644 --- a/src/remote.go +++ b/src/remote.go @@ -176,9 +176,24 @@ func (r *Remote) FindById(id string) (file *File, err error) { if f, err = req.Do(); err != nil { return } + return NewRemoteFile(f), nil } +func (r *Remote) FindByIdMulti(id string) (files []*File, err error) { + var f *File + f, err = r.FindById(id) + if err != nil { + return + } + + if f != nil { + files = append(files, f) + } + + return +} + func retryableChangeOp(fn func() (interface{}, error)) *expb.ExponentialBacker { return &expb.ExponentialBacker{ Do: fn, @@ -188,10 +203,11 @@ func retryableChangeOp(fn func() (interface{}, error)) *expb.ExponentialBacker { } } -func (r *Remote) findByPath(p string, trashed bool) (*File, error) { +func (r *Remote) findByPath(p string, trashed bool) ([]*File, error) { if rootLike(p) { - return r.FindById("root") + return r.FindByIdMulti("root") } + parts := strings.Split(p, "/") finder := r.findByPathRecv if trashed { @@ -200,11 +216,11 @@ func (r *Remote) findByPath(p string, trashed bool) (*File, error) { return finder("root", parts[1:]) } -func (r *Remote) FindByPath(p string) (file *File, err error) { +func (r *Remote) FindByPath(p string) ([]*File, error) { return r.findByPath(p, false) } -func (r *Remote) FindByPathTrashed(p string) (file *File, err error) { +func (r *Remote) FindByPathTrashed(p string) ([]*File, error) { return r.findByPath(p, true) } @@ -383,6 +399,16 @@ func (r *Remote) Touch(id string) (*File, error) { return NewRemoteFile(f), err } +func (r *Remote) TouchMulti(id string) (files []*File, err error) { + f, fErr := r.Touch(id) + if fErr != nil { + return + } + + files = append(files, f) + return +} + func toUTCString(t time.Time) string { utc := t.UTC().Round(time.Second) // Ugly but straight forward formatting as time.Parse is such a prima donna @@ -661,20 +687,45 @@ func (r *Remote) FindByPathShared(p string) (chan *File, error) { } func (r *Remote) FindMatches(mq *matchQuery) (chan *File, error) { - parent, err := r.FindByPath(mq.dirPath) + parents, err := r.FindByPath(mq.dirPath) + filesChan := make(chan *File) - if err != nil || parent == nil { + defer close(filesChan) + + if err != nil || len(parents) < 1 { close(filesChan) return filesChan, err } - req := r.service.Files.List() + done := make(chan bool) + doneCount := uint64(0) - parQuery := fmt.Sprintf("(%s in parents)", customQuote(parent.Id)) - expr := sepJoinNonEmpty(" and ", parQuery, mq.Stringer()) + for _, parent := range parents { + doneCount += 1 - req.Q(expr) - return reqDoPage(req, true, false), nil + go func(par *File) { + req := r.service.Files.List() + + parQuery := fmt.Sprintf("(%s in parents)", customQuote(par.Id)) + expr := sepJoinNonEmpty(" and ", parQuery, mq.Stringer()) + + req.Q(expr) + + matches := reqDoPage(req, true, false) + + for f := range matches { + filesChan <- f + } + + done <- true + }(parent) + } + + for i := uint64(0); i < doneCount; i++ { + <-done + } + + return filesChan, nil } func (r *Remote) findChildren(parentId string, trashed bool) chan *File { @@ -687,7 +738,7 @@ func (r *Remote) About() (about *drive.About, err error) { return r.service.About.Get().Do() } -func (r *Remote) findByPathRecvRaw(parentId string, p []string, trashed bool) (file *File, err error) { +func (r *Remote) findByPathRecvRaw(parentId string, p []string, trashed bool) (files []*File, err error) { // find the file or directory under parentId and titled with p[0] req := r.service.Files.List() // TODO: use field selectors @@ -702,33 +753,44 @@ func (r *Remote) findByPathRecvRaw(parentId string, p []string, trashed bool) (f req.Q(expr) // We only need the head file since we expect only one File to be created - req.MaxResults(1) + // req.MaxResults(1) - files, err := req.Do() + fl, err := req.Do() if err != nil { if err.Error() == ErrGoogleApiInvalidQueryHardCoded.Error() { // Send the user back the query information err = fmt.Errorf("err: %v query: `%s`", err, expr) } - return nil, err + return } - if files == nil || len(files.Items) < 1 { - return nil, ErrPathNotExists + if fl == nil || len(fl.Items) < 1 { + err = ErrPathNotExists + return } - first := files.Items[0] if len(p) == 1 { - return NewRemoteFile(first), nil + for _, f := range fl.Items { + files = append(files, NewRemoteFile(f)) + } + return files, nil } - return r.findByPathRecvRaw(first.Id, p[1:], trashed) + + for _, f := range fl.Items { + children, cErr := r.findByPathRecvRaw(f.Id, p[1:], trashed) + if cErr == nil { + files = append(files, children...) + } + } + + return files, nil } -func (r *Remote) findByPathRecv(parentId string, p []string) (file *File, err error) { +func (r *Remote) findByPathRecv(parentId string, p []string) (file []*File, err error) { return r.findByPathRecvRaw(parentId, p, false) } -func (r *Remote) findByPathTrashed(parentId string, p []string) (file *File, err error) { +func (r *Remote) findByPathTrashed(parentId string, p []string) (file []*File, err error) { return r.findByPathRecvRaw(parentId, p, true) } diff --git a/src/share.go b/src/share.go index bf83e8e3..fecf6fc1 100644 --- a/src/share.go +++ b/src/share.go @@ -128,25 +128,31 @@ var reverseRoleResolve = stringToRole() var reverseAccountTypeResolve = stringToAccountType() func (g *Commands) resolveRemotePaths(relToRootPaths []string, byId bool) (files []*File) { - var wg sync.WaitGroup - resolver := g.rem.FindByPath if byId { - resolver = g.rem.FindById + resolver = g.rem.FindByIdMulti } - wg.Add(len(relToRootPaths)) + done := make(chan bool) + doneCount := uint64(0) + for _, relToRoot := range relToRootPaths { - go func(p string, wgg *sync.WaitGroup) { - defer wgg.Done() - file, err := resolver(p) - if err != nil || file == nil { - return + doneCount += 1 + + go func(p string) { + childFiles, err := resolver(p) + if err == nil { + files = append(files, childFiles...) } - files = append(files, file) - }(relToRoot, &wg) + + done <- true + }(relToRoot) } - wg.Wait() + + for i := uint64(0); i < doneCount; i++ { + <-done + } + return files } diff --git a/src/stat.go b/src/stat.go index 6fc3104f..041e92bd 100644 --- a/src/stat.go +++ b/src/stat.go @@ -28,37 +28,37 @@ type keyValue struct { } func (g *Commands) StatById() error { - return g.statfn("statById", g.rem.FindById) + return g.statfn("statById", g.rem.FindByIdMulti) } func (g *Commands) Stat() error { return g.statfn("stat", g.rem.FindByPath) } -func (g *Commands) statfn(fname string, fn func(string) (*File, error)) error { +func (g *Commands) statfn(fname string, fn func(string) ([]*File, error)) error { for _, src := range g.opts.Sources { - f, err := fn(src) + files, err := fn(src) if err != nil { g.log.LogErrf("%s: %s err: %v\n", fname, src, err) continue } - if g.opts.Md5sum { + for _, f := range files { + if g.opts.Md5sum { + src = f.Name // forces filename if -id is used - src = f.Name // forces filename if -id is used - - // md5sum with no arguments should do md5sum * - if f.IsDir && rootLike(g.opts.Path) { - src = "" + // md5sum with no arguments should do md5sum * + if f.IsDir && rootLike(g.opts.Path) { + src = "" + } } - } - - err = g.stat(src, f, g.opts.Depth) + err = g.stat(src, f, g.opts.Depth) - if err != nil { - g.log.LogErrf("%s: %s err: %v\n", fname, src, err) - continue + if err != nil { + g.log.LogErrf("%s: %s err: %v\n", fname, src, err) + continue + } } } diff --git a/src/touch.go b/src/touch.go index 08c0f5e7..54b550b6 100644 --- a/src/touch.go +++ b/src/touch.go @@ -126,36 +126,40 @@ func (g *Commands) touch(relToRootPath, fileId string) chan *keyValue { close(fileChan) }() - f, arg := g.rem.Touch, fileId + f, arg := g.rem.TouchMulti, fileId if fileId == "" { f, arg = g.touchByPath, relToRootPath } - file, err := f(arg) + files, err := f(arg) if err != nil { kv.value = err return } - if true { // TODO: Print this out if verbosity is set - g.log.Logf("%s: %v\n", relToRootPath, file.ModTime) - } - if g.opts.Recursive && file.IsDir { - childResults := make(chan chan *keyValue) - go func() { - // Arbitrary value for rate limiter - throttle := time.Tick(1e9 * 2) - childrenChan := g.rem.findByParentIdRaw(file.Id, false, g.opts.Hidden) - for child := range childrenChan { - childResults <- g.touch(relToRootPath+"/"+child.Name, child.Id) - <-throttle - } - close(childResults) - }() + for _, file := range files { + if true { // TODO: Print this out if verbosity is set + g.log.Logf("%s: %v\n", relToRootPath, file.ModTime) + } - for childChan := range childResults { - for childFile := range childChan { - fileChan <- childFile + if g.opts.Recursive && file.IsDir { + childResults := make(chan chan *keyValue) + + go func() { + // Arbitrary value for rate limiter + throttle := time.Tick(1e9 * 2) + childrenChan := g.rem.findByParentIdRaw(file.Id, false, g.opts.Hidden) + for child := range childrenChan { + childResults <- g.touch(relToRootPath+"/"+child.Name, child.Id) + <-throttle + } + close(childResults) + }() + + for childChan := range childResults { + for childFile := range childChan { + fileChan <- childFile + } } } } @@ -163,13 +167,23 @@ func (g *Commands) touch(relToRootPath, fileId string) chan *keyValue { return fileChan } -func (g *Commands) touchByPath(relToRootPath string) (*File, error) { - file, err := g.rem.FindByPath(relToRootPath) +func (g *Commands) touchByPath(relToRootPath string) (files []*File, err error) { + files, err = g.rem.FindByPath(relToRootPath) if err != nil { - return nil, err + return } - if file == nil { - return nil, ErrPathNotExists + + if len(files) < 1 { + err = ErrPathNotExists + return } - return g.rem.Touch(file.Id) + + for _, f := range files { + file, fErr := g.rem.Touch(f.Id) + if fErr == nil && file != nil { + files = append(files, file) + } + } + + return } diff --git a/src/trash.go b/src/trash.go index 8f80dc14..c923998d 100644 --- a/src/trash.go +++ b/src/trash.go @@ -53,7 +53,7 @@ func (g *Commands) Untrash(byId bool) (err error) { } func (g *Commands) EmptyTrash() error { - rootFile, err := g.rem.FindByPath("/") + rootFiles, err := g.rem.FindByPath("/") if err != nil { return err } @@ -62,68 +62,85 @@ func (g *Commands) EmptyTrash() error { spin.play() defer spin.stop() - if g.opts.canPrompt() { - travSt := traversalSt{ - depth: -1, - file: rootFile, - headPath: "/", - inTrash: true, - mask: g.opts.TypeMask, - explicitNoPrompt: true, - } + for _, rootFile := range rootFiles { + if g.opts.canPrompt() { + travSt := traversalSt{ + depth: -1, + file: rootFile, + headPath: "/", + inTrash: true, + mask: g.opts.TypeMask, + explicitNoPrompt: true, + } - if !g.breadthFirst(travSt, spin) { - return nil + if !g.breadthFirst(travSt, spin) { + break + } } + } - g.log.Logln("This operation is irreversible. Empty trash! ") + g.log.Logln("This operation is irreversible. Empty trash! ") - if !promptForChanges() { - g.log.Logln("Aborted emptying trash") - return nil - } + if !promptForChanges() { + g.log.Logln("Aborted emptying trash") + return nil } err = g.rem.EmptyTrash() if err == nil { g.log.Logln("Successfully emptied trash") } + return err } -func (g *Commands) trasher(relToRoot string, opt *trashOpt) (*Change, error) { - var file *File +func (g *Commands) trasher(relToRoot string, opt *trashOpt) (changes []*Change, errs []error) { if relToRoot == "/" && opt.toTrash { - return nil, fmt.Errorf("Will not try to trash root.") + errs = append(errs, fmt.Errorf("Will not try to trash root.")) + return } + resolver := g.rem.FindByPathTrashed + if opt.byId { - resolver = g.rem.FindById + resolver = g.rem.FindByIdMulti } else if opt.toTrash { resolver = g.rem.FindByPath } - file, err := resolver(relToRoot) + var err error + var files []*File + + files, err = resolver(relToRoot) if err != nil { - return nil, err + errs = append(errs, err) + return } - if opt.byId { - if file.Labels != nil { - if file.Labels.Trashed == opt.toTrash { - return nil, fmt.Errorf("toTrash=%v set yet already file.Trash=%v", opt.toTrash, file.Labels.Trashed) + for _, file := range files { + if opt.byId { + if file.Labels != nil { + if file.Labels.Trashed == opt.toTrash { + errs = append(errs, fmt.Errorf("toTrash=%v set yet already file.Trash=%v", opt.toTrash, file.Labels.Trashed)) + continue + } } + + relToRoot = fmt.Sprintf("%s (%s)", relToRoot, file.Name) } - relToRoot = fmt.Sprintf("%s (%s)", relToRoot, file.Name) - } - change := &Change{Path: relToRoot, g: g} - if opt.toTrash { - change.Dest = file - } else { - change.Src = file + change := &Change{Path: relToRoot, g: g} + + if opt.toTrash { + change.Dest = file + } else { + change.Src = file + } + + changes = append(changes, change) } - return change, nil + + return } func (g *Commands) trashByMatch(inTrash, permanent bool) error { @@ -197,11 +214,11 @@ func (g *Commands) DeleteByMatch() error { func (g *Commands) reduceForTrash(args []string, opt *trashOpt) error { var cl []*Change for _, relToRoot := range args { - c, cErr := g.trasher(relToRoot, opt) + ccls, cErr := g.trasher(relToRoot, opt) if cErr != nil { g.log.LogErrf("\033[91m'%s': %v\033[00m\n", relToRoot, cErr) - } else if c != nil { - cl = append(cl, c) + } else { + cl = append(cl, ccls...) } } diff --git a/src/url.go b/src/url.go index 49c68675..087c4742 100644 --- a/src/url.go +++ b/src/url.go @@ -32,7 +32,7 @@ func (g *Commands) Url(byId bool) error { func (g *Commands) urler(byId bool) (kvChan chan *keyValue) { resolver := g.rem.FindByPath if byId { - resolver = g.rem.FindById + resolver = g.rem.FindByIdMulti } kvChan = make(chan *keyValue) @@ -41,14 +41,16 @@ func (g *Commands) urler(byId bool) (kvChan chan *keyValue) { defer close(kvChan) for _, source := range g.opts.Sources { - f, err := resolver(source) + files, err := resolver(source) - kv := keyValue{key: source, value: err} - if err == nil { - kv.value = f.Url() - } + for _, f := range files { + kv := keyValue{key: source, value: err} + if err == nil { + kv.value = f.Url() + } - kvChan <- &kv + kvChan <- &kv + } } }() From e941076c3a261f985d13463b540a5a450ab3c301 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Tue, 1 Sep 2015 23:54:47 -0600 Subject: [PATCH 2/2] synchronize by chan instead of waitGroup --- src/changes.go | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/changes.go b/src/changes.go index a54581f6..1353c4a8 100644 --- a/src/changes.go +++ b/src/changes.go @@ -19,7 +19,6 @@ import ( "os" "path/filepath" "strings" - "sync" "time" "github.com/odeke-em/drive/config" @@ -347,8 +346,8 @@ func (g *Commands) resolveChangeListRecv(clr *changeListResolve) (cl, clashes [] chunkCount += 1 } - var wg sync.WaitGroup - wg.Add(chunkCount) + doneCount := uint64(0) + doneChan := make(chan bool) clashesMap := make(map[int][]*Change) @@ -358,11 +357,19 @@ func (g *Commands) resolveChangeListRecv(clr *changeListResolve) (cl, clashes [] end = srcLen } - go g.changeSlice(clashesMap, j, &wg, clr.push, &cl, base, dirlist[i:end]) + doneCount += 1 + + go func(clm map[int][]*Change, id int, push bool, cll *[]*Change, p string, dlist []*dirList) { + g.changeSlice(clashesMap, id, push, cll, p, dlist) + doneChan <- true + }(clashesMap, j, clr.push, &cl, base, dirlist[i:end]) i += chunkSize } - wg.Wait() + + for i := uint64(0); i < doneCount; i++ { + <-doneChan + } for _, cclashes := range clashesMap { clashes = append(clashes, cclashes...) @@ -375,8 +382,7 @@ func (g *Commands) resolveChangeListRecv(clr *changeListResolve) (cl, clashes [] return cl, clashes, err } -func (g *Commands) changeSlice(clashesMap map[int][]*Change, id int, wg *sync.WaitGroup, push bool, cl *[]*Change, p string, dlist []*dirList) { - defer wg.Done() +func (g *Commands) changeSlice(clashesMap map[int][]*Change, id int, push bool, cl *[]*Change, p string, dlist []*dirList) { for _, l := range dlist { // Avoiding path.Join which normalizes '/+' to '/' var joined string @@ -395,18 +401,17 @@ func (g *Commands) changeSlice(clashesMap map[int][]*Change, id int, wg *sync.Wa } childChanges, childClashes, cErr := g.resolveChangeListRecv(clr) - if cErr == nil { - *cl = append(*cl, childChanges...) - continue - } - if cErr == ErrClashesDetected { + if len(childClashes) >= 1 { clashesMap[id] = childClashes - continue - } else if cErr != ErrPathNotExists { + } + + if cErr != nil && cErr != ErrPathNotExists { g.log.LogErrf("%s: %v\n", p, cErr) - break + continue } + + *cl = append(*cl, childChanges...) } }