From 99b8bfd38b05617e68e2c92981b3f45605c43715 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Thu, 24 Dec 2015 07:17:57 -0700 Subject: [PATCH] find*Multi version to find all clashes and matches --- src/changes.go | 30 +++++++++---- src/diff.go | 19 ++++++++- src/misc.go | 1 + src/pull.go | 34 +++++++-------- src/push.go | 61 ++++++++++++++++---------- src/remote.go | 114 ++++++++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 205 insertions(+), 54 deletions(-) diff --git a/src/changes.go b/src/changes.go index 887a7ef8..c8fc87d5 100644 --- a/src/changes.go +++ b/src/changes.go @@ -140,19 +140,33 @@ func (g *Commands) byRemoteResolve(relToRoot, fsPath string, r *File, push bool) } func (g *Commands) changeListResolve(relToRoot, fsPath string, push bool) (cl, clashes []*Change, err error) { - var r *File - resolver := g.rem.FindByPath - r, err = resolver(relToRoot) - if err != nil && err != ErrPathNotExists { - return + remotesChan := g.rem.FindByPathM(relToRoot) + iterCount := uint64(0) + noClashThreshold := uint64(1) + + for rem := range remotesChan { + if rem != nil && anyMatch(g.opts.IgnoreRegexp, rem.Name) { + return + } + + iterCount++ + + ccl, cclashes, cErr := g.byRemoteResolve(relToRoot, fsPath, rem, push) + + cl = append(cl, ccl...) + clashes = append(clashes, cclashes...) + if false && cErr != nil { + err = reComposeError(err, cErr.Error()) + } } - if r != nil && anyMatch(g.opts.IgnoreRegexp, r.Name) { - return + if iterCount > noClashThreshold && len(clashes) < 1 { + clashes = append(clashes, cl...) + // err = reComposeError(err, ErrClashesDetected.Error()) } - return g.byRemoteResolve(relToRoot, fsPath, r, push) + return } func (g *Commands) doChangeListRecv(relToRoot, fsPath string, l, r *File, push bool) (cl, clashes []*Change, err error) { diff --git a/src/diff.go b/src/diff.go index 1ad55cbf..c68b9c4a 100644 --- a/src/diff.go +++ b/src/diff.go @@ -56,19 +56,34 @@ func (g *Commands) Diff() (err error) { spin.play() defer spin.stop() + clashes := []*Change{} for _, relToRootPath := range g.opts.Sources { fsPath := g.context.AbsPathOf(relToRootPath) - ccl, _, cErr := g.changeListResolve(relToRootPath, fsPath, true) + ccl, cclashes, cErr := g.changeListResolve(relToRootPath, fsPath, true) // TODO: Show the conflicts if any + clashes = append(clashes, cclashes...) if cErr != nil { - return cErr + if cErr != ErrClashesDetected { + return cErr + } else { + err = reComposeError(err, cErr.Error()) + } } + if len(ccl) > 0 { cl = append(cl, ccl...) } } + if !g.opts.IgnoreNameClashes && len(clashes) >= 1 { + warnClashesPersist(g.log, clashes) + if err != nil { + return err + } + return ErrClashesDetected + } + spin.stop() var diffUtilPath string diff --git a/src/misc.go b/src/misc.go index 9daf5279..5195ad9b 100644 --- a/src/misc.go +++ b/src/misc.go @@ -47,6 +47,7 @@ var ( const ( MimeTypeJoiner = "-" RemoteDriveRootPath = "My Drive" + RemoteSeparator = "/" FmtTimeString = "2006-01-02T15:04:05.000Z" MsgClashesFixedNowRetry = "Clashes were fixed, please retry the operation" diff --git a/src/pull.go b/src/pull.go index 8a1eac7a..0e0bfe67 100644 --- a/src/pull.go +++ b/src/pull.go @@ -342,25 +342,24 @@ func (g *Commands) pullLikeMatchesResolver(pt pullType) (cl, clashes []*Change, } func (g *Commands) PullPiped(byId bool) (err error) { - resolver := g.rem.FindByPath + resolver := g.rem.FindByPathM if byId { - resolver = g.rem.FindById + resolver = g.rem.FindByIdM } // TODO: (@odeke-em) allow pull-trashed for _, relToRootPath := range g.opts.Sources { - rem, err := resolver(relToRootPath) - if err != nil { - return fmt.Errorf("%s: %v", relToRootPath, err) - } - if rem == nil { - continue - } + matches := resolver(relToRootPath) + for rem := range matches { + if rem == nil { + continue + } - err = g.pullAndDownload(relToRootPath, os.Stdout, rem, true) - if err != nil { - return err + err = g.pullAndDownload(relToRootPath, os.Stdout, rem, true) + if err != nil { + return err + } } } return nil @@ -407,12 +406,9 @@ func (g *Commands) pullByPath() (cl, clashes []*Change, err error) { for _, relToRootPath := range g.opts.Sources { fsPath := g.context.AbsPathOf(relToRootPath) ccl, cclashes, cErr := g.changeListResolve(relToRootPath, fsPath, false) - if cErr != nil { - if cErr != ErrClashesDetected { - return cl, clashes, cErr - } else { - clashes = append(clashes, cclashes...) - } + clashes = append(clashes, cclashes...) + if cErr != nil && cErr != ErrClashesDetected { + return cl, clashes, cErr } if len(ccl) > 0 { cl = append(cl, ccl...) @@ -484,7 +480,7 @@ func (g *Commands) playPullChanges(cl []*Change, exports []string, opMap *map[Op fn := localOpToChangerTranslator(g, c) conformingFn := func(c *Change) error { - return fn(c, exports) + return fn(c, exports) } if fn == nil { diff --git a/src/push.go b/src/push.go index 962ab0b8..b5c7b152 100644 --- a/src/push.go +++ b/src/push.go @@ -41,13 +41,13 @@ func (g *Commands) Push() (err error) { var cl []*Change g.log.Logln("Resolving...") - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, os.Kill) spin := g.playabler() spin.play() // To Ensure mount points are cleared in the event of external exceptions + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, os.Kill) go func() { _ = <-c spin.stop() @@ -55,20 +55,16 @@ func (g *Commands) Push() (err error) { os.Exit(1) }() - // TODO: Look at clashes? clashes := []*Change{} for _, relToRootPath := range g.opts.Sources { fsPath := g.context.AbsPathOf(relToRootPath) ccl, cclashes, cErr := g.changeListResolve(relToRootPath, fsPath, true) - if cErr != nil { - if cErr == ErrClashesDetected { - clashes = append(clashes, cclashes...) - continue - } else { - spin.stop() - return cErr - } + + clashes = append(clashes, cclashes...) + if cErr != nil && cErr != ErrClashesDetected { + spin.stop() + return cErr } if len(ccl) > 0 { cl = append(cl, ccl...) @@ -330,27 +326,46 @@ func (g *Commands) playPushChanges(cl []*Change, opMap *map[Operation]sizeCounte } func lonePush(g *Commands, parent, absPath, path string) (cl, clashes []*Change, err error) { - r, err := g.rem.FindByPath(absPath) - if err != nil && err != ErrPathNotExists { - return - } + remotesChan := g.rem.FindByPathM(absPath) + iterCount := uint64(0) + noClashThreshold := uint64(1) var l *File localinfo, _ := os.Stat(path) if localinfo != nil { l = NewLocalFile(path, localinfo) } - clr := &changeListResolve{ - push: true, - dir: parent, - base: absPath, - remote: r, - local: l, - depth: g.opts.Depth, + for r := range remotesChan { + if r == nil { + continue + } + + iterCount++ + + clr := &changeListResolve{ + push: true, + dir: parent, + base: absPath, + remote: r, + local: l, + depth: g.opts.Depth, + } + + ccl, cclashes, cErr := g.resolveChangeListRecv(clr) + cl = append(cl, ccl...) + clashes = append(clashes, cclashes...) + if cErr != nil { + err = reComposeError(err, cErr.Error()) + } } - return g.resolveChangeListRecv(clr) + if iterCount > noClashThreshold && len(clashes) < 1 { + clashes = append(clashes, cl...) + err = reComposeError(err, ErrClashesDetected.Error()) + } + + return } func (g *Commands) pathSplitter(absPath string) (dir, base string) { diff --git a/src/remote.go b/src/remote.go index cfde1978..d4da279b 100644 --- a/src/remote.go +++ b/src/remote.go @@ -213,6 +213,22 @@ func (r *Remote) FindBackPaths(id string) (backPaths []string, err error) { return } +func wrapInChan(f *File, err error) chan *File { + ch := make(chan *File) + + go func() { + defer close(ch) + ch <- f + }() + + return ch +} + +func (r *Remote) FindByIdM(id string) chan *File { + f, err := r.FindById(id) + return wrapInChan(f, err) +} + func (r *Remote) FindById(id string) (file *File, err error) { req := r.service.Files.Get(id) var f *drive.File @@ -231,6 +247,20 @@ func retryableChangeOp(fn func() (interface{}, error), debug bool) *expb.Exponen } } +func (r *Remote) findByPathM(p string, trashed bool) chan *File { + if rootLike(p) { + return r.FindByIdM("root") + } + + parts := strings.Split(p, RemoteSeparator) + finder := r.findByPathRecvM + if trashed { + finder = r.findByPathTrashedM + } + + return finder("root", parts[1:]) +} + func (r *Remote) findByPath(p string, trashed bool) (*File, error) { if rootLike(p) { return r.FindById("root") @@ -247,10 +277,18 @@ func (r *Remote) FindByPath(p string) (file *File, err error) { return r.findByPath(p, false) } +func (r *Remote) FindByPathM(p string) chan *File { + return r.findByPathM(p, false) +} + func (r *Remote) FindByPathTrashed(p string) (file *File, err error) { return r.findByPath(p, true) } +func (r *Remote) FindByPathTrashedM(p string) chan *File { + return r.findByPathM(p, true) +} + func reqDoPage(req *drive.FilesListCall, hidden bool, promptOnPagination bool) chan *File { fileChan := make(chan *File) @@ -870,6 +908,70 @@ func (r *Remote) About() (about *drive.About, err error) { return r.service.About.Get().Do() } +func (r *Remote) findByPathRecvRawM(parentId string, p []string, trashed bool) chan *File { + + chanOChan := make(chan chan *File) + + go func() { + defer close(chanOChan) + if len(p) < 1 { + return + } + + first, rest := p[0], p[1:] + // find the file or directory under parentId and titled with p[0] + req := r.service.Files.List() + // TODO: use field selectors + var expr string + head := urlToPath(first, false) + if trashed { + expr = fmt.Sprintf("title = %s and trashed=true", customQuote(head)) + } else { + expr = fmt.Sprintf("%s in parents and title = %s and trashed=false", + customQuote(parentId), customQuote(head)) + } + req.Q(expr) + + resultsChan := reqDoPage(req, true, false) + if len(rest) < 1 { + chanOChan <- resultsChan + return + } + + /* + files, err := req.Do() + if err != nil { + if err.Error() == ErrGoogleApiInvalidQueryHardCoded.Error() { // Send the user back the query information + err = fmt.Errorf("err: %v query: `%s`", err, expr) + } + + return nil, err + } + */ + for f := range resultsChan { + if f == nil { + continue + } + + chanOChan <- r.findByPathRecvRawM(f.Id, p[1:], trashed) + } + }() + + resolvedResults := make(chan *File) + + go func() { + defer close(resolvedResults) + + for ch := range chanOChan { + for f := range ch { + resolvedResults <- f + } + } + }() + + return resolvedResults +} + func (r *Remote) findByPathRecvRaw(parentId string, p []string, trashed bool) (file *File, err error) { // find the file or directory under parentId and titled with p[0] req := r.service.Files.List() @@ -907,11 +1009,19 @@ func (r *Remote) findByPathRecvRaw(parentId string, p []string, trashed bool) (f return r.findByPathRecvRaw(first.Id, p[1:], trashed) } -func (r *Remote) findByPathRecv(parentId string, p []string) (file *File, err error) { +func (r *Remote) findByPathRecv(parentId string, p []string) (*File, error) { return r.findByPathRecvRaw(parentId, p, false) } -func (r *Remote) findByPathTrashed(parentId string, p []string) (file *File, err error) { +func (r *Remote) findByPathRecvM(parentId string, p []string) chan *File { + return r.findByPathRecvRawM(parentId, p, false) +} + +func (r *Remote) findByPathTrashedM(parentId string, p []string) chan *File { + return r.findByPathRecvRawM(parentId, p, true) +} + +func (r *Remote) findByPathTrashed(parentId string, p []string) (*File, error) { return r.findByPathRecvRaw(parentId, p, true) }