diff --git a/gxutil/get.go b/gxutil/get.go index a924f99..f20a04f 100644 --- a/gxutil/get.go +++ b/gxutil/get.go @@ -43,49 +43,128 @@ func (pm *PM) GetPackageTo(hash, out string) (*Package, error) { return nil, err } - outtemp := out + ".part" + if err := pm.tryFetch(hash, out); err != nil { + return nil, err + } + + err = FindPackageInDir(&pkg, out) + if err != nil { + return nil, err + } + + return &pkg, nil +} + +func (pm *PM) CacheAndLinkPackage(ref, cacheloc, out string) error { + if err := pm.tryFetch(ref, cacheloc); err != nil { + return err + } + + finfo, err := os.Lstat(out) + switch { + case err == nil: + if finfo.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(out) + if err != nil { + return err + } + + // expecting 'ref' to be '/ipfs/QmFoo/pkg' + if target != cacheloc { + panic("not handling dep changes yet") + } + + // Link already exists + return nil + } else { + // TODO: should we force these to be links? + // we want to support people just cloning packages into place here, so how should we handle it here? + panic("not yet handling non-linked packages...") + } + case os.IsNotExist(err): + // ok + default: + return err + } + + if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil { + return err + } + + return os.Symlink(cacheloc, out) +} + +func (pm *PM) tryFetch(hash, target string) error { + temp := target + ".part" + + // check if already downloaded + _, err := os.Stat(target) + if err == nil { + stump.VLog("already fetched %s", target) + return nil + } // check if a fetch was previously started and failed, cleanup if found - _, err = os.Stat(outtemp) + _, err = os.Stat(temp) if err == nil { stump.VLog("Found previously failed fetch, cleaning up...") - if err := os.RemoveAll(outtemp); err != nil { + if err := os.RemoveAll(temp); err != nil { stump.Error("cleaning up previous aborted transfer: %s", err) } } begin := time.Now() stump.VLog(" - fetching %s via ipfs api", hash) + defer func() { + stump.VLog(" - fetch finished in %s", time.Since(begin)) + }() tries := 3 for i := 0; i < tries; i++ { - if err := pm.Shell().Get(hash, outtemp); err != nil { + if err := pm.Shell().Get(hash, temp); err != nil { stump.Error("from shell.Get(): %v", err) - rmerr := os.RemoveAll(outtemp) + rmerr := os.RemoveAll(temp) if rmerr != nil { stump.Error("cleaning up temp download directory: %s", rmerr) } if i == tries-1 { - return nil, err + return err } stump.Log("retrying fetch %s after a second...", hash) time.Sleep(time.Second) } else { - if err := os.Rename(outtemp, out); err != nil { - return nil, err - } - break + /* + if err := chmodR(temp, 0444); err != nil { + return err + } + */ + return os.Rename(temp, target) } } - stump.VLog(" - fetch finished in %s", time.Since(begin)) + panic("unreachable") +} - err = FindPackageInDir(&pkg, out) - if err != nil { - return nil, err - } +func chmodR(dir string, perm os.FileMode) error { + return filepath.Walk(dir, func(p string, info os.FileInfo, err error) error { + if p == dir { + return nil + } - return &pkg, nil + if err == nil { + if info.Mode()&os.ModeSymlink != 0 { + return nil + } + + perm := perm + if info.IsDir() { + perm |= 0111 + } + + return os.Chmod(p, perm) + } + return nil + }) } func FindPackageInDir(pkg interface{}, dir string) error { diff --git a/gxutil/lckfile.go b/gxutil/lckfile.go new file mode 100644 index 0000000..a96690e --- /dev/null +++ b/gxutil/lckfile.go @@ -0,0 +1,38 @@ +package gxutil + +import ( + "encoding/json" + "fmt" + "os" +) + +const LockVersion = 1 + +type LockFile struct { + Lock + LockVersion int `json:"lockVersion"` +} + +type Lock struct { + Language string `json:"language,omitempty"` + + Ref string `json:"ref,omitempty"` + Deps map[string]map[string]Lock `json:"deps,omitempty"` +} + +func LoadLockFile(lck *LockFile, fname string) error { + fi, err := os.Open(fname) + if err != nil { + return err + } + + if err := json.NewDecoder(fi).Decode(lck); err != nil { + return err + } + + if lck.LockVersion != LockVersion { + return fmt.Errorf("unsupported lockfile version: %d", lck.LockVersion) + } + + return nil +} diff --git a/gxutil/pm.go b/gxutil/pm.go index 2c99ba2..a21337d 100644 --- a/gxutil/pm.go +++ b/gxutil/pm.go @@ -11,6 +11,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "time" sh "github.com/ipfs/go-ipfs-api" @@ -22,6 +23,7 @@ import ( const GxVersion = "0.12.1" const PkgFileName = "package.json" +const LckFileName = "gx-lock.json" var installPathsCache map[string]string var binarySuffix string @@ -150,6 +152,108 @@ func isTempError(err error) bool { return strings.Contains(err.Error(), "too many open files") } +type DepWork struct { + CacheDir string + LinkDir string + Dep string + Ref string +} + +// InstallLock recursively installs all dependencies for the given lockfile +func (pm *PM) InstallLock(lck Lock, cwd string) error { + lockList := []Lock{lck} + + maxWorkers := 20 + workers := make(chan DepWork, maxWorkers) + + var wg sync.WaitGroup + var lk sync.Mutex + var firstError error + + for i := 0; i < maxWorkers; i++ { + wg.Add(1) + go func() { + for work := range workers { + pm.ProgMeter.AddEntry(work.Ref, work.Dep, "[fetch] "+work.Ref) + + cacheloc := filepath.Join(work.CacheDir, work.Ref) + linkloc := filepath.Join(work.LinkDir, work.Dep) + + if err := pm.CacheAndLinkPackage(work.Ref, cacheloc, linkloc); err != nil { + pm.ProgMeter.Error(work.Ref, err.Error()) + + lk.Lock() + if firstError == nil { + firstError = err + } + lk.Unlock() + + continue + } + + pm.ProgMeter.Finish(work.Ref) + } + + wg.Done() + }() + } + + for { + if len(lockList) == 0 { + break + } + + curr := lockList[0] + lockList = lockList[1:] + + newLocks, err := pm.installLock(curr, cwd, workers) + if err != nil { + return err + } + + if firstError == nil { + lockList = append(lockList, newLocks...) + } + } + + close(workers) + wg.Wait() + + return firstError +} + +func (pm *PM) installLock(lck Lock, cwd string, workers chan<- DepWork) ([]Lock, error) { + // Install all the direct dependencies for this lock + + // Each lock contains a mapping of languages to their own dependencies + returnList := []Lock{} + + for lang, langdeps := range lck.Deps { + ipath, err := InstallPath(lang, cwd, false) + if err != nil { + return []Lock{}, err + } + + pm.ProgMeter.AddTodos(len(langdeps)) + + for dep, deplock := range langdeps { + if deplock.Deps != nil { + returnList = append(returnList, deplock) + } + + workers <- DepWork{ + CacheDir: filepath.Join(cwd, ".gx", "cache"), + LinkDir: ipath, + Dep: dep, + Ref: deplock.Ref, + } + + } + } + + return returnList, nil +} + func (pm *PM) SetProgMeter(meter *prog.ProgMeter) { pm.ProgMeter = meter } diff --git a/main.go b/main.go index bd3a0e3..ab04ef4 100644 --- a/main.go +++ b/main.go @@ -111,6 +111,7 @@ func main() { DiffCommand, InitCommand, InstallCommand, + LockInstallCommand, PublishCommand, ReleaseCommand, RepoCommand, @@ -822,6 +823,36 @@ var depCheckCommand = cli.Command{ }, } +var LockInstallCommand = cli.Command{ + Name: "lock-install", + Usage: "Install deps from lockfile into vendor", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "nofancy", + Usage: "write minimal output", + }, + }, + Action: func(c *cli.Context) error { + cwd, err := os.Getwd() + if err != nil { + return err + } + + var lck gx.LockFile + if err := gx.LoadLockFile(&lck, filepath.Join(cwd, gx.LckFileName)); err != nil { + return err + } + + pm.ProgMeter = progmeter.NewProgMeter(c.Bool("nofancy")) + + if err := pm.InstallLock(lck.Lock, cwd); err != nil { + return fmt.Errorf("install deps: %s", err) + } + + return nil + }, +} + var CleanCommand = cli.Command{ Name: "clean", Usage: "cleanup unused packages in vendor directory",