Basic lockfile support

gxutil/get.go

@@ -43,49 +43,128 @@ func (pm *PM) GetPackageTo(hash, out string) (*Package, error) {
 		return nil, err
 	}
 
-	outtemp := out + ".part"
+	if err := pm.tryFetch(hash, out); err != nil {
+		return nil, err
+	}
+
+	err = FindPackageInDir(&pkg, out)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pkg, nil
+}
+
+func (pm *PM) CacheAndLinkPackage(ref, cacheloc, out string) error {
+	if err := pm.tryFetch(ref, cacheloc); err != nil {
+		return err
+	}
+
+	finfo, err := os.Lstat(out)
+	switch {
+	case err == nil:
+		if finfo.Mode()&os.ModeSymlink != 0 {
+			target, err := os.Readlink(out)
+			if err != nil {
+				return err
+			}
+
+			// expecting 'ref' to be '/ipfs/QmFoo/pkg'
+			if target != cacheloc {
+				panic("not handling dep changes yet")
+			}
+
+			// Link already exists
+			return nil
+		} else {
+			// TODO: should we force these to be links?
+			// we want to support people just cloning packages into place here, so how should we handle it here?
+			panic("not yet handling non-linked packages...")
+		}
+	case os.IsNotExist(err):
+		// ok
+	default:
+		return err
+	}
+
+	if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
+		return err
+	}
+
+	return os.Symlink(cacheloc, out)
+}
+
+func (pm *PM) tryFetch(hash, target string) error {
+	temp := target + ".part"
+
+	// check if already downloaded
+	_, err := os.Stat(target)
+	if err == nil {
+		stump.VLog("already fetched %s", target)
+		return nil
+	}
 
 	// check if a fetch was previously started and failed, cleanup if found
-	_, err = os.Stat(outtemp)
+	_, err = os.Stat(temp)
 	if err == nil {
 		stump.VLog("Found previously failed fetch, cleaning up...")
-		if err := os.RemoveAll(outtemp); err != nil {
+		if err := os.RemoveAll(temp); err != nil {
 			stump.Error("cleaning up previous aborted transfer: %s", err)
 		}
 	}
 
 	begin := time.Now()
 	stump.VLog("  - fetching %s via ipfs api", hash)
+	defer func() {
+		stump.VLog("  - fetch finished in %s", time.Since(begin))
+	}()
 	tries := 3
 	for i := 0; i < tries; i++ {
-		if err := pm.Shell().Get(hash, outtemp); err != nil {
+		if err := pm.Shell().Get(hash, temp); err != nil {
 			stump.Error("from shell.Get(): %v", err)
 
-			rmerr := os.RemoveAll(outtemp)
+			rmerr := os.RemoveAll(temp)
 			if rmerr != nil {
 				stump.Error("cleaning up temp download directory: %s", rmerr)
 			}
 
 			if i == tries-1 {
-				return nil, err
+				return err
 			}
 			stump.Log("retrying fetch %s after a second...", hash)
 			time.Sleep(time.Second)
 		} else {
-			if err := os.Rename(outtemp, out); err != nil {
-				return nil, err
-			}
-			break
+			/*
+				if err := chmodR(temp, 0444); err != nil {
+					return err
+				}
+			*/
+			return os.Rename(temp, target)
 		}
 	}
-	stump.VLog("  - fetch finished in %s", time.Since(begin))
+	panic("unreachable")
+}
 
-	err = FindPackageInDir(&pkg, out)
-	if err != nil {
-		return nil, err
-	}
+func chmodR(dir string, perm os.FileMode) error {
+	return filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
+		if p == dir {
+			return nil
+		}
 
-	return &pkg, nil
+		if err == nil {
+			if info.Mode()&os.ModeSymlink != 0 {
+				return nil
+			}
+
+			perm := perm
+			if info.IsDir() {
+				perm |= 0111
+			}
+
+			return os.Chmod(p, perm)
+		}
+		return nil
+	})
 }
 
 func FindPackageInDir(pkg interface{}, dir string) error {

gxutil/lckfile.go

@@ -0,0 +1,38 @@
+package gxutil
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+)
+
+const LockVersion = 1
+
+type LockFile struct {
+	Lock
+	LockVersion int `json:"lockVersion"`
+}
+
+type Lock struct {
+	Language string `json:"language,omitempty"`
+
+	Ref  string                     `json:"ref,omitempty"`
+	Deps map[string]map[string]Lock `json:"deps,omitempty"`
+}
+
+func LoadLockFile(lck *LockFile, fname string) error {
+	fi, err := os.Open(fname)
+	if err != nil {
+		return err
+	}
+
+	if err := json.NewDecoder(fi).Decode(lck); err != nil {
+		return err
+	}
+
+	if lck.LockVersion != LockVersion {
+		return fmt.Errorf("unsupported lockfile version: %d", lck.LockVersion)
+	}
+
+	return nil
+}

gxutil/pm.go

@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
 	"time"
 
 	sh "github.com/ipfs/go-ipfs-api"
@@ -22,6 +23,7 @@ import (
 const GxVersion = "0.12.1"
 
 const PkgFileName = "package.json"
+const LckFileName = "gx-lock.json"
 
 var installPathsCache map[string]string
 var binarySuffix string
@@ -150,6 +152,108 @@ func isTempError(err error) bool {
 	return strings.Contains(err.Error(), "too many open files")
 }
 
+type DepWork struct {
+	CacheDir string
+	LinkDir  string
+	Dep      string
+	Ref      string
+}
+
+// InstallLock recursively installs all dependencies for the given lockfile
+func (pm *PM) InstallLock(lck Lock, cwd string) error {
+	lockList := []Lock{lck}
+
+	maxWorkers := 20
+	workers := make(chan DepWork, maxWorkers)
+
+	var wg sync.WaitGroup
+	var lk sync.Mutex
+	var firstError error
+
+	for i := 0; i < maxWorkers; i++ {
+		wg.Add(1)
+		go func() {
+			for work := range workers {
+				pm.ProgMeter.AddEntry(work.Ref, work.Dep, "[fetch]   <ELAPSED>"+work.Ref)
+
+				cacheloc := filepath.Join(work.CacheDir, work.Ref)
+				linkloc := filepath.Join(work.LinkDir, work.Dep)
+
+				if err := pm.CacheAndLinkPackage(work.Ref, cacheloc, linkloc); err != nil {
+					pm.ProgMeter.Error(work.Ref, err.Error())
+
+					lk.Lock()
+					if firstError == nil {
+						firstError = err
+					}
+					lk.Unlock()
+
+					continue
+				}
+
+				pm.ProgMeter.Finish(work.Ref)
+			}
+
+			wg.Done()
+		}()
+	}
+
+	for {
+		if len(lockList) == 0 {
+			break
+		}
+
+		curr := lockList[0]
+		lockList = lockList[1:]
+
+		newLocks, err := pm.installLock(curr, cwd, workers)
+		if err != nil {
+			return err
+		}
+
+		if firstError == nil {
+			lockList = append(lockList, newLocks...)
+		}
+	}
+
+	close(workers)
+	wg.Wait()
+
+	return firstError
+}
+
+func (pm *PM) installLock(lck Lock, cwd string, workers chan<- DepWork) ([]Lock, error) {
+	// Install all the direct dependencies for this lock
+
+	// Each lock contains a mapping of languages to their own dependencies
+	returnList := []Lock{}
+
+	for lang, langdeps := range lck.Deps {
+		ipath, err := InstallPath(lang, cwd, false)
+		if err != nil {
+			return []Lock{}, err
+		}
+
+		pm.ProgMeter.AddTodos(len(langdeps))
+
+		for dep, deplock := range langdeps {
+			if deplock.Deps != nil {
+				returnList = append(returnList, deplock)
+			}
+
+			workers <- DepWork{
+				CacheDir: filepath.Join(cwd, ".gx", "cache"),
+				LinkDir:  ipath,
+				Dep:      dep,
+				Ref:      deplock.Ref,
+			}
+
+		}
+	}
+
+	return returnList, nil
+}
+
 func (pm *PM) SetProgMeter(meter *prog.ProgMeter) {
 	pm.ProgMeter = meter
 }

main.go

@@ -111,6 +111,7 @@ func main() {
 		DiffCommand,
 		InitCommand,
 		InstallCommand,
+		LockInstallCommand,
 		PublishCommand,
 		ReleaseCommand,
 		RepoCommand,
@@ -822,6 +823,36 @@ var depCheckCommand = cli.Command{
 	},
 }
 
+var LockInstallCommand = cli.Command{
+	Name:  "lock-install",
+	Usage: "Install deps from lockfile into vendor",
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "nofancy",
+			Usage: "write minimal output",
+		},
+	},
+	Action: func(c *cli.Context) error {
+		cwd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+
+		var lck gx.LockFile
+		if err := gx.LoadLockFile(&lck, filepath.Join(cwd, gx.LckFileName)); err != nil {
+			return err
+		}
+
+		pm.ProgMeter = progmeter.NewProgMeter(c.Bool("nofancy"))
+
+		if err := pm.InstallLock(lck.Lock, cwd); err != nil {
+			return fmt.Errorf("install deps: %s", err)
+		}
+
+		return nil
+	},
+}
+
 var CleanCommand = cli.Command{
 	Name:  "clean",
 	Usage: "cleanup unused packages in vendor directory",