feat(helm): implement new index format

This implements a new index file format for repository indices. It also
implements a new format for requirements.yaml.

Breaking change: This will break all previous versions of Helm, and will
impact helm search, repo, serve, and fetch functions.

Closes helm#1197

Makefile

@@ -62,7 +62,7 @@ test: test-unit
 
 .PHONY: test-unit
 test-unit:
-	$(GO) test $(GOFLAGS) -run $(TESTS) $(PKG) $(TESTFLAGS)
+	HELM_HOME=/no/such/dir $(GO) test $(GOFLAGS) -run $(TESTS) $(PKG) $(TESTFLAGS)
 
 .PHONY: test-style
 test-style:

cmd/helm/dependency_build_test.go

@@ -108,8 +108,12 @@ func TestDependencyBuildCmd(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if h := i.Entries["reqtest-0.1.0"].Digest; h != hash {
+	reqver := i.Entries["reqtest"][0]
+	if h := reqver.Digest; h != hash {
 		t.Errorf("Failed hash match: expected %s, got %s", hash, h)
 	}
+	if v := reqver.Version; v != "0.1.0" {
+		t.Errorf("mismatched versions. Expected %q, got %q", "0.1.0", v)
+	}
 
 }

cmd/helm/dependency_update_test.go

@@ -90,7 +90,8 @@ func TestDependencyUpdateCmd(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if h := i.Entries["reqtest-0.1.0"].Digest; h != hash {
+	reqver := i.Entries["reqtest"][0]
+	if h := reqver.Digest; h != hash {
 		t.Errorf("Failed hash match: expected %s, got %s", hash, h)
 	}
 

cmd/helm/downloader/chart_downloader.go

@@ -69,7 +69,7 @@ type ChartDownloader struct {
 // For VerifyNever and VerifyIfPossible, the Verification may be empty.
 func (c *ChartDownloader) DownloadTo(ref string, dest string) (*provenance.Verification, error) {
 	// resolve URL
-	u, err := c.ResolveChartRef(ref)
+	u, err := c.ResolveChartVersion(ref)
 	if err != nil {
 		return nil, err
 	}
@@ -111,18 +111,18 @@ func (c *ChartDownloader) DownloadTo(ref string, dest string) (*provenance.Verif
 	return ver, nil
 }
 
-// ResolveChartRef resolves a chart reference to a URL.
+// ResolveChartVersion resolves a chart reference to a URL.
 //
 // A reference may be an HTTP URL, a 'reponame/chartname' reference, or a local path.
-func (c *ChartDownloader) ResolveChartRef(ref string) (*url.URL, error) {
+func (c *ChartDownloader) ResolveChartVersion(ref string) (*url.URL, error) {
 	// See if it's already a full URL.
 	u, err := url.ParseRequestURI(ref)
 	if err == nil {
 		// If it has a scheme and host and path, it's a full URL
 		if u.IsAbs() && len(u.Host) > 0 && len(u.Path) > 0 {
 			return u, nil
 		}
-		return u, fmt.Errorf("Invalid chart url format: %s", ref)
+		return u, fmt.Errorf("invalid chart url format: %s", ref)
 	}
 
 	r, err := repo.LoadRepositoriesFile(c.HelmHome.RepositoryFile())
@@ -133,15 +133,29 @@ func (c *ChartDownloader) ResolveChartRef(ref string) (*url.URL, error) {
 	// See if it's of the form: repo/path_to_chart
 	p := strings.Split(ref, "/")
 	if len(p) > 1 {
-		if baseURL, ok := r.Repositories[p[0]]; ok {
-			if !strings.HasSuffix(baseURL, "/") {
-				baseURL = baseURL + "/"
-			}
-			return url.ParseRequestURI(baseURL + strings.Join(p[1:], "/"))
+		rf, err := findRepoEntry(p[0], r.Repositories)
+		if err != nil {
+			return u, err
+		}
+		if rf.URL == "" {
+			return u, fmt.Errorf("no URL found for repository %q", p[0])
+		}
+		baseURL := rf.URL
+		if !strings.HasSuffix(baseURL, "/") {
+			baseURL = baseURL + "/"
+		}
+		return url.ParseRequestURI(baseURL + strings.Join(p[1:], "/"))
+	}
+	return u, fmt.Errorf("invalid chart url format: %s", ref)
+}
+
+func findRepoEntry(name string, repos []*repo.Entry) (*repo.Entry, error) {
+	for _, re := range repos {
+		if re.Name == name {
+			return re, nil
 		}
-		return u, fmt.Errorf("No such repo: %s", p[0])
 	}
-	return u, fmt.Errorf("Invalid chart url format: %s", ref)
+	return nil, fmt.Errorf("no repo named %q", name)
 }
 
 // VerifyChart takes a path to a chart archive and a keyring, and verifies the chart.

cmd/helm/downloader/chart_downloader_test.go

@@ -47,7 +47,7 @@ func TestResolveChartRef(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		u, err := c.ResolveChartRef(tt.ref)
+		u, err := c.ResolveChartVersion(tt.ref)
 		if err != nil {
 			if tt.fail {
 				continue

cmd/helm/downloader/manager.go

@@ -175,8 +175,7 @@ func (m *Manager) downloadAll(deps []*chartutil.Dependency) error {
 	for _, dep := range deps {
 		fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository)
 
-		target := fmt.Sprintf("%s-%s", dep.Name, dep.Version)
-		churl, err := findChartURL(target, dep.Repository, repos)
+		churl, err := findChartURL(dep.Name, dep.Repository, repos)
 		if err != nil {
 			fmt.Fprintf(m.Out, "WARNING: %s (skipped)", err)
 			continue
@@ -207,7 +206,7 @@ func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error {
 			found = true
 		} else {
 			for _, repo := range repos {
-				if urlsAreEqual(repo, dd.Repository) {
+				if urlsAreEqual(repo.URL, dd.Repository) {
 					found = true
 				}
 			}
@@ -236,25 +235,23 @@ func (m *Manager) UpdateRepositories() error {
 	return nil
 }
 
-func (m *Manager) parallelRepoUpdate(repos map[string]string) {
+func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) {
 	out := m.Out
 	fmt.Fprintln(out, "Hang tight while we grab the latest from your chart repositories...")
 	var wg sync.WaitGroup
-	for name, url := range repos {
+	for _, re := range repos {
 		wg.Add(1)
 		go func(n, u string) {
-			err := repo.DownloadIndexFile(n, u, m.HelmHome.CacheIndex(n))
-			if err != nil {
-				updateErr := fmt.Sprintf("...Unable to get an update from the %q chart repository: %s", n, err)
-				fmt.Fprintln(out, updateErr)
+			if err := repo.DownloadIndexFile(n, u, m.HelmHome.CacheIndex(n)); err != nil {
+				fmt.Fprintf(out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", n, u, err)
 			} else {
 				fmt.Fprintf(out, "...Successfully got an update from the %q chart repository\n", n)
 			}
 			wg.Done()
-		}(name, url)
+		}(re.Name, re.URL)
 	}
 	wg.Wait()
-	fmt.Fprintln(out, "Update Complete. Happy Helming!")
+	fmt.Fprintln(out, "Update Complete. ⎈Happy Helming!⎈")
 }
 
 // urlsAreEqual normalizes two URLs and then compares for equality.
@@ -280,7 +277,15 @@ func findChartURL(name, repourl string, repos map[string]*repo.ChartRepository)
 		if urlsAreEqual(repourl, cr.URL) {
 			for ename, entry := range cr.IndexFile.Entries {
 				if ename == name {
-					return entry.URL, nil
+					for _, verEntry := range entry {
+						if len(verEntry.URLs) == 0 {
+							// Not totally sure what to do here. Returning an
+							// error is the strictest option. Skipping it might
+							// be preferable.
+							return "", fmt.Errorf("chart %q has no download URL", name)
+						}
+						return verEntry.URLs[0], nil
+					}
 				}
 			}
 		}
@@ -302,16 +307,16 @@ func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, err
 		return indices, fmt.Errorf("failed to load %s: %s", repoyaml, err)
 	}
 
-	// localName: chartRepo
-	for lname, url := range rf.Repositories {
+	for _, re := range rf.Repositories {
+		lname := re.Name
 		cacheindex := m.HelmHome.CacheIndex(lname)
 		index, err := repo.LoadIndexFile(cacheindex)
 		if err != nil {
 			return indices, err
 		}
 
 		cr := &repo.ChartRepository{
-			URL:       url,
+			URL:       re.URL,
 			IndexFile: index,
 		}
 		indices[lname] = cr

cmd/helm/downloader/testdata/helmhome/repository/cache/kubernetes-charts-index.yaml

@@ -1,38 +1,46 @@
-alpine-0.1.0:
-  name: alpine
-  url: http://storage.googleapis.com/kubernetes-charts/alpine-0.1.0.tgz
-  created: 2016-09-06 21:58:44.211261566 +0000 UTC
-  checksum: 0e6661f193211d7a5206918d42f5c2a9470b737d
-  chartfile:
-    name: alpine
-    home: https://k8s.io/helm
-    sources:
-    - https://github.com/kubernetes/helm
-    version: 0.1.0
-    description: Deploy a basic Alpine Linux pod
-    keywords: []
-    maintainers: []
-    engine: ""
-    icon: ""
-mariadb-0.3.0:
-  name: mariadb
-  url: http://storage.googleapis.com/kubernetes-charts/mariadb-0.3.0.tgz
-  created: 2016-09-06 21:58:44.211870222 +0000 UTC
-  checksum: 65229f6de44a2be9f215d11dbff311673fc8ba56
-  chartfile:
-    name: mariadb
-    home: https://mariadb.org
-    sources:
-    - https://github.com/bitnami/bitnami-docker-mariadb
-    version: 0.3.0
-    description: Chart for MariaDB
-    keywords:
-    - mariadb
-    - mysql
-    - database
-    - sql
-    maintainers:
-    - name: Bitnami
-      email: containers@bitnami.com
-    engine: gotpl
-    icon: ""
+apiVersion: v1
+entries:
+  alpine:
+    - name: alpine
+      url: http://storage.googleapis.com/kubernetes-charts/alpine-0.1.0.tgz
+      checksum: 0e6661f193211d7a5206918d42f5c2a9470b737d
+      home: https://k8s.io/helm
+      sources:
+      - https://github.com/kubernetes/helm
+      version: 0.1.0
+      description: Deploy a basic Alpine Linux pod
+      keywords: []
+      maintainers: []
+      engine: ""
+      icon: ""
+    - name: alpine
+      url: http://storage.googleapis.com/kubernetes-charts/alpine-0.2.0.tgz
+      checksum: 0e6661f193211d7a5206918d42f5c2a9470b737d
+      home: https://k8s.io/helm
+      sources:
+      - https://github.com/kubernetes/helm
+      version: 0.2.0
+      description: Deploy a basic Alpine Linux pod
+      keywords: []
+      maintainers: []
+      engine: ""
+      icon: ""
+  mariadb:
+    - name: mariadb
+      url: http://storage.googleapis.com/kubernetes-charts/mariadb-0.3.0.tgz
+      checksum: 65229f6de44a2be9f215d11dbff311673fc8ba56
+      home: https://mariadb.org
+      sources:
+      - https://github.com/bitnami/bitnami-docker-mariadb
+      version: 0.3.0
+      description: Chart for MariaDB
+      keywords:
+      - mariadb
+      - mysql
+      - database
+      - sql
+      maintainers:
+      - name: Bitnami
+        email: containers@bitnami.com
+      engine: gotpl
+      icon: ""

cmd/helm/downloader/testdata/helmhome/repository/repositories.yaml

@@ -1 +1,4 @@
-testing: "http://example.com"
+apiVersion: v1
+repositories:
+  - name: testing
+    url: "http://example.com"

cmd/helm/helm_test.go

@@ -22,17 +22,20 @@ import (
 	"io"
 	"io/ioutil"
 	"math/rand"
+	"os"
 	"regexp"
 	"testing"
 
 	"github.com/golang/protobuf/ptypes/timestamp"
 	"github.com/spf13/cobra"
 
+	"k8s.io/helm/cmd/helm/helmpath"
 	"k8s.io/helm/pkg/helm"
 	"k8s.io/helm/pkg/proto/hapi/chart"
 	"k8s.io/helm/pkg/proto/hapi/release"
 	rls "k8s.io/helm/pkg/proto/hapi/services"
 	"k8s.io/helm/pkg/proto/hapi/version"
+	"k8s.io/helm/pkg/repo"
 )
 
 var mockHookTemplate = `apiVersion: v1
@@ -211,7 +214,7 @@ type releaseCase struct {
 	resp     *release.Release
 }
 
-// tmpHelmHome sets up a Helm Home in a temp dir.
+// tempHelmHome sets up a Helm Home in a temp dir.
 //
 // This does not clean up the directory. You must do that yourself.
 // You  must also set helmHome yourself.
@@ -223,9 +226,59 @@ func tempHelmHome() (string, error) {
 	}
 
 	helmHome = dir
-	if err := ensureHome(); err != nil {
+	if err := ensureTestHome(helmpath.Home(helmHome)); err != nil {
 		return "n/", err
 	}
 	helmHome = oldhome
 	return dir, nil
 }
+
+// ensureTestHome creates a home directory like ensureHome, but without remote references.
+func ensureTestHome(home helmpath.Home) error {
+	configDirectories := []string{home.String(), home.Repository(), home.Cache(), home.LocalRepository()}
+	for _, p := range configDirectories {
+		if fi, err := os.Stat(p); err != nil {
+			fmt.Printf("Creating %s \n", p)
+			if err := os.MkdirAll(p, 0755); err != nil {
+				return fmt.Errorf("Could not create %s: %s", p, err)
+			}
+		} else if !fi.IsDir() {
+			return fmt.Errorf("%s must be a directory", p)
+		}
+	}
+
+	repoFile := home.RepositoryFile()
+	if fi, err := os.Stat(repoFile); err != nil {
+		fmt.Printf("Creating %s \n", repoFile)
+		rf := repo.NewRepoFile()
+		if err := rf.WriteFile(repoFile, 0644); err != nil {
+			return err
+		}
+	} else if fi.IsDir() {
+		return fmt.Errorf("%s must be a file, not a directory", repoFile)
+	}
+	if r, err := repo.LoadRepositoriesFile(repoFile); err == repo.ErrRepoOutOfDate {
+		fmt.Println("Updating repository file format...")
+		if err := r.WriteFile(repoFile, 0644); err != nil {
+			return err
+		}
+		fmt.Println("Done")
+	}
+
+	localRepoIndexFile := home.LocalRepository(localRepoIndexFilePath)
+	if fi, err := os.Stat(localRepoIndexFile); err != nil {
+		fmt.Printf("Creating %s \n", localRepoIndexFile)
+		i := repo.NewIndexFile()
+		if err := i.WriteFile(localRepoIndexFile, 0644); err != nil {
+			return err
+		}
+
+		//TODO: take this out and replace with helm update functionality
+		os.Symlink(localRepoIndexFile, cacheDirectory("local-index.yaml"))
+	} else if fi.IsDir() {
+		return fmt.Errorf("%s must be a file, not a directory", localRepoIndexFile)
+	}
+
+	fmt.Printf("$HELM_HOME has been configured at %s.\n", helmHome)
+	return nil
+}

cmd/helm/helmpath/helmhome.go

@@ -56,6 +56,9 @@ func (h Home) CacheIndex(name string) string {
 // LocalRepository returns the location to the local repo.
 //
 // The local repo is the one used by 'helm serve'
-func (h Home) LocalRepository() string {
-	return filepath.Join(string(h), "repository/local")
+//
+// If additional path elements are passed, they are appended to the returned path.
+func (h Home) LocalRepository(paths ...string) string {
+	frag := append([]string{string(h), "repository/local"}, paths...)
+	return filepath.Join(frag...)
 }