Handle yanked version in update

cmd/list.go

@@ -178,7 +178,7 @@ func showComponentList(env *environment.Environment, opt listOptions) (*listResu
 func showComponentVersions(env *environment.Environment, component string, opt listOptions) (*listResult, error) {
 	var comp *v1manifest.Component
 	var err error
-	comp, err = env.V1Repository().FetchComponentManifest(component)
+	comp, err = env.V1Repository().FetchComponentManifest(component, false)
 	if err != nil {
 		return nil, errors.Annotate(err, "failed to fetch component")
 	}

cmd/mirror.go

@@ -254,7 +254,7 @@ func newMirrorModifyCmd() *cobra.Command {
 			}
 
 			comp, ver := environment.ParseCompVersion(args[0])
-			m, err := env.V1Repository().FetchComponentManifest(comp)
+			m, err := env.V1Repository().FetchComponentManifest(comp, true)
 			if err != nil {
 				return err
 			}
@@ -334,7 +334,7 @@ func newMirrorPublishCmd() *cobra.Command {
 			cmd.Flags().Visit(func(f *pflag.Flag) {
 				flagSet.Insert(f.Name)
 			})
-			m, err := env.V1Repository().FetchComponentManifest(args[0])
+			m, err := env.V1Repository().FetchComponentManifest(args[0], true)
 			if err != nil {
 				fmt.Printf("Fetch local manifest: %s\n", err.Error())
 				fmt.Printf("Failed to load component manifest, create a new one\n")

components/playground/playground.go

@@ -642,7 +642,7 @@ func (p *Playground) bootCluster(env *environment.Environment, options *bootOpti
 	}
 
 	if options.version == "" {
-		version, _, err := env.V1Repository().LatestStableVersion("tidb")
+		version, _, err := env.V1Repository().LatestStableVersion("tidb", false)
 		if err != nil {
 			return err
 		}

pkg/exec/run.go

@@ -146,7 +146,7 @@ func PrepareCommand(
 	}
 
 	if version.IsEmpty() && len(checkUpdate) > 0 && checkUpdate[0] {
-		latestV, _, err := env.V1Repository().LatestStableVersion(component)
+		latestV, _, err := env.V1Repository().LatestStableVersion(component, true)
 		if err != nil {
 			return nil, err
 		}

pkg/repository/clone_mirror.go

@@ -247,7 +247,7 @@ func cloneComponents(repo *V1Repository,
 	options CloneOptions) (map[string]*v1manifest.Component, error) {
 	compManifests := map[string]*v1manifest.Component{}
 	for _, name := range components {
-		manifest, err := repo.FetchComponentManifest(name)
+		manifest, err := repo.FetchComponentManifest(name, true)
 		if err != nil {
 			return nil, errors.Annotatef(err, "fetch component '%s' manifest failed", name)
 		}

pkg/repository/v1_repository.go

@@ -103,9 +103,9 @@ func (r *V1Repository) UpdateComponents(specs []ComponentSpec) error {
 
 	var errs []string
 	for _, spec := range specs {
-		manifest, err := r.updateComponentManifest(spec.ID)
+		manifest, err := r.updateComponentManifest(spec.ID, false)
 		if err != nil {
-			if err == errUnknownComponent {
+			if errors.Cause(err) == errUnknownComponent {
 				fmt.Println(color.YellowString("The component `%s` not found (may be deleted from repository); skipped", spec.ID))
 			} else {
 				errs = append(errs, err.Error())
@@ -416,7 +416,7 @@ func (r *V1Repository) updateLocalIndex(snapshot *v1manifest.Snapshot) error {
 }
 
 // Precondition: the snapshot and index manifests exist and are up to date.
-func (r *V1Repository) updateComponentManifest(id string) (*v1manifest.Component, error) {
+func (r *V1Repository) updateComponentManifest(id string, withYanked bool) (*v1manifest.Component, error) {
 	defer func(start time.Time) {
 		verbose.Log("update component '%s' manifest finished in %s", id, time.Since(start))
 	}(time.Now())
@@ -427,9 +427,16 @@ func (r *V1Repository) updateComponentManifest(id string) (*v1manifest.Component
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
-	item, ok := index.Components[id]
+	var components map[string]v1manifest.ComponentItem
+	if withYanked {
+		components = index.ComponentListWithYanked()
+	} else {
+		components = index.ComponentList()
+	}
+
+	item, ok := components[id]
 	if !ok {
-		return nil, errUnknownComponent
+		return nil, errors.AddStack(errUnknownComponent)
 	}
 	var snapshot v1manifest.Snapshot
 	_, _, err = r.local.LoadManifest(&snapshot)
@@ -639,8 +646,8 @@ func (r *V1Repository) UpdateComponentManifests() error {
 	}
 
 	for name := range index.Components {
-		_, err = r.updateComponentManifest(name)
-		if err != nil {
+		_, err = r.updateComponentManifest(name, false)
+		if err != nil && errors.Cause(err) != errUnknownComponent {
 			return err
 		}
 	}
@@ -649,18 +656,18 @@ func (r *V1Repository) UpdateComponentManifests() error {
 }
 
 // FetchComponentManifest fetch the component manifest.
-func (r *V1Repository) FetchComponentManifest(id string) (com *v1manifest.Component, err error) {
+func (r *V1Repository) FetchComponentManifest(id string, withYanked bool) (com *v1manifest.Component, err error) {
 	err = r.ensureManifests()
 	if err != nil {
 		return nil, errors.AddStack(err)
 	}
 
-	return r.updateComponentManifest(id)
+	return r.updateComponentManifest(id, withYanked)
 }
 
 // ComponentVersion returns version item of a component
 func (r *V1Repository) ComponentVersion(id, version string, includeYanked bool) (*v1manifest.VersionItem, error) {
-	manifest, err := r.FetchComponentManifest(id)
+	manifest, err := r.FetchComponentManifest(id, includeYanked)
 	if err != nil {
 		return nil, err
 	}
@@ -675,8 +682,8 @@ func (r *V1Repository) ComponentVersion(id, version string, includeYanked bool)
 }
 
 // LatestStableVersion returns the latest stable version of specific component
-func (r *V1Repository) LatestStableVersion(id string) (v0manifest.Version, *v1manifest.VersionItem, error) {
-	com, err := r.FetchComponentManifest(id)
+func (r *V1Repository) LatestStableVersion(id string, withYanked bool) (v0manifest.Version, *v1manifest.VersionItem, error) {
+	com, err := r.FetchComponentManifest(id, withYanked)
 	if err != nil {
 		return "", nil, err
 	}
@@ -722,7 +729,7 @@ func (r *V1Repository) BinaryPath(installPath string, componentID string, versio
 		return "", err
 	}
 	if component == nil {
-		component, err = r.FetchComponentManifest(componentID)
+		component, err = r.FetchComponentManifest(componentID, true)
 		if err != nil {
 			return "", err
 		}

pkg/repository/v1_repository_test.go

@@ -25,6 +25,7 @@ import (
 
 	cjson "github.com/gibson042/canonicaljson-go"
 	"github.com/google/uuid"
+	"github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/localdata"
 	"github.com/pingcap/tiup/pkg/repository/crypto"
 	"github.com/pingcap/tiup/pkg/repository/v1manifest"
@@ -275,6 +276,33 @@ func TestUpdateIndex(t *testing.T) {
 	// TODO test that invalid signature of snapshot causes an error
 }
 
+func TestYanked(t *testing.T) {
+	mirror := MockMirror{
+		Resources: map[string]string{},
+	}
+	local := v1manifest.NewMockManifests()
+	_ = setNewRoot(t, local)
+	repo := NewV1Repo(&mirror, Options{}, local)
+
+	index, indexPriv := indexManifest(t)
+	snapshot := snapshotManifest()
+	bar := componentManifest()
+	serBar := serialize(t, bar, indexPriv)
+	mirror.Resources["/7.bar.json"] = serBar
+	local.Manifests[v1manifest.ManifestFilenameSnapshot] = &v1manifest.Manifest{Signed: snapshot}
+	local.Manifests[v1manifest.ManifestFilenameIndex] = &v1manifest.Manifest{Signed: index}
+
+	// Test yanked version
+	updated, err := repo.updateComponentManifest("bar", true)
+	assert.Nil(t, err)
+	_, ok := updated.VersionList("plat/form")["v2.0.3"]
+	assert.False(t, ok)
+
+	_, err = repo.updateComponentManifest("bar", false)
+	assert.NotNil(t, err)
+	assert.Equal(t, errors.Cause(err), errUnknownComponent)
+}
+
 func TestUpdateComponent(t *testing.T) {
 	mirror := MockMirror{
 		Resources: map[string]string{},
@@ -292,7 +320,7 @@ func TestUpdateComponent(t *testing.T) {
 	local.Manifests[v1manifest.ManifestFilenameIndex] = &v1manifest.Manifest{Signed: index}
 
 	// Test happy path
-	updated, err := repo.updateComponentManifest("foo")
+	updated, err := repo.updateComponentManifest("foo", false)
 	assert.Nil(t, err)
 	t.Logf("%+v", err)
 	assert.NotNil(t, updated)
@@ -303,13 +331,13 @@ func TestUpdateComponent(t *testing.T) {
 	oldFoo.Version = 8
 	local.Manifests["foo.json"] = &v1manifest.Manifest{Signed: oldFoo}
 	local.Saved = []string{}
-	updated, err = repo.updateComponentManifest("foo")
+	updated, err = repo.updateComponentManifest("foo", false)
 	assert.NotNil(t, err)
 	assert.Nil(t, updated)
 	assert.Empty(t, local.Saved)
 
 	// Test that id missing from index causes an error
-	updated, err = repo.updateComponentManifest("bar")
+	updated, err = repo.updateComponentManifest("bar", false)
 	assert.NotNil(t, err)
 	assert.Nil(t, updated)
 	assert.Empty(t, local.Saved)
@@ -598,6 +626,7 @@ func snapshotManifest() *v1manifest.Snapshot {
 			v1manifest.ManifestURLRoot:  {Version: 42},
 			v1manifest.ManifestURLIndex: {Version: 5},
 			"/foo.json":                 {Version: 7},
+			"/bar.json":                 {Version: 7},
 		},
 	}
 }
@@ -613,7 +642,10 @@ func componentManifest() *v1manifest.Component {
 		ID:          "foo",
 		Description: "foo does stuff",
 		Platforms: map[string]map[string]v1manifest.VersionItem{
-			"plat/form": {"v2.0.1": versionItem()},
+			"plat/form": {
+				"v2.0.1": versionItem(),
+				"v2.0.3": versionItem3(),
+			},
 		},
 	}
 }
@@ -638,6 +670,17 @@ func versionItem2() v1manifest.VersionItem {
 	}
 }
 
+func versionItem3() v1manifest.VersionItem {
+	return v1manifest.VersionItem{
+		URL:    "/foo-2.0.3.tar.gz",
+		Yanked: true,
+		FileHash: v1manifest.FileHash{
+			Hashes: map[string]string{v1manifest.SHA256: "5abe91bc22039c15c05580062357be7ab0bfd7968582a118fbb4eb817ddc2e76"},
+			Length: 12,
+		},
+	}
+}
+
 func indexManifest(t *testing.T) (*v1manifest.Index, crypto.PrivKey) {
 	info, keyID, priv, err := v1manifest.FreshKeyInfo()
 	assert.Nil(t, err)
@@ -661,11 +704,18 @@ func indexManifest(t *testing.T) (*v1manifest.Index, crypto.PrivKey) {
 			Keys:      map[string]*v1manifest.KeyInfo{keyID: info},
 			Threshold: 1,
 		}},
-		Components: map[string]v1manifest.ComponentItem{"foo": {
-			Yanked: false,
-			Owner:  "bar",
-			URL:    "/foo.json",
-		}},
+		Components: map[string]v1manifest.ComponentItem{
+			"foo": {
+				Yanked: false,
+				Owner:  "bar",
+				URL:    "/foo.json",
+			},
+			"bar": {
+				Yanked: true,
+				Owner:  "bar",
+				URL:    "/bar.json",
+			},
+		},
 		DefaultComponents: []string{},
 	}, priv
 }