diff --git a/cmd/list.go b/cmd/list.go index 6619e039b3..3ffc8e12f9 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -178,7 +178,7 @@ func showComponentList(env *environment.Environment, opt listOptions) (*listResu func showComponentVersions(env *environment.Environment, component string, opt listOptions) (*listResult, error) { var comp *v1manifest.Component var err error - comp, err = env.V1Repository().FetchComponentManifest(component) + comp, err = env.V1Repository().FetchComponentManifest(component, false) if err != nil { return nil, errors.Annotate(err, "failed to fetch component") } diff --git a/cmd/mirror.go b/cmd/mirror.go index 290ca4ea90..abfedaa024 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -254,7 +254,7 @@ func newMirrorModifyCmd() *cobra.Command { } comp, ver := environment.ParseCompVersion(args[0]) - m, err := env.V1Repository().FetchComponentManifest(comp) + m, err := env.V1Repository().FetchComponentManifest(comp, true) if err != nil { return err } @@ -334,7 +334,7 @@ func newMirrorPublishCmd() *cobra.Command { cmd.Flags().Visit(func(f *pflag.Flag) { flagSet.Insert(f.Name) }) - m, err := env.V1Repository().FetchComponentManifest(args[0]) + m, err := env.V1Repository().FetchComponentManifest(args[0], true) if err != nil { fmt.Printf("Fetch local manifest: %s\n", err.Error()) fmt.Printf("Failed to load component manifest, create a new one\n") diff --git a/components/playground/playground.go b/components/playground/playground.go index 59f9c1a210..4393ac8958 100755 --- a/components/playground/playground.go +++ b/components/playground/playground.go @@ -642,7 +642,7 @@ func (p *Playground) bootCluster(env *environment.Environment, options *bootOpti } if options.version == "" { - version, _, err := env.V1Repository().LatestStableVersion("tidb") + version, _, err := env.V1Repository().LatestStableVersion("tidb", false) if err != nil { return err } diff --git a/pkg/exec/run.go b/pkg/exec/run.go index b4148e38c7..1714007576 100644 --- a/pkg/exec/run.go +++ b/pkg/exec/run.go @@ -146,7 +146,7 @@ func PrepareCommand( } if version.IsEmpty() && len(checkUpdate) > 0 && checkUpdate[0] { - latestV, _, err := env.V1Repository().LatestStableVersion(component) + latestV, _, err := env.V1Repository().LatestStableVersion(component, true) if err != nil { return nil, err } diff --git a/pkg/repository/clone_mirror.go b/pkg/repository/clone_mirror.go index 05523f6035..bf9eb0bb08 100644 --- a/pkg/repository/clone_mirror.go +++ b/pkg/repository/clone_mirror.go @@ -247,7 +247,7 @@ func cloneComponents(repo *V1Repository, options CloneOptions) (map[string]*v1manifest.Component, error) { compManifests := map[string]*v1manifest.Component{} for _, name := range components { - manifest, err := repo.FetchComponentManifest(name) + manifest, err := repo.FetchComponentManifest(name, true) if err != nil { return nil, errors.Annotatef(err, "fetch component '%s' manifest failed", name) } diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index bdbc667a16..f56a3166c4 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -103,9 +103,9 @@ func (r *V1Repository) UpdateComponents(specs []ComponentSpec) error { var errs []string for _, spec := range specs { - manifest, err := r.updateComponentManifest(spec.ID) + manifest, err := r.updateComponentManifest(spec.ID, false) if err != nil { - if err == errUnknownComponent { + if errors.Cause(err) == errUnknownComponent { fmt.Println(color.YellowString("The component `%s` not found (may be deleted from repository); skipped", spec.ID)) } else { errs = append(errs, err.Error()) @@ -416,7 +416,7 @@ func (r *V1Repository) updateLocalIndex(snapshot *v1manifest.Snapshot) error { } // Precondition: the snapshot and index manifests exist and are up to date. -func (r *V1Repository) updateComponentManifest(id string) (*v1manifest.Component, error) { +func (r *V1Repository) updateComponentManifest(id string, withYanked bool) (*v1manifest.Component, error) { defer func(start time.Time) { verbose.Log("update component '%s' manifest finished in %s", id, time.Since(start)) }(time.Now()) @@ -427,9 +427,16 @@ func (r *V1Repository) updateComponentManifest(id string) (*v1manifest.Component if err != nil { return nil, errors.Trace(err) } - item, ok := index.Components[id] + var components map[string]v1manifest.ComponentItem + if withYanked { + components = index.ComponentListWithYanked() + } else { + components = index.ComponentList() + } + + item, ok := components[id] if !ok { - return nil, errUnknownComponent + return nil, errors.AddStack(errUnknownComponent) } var snapshot v1manifest.Snapshot _, _, err = r.local.LoadManifest(&snapshot) @@ -639,8 +646,8 @@ func (r *V1Repository) UpdateComponentManifests() error { } for name := range index.Components { - _, err = r.updateComponentManifest(name) - if err != nil { + _, err = r.updateComponentManifest(name, false) + if err != nil && errors.Cause(err) != errUnknownComponent { return err } } @@ -649,18 +656,18 @@ func (r *V1Repository) UpdateComponentManifests() error { } // FetchComponentManifest fetch the component manifest. -func (r *V1Repository) FetchComponentManifest(id string) (com *v1manifest.Component, err error) { +func (r *V1Repository) FetchComponentManifest(id string, withYanked bool) (com *v1manifest.Component, err error) { err = r.ensureManifests() if err != nil { return nil, errors.AddStack(err) } - return r.updateComponentManifest(id) + return r.updateComponentManifest(id, withYanked) } // ComponentVersion returns version item of a component func (r *V1Repository) ComponentVersion(id, version string, includeYanked bool) (*v1manifest.VersionItem, error) { - manifest, err := r.FetchComponentManifest(id) + manifest, err := r.FetchComponentManifest(id, includeYanked) if err != nil { return nil, err } @@ -675,8 +682,8 @@ func (r *V1Repository) ComponentVersion(id, version string, includeYanked bool) } // LatestStableVersion returns the latest stable version of specific component -func (r *V1Repository) LatestStableVersion(id string) (v0manifest.Version, *v1manifest.VersionItem, error) { - com, err := r.FetchComponentManifest(id) +func (r *V1Repository) LatestStableVersion(id string, withYanked bool) (v0manifest.Version, *v1manifest.VersionItem, error) { + com, err := r.FetchComponentManifest(id, withYanked) if err != nil { return "", nil, err } @@ -722,7 +729,7 @@ func (r *V1Repository) BinaryPath(installPath string, componentID string, versio return "", err } if component == nil { - component, err = r.FetchComponentManifest(componentID) + component, err = r.FetchComponentManifest(componentID, true) if err != nil { return "", err } diff --git a/pkg/repository/v1_repository_test.go b/pkg/repository/v1_repository_test.go index b02d2a81c8..896662fa58 100644 --- a/pkg/repository/v1_repository_test.go +++ b/pkg/repository/v1_repository_test.go @@ -25,6 +25,7 @@ import ( cjson "github.com/gibson042/canonicaljson-go" "github.com/google/uuid" + "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/localdata" "github.com/pingcap/tiup/pkg/repository/crypto" "github.com/pingcap/tiup/pkg/repository/v1manifest" @@ -275,6 +276,33 @@ func TestUpdateIndex(t *testing.T) { // TODO test that invalid signature of snapshot causes an error } +func TestYanked(t *testing.T) { + mirror := MockMirror{ + Resources: map[string]string{}, + } + local := v1manifest.NewMockManifests() + _ = setNewRoot(t, local) + repo := NewV1Repo(&mirror, Options{}, local) + + index, indexPriv := indexManifest(t) + snapshot := snapshotManifest() + bar := componentManifest() + serBar := serialize(t, bar, indexPriv) + mirror.Resources["/7.bar.json"] = serBar + local.Manifests[v1manifest.ManifestFilenameSnapshot] = &v1manifest.Manifest{Signed: snapshot} + local.Manifests[v1manifest.ManifestFilenameIndex] = &v1manifest.Manifest{Signed: index} + + // Test yanked version + updated, err := repo.updateComponentManifest("bar", true) + assert.Nil(t, err) + _, ok := updated.VersionList("plat/form")["v2.0.3"] + assert.False(t, ok) + + _, err = repo.updateComponentManifest("bar", false) + assert.NotNil(t, err) + assert.Equal(t, errors.Cause(err), errUnknownComponent) +} + func TestUpdateComponent(t *testing.T) { mirror := MockMirror{ Resources: map[string]string{}, @@ -292,7 +320,7 @@ func TestUpdateComponent(t *testing.T) { local.Manifests[v1manifest.ManifestFilenameIndex] = &v1manifest.Manifest{Signed: index} // Test happy path - updated, err := repo.updateComponentManifest("foo") + updated, err := repo.updateComponentManifest("foo", false) assert.Nil(t, err) t.Logf("%+v", err) assert.NotNil(t, updated) @@ -303,13 +331,13 @@ func TestUpdateComponent(t *testing.T) { oldFoo.Version = 8 local.Manifests["foo.json"] = &v1manifest.Manifest{Signed: oldFoo} local.Saved = []string{} - updated, err = repo.updateComponentManifest("foo") + updated, err = repo.updateComponentManifest("foo", false) assert.NotNil(t, err) assert.Nil(t, updated) assert.Empty(t, local.Saved) // Test that id missing from index causes an error - updated, err = repo.updateComponentManifest("bar") + updated, err = repo.updateComponentManifest("bar", false) assert.NotNil(t, err) assert.Nil(t, updated) assert.Empty(t, local.Saved) @@ -598,6 +626,7 @@ func snapshotManifest() *v1manifest.Snapshot { v1manifest.ManifestURLRoot: {Version: 42}, v1manifest.ManifestURLIndex: {Version: 5}, "/foo.json": {Version: 7}, + "/bar.json": {Version: 7}, }, } } @@ -613,7 +642,10 @@ func componentManifest() *v1manifest.Component { ID: "foo", Description: "foo does stuff", Platforms: map[string]map[string]v1manifest.VersionItem{ - "plat/form": {"v2.0.1": versionItem()}, + "plat/form": { + "v2.0.1": versionItem(), + "v2.0.3": versionItem3(), + }, }, } } @@ -638,6 +670,17 @@ func versionItem2() v1manifest.VersionItem { } } +func versionItem3() v1manifest.VersionItem { + return v1manifest.VersionItem{ + URL: "/foo-2.0.3.tar.gz", + Yanked: true, + FileHash: v1manifest.FileHash{ + Hashes: map[string]string{v1manifest.SHA256: "5abe91bc22039c15c05580062357be7ab0bfd7968582a118fbb4eb817ddc2e76"}, + Length: 12, + }, + } +} + func indexManifest(t *testing.T) (*v1manifest.Index, crypto.PrivKey) { info, keyID, priv, err := v1manifest.FreshKeyInfo() assert.Nil(t, err) @@ -661,11 +704,18 @@ func indexManifest(t *testing.T) (*v1manifest.Index, crypto.PrivKey) { Keys: map[string]*v1manifest.KeyInfo{keyID: info}, Threshold: 1, }}, - Components: map[string]v1manifest.ComponentItem{"foo": { - Yanked: false, - Owner: "bar", - URL: "/foo.json", - }}, + Components: map[string]v1manifest.ComponentItem{ + "foo": { + Yanked: false, + Owner: "bar", + URL: "/foo.json", + }, + "bar": { + Yanked: true, + Owner: "bar", + URL: "/bar.json", + }, + }, DefaultComponents: []string{}, }, priv }