From 8002fb3200401fcb8ee7bd6d92c99e0aa6aa76d8 Mon Sep 17 00:00:00 2001 From: lucklove Date: Mon, 15 Mar 2021 17:12:04 +0800 Subject: [PATCH 1/5] Feature: cache timestamp.json to accelerate --- pkg/repository/v1_repository.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index bc5a6de573..833485cc18 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -48,8 +48,9 @@ var ErrUnknownVersion = errors.New("unknown version") // V1Repository represents a remote repository viewed with the v1 manifest design. type V1Repository struct { Options - mirror Mirror - local v1manifest.LocalManifests + mirror Mirror + local v1manifest.LocalManifests + timestamp *v1manifest.Manifest } // ComponentSpec describes a component a user would like to have or use. @@ -297,6 +298,11 @@ func FnameWithVersion(fname string, version uint) string { } func (r *V1Repository) updateLocalRoot() error { + // There is no need to update root.json if other manifest not changed + if r.timestamp != nil { + return nil + } + defer func(start time.Time) { verbose.Log("Update local root finished in %s", time.Since(start)) }(time.Now()) @@ -493,12 +499,23 @@ func (r *V1Repository) DownloadComponent(item *v1manifest.VersionItem, target st return nil } +// PurgeTimestamp remove timestamp cache from repository +func (r *V1Repository) PurgeTimestamp() { + r.timestamp = nil +} + // FetchTimestamp downloads the timestamp file, validates it, and checks if the snapshot hash in it // has the same value of our local one. (not hashing the snapshot file itself) // Return weather the manifest is changed compared to the one in local ts and the FileHash of snapshot. func (r *V1Repository) fetchTimestamp() (changed bool, manifest *v1manifest.Manifest, err error) { + // check cache first + if r.timestamp != nil { + return false, r.timestamp, nil + } + defer func(start time.Time) { verbose.Log("Fetch timestamp finished in %s", time.Since(start)) + r.timestamp = manifest }(time.Now()) var ts v1manifest.Timestamp From 87ceb7219ee2a70ef2b6e4b0f336d2d6317d772b Mon Sep 17 00:00:00 2001 From: lucklove Date: Mon, 15 Mar 2021 17:38:40 +0800 Subject: [PATCH 2/5] Purge timestamp for publish command --- cmd/mirror.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/mirror.go b/cmd/mirror.go index e6afb70255..c4fba83c7d 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -509,6 +509,7 @@ func doPublish( flagSet set.StringSet, ) error { env := environment.GlobalEnv() + env.V1Repository().PurgeTimestamp() m, err := env.V1Repository().FetchComponentManifest(component, true) if err != nil { if perrs.Cause(err) == repository.ErrUnknownComponent { From 9f2d7c2a5a7128c280e9cae44ff94a6b7e034c6b Mon Sep 17 00:00:00 2001 From: lucklove Date: Tue, 16 Mar 2021 12:07:19 +0800 Subject: [PATCH 3/5] Fix test --- pkg/repository/v1_repository_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/repository/v1_repository_test.go b/pkg/repository/v1_repository_test.go index eaff3b6b73..5539fb3fa1 100644 --- a/pkg/repository/v1_repository_test.go +++ b/pkg/repository/v1_repository_test.go @@ -142,6 +142,7 @@ func TestCheckTimestamp(t *testing.T) { expiredTimestamp := timestampManifest() expiredTimestamp.Expires = "2000-05-12T04:51:08Z" mirror.Resources[v1manifest.ManifestURLTimestamp] = serialize(t, expiredTimestamp) + repo.PurgeTimestamp() _, _, err = repo.fetchTimestamp() assert.NotNil(t, err) @@ -390,6 +391,7 @@ func TestEnsureManifests(t *testing.T) { mirror.Resources[v1manifest.ManifestURLTimestamp] = serialize(t, ts, priv2) local.Saved = []string{} + repo.PurgeTimestamp() err = repo.ensureManifests() assert.Nil(t, err) assert.Contains(t, local.Saved, v1manifest.ManifestFilenameRoot) @@ -403,6 +405,7 @@ func TestEnsureManifests(t *testing.T) { mirror.Resources[v1manifest.ManifestURLSnapshot] = snapStr mirror.Resources[v1manifest.ManifestURLTimestamp] = serialize(t, ts, priv) + repo.PurgeTimestamp() err = repo.ensureManifests() assert.NotNil(t, err) } @@ -522,6 +525,7 @@ func TestUpdateComponents(t *testing.T) { ts.Version++ mirror.Resources[v1manifest.ManifestURLSnapshot] = snapStr mirror.Resources[v1manifest.ManifestURLTimestamp] = serialize(t, ts, priv) + repo.PurgeTimestamp() err = repo.UpdateComponents([]ComponentSpec{{ ID: "foo", TargetDir: "/tmp/mock-mock", From 929929cad1df2dbf73d329a996fe4e1aab93b4f8 Mon Sep 17 00:00:00 2001 From: lucklove Date: Tue, 16 Mar 2021 15:18:47 +0800 Subject: [PATCH 4/5] Add test --- pkg/repository/v1_repository_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/repository/v1_repository_test.go b/pkg/repository/v1_repository_test.go index 5539fb3fa1..a8d7c1f070 100644 --- a/pkg/repository/v1_repository_test.go +++ b/pkg/repository/v1_repository_test.go @@ -155,6 +155,30 @@ func TestCheckTimestamp(t *testing.T) { // TODO test that a bad signature causes an error } +func TestCacheTimestamp(t *testing.T) { + mirror := MockMirror{ + Resources: map[string]string{}, + } + local := v1manifest.NewMockManifests() + privk := setNewRoot(t, local) + repo := NewV1Repo(&mirror, Options{}, local) + + repoTimestamp := timestampManifest() + mirror.Resources[v1manifest.ManifestURLTimestamp] = serialize(t, repoTimestamp, privk) + changed, _, err := repo.fetchTimestamp() + assert.Nil(t, err) + assert.True(t, changed) + + delete(mirror.Resources, v1manifest.ManifestURLTimestamp) + changed, _, err = repo.fetchTimestamp() + assert.Nil(t, err) + assert.False(t, changed) + + repo.PurgeTimestamp() + changed, _, err = repo.fetchTimestamp() + assert.NotNil(t, err) +} + func TestUpdateLocalSnapshot(t *testing.T) { mirror := MockMirror{ Resources: map[string]string{}, From 0563abc9c63bc0fb15989efcac8fe39ad17cf443 Mon Sep 17 00:00:00 2001 From: lucklove Date: Tue, 16 Mar 2021 15:54:29 +0800 Subject: [PATCH 5/5] Fix test --- pkg/repository/v1_repository_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/repository/v1_repository_test.go b/pkg/repository/v1_repository_test.go index a8d7c1f070..4326a3e0dd 100644 --- a/pkg/repository/v1_repository_test.go +++ b/pkg/repository/v1_repository_test.go @@ -175,7 +175,7 @@ func TestCacheTimestamp(t *testing.T) { assert.False(t, changed) repo.PurgeTimestamp() - changed, _, err = repo.fetchTimestamp() + _, _, err = repo.fetchTimestamp() assert.NotNil(t, err) }