Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Display image size for multiarch container images #23821

Merged
merged 4 commits into from
Apr 2, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions models/migrations/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,8 @@ var migrations = []Migration{
NewMigration("Add version column to action_runner table", v1_20.AddVersionToActionRunner),
// v249 -> v250
NewMigration("Improve Action table indices v3", v1_20.ImproveActionTableIndices),
// v250 -> v251
NewMigration("Change Container Metadata", v1_20.ChangeContainerMetadataMultiArch),
}

// GetCurrentDBVersion returns the current db version
Expand Down
135 changes: 135 additions & 0 deletions models/migrations/v1_20/v250.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package v1_20 //nolint

import (
"strings"

"code.gitea.io/gitea/modules/json"

"xorm.io/xorm"
)

func ChangeContainerMetadataMultiArch(x *xorm.Engine) error {
sess := x.NewSession()
defer sess.Close()

if err := sess.Begin(); err != nil {
return err
}

type PackageVersion struct {
ID int64 `xorm:"pk"`
MetadataJSON string `xorm:"metadata_json"`
}

type PackageBlob struct{}

// Get all relevant packages (manifest list images have a container.manifest.reference property)

var pvs []*PackageVersion
err := sess.
Table("package_version").
Select("id, metadata_json").
Where("id IN (SELECT DISTINCT ref_id FROM package_property WHERE ref_type = 0 AND name = 'container.manifest.reference')").
Find(&pvs)
if err != nil {
return err
}

type MetadataOld struct {
Type string `json:"type"`
IsTagged bool `json:"is_tagged"`
Platform string `json:"platform,omitempty"`
Description string `json:"description,omitempty"`
Authors []string `json:"authors,omitempty"`
Licenses string `json:"license,omitempty"`
ProjectURL string `json:"project_url,omitempty"`
RepositoryURL string `json:"repository_url,omitempty"`
DocumentationURL string `json:"documentation_url,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
ImageLayers []string `json:"layer_creation,omitempty"`
MultiArch map[string]string `json:"multiarch,omitempty"`
}

type Manifest struct {
Platform string `json:"platform"`
Digest string `json:"digest"`
Size int64 `json:"size"`
}

type MetadataNew struct {
Type string `json:"type"`
IsTagged bool `json:"is_tagged"`
Platform string `json:"platform,omitempty"`
Description string `json:"description,omitempty"`
Authors []string `json:"authors,omitempty"`
Licenses string `json:"license,omitempty"`
ProjectURL string `json:"project_url,omitempty"`
RepositoryURL string `json:"repository_url,omitempty"`
DocumentationURL string `json:"documentation_url,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
ImageLayers []string `json:"layer_creation,omitempty"`
Manifests []*Manifest `json:"manifests,omitempty"`
}

for _, pv := range pvs {
var old *MetadataOld
if err := json.Unmarshal([]byte(pv.MetadataJSON), &old); err != nil {
return err
}

// Calculate the size of every contained manifest

manifests := make([]*Manifest, 0, len(old.MultiArch))
for platform, digest := range old.MultiArch {
size, err := sess.
Table("package_blob").
Join("INNER", "package_file", "package_blob.id = package_file.blob_id").
Join("INNER", "package_version pv", "pv.id = package_file.version_id").
Join("INNER", "package_version pv2", "pv2.package_id = pv.package_id").
Where("pv.lower_version = ? AND pv2.id = ?", strings.ToLower(digest), pv.ID).
SumInt(new(PackageBlob), "size")
if err != nil {
return err
}

manifests = append(manifests, &Manifest{
Platform: platform,
Digest: digest,
Size: size,
})
}

// Convert to new metadata format

new := &MetadataNew{
Type: old.Type,
IsTagged: old.IsTagged,
Platform: old.Platform,
Description: old.Description,
Authors: old.Authors,
Licenses: old.Licenses,
ProjectURL: old.ProjectURL,
RepositoryURL: old.RepositoryURL,
DocumentationURL: old.DocumentationURL,
Labels: old.Labels,
ImageLayers: old.ImageLayers,
Manifests: manifests,
}

metadataJSON, err := json.Marshal(new)
if err != nil {
return err
}

pv.MetadataJSON = string(metadataJSON)

if _, err := sess.ID(pv.ID).Update(pv); err != nil {
return err
}
}

return sess.Commit()
}
8 changes: 7 additions & 1 deletion modules/packages/container/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,13 @@ type Metadata struct {
DocumentationURL string `json:"documentation_url,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
ImageLayers []string `json:"layer_creation,omitempty"`
MultiArch map[string]string `json:"multiarch,omitempty"`
Manifests []*Manifest `json:"manifests,omitempty"`
}

type Manifest struct {
Platform string `json:"platform"`
Digest string `json:"digest"`
Size int64 `json:"size"`
}

// ParseImageConfig parses the metadata of an image config
Expand Down
2 changes: 1 addition & 1 deletion modules/packages/container/metadata_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func TestParseImageConfig(t *testing.T) {
},
metadata.Labels,
)
assert.Empty(t, metadata.MultiArch)
assert.Empty(t, metadata.Manifests)

configHelm := `{"description":"` + description + `", "home": "` + projectURL + `", "sources": ["` + repositoryURL + `"], "maintainers":[{"name":"` + author + `"}]}`

Expand Down
21 changes: 16 additions & 5 deletions routers/api/packages/container/manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func processImageManifestIndex(mci *manifestCreationInfo, buf *packages_module.H

metadata := &container_module.Metadata{
Type: container_module.TypeOCI,
MultiArch: make(map[string]string),
Manifests: make([]*container_module.Manifest, 0, len(index.Manifests)),
}

for _, manifest := range index.Manifests {
Expand All @@ -233,7 +233,7 @@ func processImageManifestIndex(mci *manifestCreationInfo, buf *packages_module.H
}
}

_, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
pfd, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
OwnerID: mci.Owner.ID,
Image: mci.Image,
Digest: string(manifest.Digest),
Expand All @@ -246,7 +246,18 @@ func processImageManifestIndex(mci *manifestCreationInfo, buf *packages_module.H
return err
}

metadata.MultiArch[platform] = string(manifest.Digest)
size, err := packages_model.CalculateFileSize(ctx, &packages_model.PackageFileSearchOptions{
VersionID: pfd.File.VersionID,
})
if err != nil {
return err
}

metadata.Manifests = append(metadata.Manifests, &container_module.Manifest{
Platform: platform,
Digest: string(manifest.Digest),
Size: size,
})
}

pv, err := createPackageAndVersion(ctx, mci, metadata)
Expand Down Expand Up @@ -369,8 +380,8 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met
return nil, err
}
}
for _, digest := range metadata.MultiArch {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, digest); err != nil {
for _, manifest := range metadata.Manifests {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, manifest.Digest); err != nil {
log.Error("Error setting package version property: %v", err)
return nil, err
}
Expand Down
28 changes: 18 additions & 10 deletions templates/package/content/container.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,27 @@
</div>
</div>
</div>
{{if .PackageDescriptor.Metadata.MultiArch}}
{{if .PackageDescriptor.Metadata.Manifests}}
<h4 class="ui top attached header">{{.locale.Tr "packages.container.multi_arch"}}</h4>
<div class="ui attached segment">
<div class="ui form">
{{range $arch, $digest := .PackageDescriptor.Metadata.MultiArch}}
<div class="field">
<label>{{svg "octicon-terminal"}} {{$arch}}</label>
{{if eq $.PackageDescriptor.Metadata.Type "oci"}}
<div class="markup"><pre class="code-block"><code>docker pull {{$.RegistryHost}}/{{$.PackageDescriptor.Owner.LowerName}}/{{$.PackageDescriptor.Package.LowerName}}@{{$digest}}</code></pre></div>
<table class="ui very basic compact table">
<thead>
<tr>
<th>{{.locale.Tr "packages.container.digest"}}</th>
<th>{{.locale.Tr "packages.container.multi_arch"}}</th>
<th>{{.locale.Tr "admin.packages.size"}}</th>
</tr>
</thead>
<tbody>
{{range .PackageDescriptor.Metadata.Manifests}}
<tr>
<td><a href="{{$.PackageDescriptor.PackageWebLink}}/{{PathEscape .Digest}}">{{.Digest}}</a></td>
<td>{{.Platform}}</td>
<td>{{FileSize .Size}}</td>
</tr>
{{end}}
</div>
{{end}}
</div>
</tbody>
</table>
</div>
{{end}}
{{if .PackageDescriptor.Metadata.Description}}
Expand Down
2 changes: 2 additions & 0 deletions templates/package/view.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,9 @@
{{template "package/metadata/rubygems" .}}
{{template "package/metadata/swift" .}}
{{template "package/metadata/vagrant" .}}
{{if not (and (eq .PackageDescriptor.Package.Type "container") .PackageDescriptor.Metadata.Manifests)}}
<div class="item">{{svg "octicon-database" 16 "gt-mr-3"}} {{FileSize .PackageDescriptor.CalculateBlobSize}}</div>
{{end}}
</div>
{{if not (eq .PackageDescriptor.Package.Type "container")}}
<div class="ui divider"></div>
Expand Down
22 changes: 17 additions & 5 deletions tests/integration/api_packages_container_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ func TestPackageContainer(t *testing.T) {
metadata := pd.Metadata.(*container_module.Metadata)
assert.Equal(t, container_module.TypeOCI, metadata.Type)
assert.Len(t, metadata.ImageLayers, 2)
assert.Empty(t, metadata.MultiArch)
assert.Empty(t, metadata.Manifests)

assert.Len(t, pd.Files, 3)
for _, pfd := range pd.Files {
Expand Down Expand Up @@ -462,10 +462,22 @@ func TestPackageContainer(t *testing.T) {
assert.IsType(t, &container_module.Metadata{}, pd.Metadata)
metadata := pd.Metadata.(*container_module.Metadata)
assert.Equal(t, container_module.TypeOCI, metadata.Type)
assert.Contains(t, metadata.MultiArch, "linux/arm/v7")
assert.Equal(t, manifestDigest, metadata.MultiArch["linux/arm/v7"])
assert.Contains(t, metadata.MultiArch, "linux/arm64/v8")
assert.Equal(t, untaggedManifestDigest, metadata.MultiArch["linux/arm64/v8"])
assert.Len(t, metadata.Manifests, 2)
assert.Condition(t, func() bool {
for _, m := range metadata.Manifests {
switch m.Platform {
case "linux/arm/v7":
assert.Equal(t, manifestDigest, m.Digest)
assert.EqualValues(t, 1524, m.Size)
case "linux/arm64/v8":
assert.Equal(t, untaggedManifestDigest, m.Digest)
assert.EqualValues(t, 1514, m.Size)
default:
return false
}
}
return true
})

assert.Len(t, pd.Files, 1)
assert.True(t, pd.Files[0].File.IsLead)
Expand Down