Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix container blob mount #22226

Merged
merged 5 commits into from
Jan 16, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions models/packages/container/search.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type BlobSearchOptions struct {
Digest string
Tag string
IsManifest bool
Repository string
}

func (opts *BlobSearchOptions) toConds() builder.Cond {
Expand Down Expand Up @@ -53,6 +54,15 @@ func (opts *BlobSearchOptions) toConds() builder.Cond {

cond = cond.And(builder.In("package_file.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")))
}
if opts.Repository != "" {
var propsCond builder.Cond = builder.Eq{
"package_property.ref_type": packages.PropertyTypePackage,
"package_property.name": container_module.PropertyRepository,
"package_property.value": opts.Repository,
}

cond = cond.And(builder.In("package.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")))
}

return cond
}
Expand Down
129 changes: 76 additions & 53 deletions routers/api/packages/container/blob.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,60 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic

contentStore := packages_module.NewContentStore()

uploadVersion, err := getOrCreateUploadVersion(pi)
if err != nil {
return nil, err
}

err = db.WithTx(db.DefaultContext, func(ctx context.Context) error {
pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
if err != nil {
log.Error("Error inserting package blob: %v", err)
return err
}
// FIXME: Workaround to be removed in v1.20
// https://github.com/go-gitea/gitea/issues/19586
if exists {
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
exists = false
}
}
if !exists {
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
log.Error("Error saving package blob in content store: %v", err)
return err
}
}

return createFileForBlob(ctx, uploadVersion, pb)
})
if err != nil {
if !exists {
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
log.Error("Error deleting package blob from content store: %v", err)
}
}
return nil, err
}

return pb, nil
}

// mountBlob mounts the specific blob to a different package
func mountBlob(pi *packages_service.PackageInfo, pb *packages_model.PackageBlob) error {
uploadVersion, err := getOrCreateUploadVersion(pi)
if err != nil {
return err
}

return db.WithTx(db.DefaultContext, func(ctx context.Context) error {
return createFileForBlob(ctx, uploadVersion, pb)
})
}

func getOrCreateUploadVersion(pi *packages_service.PackageInfo) (*packages_model.PackageVersion, error) {
var uploadVersion *packages_model.PackageVersion

// FIXME: Replace usage of mutex with database transaction
Expand Down Expand Up @@ -83,66 +137,35 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
return nil
})
uploadVersionMutex.Unlock()
if err != nil {
return nil, err
}

err = db.WithTx(db.DefaultContext, func(ctx context.Context) error {
pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
if err != nil {
log.Error("Error inserting package blob: %v", err)
return err
}
// FIXME: Workaround to be removed in v1.20
// https://github.com/go-gitea/gitea/issues/19586
if exists {
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
exists = false
}
}
if !exists {
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
log.Error("Error saving package blob in content store: %v", err)
return err
}
}

filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))
return uploadVersion, err
}

pf := &packages_model.PackageFile{
VersionID: uploadVersion.ID,
BlobID: pb.ID,
Name: filename,
LowerName: filename,
CompositeKey: packages_model.EmptyFileKey,
}
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
if err == packages_model.ErrDuplicatePackageFile {
return nil
}
log.Error("Error inserting package file: %v", err)
return err
}
func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, pb *packages_model.PackageBlob) error {
filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))

if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, container_module.PropertyDigest, digestFromPackageBlob(pb)); err != nil {
log.Error("Error setting package file property: %v", err)
return err
pf := &packages_model.PackageFile{
VersionID: pv.ID,
BlobID: pb.ID,
Name: filename,
LowerName: filename,
CompositeKey: packages_model.EmptyFileKey,
}
var err error
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
if err == packages_model.ErrDuplicatePackageFile {
return nil
}
log.Error("Error inserting package file: %v", err)
return err
}

return nil
})
if err != nil {
if !exists {
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
log.Error("Error deleting package blob from content store: %v", err)
}
}
return nil, err
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, container_module.PropertyDigest, digestFromPackageBlob(pb)); err != nil {
log.Error("Error setting package file property: %v", err)
return err
}

return pb, nil
return nil
}

func deleteBlob(ownerID int64, image, digest string) error {
Expand Down
9 changes: 7 additions & 2 deletions routers/api/packages/container/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,10 +195,15 @@ func InitiateUploadBlob(ctx *context.Context) {
from := ctx.FormTrim("from")
if mount != "" {
blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
Image: from,
Digest: mount,
Repository: from,
Digest: mount,
})
if blob != nil {
if err := mountBlob(&packages_service.PackageInfo{Owner: ctx.Package.Owner, Name: image}, blob.Blob); err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}

setResponseHeaders(ctx.Resp, &containerHeaders{
Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, mount),
ContentDigest: mount,
Expand Down
2 changes: 1 addition & 1 deletion templates/package/settings.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
{{.locale.Tr "packages.settings.delete"}}
</div>
<div class="content">
<div class="ui warning message text left">
<div class="ui warning message text left word-break">
{{.locale.Tr "packages.settings.delete.notice" .PackageDescriptor.Package.Name .PackageDescriptor.Version.Version}}
</div>
<form class="ui form" action="{{.Link}}" method="post">
Expand Down
41 changes: 26 additions & 15 deletions tests/integration/api_packages_container_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,32 @@ func TestPackageContainer(t *testing.T) {
})
})

t.Run("UploadBlob/Mount", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()

req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, unknownDigest))
addTokenAuthHeader(req, userToken)
MakeRequest(t, req, http.StatusAccepted)

req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, blobDigest))
addTokenAuthHeader(req, userToken)
resp := MakeRequest(t, req, http.StatusCreated)

assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))

req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s&from=%s", url, unknownDigest, "unknown/image"))
addTokenAuthHeader(req, userToken)
MakeRequest(t, req, http.StatusAccepted)

req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s&from=%s/%s", url, blobDigest, user.Name, image))
addTokenAuthHeader(req, userToken)
resp = MakeRequest(t, req, http.StatusCreated)

assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
})

for _, tag := range tags {
t.Run(fmt.Sprintf("[Tag:%s]", tag), func(t *testing.T) {
t.Run("UploadManifest", func(t *testing.T) {
Expand Down Expand Up @@ -444,21 +470,6 @@ func TestPackageContainer(t *testing.T) {
assert.Equal(t, indexManifestDigest, pd.Files[0].Properties.GetByName(container_module.PropertyDigest))
})

t.Run("UploadBlob/Mount", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()

req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, unknownDigest))
addTokenAuthHeader(req, userToken)
MakeRequest(t, req, http.StatusAccepted)

req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, blobDigest))
addTokenAuthHeader(req, userToken)
resp := MakeRequest(t, req, http.StatusCreated)

assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
})

t.Run("HeadBlob", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()

Expand Down