From ca1215b436dbaef015a7e92c0f0c121871077846 Mon Sep 17 00:00:00 2001 From: yzewei Date: Fri, 22 Dec 2023 11:04:21 +0800 Subject: [PATCH 1/7] add loongarch support for buildkit archutil Signed-off-by: yzewei (cherry picked from commit 736c35ad3561f709876bfcb52e2644f71b05c805) --- hack/dockerfiles/archutil.Dockerfile | 11 +++++++++-- util/archutil/detect.go | 10 ++++++++++ util/archutil/fixtures/exit.loongarch64.s | 6 ++++++ util/archutil/loong64_binary.go | 9 +++++++++ util/archutil/loong64_check.go | 8 ++++++++ util/archutil/loong64_check_loong64.go | 7 +++++++ 6 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 util/archutil/fixtures/exit.loongarch64.s create mode 100644 util/archutil/loong64_binary.go create mode 100644 util/archutil/loong64_check.go create mode 100644 util/archutil/loong64_check_loong64.go diff --git a/hack/dockerfiles/archutil.Dockerfile b/hack/dockerfiles/archutil.Dockerfile index 20af69af3e26..3b6a875c0cea 100644 --- a/hack/dockerfiles/archutil.Dockerfile +++ b/hack/dockerfiles/archutil.Dockerfile @@ -15,7 +15,8 @@ RUN apt-get update && apt-get --no-install-recommends install -y git binutils \ binutils-s390x-linux-gnu \ binutils-powerpc64le-linux-gnu \ binutils-mips64el-linux-gnuabi64 \ - binutils-mips64-linux-gnuabi64 + binutils-mips64-linux-gnuabi64 \ + binutils-loongarch64-linux-gnu WORKDIR /src FROM base AS exit-amd64 @@ -58,6 +59,10 @@ FROM base AS exit-mips64 COPY util/archutil/fixtures/exit.mips64.s . RUN mips64-linux-gnuabi64-as --noexecstack -o exit.o exit.mips64.s && mips64-linux-gnuabi64-ld -o exit -s exit.o && mips64-linux-gnuabi64-strip --strip-unneeded exit +FROM base AS exit-loong64 +COPY util/archutil/fixtures/exit.loongarch64.s . +RUN loongarch64-linux-gnu-as --noexecstack -o exit.o exit.loongarch64.s && loongarch64-linux-gnu-ld -o exit -s exit.o && loongarch64-linux-gnu-strip --strip-unneeded exit + FROM scratch AS exits COPY --from=exit-amd64 /src/exit amd64 COPY --from=exit-386 /src/exit 386 @@ -69,6 +74,7 @@ COPY --from=exit-ppc64 /src/exit ppc64 COPY --from=exit-ppc64le /src/exit ppc64le COPY --from=exit-mips64le /src/exit mips64le COPY --from=exit-mips64 /src/exit mips64 +COPY --from=exit-loong64 /src/exit loong64 FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS generate WORKDIR /go/src/github.com/moby/buildkit @@ -86,7 +92,8 @@ RUN --mount=type=bind,target=.,rw \ bin/archutil/ppc64 \ bin/archutil/ppc64le \ bin/archutil/mips64le \ - bin/archutil/mips64 + bin/archutil/mips64 \ + bin/archutil/loong64 tree -nh bin/archutil cp bin/archutil/*_binary.go /out EOT diff --git a/util/archutil/detect.go b/util/archutil/detect.go index 7826441271fd..5808333f91b5 100644 --- a/util/archutil/detect.go +++ b/util/archutil/detect.go @@ -78,6 +78,11 @@ func SupportedPlatforms(noCache bool) []ocispecs.Platform { arr = append(arr, linux(p)) } } + if p := "loong64"; def.Architecture != p { + if _, err := loong64Supported(); err == nil { + arr = append(arr, linux(p)) + } + } if p := "arm"; def.Architecture != p { if _, err := armSupported(); err == nil { p := linux("arm") @@ -144,6 +149,11 @@ func WarnIfUnsupported(pfs []ocispecs.Platform) { printPlatformWarning(p, err) } } + if p.Architecture == "loong64" { + if _, err := loong64Supported(); err != nil { + printPlatformWarning(p, err) + } + } if p.Architecture == "arm" { if _, err := armSupported(); err != nil { printPlatformWarning(p, err) diff --git a/util/archutil/fixtures/exit.loongarch64.s b/util/archutil/fixtures/exit.loongarch64.s new file mode 100644 index 000000000000..478cd622c69c --- /dev/null +++ b/util/archutil/fixtures/exit.loongarch64.s @@ -0,0 +1,6 @@ + .global _start + .text +_start: + li.w $a0,0 + li.w $a7,93 + syscall 0 diff --git a/util/archutil/loong64_binary.go b/util/archutil/loong64_binary.go new file mode 100644 index 000000000000..01d4454310df --- /dev/null +++ b/util/archutil/loong64_binary.go @@ -0,0 +1,9 @@ +//go:build !loong64 +// +build !loong64 + +package archutil + +// This file is generated by running "make archutil". +// Do not edit manually. + +const Binaryloong64 = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xaa\x77\xf5\x71\x63\x62\x64\x64\x80\x01\x26\x06\x08\x6f\x03\x03\x83\x02\x88\x76\x80\x8a\x5f\x80\xd2\xce\x60\x31\x0b\x06\x26\x06\x07\x06\x66\x06\x26\x06\x90\x1a\x56\x06\x14\xa0\xc0\x88\x44\xef\x81\x0a\xc2\x68\x98\x81\x81\x4f\x4b\x52\xd8\x18\x88\x07\x02\x50\x9a\x85\x41\x94\x81\xbb\xa4\x91\x99\x81\x41\x9b\x81\x41\xaf\x38\xa3\xb8\xa4\xa8\x24\x31\x89\x41\xaf\x24\xb5\xa2\x84\x81\x0a\x80\x9b\x81\x01\xec\x27\x98\xdb\x60\xe1\xb0\x01\xca\xe7\x41\x53\xcf\x88\x85\xcf\x8c\xc5\x5c\x98\xff\x05\x09\xe8\x07\x04\x00\x00\xff\xff\x5f\xc5\x9b\x9d\x90\x01\x00\x00" diff --git a/util/archutil/loong64_check.go b/util/archutil/loong64_check.go new file mode 100644 index 000000000000..9bc966ce245a --- /dev/null +++ b/util/archutil/loong64_check.go @@ -0,0 +1,8 @@ +//go:build !loong64 +// +build !loong64 + +package archutil + +func loong64Supported() (string, error) { + return check("loong64", Binaryloong64) +} diff --git a/util/archutil/loong64_check_loong64.go b/util/archutil/loong64_check_loong64.go new file mode 100644 index 000000000000..20f4764ff590 --- /dev/null +++ b/util/archutil/loong64_check_loong64.go @@ -0,0 +1,7 @@ +//go:build loong64 + +package archutil + +func loong64Supported() (string, error) { + return "", nil +} From 262e626e858e4681823ed3713e3b905157d29761 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Sun, 18 Feb 2024 10:53:10 +0100 Subject: [PATCH 2/7] ci(validate): temporarily disable archutil-arm64 job Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> (cherry picked from commit 0356194de0e1a32082c08519e23ae7b6b6e0ea0e) --- .github/workflows/validate.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e276c8e72557..085ccb2aa553 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -63,6 +63,9 @@ jobs: archutil-arm64: runs-on: ubuntu-22.04 + # TODO: enable when binutils-loongarch64-linux-gnu pkg is available for aarch64 arch + # https://github.com/moby/buildkit/pull/4392#issuecomment-1938223235 + if: false steps: - name: Checkout From 0567a816707cd9497f53380fd9152106fc0261e4 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 1 Mar 2024 12:14:05 +0100 Subject: [PATCH 3/7] vendor: update github.com/tonistiigi/fsutil to 7525a1af2bb5 Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> (cherry picked from commit 9944a5c47598dda0fc293920d693a1adb7b29cc7) --- go.mod | 2 +- go.sum | 4 ++-- vendor/github.com/tonistiigi/fsutil/send.go | 17 ++++++++++------- vendor/modules.txt | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index be45ed8e2ca9..d71b29ad7e22 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spdx/tools-golang v0.5.3 github.com/stretchr/testify v1.8.4 - github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6 + github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598 github.com/tonistiigi/go-archvariant v1.0.0 github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea diff --git a/go.sum b/go.sum index 544f6bd877f7..2d9dd82c7da2 100644 --- a/go.sum +++ b/go.sum @@ -405,8 +405,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6 h1:v9u6pmdUkarXL/1S/6LGcG9wsiBLd9N/WyJq/Y9WPcg= -github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM= +github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 h1:oZS8KCqAg62sxJkEq/Ppzqrb6EooqzWtL8Oaex7bc5c= +github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM= github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598 h1:DA/NDC0YbMdnfcOSUzAnbUZE6dSM54d+0hrBqG+bOfs= github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598/go.mod h1:anhKd3mnC1shAbQj1Q4IJ+w6xqezxnyDYlx/yKa7IXM= github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0= diff --git a/vendor/github.com/tonistiigi/fsutil/send.go b/vendor/github.com/tonistiigi/fsutil/send.go index 6a935ed06c03..ba97ef7ad0ed 100644 --- a/vendor/github.com/tonistiigi/fsutil/send.go +++ b/vendor/github.com/tonistiigi/fsutil/send.go @@ -43,13 +43,14 @@ type sendHandle struct { } type sender struct { - conn Stream - fs FS - files map[uint32]string - mu sync.RWMutex - progressCb func(int, bool) - progressCurrent int - sendpipeline chan *sendHandle + conn Stream + fs FS + files map[uint32]string + mu sync.RWMutex + progressCb func(int, bool) + progressCurrent int + progressCurrentMu sync.Mutex + sendpipeline chan *sendHandle } func (s *sender) run(ctx context.Context) error { @@ -112,6 +113,8 @@ func (s *sender) run(ctx context.Context) error { func (s *sender) updateProgress(size int, last bool) { if s.progressCb != nil { + s.progressCurrentMu.Lock() + defer s.progressCurrentMu.Unlock() s.progressCurrent += size s.progressCb(s.progressCurrent, last) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 7a7a9ae295a4..a269d2aa0ac9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -765,7 +765,7 @@ github.com/spdx/tools-golang/spdx/v2/v2_3 ## explicit; go 1.20 github.com/stretchr/testify/assert github.com/stretchr/testify/require -# github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6 +# github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 ## explicit; go 1.20 github.com/tonistiigi/fsutil github.com/tonistiigi/fsutil/copy From 9195a118d5d473dbba4355c230ca736b0933a04e Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 3 Mar 2024 08:52:38 -0800 Subject: [PATCH 4/7] util: typo fix Signed-off-by: Tonis Tiigi (cherry picked from commit c35d98cef913769483043b465d6b42b5ab1198c2) --- util/system/path.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/system/path.go b/util/system/path.go index 94f9a826f24c..422651ecea5c 100644 --- a/util/system/path.go +++ b/util/system/path.go @@ -27,7 +27,7 @@ func DefaultPathEnv(os string) string { // NormalizePath cleans the path based on the operating system the path is meant for. // It takes into account a potential parent path, and will join the path to the parent -// if the path is relative. Additionally, it will apply the folliwing rules: +// if the path is relative. Additionally, it will apply the following rules: // - always return an absolute path // - always strip drive letters for Windows paths // - optionally keep the trailing slashes on paths From d959049b3ffcb2a57539ceddf9b659402d1c33fe Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 3 Mar 2024 08:52:55 -0800 Subject: [PATCH 5/7] dockerfile: allow pivot point for --parents flag This replicates similar functionality in rsync. Signed-off-by: Tonis Tiigi (cherry picked from commit 6f8bd4cb0e7ccf773d692beb9bc89032099881b2) --- frontend/dockerfile/dockerfile2llb/convert.go | 26 +++-- .../dockerfile/dockerfile_parents_test.go | 106 ++++++++++++++++++ 2 files changed, 125 insertions(+), 7 deletions(-) diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 531aa09484bd..b79407955542 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -1252,18 +1252,30 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { a = a.Copy(st, f, dest, opts...) } } else { + var patterns []string + if cfg.parents { + // detect optional pivot point + parent, pattern, ok := strings.Cut(src, "/./") + if !ok { + pattern = src + src = "/" + } else { + src = parent + } + + pattern, err = system.NormalizePath("/", pattern, d.platform.OS, false) + if err != nil { + return errors.Wrap(err, "removing drive letter") + } + + patterns = []string{strings.TrimPrefix(pattern, "/")} + } + src, err = system.NormalizePath("/", src, d.platform.OS, false) if err != nil { return errors.Wrap(err, "removing drive letter") } - var patterns []string - if cfg.parents { - path := strings.TrimPrefix(src, "/") - patterns = []string{path} - src = "/" - } - opts := append([]llb.CopyOption{&llb.CopyInfo{ Mode: mode, FollowSymlinks: true, diff --git a/frontend/dockerfile/dockerfile_parents_test.go b/frontend/dockerfile/dockerfile_parents_test.go index 4400a27781eb..47f06e803924 100644 --- a/frontend/dockerfile/dockerfile_parents_test.go +++ b/frontend/dockerfile/dockerfile_parents_test.go @@ -18,6 +18,7 @@ import ( var parentsTests = integration.TestFuncs( testCopyParents, + testCopyRelativeParents, ) func init() { @@ -75,3 +76,108 @@ COPY --parents foo1/foo2/ba* . require.NoError(t, err) require.Equal(t, "testing2", string(dt)) } + +func testCopyRelativeParents(t *testing.T, sb integration.Sandbox) { + f := getFrontend(t, sb) + + dockerfile := []byte(` +FROM alpine AS base +WORKDIR /test +RUN < Date: Sun, 3 Mar 2024 09:09:16 -0800 Subject: [PATCH 6/7] dockerfile: add docs for --parents Signed-off-by: Tonis Tiigi (cherry picked from commit d1446a4748e692c99a388a2372cd1b7e79f760b5) --- frontend/dockerfile/docs/reference.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/frontend/dockerfile/docs/reference.md b/frontend/dockerfile/docs/reference.md index 9b29bde90e28..1916f386209a 100644 --- a/frontend/dockerfile/docs/reference.md +++ b/frontend/dockerfile/docs/reference.md @@ -1596,8 +1596,28 @@ COPY --parents ./x/a.txt ./y/a.txt /parents/ # /parents/y/a.txt ``` -This behavior is analogous to the [Linux `cp` utility's](https://www.man7.org/linux/man-pages/man1/cp.1.html) -`--parents` flag. +This behavior is similar to the [Linux `cp` utility's](https://www.man7.org/linux/man-pages/man1/cp.1.html) +`--parents` or [`rsync`](https://man7.org/linux/man-pages/man1/rsync.1.html) `--relative` flag. + +As with Rsync, it is possible to limit which parent directories are preserved by +inserting a dot and a slash (`./`) into the source path. If such point exists, only parent +directories after it will be preserved. This may be especially useful copies between stages +with `--from` where the source paths need to be absolute. + +```dockerfile +# syntax=docker/dockerfile-upstream:master-labs +FROM scratch + +COPY --parents ./x/./y/*.txt /parents/ + +# Build context: +# ./x/y/a.txt +# ./x/y/b.txt +# +# Output: +# /parents/y/a.txt +# /parents/y/b.txt +``` Note that, without the `--parents` flag specified, any filename collision will fail the Linux `cp` operation with an explicit error message From b9e15810a0a0383c444c12330e587c2aacfdf6b7 Mon Sep 17 00:00:00 2001 From: Anthony Nandaa Date: Mon, 4 Mar 2024 15:25:00 +0300 Subject: [PATCH 7/7] fix: windows getting started guide for uniformity This adds `ContainerAdministrator` as the default user to guarantee a uniform experience on all the platforms. The previous guide would fail on WS2022 but work on WS2019 and Windows 11. The issue is being investigated here #4731 Signed-off-by: Anthony Nandaa (cherry picked from commit 2aa80d749c9c6e750b4a6e66df38e0fc8443183b) --- docs/windows.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/windows.md b/docs/windows.md index 14cc882ee17d..9f8e69a9b85e 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -89,6 +89,7 @@ Now that everything is setup, let's build a [simple _hello world_ image](https:/ ```powershell Set-Content Dockerfile @" FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 + USER ContainerAdministrator COPY hello.txt C:/ RUN echo "Goodbye!" >> hello.txt CMD ["cmd", "/C", "type C:\\hello.txt"]