[v0.12] cherry-picks for v0.12.2

.github/workflows/build.yml

@@ -24,9 +24,13 @@ env:
   REPO_SLUG: "docker/buildx-bin"
   DESTDIR: "./bin"
   TEST_CACHE_SCOPE: "test"
+  TESTFLAGS: "-v --parallel=6 --timeout=30m"
+  GOTESTSUM_FORMAT: "standard-verbose"
+  GO_VERSION: "1.21.6"
+  GOTESTSUM_VERSION: "v1.9.0"  # same as one in Dockerfile
 
 jobs:
-  prepare-test:
+  prepare-test-integration:
     runs-on: ubuntu-22.04
     steps:
       -
@@ -51,14 +55,12 @@ jobs:
             *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
             *.cache-to=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
 
-  test:
+  test-integration:
     runs-on: ubuntu-22.04
     needs:
-      - prepare-test
+      - prepare-test-integration
     env:
-      TESTFLAGS: "-v --parallel=6 --timeout=30m"
       TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m"
-      GOTESTSUM_FORMAT: "standard-verbose"
       TEST_IMAGE_BUILD: "0"
       TEST_IMAGE_ID: "buildx-tests"
     strategy:
@@ -71,9 +73,6 @@ jobs:
           - remote
         pkg:
           - ./tests
-        include:
-          - pkg: ./...
-            skip-integration-tests: 1
     steps:
       -
         name: Checkout
@@ -101,19 +100,19 @@ jobs:
       -
         name: Test
         run: |
-          export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')
+          export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')
           ./hack/test
         env:
           TEST_DOCKERD: "${{ startsWith(matrix.worker, 'docker') && '1' || '0' }}"
           TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker\\+containerd') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
           TESTPKGS: "${{ matrix.pkg }}"
-          SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}"
       -
         name: Send to Codecov
         if: always()
         uses: codecov/codecov-action@v3
         with:
           directory: ./bin/testreports
+          flags: integration
       -
         name: Generate annotations
         if: always()
@@ -128,6 +127,75 @@ jobs:
           name: test-reports
           path: ./bin/testreports
 
+  test-unit:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-22.04
+          - macos-12
+          - windows-2022
+    env:
+      SKIP_INTEGRATION_TESTS: 1
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v4
+      -
+        name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "${{ env.GO_VERSION }}"
+      -
+        name: Prepare
+        run: |
+          testreportsBaseDir=./bin/testreports
+          testreportsDir=$testreportsBaseDir/unit-${{ matrix.os }}
+          echo "TESTREPORTS_BASEDIR=$testreportsBaseDir" >> $GITHUB_ENV
+          echo "TESTREPORTS_DIR=$testreportsDir" >> $GITHUB_ENV
+          mkdir -p $testreportsDir
+        shell: bash
+      -
+        name: Install gotestsum
+        run: |
+          go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }}
+      -
+        name: Test
+        env:
+          TMPDIR: ${{ runner.temp }}
+        run: |
+          gotestsum \
+            --jsonfile="${{ env.TESTREPORTS_DIR }}/go-test-report.json" \
+            --junitfile="${{ env.TESTREPORTS_DIR }}/junit-report.xml" \
+            --packages="./..." \
+            -- \
+              "-mod=vendor" \
+              "-coverprofile" "${{ env.TESTREPORTS_DIR }}/coverage.txt" \
+              "-covermode" "atomic" ${{ env.TESTFLAGS }}
+        shell: bash
+      -
+        name: Send to Codecov
+        if: always()
+        uses: codecov/codecov-action@v3
+        with:
+          directory: ${{ env.TESTREPORTS_DIR }}
+          env_vars: RUNNER_OS
+          flags: unit
+      -
+        name: Generate annotations
+        if: always()
+        uses: crazy-max/.github/.github/actions/gotest-annotations@1a64ea6d01db9a48aa61954cb20e265782c167d9
+        with:
+          directory: ${{ env.TESTREPORTS_DIR }}
+      -
+        name: Upload test reports
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-reports
+          path: ${{ env.TESTREPORTS_BASEDIR }}
+
   prepare-binaries:
     runs-on: ubuntu-22.04
     outputs:
@@ -192,7 +260,8 @@ jobs:
   bin-image:
     runs-on: ubuntu-22.04
     needs:
-      - test
+      - test-integration
+      - test-unit
     if: ${{ github.event_name != 'pull_request' && github.repository == 'docker/buildx' }}
     steps:
       -
@@ -244,7 +313,8 @@ jobs:
   release:
     runs-on: ubuntu-22.04
     needs:
-      - test
+      - test-integration
+      - test-unit
       - binaries
     steps:
       -

.github/workflows/docs-upstream.yml

@@ -45,18 +45,18 @@ jobs:
           DOCS_FORMATS: yaml
       -
         name: Upload reference YAML docs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: docs-yaml
           path: /tmp/buildx-docs/out/reference
           retention-days: 1
 
   validate:
-    uses: docker/docs/.github/workflows/validate-upstream.yml@e864c797e391fa35aecf1ddcecdbdeb683546424  # pin for artifact v3 support
+    uses: docker/docs/.github/workflows/validate-upstream.yml@6b73b05acb21edf7995cc5b3c6672d8e314cee7a  # pin for artifact v4 support: https://github.com/docker/docs/pull/19220
     needs:
       - docs-yaml
     with:
       module-name: docker/buildx
       data-files-id: docs-yaml
       data-files-folder: buildx
-      data-files-placeholder-folder: engine/reference/commandline
+      create-placeholder-stubs: true

build/git.go

@@ -45,9 +45,9 @@ func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath st
 	if filepath.IsAbs(contextPath) {
 		wd = contextPath
 	} else {
-		cwd, _ := os.Getwd()
-		wd, _ = filepath.Abs(filepath.Join(cwd, contextPath))
+		wd, _ = filepath.Abs(filepath.Join(getWd(), contextPath))
 	}
+	wd = gitutil.SanitizePath(wd)
 
 	gitc, err := gitutil.New(gitutil.WithContext(ctx), gitutil.WithWorkingDir(wd))
 	if err != nil {
@@ -101,8 +101,7 @@ func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath st
 				dockerfilePath = filepath.Join(wd, "Dockerfile")
 			}
 			if !filepath.IsAbs(dockerfilePath) {
-				cwd, _ := os.Getwd()
-				dockerfilePath = filepath.Join(cwd, dockerfilePath)
+				dockerfilePath = filepath.Join(getWd(), dockerfilePath)
 			}
 			dockerfilePath, _ = filepath.Rel(root, dockerfilePath)
 			if !strings.HasPrefix(dockerfilePath, "..") {
@@ -113,3 +112,11 @@ func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath st
 
 	return
 }
+
+func getWd() string {
+	wd, _ := os.Getwd()
+	if lp, err := getLongPathName(wd); err == nil {
+		return lp
+	}
+	return wd
+}

build/git_unix.go

@@ -0,0 +1,9 @@
+//go:build !windows
+// +build !windows
+
+package build
+
+// getLongPathName is a no-op on non-Windows platforms.
+func getLongPathName(path string) (string, error) {
+	return path, nil
+}

build/git_windows.go

@@ -0,0 +1,26 @@
+package build
+
+import "golang.org/x/sys/windows"
+
+// getLongPathName converts Windows short pathnames to full pathnames.
+// For example C:\Users\ADMIN~1 --> C:\Users\Administrator.
+func getLongPathName(path string) (string, error) {
+	// See https://groups.google.com/forum/#!topic/golang-dev/1tufzkruoTg
+	p, err := windows.UTF16FromString(path)
+	if err != nil {
+		return "", err
+	}
+	b := p // GetLongPathName says we can reuse buffer
+	n, err := windows.GetLongPathName(&p[0], &b[0], uint32(len(b)))
+	if err != nil {
+		return "", err
+	}
+	if n > uint32(len(b)) {
+		b = make([]uint16, n)
+		_, err = windows.GetLongPathName(&p[0], &b[0], uint32(len(b)))
+		if err != nil {
+			return "", err
+		}
+	}
+	return windows.UTF16ToString(b), nil
+}

commands/bake.go

@@ -141,7 +141,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions, cFlags com
 			if err == nil {
 				err = err1
 			}
-			if err == nil && progressMode != progressui.QuietMode {
+			if err == nil && progressMode != progressui.QuietMode && progressMode != progressui.RawJSONMode {
 				desktop.PrintBuildDetails(os.Stderr, printer.BuildRefs(), term)
 			}
 		}
@@ -295,13 +295,17 @@ func saveLocalStateGroup(dockerCli command.Cli, ref string, lsg localstate.State
 }
 
 func readBakeFiles(ctx context.Context, nodes []builder.Node, url string, names []string, stdin io.Reader, pw progress.Writer) (files []bake.File, inp *bake.Input, err error) {
-	var lnames []string
-	var rnames []string
+	var lnames []string // local
+	var rnames []string // remote
+	var anames []string // both
 	for _, v := range names {
 		if strings.HasPrefix(v, "cwd://") {
-			lnames = append(lnames, strings.TrimPrefix(v, "cwd://"))
+			tname := strings.TrimPrefix(v, "cwd://")
+			lnames = append(lnames, tname)
+			anames = append(anames, tname)
 		} else {
 			rnames = append(rnames, v)
+			anames = append(anames, v)
 		}
 	}
 
@@ -320,7 +324,7 @@ func readBakeFiles(ctx context.Context, nodes []builder.Node, url string, names
 			if url != "" {
 				lfiles, err = bake.ReadLocalFiles(lnames, stdin, sub)
 			} else {
-				lfiles, err = bake.ReadLocalFiles(append(lnames, rnames...), stdin, sub)
+				lfiles, err = bake.ReadLocalFiles(anames, stdin, sub)
 			}
 			return nil
 		})

commands/build.go

@@ -288,10 +288,13 @@ func runBuild(dockerCli command.Cli, options buildOptions) (err error) {
 		return retErr
 	}
 
-	if progressMode != progressui.QuietMode {
-		desktop.PrintBuildDetails(os.Stderr, printer.BuildRefs(), term)
-	} else {
+	switch progressMode {
+	case progressui.RawJSONMode:
+		// no additional display
+	case progressui.QuietMode:
 		fmt.Println(getImageID(resp.ExporterResponse))
+	default:
+		desktop.PrintBuildDetails(os.Stderr, printer.BuildRefs(), term)
 	}
 	if options.imageIDFile != "" {
 		if err := os.WriteFile(options.imageIDFile, []byte(getImageID(resp.ExporterResponse)), 0644); err != nil {
@@ -749,7 +752,7 @@ func dockerUlimitToControllerUlimit(u *dockeropts.UlimitOpt) *controllerapi.Ulim
 }
 
 func printWarnings(w io.Writer, warnings []client.VertexWarning, mode progressui.DisplayMode) {
-	if len(warnings) == 0 || mode == progressui.QuietMode {
+	if len(warnings) == 0 || mode == progressui.QuietMode || mode == progressui.RawJSONMode {
 		return
 	}
 	fmt.Fprintf(w, "\n ")

docs/reference/buildx_build.md

@@ -33,7 +33,7 @@ Start a build
 | [`--metadata-file`](#metadata-file)                                                                                                                | `string`      |           | Write build result metadata to the file                                                             |
 | `--network`                                                                                                                                        | `string`      | `default` | Set the networking mode for the `RUN` instructions during build                                     |
 | `--no-cache`                                                                                                                                       |               |           | Do not use cache when building the image                                                            |
-| `--no-cache-filter`                                                                                                                                | `stringArray` |           | Do not cache specified stages                                                                       |
+| [`--no-cache-filter`](#no-cache-filter)                                                                                                            | `stringArray` |           | Do not cache specified stages                                                                       |
 | [`-o`](#output), [`--output`](#output)                                                                                                             | `stringArray` |           | Output destination (format: `type=local,dest=path`)                                                 |
 | [`--platform`](#platform)                                                                                                                          | `stringArray` |           | Set target platform for build                                                                       |
 | `--print`                                                                                                                                          | `string`      |           | Print result of information request (e.g., outline, targets)                                        |
@@ -341,6 +341,61 @@ $ cat metadata.json
 }
 ```
 
+### <a name="no-cache-filter"></a> Ignore build cache for specific stages (--no-cache-filter)
+
+The `--no-cache-filter` lets you specify one or more stages of a multi-stage
+Dockerfile for which build cache should be ignored. To specify multiple stages,
+use a comma-separated syntax:
+
+```console
+$ docker buildx build --no-cache-filter stage1,stage2,stage3 .
+```
+
+For example, the following Dockerfile contains four stages:
+
+- `base`
+- `install`
+- `test`
+- `release`
+
+```dockerfile
+# syntax=docker/dockerfile:1
+
+FROM oven/bun:1 as base
+WORKDIR /app
+
+FROM base AS install
+WORKDIR /temp/dev
+RUN --mount=type=bind,source=package.json,target=package.json \
+    --mount=type=bind,source=bun.lockb,target=bun.lockb \
+    bun install --frozen-lockfile
+
+FROM base AS test
+COPY --from=install /temp/dev/node_modules node_modules
+COPY . .
+RUN bun test
+
+FROM base AS release
+ENV NODE_ENV=production
+COPY --from=install /temp/dev/node_modules node_modules
+COPY . .
+ENTRYPOINT ["bun", "run", "index.js"]
+```
+
+To ignore the cache for the `install` stage:
+
+```console
+$ docker buildx build --no-cache-filter install .
+```
+
+To ignore the cache the `install` and `release` stages:
+
+```console
+$ docker buildx build --no-cache-filter install,release .
+```
+
+The arguments for the `--no-cache-filter` flag must be names of stages.
+
 ### <a name="output"></a> Set the export action for the build result (-o, --output)
 
 ```text