Updated containerd1.7; google.golang.org/protobuf

Update to containerd 1.7 and move to `google.golang.org/protobuf`
from `github.com/gogo/protobuf/gogoproto`.

These two changes are intertwined, since containerd 1.7 changes its
ttrpc task server definitions and protobuff generation (as well as some
other API changes).
Upgrading to `google.golang.org/protobuf` also requires updating
the `containerd/cgroups` dependency to v3
(`github.com/containerd/cgroups/v3/cgroup1/stats/`).

The new `protoc-gen-go-grpc` generators do not allow directives such as
`gogoproto.customname`, so the `go-fix-acronym` command is used to
update acronym customization (similar to what containerd does).

Added an `Update-Proto.ps1` script to re-gerenate protobuf files locally
and in GitHub CI.

Add `protobuild` and protobuff `grpc` and `ttrpc` generators to
`tools.go` so they are tracked and vendored, and can be trivially
installed via `go install`.

Add empty `after` section to Protobuild to suppress errors about missing
directories `usr/local/include` and `/usr/include`, which are defaults.

Add protobuf file to gitignore so it does not affect `git diff` results.

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

.github/workflows/ci.yml

@@ -49,48 +49,56 @@ jobs:
 
   protos:
     runs-on: "windows-2019"
+    env:
+      # translating from github.com/Microsoft/hcsshim/<path> to path is easier if the package path is under GOPATH/src
+      GOPATH: '${{ github.workspace }}\go'
+
     steps:
       - uses: actions/setup-go@v4
         with:
           go-version: ${{ env.GO_VERSION }}
+
+      # protobuild requires the code to be in $GOPATH to translate from github.com/Microsoft/hcsshim
+      # to the correct path on disk
       - uses: actions/checkout@v3
         with:
-          path: "go/src/github.com/Microsoft/hcsshim"
-      # Install protoc-gen-gogoctrd in D:\bin
-      - uses: actions/checkout@v3
-        with:
-          repository: containerd/containerd
-          ref: v1.6.2
-          path: "containerd"
-      - name: Install protoc-gen-gogoctrd
-        shell: powershell
-        run: |
-          cd containerd
-          go build ./cmd/protoc-gen-gogoctrd
-          mkdir D:\bin
-          mv protoc-gen-gogoctrd.exe D:\bin
+          path: "${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim"
+
       # Install protoc in D:\bin
       - name: Install protoc
         shell: powershell
         run: |
-          Invoke-WebRequest -OutFile protoc.zip -Uri https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-win32.zip
+          $VerbosePreference = 'Continue'
+          $ErrorActionPreference = 'Stop'
+
+          $protocVersion = '3.19.4'
+          $url = "https://github.com/protocolbuffers/protobuf/releases/download/v${protocVersion}/protoc-${protocVersion}-win32.zip"
+          curl.exe -L --no-progress-meter -o protoc.zip $url
+
           Expand-Archive -Path protoc.zip -DestinationPath .
-          mv include go/src/github.com/Microsoft/hcsshim/protobuf
-          mv bin\protoc.exe D:\bin
-      - name: Run Protobuild
+
+          mkdir -f ${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim/protobuf
+          mv include/* ${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim/protobuf
+
+          # put protoc in GOBIN to make things easier
+          $bin = Join-Path (go env GOPATH) 'bin'
+          mkdir -f $bin
+          mv bin\protoc.exe $bin
+
+          $bin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
+      - name: Run protobuild
         shell: powershell
         run: |
-          go install github.com/containerd/protobuild@v0.2.0
-          cd go\src\github.com\Microsoft\hcsshim
-
-          $Env:Path += ";D:\bin;" + $Env:GOPATH + "\bin"
-          protobuild $(go list ./... | grep -v /vendor/)
+          Write-Output "::group::protobuild"
+          .\scripts\Update-Proto.ps1
+          Write-Output "::endgroup::"
 
+          git add --all --intent-to-add .
+          Write-Output "::group::git diff"
           git diff --exit-code
-        env:
-          GOPATH: '${{ github.workspace }}\go'
-          GOFLAGS: ""
-          GOPROXY: ""
+          Write-Output "::endgroup::"
+        working-directory: "${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim"
 
   verify-vendor:
     runs-on: "windows-2019"
@@ -262,6 +270,7 @@ jobs:
         with:
           path: src/github.com/containerd/containerd
           repository: "containerd/containerd"
+          ref: "release/1.7"
 
       - uses: actions/checkout@v3
         with:

.gitignore

@@ -37,6 +37,9 @@ rootfs-conv/*
 deps/*
 out/*
 
+# protobuf files
+/protobuf/*
+
 # test results
 test/results
 

Protobuild.toml

@@ -1,48 +1,25 @@
-version = "1"
-generator = "gogoctrd"
-plugins = ["grpc", "fieldpath"]
+version = "2"
+generators = ["go", "go-grpc"]
 
-# Control protoc include paths. Below are usually some good defaults, but feel
-# free to try it without them if it works for your project.
+# Control protoc include paths.
 [includes]
-  # Include paths that will be added before all others. Typically, you want to
-  # treat the root of the project as an include, but this may not be necessary.
   before = ["./protobuf"]
 
-  # Paths that should be treated as include roots in relation to the vendor
-  # directory. These will be calculated with the vendor directory nearest the
-  # target package.
-  packages = ["github.com/gogo/protobuf"]
+  # defaults are "/usr/local/include" and "/usr/include", which don't exist on Windows.
+  # override defaults to supress errors about non-existant directories.
+  after = []
 
-# This section maps protobuf imports to Go packages. These will become
-# `-M` directives in the call to the go protobuf generator.
+# This section maps protobuf imports to Go packages.
 [packages]
-  "gogoproto/gogo.proto" = "github.com/gogo/protobuf/gogoproto"
-  "google/protobuf/any.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/empty.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/struct.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/descriptor.proto" = "github.com/gogo/protobuf/protoc-gen-gogo/descriptor"
-  "google/protobuf/field_mask.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/timestamp.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/duration.proto" = "github.com/gogo/protobuf/types"
-  "github/containerd/cgroups/stats/v1/metrics.proto" = "github.com/containerd/cgroups/stats/v1"
+  # github.com/containerd/cgroups protofiles still list their go path as "github.com/containerd/cgroups/cgroup1/stats"
+  "github.com/containerd/cgroups/v3/cgroup1/stats/metrics.proto" = "github.com/containerd/cgroups/v3/cgroup1/stats"
 
 [[overrides]]
-prefixes = ["github.com/Microsoft/hcsshim/internal/shimdiag"]
-plugins = ["ttrpc"]
-
-[[overrides]]
-prefixes = ["github.com/Microsoft/hcsshim/internal/extendedtask"]
-plugins = ["ttrpc"]
-
-[[overrides]]
-prefixes = ["github.com/Microsoft/hcsshim/internal/computeagent"]
-plugins = ["ttrpc"]
-
-[[overrides]]
-prefixes = ["github.com/Microsoft/hcsshim/internal/ncproxyttrpc"]
-plugins = ["ttrpc"]
-
-[[overrides]]
-prefixes = ["github.com/Microsoft/hcsshim/internal/vmservice"]
-plugins = ["ttrpc"]
+prefixes = [
+  "github.com/Microsoft/hcsshim/internal/shimdiag",
+  "github.com/Microsoft/hcsshim/internal/extendedtask",
+  "github.com/Microsoft/hcsshim/internal/computeagent",
+  "github.com/Microsoft/hcsshim/internal/ncproxyttrpc",
+  "github.com/Microsoft/hcsshim/internal/vmservice",
+]
+generators = ["go", "go-ttrpc"]

cmd/containerd-shim-runhcs-v1/delete.go

@@ -9,15 +9,17 @@ import (
 	"path/filepath"
 	"time"
 
+	task "github.com/containerd/containerd/api/runtime/task/v2"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
 	"github.com/Microsoft/hcsshim/internal/hcs"
 	"github.com/Microsoft/hcsshim/internal/memory"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/internal/winapi"
-	"github.com/containerd/containerd/runtime/v2/task"
-	"github.com/gogo/protobuf/proto"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
 )
 
 // LimitedRead reads at max `readLimitBytes` bytes from the file at path `filePath`. If the file has
@@ -119,7 +121,7 @@ The delete command will be executed in the container's bundle as its cwd.
 		}
 
 		if data, err := proto.Marshal(&task.DeleteResponse{
-			ExitedAt:   time.Now(),
+			ExitedAt:   timestamppb.New(time.Now()),
 			ExitStatus: 255,
 		}); err != nil {
 			return err

cmd/containerd-shim-runhcs-v1/exec.go

@@ -5,8 +5,8 @@ package main
 import (
 	"context"
 
+	task "github.com/containerd/containerd/api/runtime/task/v2"
 	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/runtime/v2/task"
 	"github.com/pkg/errors"
 )
 

cmd/containerd-shim-runhcs-v1/exec_hcs.go

@@ -8,14 +8,15 @@ import (
 	"time"
 
 	eventstypes "github.com/containerd/containerd/api/events"
+	task "github.com/containerd/containerd/api/runtime/task/v2"
 	containerd_v1_types "github.com/containerd/containerd/api/types/task"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/runtime"
-	"github.com/containerd/containerd/runtime/v2/task"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/Microsoft/hcsshim/internal/cmd"
 	"github.com/Microsoft/hcsshim/internal/cow"
@@ -151,11 +152,11 @@ func (he *hcsExec) Status() *task.StateResponse {
 	var s containerd_v1_types.Status
 	switch he.state {
 	case shimExecStateCreated:
-		s = containerd_v1_types.StatusCreated
+		s = containerd_v1_types.Status_CREATED
 	case shimExecStateRunning:
-		s = containerd_v1_types.StatusRunning
+		s = containerd_v1_types.Status_RUNNING
 	case shimExecStateExited:
-		s = containerd_v1_types.StatusStopped
+		s = containerd_v1_types.Status_STOPPED
 	}
 
 	return &task.StateResponse{
@@ -169,7 +170,7 @@ func (he *hcsExec) Status() *task.StateResponse {
 		Stderr:     he.io.StderrPath(),
 		Terminal:   he.io.Terminal(),
 		ExitStatus: he.exitStatus,
-		ExitedAt:   he.exitedAt,
+		ExitedAt:   timestamppb.New(he.exitedAt),
 	}
 }
 
@@ -492,7 +493,7 @@ func (he *hcsExec) waitForExit() {
 				ID:          he.id,
 				Pid:         uint32(he.pid),
 				ExitStatus:  he.exitStatus,
-				ExitedAt:    he.exitedAt,
+				ExitedAt:    timestamppb.New(he.exitedAt),
 			}); err != nil {
 			log.G(ctx).WithError(err).Error("failed to publish TaskExitEvent")
 		}

cmd/containerd-shim-runhcs-v1/exec_test.go

@@ -6,7 +6,8 @@ import (
 	"context"
 	"time"
 
-	"github.com/containerd/containerd/runtime/v2/task"
+	task "github.com/containerd/containerd/api/runtime/task/v2"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // newTestShimExec creates a test exec. If you are intending to make an init
@@ -47,7 +48,7 @@ func (tse *testShimExec) Status() *task.StateResponse {
 		ExecID:     tse.id,
 		Pid:        uint32(tse.pid),
 		ExitStatus: tse.status,
-		ExitedAt:   tse.at,
+		ExitedAt:   timestamppb.New(tse.at),
 	}
 }
 func (tse *testShimExec) Start(ctx context.Context) error {

cmd/containerd-shim-runhcs-v1/exec_wcow_podsandbox.go

@@ -9,12 +9,13 @@ import (
 
 	"github.com/Microsoft/hcsshim/internal/log"
 	eventstypes "github.com/containerd/containerd/api/events"
+	task "github.com/containerd/containerd/api/runtime/task/v2"
 	containerd_v1_types "github.com/containerd/containerd/api/types/task"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/runtime"
-	"github.com/containerd/containerd/runtime/v2/task"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 func newWcowPodSandboxExec(ctx context.Context, events publisher, tid, bundle string) *wcowPodSandboxExec {
@@ -100,11 +101,11 @@ func (wpse *wcowPodSandboxExec) Status() *task.StateResponse {
 	var s containerd_v1_types.Status
 	switch wpse.state {
 	case shimExecStateCreated:
-		s = containerd_v1_types.StatusCreated
+		s = containerd_v1_types.Status_CREATED
 	case shimExecStateRunning:
-		s = containerd_v1_types.StatusRunning
+		s = containerd_v1_types.Status_RUNNING
 	case shimExecStateExited:
-		s = containerd_v1_types.StatusStopped
+		s = containerd_v1_types.Status_STOPPED
 	}
 
 	return &task.StateResponse{
@@ -118,7 +119,7 @@ func (wpse *wcowPodSandboxExec) Status() *task.StateResponse {
 		Stderr:     "", // NilIO
 		Terminal:   false,
 		ExitStatus: wpse.exitStatus,
-		ExitedAt:   wpse.exitedAt,
+		ExitedAt:   timestamppb.New(wpse.exitedAt),
 	}
 }