Updated containerd1.7; google.golang.org/protobuf (#1706)

* Update containerd1.7; google.golang.org/protobuf

Update to containerd 1.7 and move to `google.golang.org/protobuf`
from `github.com/gogo/protobuf/gogoproto`.

These two changes are intertwined, since containerd 1.7 changes its
ttrpc task server definitions and protobuff generation (as well as some
other API changes).
Additionally, the task server gRPC code is imported from containerd
directly, rather than being generated here, and that code now explicitly
imports `google.golang.org/protobuf` instead of
`github.com/gogo/protobuf/gogoproto`.
Upgrading to `google.golang.org/protobuf` also requires updating
the `containerd/cgroups` dependency to v3
(`github.com/containerd/cgroups/v3/cgroup1/stats/`).

The new `protoc-gen-go-grpc` generators do not allow directives such as
`gogoproto.customname`, so the `go-fix-acronym` command is used to
update acronym customization (which is what containerd does).

Updated `Protobuild.toml` to specify new generators.

Added an `Update-Proto.ps1` script to re-generate protobuf files locally
and in GitHub CI.

Add `protobuild` and protobuff `grpc` and `ttrpc` generators to
`tools.go` so they are tracked and vendored, and can be trivially
installed via `go install`.

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

* Vendor protobuf import changes

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

---------

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

.github/workflows/ci.yml

@@ -60,50 +60,19 @@ jobs:
       GOPATH: '${{ github.workspace }}\go'
 
     steps:
+      # protobuild requires the code to be in $GOPATH to translate from github.com/Microsoft/hcsshim
+      # to the correct path on disk
       - name: Checkout hcsshim
         uses: actions/checkout@v3
         with:
-          path: go/src/github.com/Microsoft/hcsshim
+          path: "${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim"
 
       - name: Install go
         uses: actions/setup-go@v4
         with:
           go-version: ${{ env.GO_VERSION }}
           cache-dependency-path: go/src/github.com/Microsoft/hcsshim/go.sum
 
-      - name: Get containerd ref
-        shell: powershell
-        run: |
-          $v = go list -m -f '{{ .Version }}' 'github.com/containerd/containerd' 2>&1
-          if ( $LASTEXITCODE ) {
-            Write-Output '::error::Could not retrieve containerd version.'
-            exit $LASTEXITCODE
-          }
-
-          Write-Output "containerd ref is: $v"
-          "containerd_ref=$v" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-        working-directory: go/src/github.com/Microsoft/hcsshim
-
-      - name: Checkout containerd
-        uses: actions/checkout@v3
-        with:
-          repository: containerd/containerd
-          path: "containerd"
-          ref: "${{ env.containerd_ref }}"
-
-      - name: Install protobuild and protoc-gen-gogoctrd
-        shell: powershell
-        run: |
-          # not actually GOBIN
-          $goBin = Join-Path (go env GOPATH) 'bin'
-          mkdir -f $goBin
-          $goBin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-
-          go install github.com/containerd/protobuild@v0.2.0
-
-          cd containerd
-          go build -o $goBin ./cmd/protoc-gen-gogoctrd
-
       - name: Install protoc
         shell: powershell
         run: |
@@ -122,25 +91,25 @@ jobs:
           mkdir -f ${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim/protobuf
           mv include/* ${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim/protobuf
 
-          # put protoc in GOPATH to make things easier
-          mv bin\protoc.exe (Join-Path (go env GOPATH) 'bin')
+          # put protoc in GOBIN to make things easier
+          $bin = Join-Path (go env GOPATH) 'bin'
+          mkdir -f $bin
+          mv bin\protoc.exe $bin
+
+          $bin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Run Protobuild
+      - name: Run protobuild
         shell: powershell
         run: |
           Write-Output "::group::protobuild"
-          protobuild $(go list ./... | grep -v /vendor/)
+          .\scripts\Update-Proto.ps1
           Write-Output "::endgroup::"
-          if ( $LASTEXITCODE ) {
-            Write-Output '::error::Failed to run protobuild.'
-            exit $LASTEXITCODE
-          }
 
-          Write-Output "::group::git diff"
           # look for any new files not previously tracked
           git add --all --intent-to-add .
+          Write-Output "::group::git diff"
           git diff --exit-code
           Write-Output "::endgroup::"
         working-directory: "${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim"

.golangci.yml

@@ -135,9 +135,9 @@ issues:
       linters:
         - stylecheck
       Text: "ST1003:"
-    
+
     # v0 APIs are deprecated, but still retained for backwards compatability
     - path: cmd\\ncproxy\\
       linters:
         - staticcheck
-      text: "^SA1019: (ncproxygrpcv0|nodenetsvcV0)"
+      text: "^SA1019: .*(ncproxygrpc|nodenetsvc)[/]?v0"

Protobuild.toml

@@ -1,52 +1,25 @@
-version = "1"
-generator = "gogoctrd"
-plugins = ["grpc", "fieldpath"]
+version = "2"
+generators = ["go", "go-grpc"]
 
-# Control protoc include paths. Below are usually some good defaults, but feel
-# free to try it without them if it works for your project.
+# Control protoc include paths.
 [includes]
-  # Include paths that will be added before all others. Typically, you want to
-  # treat the root of the project as an include, but this may not be necessary.
   before = ["./protobuf"]
 
   # defaults are "/usr/local/include" and "/usr/include", which don't exist on Windows.
   # override defaults to supress errors about non-existant directories.
   after = []
 
-  # Paths that should be treated as include roots in relation to the vendor
-  # directory. These will be calculated with the vendor directory nearest the
-  # target package.
-  packages = ["github.com/gogo/protobuf"]
-
-# This section maps protobuf imports to Go packages. These will become
-# `-M` directives in the call to the go protobuf generator.
+# This section maps protobuf imports to Go packages.
 [packages]
-  "gogoproto/gogo.proto" = "github.com/gogo/protobuf/gogoproto"
-  "google/protobuf/any.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/empty.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/struct.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/descriptor.proto" = "github.com/gogo/protobuf/protoc-gen-gogo/descriptor"
-  "google/protobuf/field_mask.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/timestamp.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/duration.proto" = "github.com/gogo/protobuf/types"
-  "github/containerd/cgroups/stats/v1/metrics.proto" = "github.com/containerd/cgroups/stats/v1"
-
-[[overrides]]
-prefixes = ["github.com/Microsoft/hcsshim/internal/shimdiag"]
-plugins = ["ttrpc"]
-
-[[overrides]]
-prefixes = ["github.com/Microsoft/hcsshim/internal/extendedtask"]
-plugins = ["ttrpc"]
-
-[[overrides]]
-prefixes = ["github.com/Microsoft/hcsshim/internal/computeagent"]
-plugins = ["ttrpc"]
-
-[[overrides]]
-prefixes = ["github.com/Microsoft/hcsshim/internal/ncproxyttrpc"]
-plugins = ["ttrpc"]
+  # github.com/containerd/cgroups protofiles still list their go path as "github.com/containerd/cgroups/cgroup1/stats"
+  "github.com/containerd/cgroups/v3/cgroup1/stats/metrics.proto" = "github.com/containerd/cgroups/v3/cgroup1/stats"
 
 [[overrides]]
-prefixes = ["github.com/Microsoft/hcsshim/internal/vmservice"]
-plugins = ["ttrpc"]
+prefixes = [
+  "github.com/Microsoft/hcsshim/internal/shimdiag",
+  "github.com/Microsoft/hcsshim/internal/extendedtask",
+  "github.com/Microsoft/hcsshim/internal/computeagent",
+  "github.com/Microsoft/hcsshim/internal/ncproxyttrpc",
+  "github.com/Microsoft/hcsshim/internal/vmservice",
+]
+generators = ["go", "go-ttrpc"]

cmd/containerd-shim-runhcs-v1/delete.go

@@ -9,15 +9,17 @@ import (
 	"path/filepath"
 	"time"
 
+	task "github.com/containerd/containerd/api/runtime/task/v2"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
 	"github.com/Microsoft/hcsshim/internal/hcs"
 	"github.com/Microsoft/hcsshim/internal/memory"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/internal/winapi"
-	"github.com/containerd/containerd/runtime/v2/task"
-	"github.com/gogo/protobuf/proto"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
 )
 
 // LimitedRead reads at max `readLimitBytes` bytes from the file at path `filePath`. If the file has
@@ -122,7 +124,7 @@ The delete command will be executed in the container's bundle as its cwd.
 		}
 
 		if data, err := proto.Marshal(&task.DeleteResponse{
-			ExitedAt:   time.Now(),
+			ExitedAt:   timestamppb.New(time.Now()),
 			ExitStatus: 255,
 		}); err != nil {
 			return err

cmd/containerd-shim-runhcs-v1/exec.go

@@ -5,8 +5,8 @@ package main
 import (
 	"context"
 
+	task "github.com/containerd/containerd/api/runtime/task/v2"
 	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/runtime/v2/task"
 	"github.com/pkg/errors"
 )
 

cmd/containerd-shim-runhcs-v1/exec_hcs.go

@@ -8,14 +8,15 @@ import (
 	"time"
 
 	eventstypes "github.com/containerd/containerd/api/events"
+	task "github.com/containerd/containerd/api/runtime/task/v2"
 	containerd_v1_types "github.com/containerd/containerd/api/types/task"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/runtime"
-	"github.com/containerd/containerd/runtime/v2/task"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/Microsoft/hcsshim/internal/cmd"
 	"github.com/Microsoft/hcsshim/internal/cow"
@@ -151,11 +152,11 @@ func (he *hcsExec) Status() *task.StateResponse {
 	var s containerd_v1_types.Status
 	switch he.state {
 	case shimExecStateCreated:
-		s = containerd_v1_types.StatusCreated
+		s = containerd_v1_types.Status_CREATED
 	case shimExecStateRunning:
-		s = containerd_v1_types.StatusRunning
+		s = containerd_v1_types.Status_RUNNING
 	case shimExecStateExited:
-		s = containerd_v1_types.StatusStopped
+		s = containerd_v1_types.Status_STOPPED
 	}
 
 	return &task.StateResponse{
@@ -169,7 +170,7 @@ func (he *hcsExec) Status() *task.StateResponse {
 		Stderr:     he.io.StderrPath(),
 		Terminal:   he.io.Terminal(),
 		ExitStatus: he.exitStatus,
-		ExitedAt:   he.exitedAt,
+		ExitedAt:   timestamppb.New(he.exitedAt),
 	}
 }
 
@@ -492,7 +493,7 @@ func (he *hcsExec) waitForExit() {
 				ID:          he.id,
 				Pid:         uint32(he.pid),
 				ExitStatus:  he.exitStatus,
-				ExitedAt:    he.exitedAt,
+				ExitedAt:    timestamppb.New(he.exitedAt),
 			}); err != nil {
 			log.G(ctx).WithError(err).Error("failed to publish TaskExitEvent")
 		}

cmd/containerd-shim-runhcs-v1/exec_test.go

@@ -6,7 +6,8 @@ import (
 	"context"
 	"time"
 
-	"github.com/containerd/containerd/runtime/v2/task"
+	task "github.com/containerd/containerd/api/runtime/task/v2"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // newTestShimExec creates a test exec. If you are intending to make an init
@@ -47,7 +48,7 @@ func (tse *testShimExec) Status() *task.StateResponse {
 		ExecID:     tse.id,
 		Pid:        uint32(tse.pid),
 		ExitStatus: tse.status,
-		ExitedAt:   tse.at,
+		ExitedAt:   timestamppb.New(tse.at),
 	}
 }
 func (tse *testShimExec) Start(ctx context.Context) error {

cmd/containerd-shim-runhcs-v1/exec_wcow_podsandbox.go

@@ -9,12 +9,13 @@ import (
 
 	"github.com/Microsoft/hcsshim/internal/log"
 	eventstypes "github.com/containerd/containerd/api/events"
+	task "github.com/containerd/containerd/api/runtime/task/v2"
 	containerd_v1_types "github.com/containerd/containerd/api/types/task"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/runtime"
-	"github.com/containerd/containerd/runtime/v2/task"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 func newWcowPodSandboxExec(ctx context.Context, events publisher, tid, bundle string) *wcowPodSandboxExec {
@@ -100,11 +101,11 @@ func (wpse *wcowPodSandboxExec) Status() *task.StateResponse {
 	var s containerd_v1_types.Status
 	switch wpse.state {
 	case shimExecStateCreated:
-		s = containerd_v1_types.StatusCreated
+		s = containerd_v1_types.Status_CREATED
 	case shimExecStateRunning:
-		s = containerd_v1_types.StatusRunning
+		s = containerd_v1_types.Status_RUNNING
 	case shimExecStateExited:
-		s = containerd_v1_types.StatusStopped
+		s = containerd_v1_types.Status_STOPPED
 	}
 
 	return &task.StateResponse{
@@ -118,7 +119,7 @@ func (wpse *wcowPodSandboxExec) Status() *task.StateResponse {
 		Stderr:     "", // NilIO
 		Terminal:   false,
 		ExitStatus: wpse.exitStatus,
-		ExitedAt:   wpse.exitedAt,
+		ExitedAt:   timestamppb.New(wpse.exitedAt),
 	}
 }