Update to linkerd2-proxy-api v0.5 and tonic v0.7 (#1596)

With tonic v0.7, we now have the ability to provide a fixed PROTOC
binary (rather than building/fetching a protoc implementation at
build-time).

This change updates the `linkerd-transport-header` and
`opencensus-proto` crates to use statically-generated sources,
eliminating the need for a `protoc` binary at build-time. Each crate
includes a `bootstrap` test that fails if the generated sources differ
from what is checked into git. These tests can also be used to
regenerate sources when the protobuf (or tonic generation) changes.

A local `install-protoc` action is added that fetches a protoc binary
and configures the `PROTOC_NO_VENDOR` and `PROTOC` environment
variables. This action is used by the check-all, check-each, and test
workflows.

Signed-off-by: Oliver Gould <ver@buoyant.io>

.devcontainer/Dockerfile

@@ -13,6 +13,26 @@ ARG CARGO_DENY_VERSION=0.11.1
 RUN curl --proto '=https' --tlsv1.3 -vsSfL "https://github.com/EmbarkStudios/cargo-deny/releases/download/${CARGO_DENY_VERSION}/cargo-deny-${CARGO_DENY_VERSION}-x86_64-unknown-linux-musl.tar.gz" \
     | tar zvxf - --strip-components=1 -C /usr/local/bin "cargo-deny-${CARGO_DENY_VERSION}-x86_64-unknown-linux-musl/cargo-deny"
 
+FROM docker.io/rust:${RUST_VERSION}-bullseye as yq
+ARG YQ_VERSION=v4.2.0
+RUN curl --proto '=https' --tlsv1.3 -vsSfLo /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \
+    && chmod +x /usr/local/bin/yq
+
+FROM docker.io/rust:${RUST_VERSION}-bullseye as nightly
+RUN rustup toolchain add nightly
+
+FROM nightly as fuzz
+RUN cargo +nightly install cargo-fuzz
+
+FROM docker.io/rust:${RUST_VERSION}-bullseye as protoc
+ARG PROTOC_VERSION=v3.20.0
+WORKDIR /tmp
+RUN arch="$(uname -m)" ; \
+    version="$PROTOC_VERSION" ; \
+    curl --proto '=https' --tlsv1.3 -vsSfLo protoc.zip  "https://github.com/google/protobuf/releases/download/$version/protoc-${version#v}-linux-$arch.zip" && \
+    unzip protoc.zip bin/protoc && \
+    chmod 755 bin/protoc
+
 #
 # Main image
 #
@@ -58,5 +78,9 @@ COPY --from=cargo-deny /usr/local/bin/cargo-deny /usr/local/bin/cargo-deny
 COPY --from=k3d /usr/local/bin/k3d /usr/local/bin/k3d
 COPY --from=kubectl /usr/local/bin/kubectl /usr/local/bin/kubectl
 
+COPY --from=protoc /tmp/bin/protoc /usr/local/bin/protoc
+ENV PROTOC_NO_VENDOR=1
+ENV PROTOC=/usr/local/bin/protoc
+
 ENTRYPOINT ["/usr/local/share/docker-init.sh"]
 CMD ["sleep", "infinity"]

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
 	"name": "linkerd2-proxy",
-	"image": "ghcr.io/linkerd/dev-proxy:v8",
+	"image": "ghcr.io/linkerd/dev-proxy:v9",
 	// "dockerFile": "./Dockerfile",
 	"extensions": [
 		"matklad.rust-analyzer",

.gitattributes

@@ -1 +1,3 @@
 Cargo.lock linguist-generated=false
+linkerd/transport-header/src/gen/* linguist-generated=true
+opencensus-proto/src/gen/* linguist-generated=true

.github/actions/install-protoc/action.yml

@@ -0,0 +1,33 @@
+name: install-protoc
+
+description: Install protoc and set the `PROTOC` environment variablec
+
+inputs:
+  version:
+    required: true
+    description: "Protoc version"
+    default: "v3.20.0"
+
+runs:
+  using: composite
+  steps:
+    - name: Install protoc
+      shell: bash
+      run: |
+        os=linux
+        if [ "$(uname -s)" = Darwin ]; then
+          os=osx
+        fi
+        arch="$(uname -m)"
+        version="${{ inputs.version }}"
+        tmp=$(mktemp -d -t protoc.XXX)
+        curl --fail --silent --show-error --location \
+          --proto '=https' --tlsv1.3 \
+          --output "$tmp/protoc.zip" \
+          "https://github.com/google/protobuf/releases/download/$version/protoc-${version#v}-$os-$arch.zip"
+        unzip $tmp/protoc.zip bin/protoc -d /usr/local
+        chmod 755 /usr/local/bin/protoc
+        ( echo "PROTOC_NO_VENDOR=1"
+          echo "PROTOC=/usr/local/bin/protoc"
+        ) >> $GITHUB_ENV
+

.github/workflows/check-all.yml

@@ -32,6 +32,7 @@ jobs:
           curl --proto =https --tlsv1.3 -vsSfLo /usr/local/bin/cargo-action-fmt "https://github.com/olix0r/cargo-action-fmt/releases/download/release%2F${CARGO_ACTION_FMT_VERSION}/cargo-action-fmt-x86_64-unknown-linux-gnu"
           chmod 755 /usr/local/bin/cargo-action-fmt
       - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
+      - uses: ./.github/actions/install-protoc
       - run: cargo fetch
       - run: |
           cargo check --frozen \

.github/workflows/check-each.yml

@@ -60,6 +60,9 @@ jobs:
           curl --proto =https --tlsv1.3 -vsSfLo /usr/local/bin/cargo-action-fmt "https://github.com/olix0r/cargo-action-fmt/releases/download/release%2F${CARGO_ACTION_FMT_VERSION}/cargo-action-fmt-x86_64-unknown-linux-gnu"
           chmod 755 /usr/local/bin/cargo-action-fmt
       - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
+      - name: Install protoc
+        if: matrix.crate == 'linkerd-transport-header' || matrix.crate == 'opencensus-proto'
+        uses: ./.github/actions/install-protoc
       - run: cargo fetch
       - run: cargo check -p ${{ matrix.crate }} --frozen --all-targets --message-format=json | cargo-action-fmt
 

.github/workflows/test.yml

@@ -28,6 +28,7 @@ jobs:
       image: docker://rust:1.59.0-buster
     steps:
       - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
+      - uses: ./.github/actions/install-protoc
       - run: |
           cargo test --all --no-run \
             --exclude=linkerd-app \

Cargo.lock

@@ -127,6 +127,15 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "cmake"
+version = "0.1.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.3.2"
@@ -186,7 +195,7 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73"
 dependencies = [
- "heck 0.4.0",
+ "heck",
  "proc-macro2",
  "quote",
  "syn",
@@ -373,15 +382,6 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "heck"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
-dependencies = [
- "unicode-segmentation",
-]
-
 [[package]]
 name = "heck"
 version = "0.4.0"
@@ -485,18 +485,6 @@ dependencies = [
  "tower",
 ]
 
-[[package]]
-name = "hyper-timeout"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
-dependencies = [
- "hyper",
- "pin-project-lite",
- "tokio",
- "tokio-io-timeout",
-]
-
 [[package]]
 name = "idna"
 version = "0.2.3"
@@ -1488,9 +1476,9 @@ dependencies = [
 
 [[package]]
 name = "linkerd2-proxy-api"
-version = "0.3.1"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6aaf91178c272abeaac52b2472351e73affa723bfdd0c15e147e2f975f2fbe5"
+checksum = "12c93aba8dbdc8f465de51ef08c5e1938790ea0ae7390d66b3f4d2d36c297d0b"
 dependencies = [
  "h2",
  "http",
@@ -1500,7 +1488,6 @@ dependencies = [
  "quickcheck",
  "thiserror",
  "tonic",
- "tonic-build",
 ]
 
 [[package]]
@@ -1725,6 +1712,16 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
 
+[[package]]
+name = "prettyplease"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b83ec2d0af5c5c556257ff52c9f98934e243b9fd39604bfb2a9b75ec2e97f18"
+dependencies = [
+ "proc-macro2",
+ "syn",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.37"
@@ -1748,22 +1745,24 @@ dependencies = [
 
 [[package]]
 name = "prost"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
+checksum = "1bd5316aa8f5c82add416dfbc25116b84b748a21153f512917e8143640a71bbd"
 dependencies = [
  "bytes",
  "prost-derive",
 ]
 
 [[package]]
 name = "prost-build"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
+checksum = "328f9f29b82409216decb172d81e936415d21245befa79cd34c3f29d87d1c50b"
 dependencies = [
  "bytes",
- "heck 0.3.3",
+ "cfg-if",
+ "cmake",
+ "heck",
  "itertools",
  "lazy_static",
  "log",
@@ -1778,9 +1777,9 @@ dependencies = [
 
 [[package]]
 name = "prost-derive"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
+checksum = "df35198f0777b75e9ff669737c6da5136b59dba33cf5a010a6d1cc4d56defc6f"
 dependencies = [
  "anyhow",
  "itertools",
@@ -1791,9 +1790,9 @@ dependencies = [
 
 [[package]]
 name = "prost-types"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
+checksum = "926681c118ae6e512a3ccefd4abbe5521a14f4cc1e207356d4d00c0b7f2006fd"
 dependencies = [
  "bytes",
  "prost",
@@ -2133,16 +2132,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "tokio-io-timeout"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
-dependencies = [
- "pin-project-lite",
- "tokio",
-]
-
 [[package]]
 name = "tokio-macros"
 version = "1.7.0"
@@ -2220,41 +2209,36 @@ dependencies = [
 
 [[package]]
 name = "tonic"
-version = "0.6.2"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a"
+checksum = "30fb54bf1e446f44d870d260d99957e7d11fb9d0a0f5bd1a662ad1411cc103f9"
 dependencies = [
  "async-stream",
  "async-trait",
  "base64",
  "bytes",
  "futures-core",
  "futures-util",
- "h2",
  "http",
  "http-body",
- "hyper",
- "hyper-timeout",
  "percent-encoding",
  "pin-project",
  "prost",
  "prost-derive",
- "tokio",
  "tokio-stream",
- "tokio-util 0.6.9",
- "tower",
+ "tokio-util 0.7.1",
  "tower-layer",
  "tower-service",
  "tracing",
- "tracing-futures",
 ]
 
 [[package]]
 name = "tonic-build"
-version = "0.6.2"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757"
+checksum = "4d17087af5c80e5d5fc8ba9878e60258065a0a757e35efe7a05b7904bece1943"
 dependencies = [
+ "prettyplease",
  "proc-macro2",
  "prost-build",
  "quote",
@@ -2341,16 +2325,6 @@ dependencies = [
  "valuable",
 ]
 
-[[package]]
-name = "tracing-futures"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
-dependencies = [
- "pin-project",
- "tracing",
-]
-
 [[package]]
 name = "tracing-log"
 version = "0.1.2"
@@ -2460,12 +2434,6 @@ dependencies = [
  "tinyvec",
 ]
 
-[[package]]
-name = "unicode-segmentation"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
-
 [[package]]
 name = "unicode-xid"
 version = "0.2.2"

deny.toml

@@ -17,7 +17,6 @@ ignore = []
 unlicensed = "deny"
 allow = [
     "Apache-2.0",
-    "BSD-3-Clause",
     "ISC",
     "MIT",
 ]
@@ -48,14 +47,8 @@ deny = [
     { name = "rustls", wrappers = ["tokio-rustls"] },
 ]
 skip = [
-    # Waiting on a tokio release that updates parking_lot to v0.12.
-    { name = "parking_lot", version = "0.11" },
-    { name = "parking_lot_core", version = "0.8" },
-    # Waiting on a prost-build release that includes
-    # https://github.com/tokio-rs/prost/pull/583
-    { name = "heck", version = "0.3" },
     # waiting on `h2` and `tower` releases that update h2 to v0.7
-    { name = "tokio-util", version = "0.6" }
+    { name = "tokio-util", version = "0.6" },
 ]
 skip-tree = [
     # Hasn't seen a new release since 2017. Pulls in an older version of nom.

linkerd/app/Cargo.toml

@@ -27,6 +27,6 @@ regex = "1"
 thiserror = "1"
 tokio = { version = "1", features = ["rt"] }
 tokio-stream = { version = "0.1", features = ["time", "sync"] }
-tonic = { version = "0.6", default-features = false, features = ["prost"] }
+tonic = { version = "0.7", default-features = false, features = ["prost"] }
 tower = "0.4"
 tracing = "0.1"

linkerd/app/core/Cargo.toml

@@ -61,7 +61,7 @@ serde_json = "1"
 thiserror = "1"
 tokio = { version = "1", features = ["macros", "sync", "parking_lot"] }
 tokio-stream = { version = "0.1", features = ["time"] }
-tonic = { version = "0.6", default-features = false, features = ["prost"] }
+tonic = { version = "0.7", default-features = false, features = ["prost"] }
 tracing = "0.1"
 parking_lot = "0.12"
 pin-project = "1"