Refactor cosigned to take advantage of duck typing.

With this change, the webhook can take advantage of duck typing to parse all of the "Pod Specable" types currently supported.

This also takes advantage of the `knative.dev/pkg` webhook infrastructure to reduce boilerplate and eliminate the need for `cert-manager`.

Lastly, this starts to sketch out some cosigned e2e tests to verify that things work.

Signed-off-by: Matt Moore <mattomata@gmail.com>

.github/workflows/kind-e2e-cosigned.yaml

@@ -0,0 +1,227 @@
+name: Cosigned KinD E2E
+
+on:
+  pull_request:
+    branches: [ 'main', 'release-*' ]
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./src/github.com/sigstore/cosign
+
+jobs:
+
+  e2e-tests:
+    name: e2e tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false # Keep running if one leg fails.
+      matrix:
+        k8s-version:
+        # - v1.19.11
+        # - v1.20.7
+        - v1.21.1
+
+        include:
+          # Map between K8s and KinD versions.
+          # This is attempting to make it a bit clearer what's being tested.
+          # See: https://github.com/kubernetes-sigs/kind/releases
+        # - k8s-version: v1.19.11
+        #   kind-version: v0.11.1
+        #   kind-image-sha: sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729
+        #   cluster-suffix: c${{ github.run_id }}.local
+        # - k8s-version: v1.20.7
+        #   kind-version: v0.11.1
+        #   kind-image-sha: sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9
+        #   cluster-suffix: c${{ github.run_id }}.local
+        - k8s-version: v1.21.1
+          kind-version: v0.11.1
+          kind-image-sha: sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6
+          cluster-suffix: c${{ github.run_id }}.local
+
+    env:
+      GOPATH: ${{ github.workspace }}
+      GO111MODULE: on
+      # https://github.com/google/go-containerregistry/pull/125 allows insecure registry for
+      # '*.local' hostnames. This works both for `ko` and our own tag-to-digest resolution logic,
+      # thus allowing us to test without bypassing tag-to-digest resolution.
+      REGISTRY_NAME: registry.local
+      REGISTRY_PORT: 5000
+      KO_DOCKER_REPO: registry.local:5000/cosigned
+
+    steps:
+    - name: Set up Go 1.16.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.x
+
+    - name: Install Dependencies
+      working-directory: ./
+      run: |
+        echo '::group:: install ko'
+        curl -L https://github.com/google/ko/releases/download/v0.6.0/ko_0.6.0_Linux_x86_64.tar.gz | tar xzf - ko
+        chmod +x ./ko
+        sudo mv ko /usr/local/bin
+        echo '::endgroup::'
+
+    - name: Check out code onto GOPATH
+      uses: actions/checkout@v2
+      with:
+        path: ./src/github.com/sigstore/cosign
+
+    - name: Install Cosign
+      run: |
+        go install ./cmd/cosign
+
+    # This KinD setup is based on what we use for knative/serving on GHA, and it includes several "fun"
+    # monkey wrenches (e.g. randomizing cluster suffix: `.svc.cluster.local`) to make sure we don't bake
+    # in any invalid assumptions about a particular Kubernetes configuration.
+    - name: Install KinD
+      run: |
+        set -x
+        # Disable swap otherwise memory enforcement doesn't work
+        # See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600009955324200
+        sudo swapoff -a
+        sudo rm -f /swapfile
+        # Use in-memory storage to avoid etcd server timeouts.
+        # https://kubernetes.slack.com/archives/CEKK1KTN2/p1615134111016300
+        # https://github.com/kubernetes-sigs/kind/issues/845
+        sudo mkdir -p /tmp/etcd
+        sudo mount -t tmpfs tmpfs /tmp/etcd
+        curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${{ matrix.kind-version }}/kind-$(uname)-amd64
+        chmod +x ./kind
+        sudo mv kind /usr/local/bin
+
+    - name: Configure KinD Cluster
+      run: |
+        set -x
+        # KinD configuration.
+        cat > kind.yaml <<EOF
+        apiVersion: kind.x-k8s.io/v1alpha4
+        kind: Cluster
+        # Configure registry for KinD.
+        containerdConfigPatches:
+        - |-
+          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."$REGISTRY_NAME:$REGISTRY_PORT"]
+            endpoint = ["http://$REGISTRY_NAME:$REGISTRY_PORT"]
+        # This is needed in order to support projected volumes with service account tokens.
+        # See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600268272383600
+        kubeadmConfigPatches:
+          - |
+            apiVersion: kubeadm.k8s.io/v1beta2
+            kind: ClusterConfiguration
+            metadata:
+              name: config
+            apiServer:
+              extraArgs:
+                "service-account-issuer": "kubernetes.default.svc"
+                "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
+            networking:
+              dnsDomain: "${{ matrix.cluster-suffix }}"
+        nodes:
+        - role: control-plane
+          image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
+          extraMounts:
+          - containerPath: /var/lib/etcd
+            hostPath: /tmp/etcd
+        - role: worker
+          image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
+        EOF
+
+    - name: Create KinD Cluster
+      run: |
+        set -x
+        kind create cluster --config kind.yaml
+
+    - name: Setup local registry
+      run: |
+        # Run a registry.
+        docker run -d --restart=always \
+          -p $REGISTRY_PORT:$REGISTRY_PORT --name $REGISTRY_NAME registry:2
+
+        # Connect the registry to the KinD network.
+        docker network connect "kind" $REGISTRY_NAME
+
+        # Make the $REGISTRY_NAME -> 127.0.0.1, to tell `ko` to publish to
+        # local reigstry, even when pushing $REGISTRY_NAME:$REGISTRY_PORT/some/image
+        sudo echo "127.0.0.1 $REGISTRY_NAME" | sudo tee -a /etc/hosts
+
+    - name: Install cosigned
+      run: |
+        ko apply -Bf config/
+
+        # Update the cosign verification-key secret with a proper key pair.
+        cosign generate-key-pair k8s://cosign-system/verification-key
+
+        # Wait for the webhook to come up and become Ready
+        kubectl rollout status --timeout 5m --namespace cosign-system deployments/webhook
+
+    - name: Run Tests
+      run: |
+        echo '::group:: publish test image'
+        DIGEST=$(ko publish ./cmd/sample)
+        echo '::endgroup::'
+
+        echo '::group:: test pod rejection'
+        cat > pod.yaml <<EOF
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          generateName: pod-test-
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: sample
+            image: $DIGEST
+        EOF
+        if kubectl create -f pod.yaml ; then
+          echo Failed to block Pod creation!
+          exit 1
+        fi
+        echo '::endgroup::'
+
+        echo '::group:: test job rejection'
+        # TODO(mattmoor): Test creating a job (should fail, it's not signed!)
+        echo '::endgroup::'
+
+        echo '::group:: sign test image'
+        cosign sign -key k8s://cosign-system/verification-key $DIGEST
+        echo '::endgroup::'
+
+        echo '::group:: test pod success'
+        # This time it should succeed!
+        kubectl create -f pod.yaml
+        if kubectl create -f pod.yaml ; then
+          echo Failed to create Pod with properly signed image!
+          exit 1
+        fi
+        echo '::endgroup::'
+
+        echo '::group:: test job success'
+        # TODO(mattmoor): Test creating a job (should succeed, it's signed!)
+        echo '::endgroup::'
+
+
+    - name: Collect diagnostics
+      if: ${{ failure() }}
+      run: |
+        # Add more namespaces to dump here.
+        for ns in cosign-system; do
+          kubectl get pods -n${ns}
+
+          echo '::group:: describe'
+          kubectl describe pods -n${ns}
+          echo '::endgroup::'
+
+          for x in $(kubectl get pods -n${ns} -oname); do
+
+            echo "::group:: describe $x"
+            kubectl describe -n${ns} $x
+            echo '::endgroup::'
+
+            echo "::group:: $x logs"
+            kubectl logs -n${ns} $x --all-containers
+            echo '::endgroup::'
+
+          done
+        done

.ko.yaml

@@ -14,4 +14,4 @@
 # limitations under the License.
 
 # We need a shell for a lot of redirection/piping to work
-defaultBaseImage: gcr.io/distroless/base:debug
+defaultBaseImage: gcr.io/distroless/base:debug-nonroot

cmd/cosign/cli/fulcio/fulcio.go

@@ -16,7 +16,6 @@
 package fulcio
 
 import (
-	"bytes"
 	"context"
 	"crypto"
 	"crypto/ecdsa"
@@ -28,10 +27,7 @@ import (
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"strings"
-	"sync"
 
 	"github.com/go-openapi/runtime"
 	httptransport "github.com/go-openapi/runtime/client"
@@ -43,8 +39,8 @@ import (
 	"github.com/pkg/errors"
 	"golang.org/x/term"
 
+	"github.com/sigstore/cosign/cmd/cosign/cli/fulcio/fulcioroots"
 	"github.com/sigstore/cosign/pkg/cosign"
-	"github.com/sigstore/cosign/pkg/cosign/tuf"
 	fulcioClient "github.com/sigstore/fulcio/pkg/generated/client"
 	"github.com/sigstore/fulcio/pkg/generated/client/operations"
 	"github.com/sigstore/fulcio/pkg/generated/models"
@@ -56,7 +52,6 @@ const (
 	FlowNormal = "normal"
 	FlowDevice = "device"
 	FlowToken  = "token"
-	altRoot    = "SIGSTORE_ROOT_FILE"
 )
 
 type Resp struct {
@@ -65,14 +60,9 @@ type Resp struct {
 	SCT      []byte
 }
 
-// This is the root in the fulcio project.
-//go:embed fulcio.pem
-var rootPem string
-
 // This is the CT log public key
 //go:embed ctfe.pub
 var ctPublicKey string
-var fulcioTargetStr = `fulcio.crt.pem`
 
 var (
 	// For testing
@@ -242,47 +232,6 @@ func (f *Signer) PublicKey(opts ...signature.PublicKeyOption) (crypto.PublicKey,
 
 var _ signature.Signer = &Signer{}
 
-var (
-	rootsOnce sync.Once
-	roots     *x509.CertPool
-)
-
 func GetRoots() *x509.CertPool {
-	rootsOnce.Do(func() {
-		roots = initRoots()
-	})
-	return roots
-}
-
-func initRoots() *x509.CertPool {
-	cp := x509.NewCertPool()
-	rootEnv := os.Getenv(altRoot)
-	if rootEnv != "" {
-		raw, err := ioutil.ReadFile(rootEnv)
-		if err != nil {
-			panic(fmt.Sprintf("error reading root PEM file: %s", err))
-		}
-		if !cp.AppendCertsFromPEM(raw) {
-			panic("error creating root cert pool")
-		}
-	} else {
-		// First try retrieving from TUF root. Otherwise use rootPem.
-		ctx := context.Background() // TODO: pass in context?
-		buf := tuf.ByteDestination{Buffer: &bytes.Buffer{}}
-		err := tuf.GetTarget(ctx, fulcioTargetStr, &buf)
-		if err != nil {
-			// The user may not have initialized the local root metadata. Log the error and use the embedded root.
-			fmt.Fprintln(os.Stderr, "No TUF root installed, using embedded CA certificate.")
-			if !cp.AppendCertsFromPEM([]byte(rootPem)) {
-				panic("error creating root cert pool")
-			}
-		} else {
-			// TODO: Remove the string replace when SigStore root is updated.
-			replaced := strings.ReplaceAll(buf.String(), "\n  ", "\n")
-			if !cp.AppendCertsFromPEM([]byte(replaced)) {
-				panic("error creating root cert pool")
-			}
-		}
-	}
-	return cp
+	return fulcioroots.Get()
 }