From c00d70a36217c40e5c373f995237c29eecfdf637 Mon Sep 17 00:00:00 2001 From: Kevin Lingerfelt Date: Wed, 28 Mar 2018 13:44:10 -0700 Subject: [PATCH 1/6] Add CA certificate bundle distributor to conduit install Signed-off-by: Kevin Lingerfelt --- Gopkg.lock | 5 +- cli/Dockerfile-bin | 2 +- cli/cmd/install.go | 2 + cli/cmd/install_test.go | 1 + cli/cmd/testdata/install_default.golden | 132 ++++++++++++- cli/cmd/testdata/install_output.golden | 133 ++++++++++++- cli/install/template.go | 76 +++++++- controller/Dockerfile | 2 +- controller/ca/controller.go | 241 ++++++++++++++++++++++++ controller/ca/controller_test.go | 161 ++++++++++++++++ controller/cmd/ca-distributor/main.go | 59 ++++++ pkg/k8s/labels.go | 8 + proxy-init/Dockerfile | 2 +- test/stat/stat_test.go | 2 +- web/Dockerfile | 2 +- 15 files changed, 809 insertions(+), 19 deletions(-) mode change 100755 => 100644 cli/cmd/testdata/install_default.golden create mode 100644 controller/ca/controller.go create mode 100644 controller/ca/controller_test.go create mode 100644 controller/cmd/ca-distributor/main.go diff --git a/Gopkg.lock b/Gopkg.lock index 5fa0254affa9e..0442264b56c4a 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -608,7 +608,8 @@ "util/homedir", "util/integer", "util/jsonpath", - "util/retry" + "util/retry", + "util/workqueue" ] revision = "33f2870a2b83179c823ddc90e5513f9e5fe43b38" version = "kubernetes-1.10.2" @@ -626,6 +627,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "ef19d9c67ad3dd2b03a2ecb5fcbfcc71913e3ffc4cf79e72db6a2d0565e5f622" + inputs-digest = "44c7140f52ef427aed7c94340d686bdd91a23160c39a37cb5c3bdccff380e548" solver-name = "gps-cdcl" solver-version = 1 diff --git a/cli/Dockerfile-bin b/cli/Dockerfile-bin index 37610881be8c9..53eba38ae400f 100644 --- a/cli/Dockerfile-bin +++ b/cli/Dockerfile-bin @@ -1,5 +1,5 @@ ## compile binaries -FROM gcr.io/runconduit/go-deps:c934e1ab as golang +FROM gcr.io/runconduit/go-deps:07e4dc9b as golang WORKDIR /go/src/github.com/runconduit/conduit COPY cli cli COPY controller/k8s controller/k8s diff --git a/cli/cmd/install.go b/cli/cmd/install.go index 4875015cacf98..bff288dd64154 100644 --- a/cli/cmd/install.go +++ b/cli/cmd/install.go @@ -32,6 +32,7 @@ type installConfig struct { CreatedByAnnotation string ProxyAPIPort uint EnableTLS bool + CertificateBundleName string } type installOptions struct { @@ -105,6 +106,7 @@ func validateAndBuildConfig(options *installOptions) (*installConfig, error) { CreatedByAnnotation: k8s.CreatedByAnnotation, ProxyAPIPort: options.proxyAPIPort, EnableTLS: options.enableTLS, + CertificateBundleName: k8s.CertificateBundleName, }, nil } diff --git a/cli/cmd/install_test.go b/cli/cmd/install_test.go index 25755a4bd2f06..ca1630578e332 100644 --- a/cli/cmd/install_test.go +++ b/cli/cmd/install_test.go @@ -37,6 +37,7 @@ func TestRender(t *testing.T) { CreatedByAnnotation: "CreatedByAnnotation", ProxyAPIPort: 123, EnableTLS: true, + CertificateBundleName: "CertificateBundleName", } testCases := []struct { diff --git a/cli/cmd/testdata/install_default.golden b/cli/cmd/testdata/install_default.golden old mode 100755 new mode 100644 index f0d27435ea4b8..c8cf243c6e0bc --- a/cli/cmd/testdata/install_default.golden +++ b/cli/cmd/testdata/install_default.golden @@ -12,7 +12,7 @@ metadata: name: conduit-controller namespace: conduit -### RBAC ### +### Controller RBAC ### --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -31,7 +31,6 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: conduit-controller - namespace: conduit roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -41,6 +40,46 @@ subjects: name: conduit-controller namespace: conduit +### Service Account CA ### +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: conduit-ca + namespace: conduit + +### CA RBAC ### +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: conduit-ca +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "list", "watch"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: [conduit-ca-bundle] + verbs: ["update"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "watch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: conduit-ca +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: conduit-ca +subjects: +- kind: ServiceAccount + name: conduit-ca + namespace: conduit + ### Service Account Prometheus ### --- kind: ServiceAccount @@ -49,7 +88,7 @@ metadata: name: conduit-prometheus namespace: conduit -### RBAC ### +### Prometheus RBAC ### --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -65,7 +104,6 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: conduit-prometheus - namespace: conduit roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -248,6 +286,92 @@ spec: serviceAccount: conduit-controller status: {} --- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + conduit.io/created-by: conduit/cli undefined + creationTimestamp: null + labels: + conduit.io/control-plane-component: ca-bundle-distributor + name: ca-bundle-distributor + namespace: conduit +spec: + replicas: 1 + strategy: {} + template: + metadata: + annotations: + conduit.io/created-by: conduit/cli undefined + conduit.io/proxy-version: undefined + creationTimestamp: null + labels: + conduit.io/control-plane-component: ca-bundle-distributor + conduit.io/control-plane-ns: conduit + conduit.io/proxy-deployment: ca-bundle-distributor + spec: + containers: + - args: + - ca-distributor + - -controller-namespace=conduit + - -log-level=info + - -logtostderr=true + image: gcr.io/runconduit/controller:undefined + imagePullPolicy: IfNotPresent + name: ca-distributor + resources: {} + - env: + - name: CONDUIT_PROXY_LOG + value: warn,conduit_proxy=info + - name: CONDUIT_PROXY_BIND_TIMEOUT + value: 10s + - name: CONDUIT_PROXY_CONTROL_URL + value: tcp://proxy-api.conduit.svc.cluster.local:8086 + - name: CONDUIT_PROXY_CONTROL_LISTENER + value: tcp://0.0.0.0:4190 + - name: CONDUIT_PROXY_METRICS_LISTENER + value: tcp://0.0.0.0:4191 + - name: CONDUIT_PROXY_PRIVATE_LISTENER + value: tcp://127.0.0.1:4140 + - name: CONDUIT_PROXY_PUBLIC_LISTENER + value: tcp://0.0.0.0:4143 + - name: CONDUIT_PROXY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/runconduit/proxy:undefined + imagePullPolicy: IfNotPresent + name: conduit-proxy + ports: + - containerPort: 4143 + name: conduit-proxy + - containerPort: 4191 + name: conduit-metrics + resources: {} + securityContext: + runAsUser: 2102 + initContainers: + - args: + - --incoming-proxy-port + - "4143" + - --outgoing-proxy-port + - "4140" + - --proxy-uid + - "2102" + - --inbound-ports-to-ignore + - 4190,4191 + image: gcr.io/runconduit/proxy-init:undefined + imagePullPolicy: IfNotPresent + name: conduit-init + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + privileged: false + serviceAccount: conduit-ca +status: {} +--- kind: Service apiVersion: v1 metadata: diff --git a/cli/cmd/testdata/install_output.golden b/cli/cmd/testdata/install_output.golden index 625427971a422..a71fb8ce2f1ac 100644 --- a/cli/cmd/testdata/install_output.golden +++ b/cli/cmd/testdata/install_output.golden @@ -12,7 +12,7 @@ metadata: name: conduit-controller namespace: Namespace -### RBAC ### +### Controller RBAC ### --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -31,7 +31,6 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: conduit-controller - namespace: Namespace roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -41,6 +40,46 @@ subjects: name: conduit-controller namespace: Namespace +### Service Account CA ### +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: conduit-ca + namespace: Namespace + +### CA RBAC ### +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: conduit-ca +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "list", "watch"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: [CertificateBundleName] + verbs: ["update"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "watch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: conduit-ca +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: conduit-ca +subjects: +- kind: ServiceAccount + name: conduit-ca + namespace: Namespace + ### Service Account Prometheus ### --- kind: ServiceAccount @@ -49,7 +88,7 @@ metadata: name: conduit-prometheus namespace: Namespace -### RBAC ### +### Prometheus RBAC ### --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -65,7 +104,6 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: conduit-prometheus - namespace: Namespace roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -249,6 +287,93 @@ spec: serviceAccount: conduit-controller status: {} --- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + CreatedByAnnotation: CliVersion + creationTimestamp: null + labels: + ControllerComponentLabel: ca-bundle-distributor + name: ca-bundle-distributor + namespace: Namespace +spec: + replicas: 1 + strategy: {} + template: + metadata: + annotations: + CreatedByAnnotation: CliVersion + conduit.io/created-by: conduit/cli undefined + conduit.io/proxy-version: undefined + creationTimestamp: null + labels: + ControllerComponentLabel: ca-bundle-distributor + conduit.io/control-plane-ns: Namespace + conduit.io/proxy-deployment: ca-bundle-distributor + spec: + containers: + - args: + - ca-distributor + - -controller-namespace=Namespace + - -log-level=ControllerLogLevel + - -logtostderr=true + image: ControllerImage + imagePullPolicy: ImagePullPolicy + name: ca-distributor + resources: {} + - env: + - name: CONDUIT_PROXY_LOG + value: warn,conduit_proxy=info + - name: CONDUIT_PROXY_BIND_TIMEOUT + value: 10s + - name: CONDUIT_PROXY_CONTROL_URL + value: tcp://proxy-api.Namespace.svc.cluster.local:8086 + - name: CONDUIT_PROXY_CONTROL_LISTENER + value: tcp://0.0.0.0:4190 + - name: CONDUIT_PROXY_METRICS_LISTENER + value: tcp://0.0.0.0:4191 + - name: CONDUIT_PROXY_PRIVATE_LISTENER + value: tcp://127.0.0.1:4140 + - name: CONDUIT_PROXY_PUBLIC_LISTENER + value: tcp://0.0.0.0:4143 + - name: CONDUIT_PROXY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/runconduit/proxy:undefined + imagePullPolicy: IfNotPresent + name: conduit-proxy + ports: + - containerPort: 4143 + name: conduit-proxy + - containerPort: 4191 + name: conduit-metrics + resources: {} + securityContext: + runAsUser: 2102 + initContainers: + - args: + - --incoming-proxy-port + - "4143" + - --outgoing-proxy-port + - "4140" + - --proxy-uid + - "2102" + - --inbound-ports-to-ignore + - 4190,4191 + image: gcr.io/runconduit/proxy-init:undefined + imagePullPolicy: IfNotPresent + name: conduit-init + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + privileged: false + serviceAccount: conduit-ca +status: {} +--- kind: Service apiVersion: v1 metadata: diff --git a/cli/install/template.go b/cli/install/template.go index 765508e9c3e70..7231ca50d267a 100644 --- a/cli/install/template.go +++ b/cli/install/template.go @@ -15,7 +15,7 @@ metadata: name: conduit-controller namespace: {{.Namespace}} -### RBAC ### +### Controller RBAC ### --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -34,7 +34,6 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: conduit-controller - namespace: {{.Namespace}} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -44,6 +43,46 @@ subjects: name: conduit-controller namespace: {{.Namespace}} +### Service Account CA ### +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: conduit-ca + namespace: {{.Namespace}} + +### CA RBAC ### +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: conduit-ca +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "list", "watch"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: [{{.CertificateBundleName}}] + verbs: ["update"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "watch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: conduit-ca +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: conduit-ca +subjects: +- kind: ServiceAccount + name: conduit-ca + namespace: {{.Namespace}} + ### Service Account Prometheus ### --- kind: ServiceAccount @@ -52,7 +91,7 @@ metadata: name: conduit-prometheus namespace: {{.Namespace}} -### RBAC ### +### Prometheus RBAC ### --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -68,7 +107,6 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: conduit-prometheus - namespace: {{.Namespace}} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -191,6 +229,36 @@ spec: - "-log-level={{.ControllerLogLevel}}" - "-logtostderr=true" +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: ca-bundle-distributor + namespace: {{.Namespace}} + labels: + {{.ControllerComponentLabel}}: ca-bundle-distributor + annotations: + {{.CreatedByAnnotation}}: {{.CliVersion}} +spec: + replicas: {{.ControllerReplicas}} + template: + metadata: + labels: + {{.ControllerComponentLabel}}: ca-bundle-distributor + annotations: + {{.CreatedByAnnotation}}: {{.CliVersion}} + spec: + serviceAccount: conduit-ca + containers: + - name: ca-distributor + image: {{.ControllerImage}} + imagePullPolicy: {{.ImagePullPolicy}} + args: + - "ca-distributor" + - "-controller-namespace={{.Namespace}}" + - "-log-level={{.ControllerLogLevel}}" + - "-logtostderr=true" + ### Web ### --- kind: Service diff --git a/controller/Dockerfile b/controller/Dockerfile index 2396a012e81c2..84ca996f87a3b 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -1,5 +1,5 @@ ## compile controller services -FROM gcr.io/runconduit/go-deps:c934e1ab as golang +FROM gcr.io/runconduit/go-deps:07e4dc9b as golang WORKDIR /go/src/github.com/runconduit/conduit COPY controller/gen controller/gen COPY pkg pkg diff --git a/controller/ca/controller.go b/controller/ca/controller.go new file mode 100644 index 0000000000000..f6b64e31ee7a1 --- /dev/null +++ b/controller/ca/controller.go @@ -0,0 +1,241 @@ +package ca + +import ( + "fmt" + "time" + + "github.com/runconduit/conduit/pkg/k8s" + log "github.com/sirupsen/logrus" + "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" +) + +type CertificateController struct { + client kubernetes.Interface + namespace string + + syncHandler func(key string) error + + podLister corelisters.PodLister + podListerSynced cache.InformerSynced + + configMapLister corelisters.ConfigMapLister + configMapListerSynced cache.InformerSynced + + queue workqueue.RateLimitingInterface +} + +func NewCertificateController( + client kubernetes.Interface, + conduitNamespace string, + podInformer coreinformers.PodInformer, + configMapInformer coreinformers.ConfigMapInformer, +) *CertificateController { + c := &CertificateController{ + client: client, + namespace: conduitNamespace, + podLister: podInformer.Lister(), + podListerSynced: podInformer.Informer().HasSynced, + configMapLister: configMapInformer.Lister(), + configMapListerSynced: configMapInformer.Informer().HasSynced, + queue: workqueue.NewNamedRateLimitingQueue( + workqueue.DefaultControllerRateLimiter(), "certificates"), + } + + podInformer.Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.handlePodUpdate(obj.(*v1.Pod)) + }, + UpdateFunc: func(_, obj interface{}) { + c.handlePodUpdate(obj.(*v1.Pod)) + }, + }, + ) + + configMapInformer.Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.handleConfigMapUpdate(obj.(*v1.ConfigMap)) + }, + UpdateFunc: func(_, obj interface{}) { + c.handleConfigMapUpdate(obj.(*v1.ConfigMap)) + }, + DeleteFunc: c.handleConfigMapDelete, + }, + ) + + c.syncHandler = c.syncNamespace + + return c +} + +func (c *CertificateController) Run(stopCh <-chan struct{}) error { + defer runtime.HandleCrash() + defer c.queue.ShutDown() + + log.Info("starting certificate controller") + defer log.Info("shutting down certificate controller") + + if !cache.WaitForCacheSync(stopCh, c.podListerSynced, c.configMapListerSynced) { + return fmt.Errorf("timed out waiting for cache to sync") + } + log.Info("caches are synced") + + go wait.Until(c.worker, time.Second, stopCh) + + <-stopCh + return nil +} + +func (c *CertificateController) worker() { + for c.processNextWorkItem() { + } +} + +func (c *CertificateController) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + + err := c.syncHandler(key.(string)) + if err != nil { + log.Errorf("error syncing config map: %s", err) + c.queue.AddRateLimited(key) + return true + } + + c.queue.Forget(key) + return true +} + +func (c *CertificateController) syncNamespace(ns string) error { + conduitConfigMap, err := c.configMapLister.ConfigMaps(c.namespace). + Get(k8s.CertificateBundleName) + if apierrors.IsNotFound(err) { + log.Warnf("configmap [%s] not found in namespace [%s]", + k8s.CertificateBundleName, c.namespace) + return nil + } + if err != nil { + return err + } + + configMap := &v1.ConfigMap{ + ObjectMeta: meta.ObjectMeta{Name: k8s.CertificateBundleName}, + Data: conduitConfigMap.Data, + } + + log.Debugf("adding configmap [%s] to namespace [%s]", + k8s.CertificateBundleName, ns) + _, err = c.client.CoreV1().ConfigMaps(ns).Create(configMap) + if apierrors.IsAlreadyExists(err) { + _, err = c.client.CoreV1().ConfigMaps(ns).Update(configMap) + } + + return err +} + +func (c *CertificateController) handlePodUpdate(pod *v1.Pod) { + if c.isInjectedPod(pod) && !c.filterNamespace(pod.Namespace) { + c.queue.Add(pod.Namespace) + } +} + +func (c *CertificateController) handleConfigMapUpdate(cm *v1.ConfigMap) { + if cm.Namespace == c.namespace && cm.Name == k8s.CertificateBundleName { + namespaces, err := c.getInjectedNamespaces() + if err != nil { + log.Errorf("error getting namespaces: %s", err) + return + } + + for _, ns := range namespaces { + c.queue.Add(ns) + } + } +} + +func (c *CertificateController) handleConfigMapDelete(obj interface{}) { + configMap, ok := obj.(*v1.ConfigMap) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + log.Warnf("couldn't get object from tombstone: %+v", obj) + return + } + configMap, ok = tombstone.Obj.(*v1.ConfigMap) + if !ok { + log.Warnf("object is not a configmap: %+v", tombstone.Obj) + return + } + } + + if configMap.Name == k8s.CertificateBundleName && configMap.Namespace != c.namespace { + injected, err := c.isInjectedNamespace(configMap.Namespace) + if err != nil { + log.Errorf("error getting pods in namespace [%s]: %s", configMap.Namespace, err) + return + } + if injected { + log.Infof("configmap [%s] in namespace [%s] deleted; recreating it", + k8s.CertificateBundleName, configMap.Namespace) + c.queue.Add(configMap.Namespace) + } + } +} + +func (c *CertificateController) getInjectedNamespaces() ([]string, error) { + pods, err := c.podLister.List(labels.Everything()) + if err != nil { + return nil, err + } + + namespaces := make(sets.String) + for _, pod := range pods { + if !c.filterNamespace(pod.Namespace) && c.isInjectedPod(pod) { + namespaces.Insert(pod.Namespace) + } + } + + return namespaces.List(), nil +} + +func (c *CertificateController) filterNamespace(ns string) bool { + for _, filter := range []string{c.namespace, "kube-system", "kube-public"} { + if ns == filter { + return true + } + } + return false +} + +func (c *CertificateController) isInjectedNamespace(ns string) (bool, error) { + pods, err := c.podLister.Pods(ns).List(labels.Everything()) + if err != nil { + return false, err + } + for _, pod := range pods { + if c.isInjectedPod(pod) { + return true, nil + } + } + return false, nil +} + +func (c *CertificateController) isInjectedPod(pod *v1.Pod) bool { + _, ok := pod.Annotations[k8s.CreatedByAnnotation] + return ok +} diff --git a/controller/ca/controller_test.go b/controller/ca/controller_test.go new file mode 100644 index 0000000000000..b3510df06fcd9 --- /dev/null +++ b/controller/ca/controller_test.go @@ -0,0 +1,161 @@ +package ca + +import ( + "testing" + "time" + + "github.com/runconduit/conduit/pkg/k8s" + "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" +) + +var ( + conduitNS = "conduitest" + + conduitNamespace = &v1.Namespace{ + ObjectMeta: meta.ObjectMeta{Name: conduitNS}, + } + + conduitConfigMap = &v1.ConfigMap{ + ObjectMeta: meta.ObjectMeta{ + Namespace: conduitNS, + Name: k8s.CertificateBundleName, + }, + } + + injectedNS = "injecttest" + + injectedNamespace = &v1.Namespace{ + ObjectMeta: meta.ObjectMeta{Name: injectedNS}, + } + + injectedConfigMap = &v1.ConfigMap{ + ObjectMeta: meta.ObjectMeta{ + Namespace: injectedNS, + Name: k8s.CertificateBundleName, + }, + } + + injectedPod = &v1.Pod{ + ObjectMeta: meta.ObjectMeta{ + Namespace: injectedNS, + Annotations: map[string]string{ + k8s.CreatedByAnnotation: "conduit", + }, + }, + } +) + +func TestCertificateController(t *testing.T) { + t.Run("creates new configmap in injected namespace on pod add", func(t *testing.T) { + controller, sharedInformers, client, synced := new(injectedNamespace) + stopCh := run(controller, sharedInformers) + defer close(stopCh) + + controller.handlePodUpdate(injectedPod) + + select { + case <-synced: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for sync") + } + + action := client.Actions()[len(client.Actions())-1] + + if !action.Matches("create", "configmaps") { + t.Fatalf("expected action to be configmap create, got: %+v", action) + } + + if action.GetNamespace() != injectedNS { + t.Fatalf("expected action to happen in [%s] namespace, got [%s] namespace", + injectedNS, action.GetNamespace()) + } + }) + + t.Run("updates configmap in injected namespace on configmap update", func(t *testing.T) { + controller, sharedInformers, client, synced := new( + injectedNamespace, injectedPod, injectedConfigMap) + stopCh := run(controller, sharedInformers) + defer close(stopCh) + + controller.handleConfigMapUpdate(conduitConfigMap) + + select { + case <-synced: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for sync") + } + + action := client.Actions()[len(client.Actions())-1] + + if !action.Matches("update", "configmaps") { + t.Fatalf("expected action to be configmap update, got: %+v", action) + } + + if action.GetNamespace() != injectedNS { + t.Fatalf("expected action to happen in [%s] namespace, got [%s] namespace", + injectedNS, action.GetNamespace()) + } + }) + + t.Run("re-adds configmap in injected namespace on configmap delete", func(t *testing.T) { + controller, sharedInformers, client, synced := new( + injectedNamespace, injectedPod, injectedConfigMap) + stopCh := run(controller, sharedInformers) + defer close(stopCh) + + controller.handleConfigMapDelete(injectedConfigMap) + + select { + case <-synced: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for sync") + } + + action := client.Actions()[len(client.Actions())-1] + + // we expect an update instead of a create here because we didn't actually + // delete the config map; we just triggered a delete event + if !action.Matches("update", "configmaps") { + t.Fatalf("expected action to be configmap update, got: %+v", action) + } + + if action.GetNamespace() != injectedNS { + t.Fatalf("expected action to happen in [%s] namespace, got [%s] namespace", + injectedNS, action.GetNamespace()) + } + }) +} + +func new(fixtures ...runtime.Object) (*CertificateController, informers.SharedInformerFactory, *fake.Clientset, chan bool) { + withConduit := append([]runtime.Object{conduitNamespace, conduitConfigMap}, fixtures...) + clientSet := fake.NewSimpleClientset(withConduit...) + sharedInformers := informers.NewSharedInformerFactory(clientSet, 10*time.Minute) + controller := NewCertificateController( + clientSet, + conduitNS, + sharedInformers.Core().V1().Pods(), + sharedInformers.Core().V1().ConfigMaps(), + ) + + synced := make(chan bool, 1) + controller.syncHandler = func(key string) error { + err := controller.syncNamespace(key) + synced <- true + return err + } + + return controller, sharedInformers, clientSet, synced +} + +func run(c *CertificateController, i informers.SharedInformerFactory) chan struct{} { + stopCh := make(chan struct{}) + i.Start(stopCh) + go c.Run(stopCh) + cache.WaitForCacheSync(stopCh, c.podListerSynced, c.configMapListerSynced) + return stopCh +} diff --git a/controller/cmd/ca-distributor/main.go b/controller/cmd/ca-distributor/main.go new file mode 100644 index 0000000000000..d5e60cd3d7167 --- /dev/null +++ b/controller/cmd/ca-distributor/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "flag" + "os" + "os/signal" + "syscall" + "time" + + "github.com/runconduit/conduit/controller/ca" + "github.com/runconduit/conduit/controller/k8s" + "github.com/runconduit/conduit/pkg/version" + log "github.com/sirupsen/logrus" + "k8s.io/client-go/informers" +) + +const configMapName = "conduit-ca-bundle" + +func main() { + controllerNamespace := flag.String("controller-namespace", "conduit", "namespace in which Conduit is installed") + kubeConfigPath := flag.String("kubeconfig", "", "path to kube config") + logLevel := flag.String("log-level", log.InfoLevel.String(), "log level, must be one of: panic, fatal, error, warn, info, debug") + printVersion := version.VersionFlag() + flag.Parse() + + // set global log level + level, err := log.ParseLevel(*logLevel) + if err != nil { + log.Fatalf("invalid log-level: %s", *logLevel) + } + log.SetLevel(level) + + version.MaybePrintVersionAndExit(*printVersion) + + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + + clientSet, err := k8s.NewClientSet(*kubeConfigPath) + if err != nil { + log.Fatalf(err.Error()) + } + + sharedInformers := informers.NewSharedInformerFactory(clientSet, 10*time.Minute) + controller := ca.NewCertificateController( + clientSet, + *controllerNamespace, + sharedInformers.Core().V1().Pods(), + sharedInformers.Core().V1().ConfigMaps(), + ) + stopCh := make(chan struct{}) + + sharedInformers.Start(stopCh) + go controller.Run(stopCh) + + <-stop + + log.Info("shutting down") + close(stopCh) +} diff --git a/pkg/k8s/labels.go b/pkg/k8s/labels.go index 1d79c6b6c5f86..928d135a1899e 100644 --- a/pkg/k8s/labels.go +++ b/pkg/k8s/labels.go @@ -62,6 +62,14 @@ const ( // ProxyVersionAnnotation indicates the version of the injected data plane // (e.g. v0.1.3). ProxyVersionAnnotation = "conduit.io/proxy-version" + + /* + * Component Names + */ + + // CertificateBundleName is the name of the ConfigMap that holds the root + // certificate + CertificateBundleName = "conduit-ca-bundle" ) var proxyLabels = []string{ diff --git a/proxy-init/Dockerfile b/proxy-init/Dockerfile index 70390158e1872..040fff86238cc 100644 --- a/proxy-init/Dockerfile +++ b/proxy-init/Dockerfile @@ -1,5 +1,5 @@ ## compile proxy-init utility -FROM gcr.io/runconduit/go-deps:c934e1ab as golang +FROM gcr.io/runconduit/go-deps:07e4dc9b as golang WORKDIR /go/src/github.com/runconduit/conduit COPY ./proxy-init ./proxy-init RUN CGO_ENABLED=0 GOOS=linux go install -v -installsuffix cgo ./proxy-init/ diff --git a/test/stat/stat_test.go b/test/stat/stat_test.go index 9afe79431d92d..d0407efad60cf 100644 --- a/test/stat/stat_test.go +++ b/test/stat/stat_test.go @@ -75,7 +75,7 @@ func parseRows(out string) (map[string]*rowStat, error) { rows := strings.Split(out, "\n") rows = rows[1 : len(rows)-1] // strip header and trailing newline - expectedRowCount := 4 + expectedRowCount := 5 if len(rows) != expectedRowCount { return nil, fmt.Errorf( "Expected [%d] rows in stat output, got [%d]; full output:\n%s", diff --git a/web/Dockerfile b/web/Dockerfile index 35a6dff2218fa..0918421c53f87 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -24,7 +24,7 @@ ENV NODE_ENV production RUN $ROOT/bin/web build ## compile go server -FROM gcr.io/runconduit/go-deps:c934e1ab as golang +FROM gcr.io/runconduit/go-deps:07e4dc9b as golang WORKDIR /go/src/github.com/runconduit/conduit COPY web web COPY controller controller From c57ac907e7ad69f300ddae3ff424d7558309f662 Mon Sep 17 00:00:00 2001 From: Kevin Lingerfelt Date: Tue, 12 Jun 2018 17:26:53 -0700 Subject: [PATCH 2/6] Update ca-distributor to use shared informers Signed-off-by: Kevin Lingerfelt --- Gopkg.lock | 2 +- cli/Dockerfile-bin | 2 +- cli/cmd/testdata/install_default.golden | 11 +- cli/cmd/testdata/install_output.golden | 11 +- cli/install/template.go | 11 +- controller/Dockerfile | 2 +- controller/ca/controller.go | 104 ++++++---------- controller/ca/controller_test.go | 156 ++++++++++++++---------- controller/cmd/ca-distributor/main.go | 29 +++-- proxy-init/Dockerfile | 2 +- web/Dockerfile | 2 +- 11 files changed, 174 insertions(+), 158 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 0442264b56c4a..58ce8e011d377 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -627,6 +627,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "44c7140f52ef427aed7c94340d686bdd91a23160c39a37cb5c3bdccff380e548" + inputs-digest = "a46330526b3d860daa1507f55bf062b7e132e324f5d9b0a469857e972c907488" solver-name = "gps-cdcl" solver-version = 1 diff --git a/cli/Dockerfile-bin b/cli/Dockerfile-bin index 53eba38ae400f..2daa07c92dc4b 100644 --- a/cli/Dockerfile-bin +++ b/cli/Dockerfile-bin @@ -1,5 +1,5 @@ ## compile binaries -FROM gcr.io/runconduit/go-deps:07e4dc9b as golang +FROM gcr.io/runconduit/go-deps:bca359b1 as golang WORKDIR /go/src/github.com/runconduit/conduit COPY cli cli COPY controller/k8s controller/k8s diff --git a/cli/cmd/testdata/install_default.golden b/cli/cmd/testdata/install_default.golden index c8cf243c6e0bc..1ea17181efaef 100644 --- a/cli/cmd/testdata/install_default.golden +++ b/cli/cmd/testdata/install_default.golden @@ -23,7 +23,7 @@ rules: resources: ["deployments", "replicasets"] verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers"] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] verbs: ["list", "get", "watch"] --- @@ -57,14 +57,17 @@ metadata: rules: - apiGroups: [""] resources: ["configmaps"] - verbs: ["create", "list", "watch"] + verbs: ["create"] - apiGroups: [""] resources: ["configmaps"] resourceNames: [conduit-ca-bundle] verbs: ["update"] +- apiGroups: ["extensions", "apps"] + resources: ["deployments", "replicasets"] + verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods"] - verbs: ["list", "watch"] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] + verbs: ["list", "get", "watch"] --- kind: ClusterRoleBinding diff --git a/cli/cmd/testdata/install_output.golden b/cli/cmd/testdata/install_output.golden index a71fb8ce2f1ac..53e87f6af1ad5 100644 --- a/cli/cmd/testdata/install_output.golden +++ b/cli/cmd/testdata/install_output.golden @@ -23,7 +23,7 @@ rules: resources: ["deployments", "replicasets"] verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers"] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] verbs: ["list", "get", "watch"] --- @@ -57,14 +57,17 @@ metadata: rules: - apiGroups: [""] resources: ["configmaps"] - verbs: ["create", "list", "watch"] + verbs: ["create"] - apiGroups: [""] resources: ["configmaps"] resourceNames: [CertificateBundleName] verbs: ["update"] +- apiGroups: ["extensions", "apps"] + resources: ["deployments", "replicasets"] + verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods"] - verbs: ["list", "watch"] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] + verbs: ["list", "get", "watch"] --- kind: ClusterRoleBinding diff --git a/cli/install/template.go b/cli/install/template.go index 7231ca50d267a..05f4be90b2aa4 100644 --- a/cli/install/template.go +++ b/cli/install/template.go @@ -26,7 +26,7 @@ rules: resources: ["deployments", "replicasets"] verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers"] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] verbs: ["list", "get", "watch"] --- @@ -60,14 +60,17 @@ metadata: rules: - apiGroups: [""] resources: ["configmaps"] - verbs: ["create", "list", "watch"] + verbs: ["create"] - apiGroups: [""] resources: ["configmaps"] resourceNames: [{{.CertificateBundleName}}] verbs: ["update"] +- apiGroups: ["extensions", "apps"] + resources: ["deployments", "replicasets"] + verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods"] - verbs: ["list", "watch"] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] + verbs: ["list", "get", "watch"] --- kind: ClusterRoleBinding diff --git a/controller/Dockerfile b/controller/Dockerfile index 84ca996f87a3b..26662f01413f3 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -1,5 +1,5 @@ ## compile controller services -FROM gcr.io/runconduit/go-deps:07e4dc9b as golang +FROM gcr.io/runconduit/go-deps:bca359b1 as golang WORKDIR /go/src/github.com/runconduit/conduit COPY controller/gen controller/gen COPY pkg pkg diff --git a/controller/ca/controller.go b/controller/ca/controller.go index f6b64e31ee7a1..50923403fcb66 100644 --- a/controller/ca/controller.go +++ b/controller/ca/controller.go @@ -1,10 +1,10 @@ package ca import ( - "fmt" "time" - "github.com/runconduit/conduit/pkg/k8s" + "github.com/runconduit/conduit/controller/k8s" + pkgK8s "github.com/runconduit/conduit/pkg/k8s" log "github.com/sirupsen/logrus" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -13,64 +13,36 @@ import ( "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - coreinformers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes" - corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" ) type CertificateController struct { - client kubernetes.Interface - namespace string - + namespace string + k8sAPI *k8s.API syncHandler func(key string) error - - podLister corelisters.PodLister - podListerSynced cache.InformerSynced - - configMapLister corelisters.ConfigMapLister - configMapListerSynced cache.InformerSynced - - queue workqueue.RateLimitingInterface + queue workqueue.RateLimitingInterface } -func NewCertificateController( - client kubernetes.Interface, - conduitNamespace string, - podInformer coreinformers.PodInformer, - configMapInformer coreinformers.ConfigMapInformer, -) *CertificateController { +func NewCertificateController(conduitNamespace string, k8sAPI *k8s.API) *CertificateController { c := &CertificateController{ - client: client, - namespace: conduitNamespace, - podLister: podInformer.Lister(), - podListerSynced: podInformer.Informer().HasSynced, - configMapLister: configMapInformer.Lister(), - configMapListerSynced: configMapInformer.Informer().HasSynced, + namespace: conduitNamespace, + k8sAPI: k8sAPI, queue: workqueue.NewNamedRateLimitingQueue( workqueue.DefaultControllerRateLimiter(), "certificates"), } - podInformer.Informer().AddEventHandler( + k8sAPI.Pod.Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - c.handlePodUpdate(obj.(*v1.Pod)) - }, - UpdateFunc: func(_, obj interface{}) { - c.handlePodUpdate(obj.(*v1.Pod)) - }, + AddFunc: c.handlePodAdd, + UpdateFunc: c.handlePodUpdate, }, ) - configMapInformer.Informer().AddEventHandler( + k8sAPI.CM.Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - c.handleConfigMapUpdate(obj.(*v1.ConfigMap)) - }, - UpdateFunc: func(_, obj interface{}) { - c.handleConfigMapUpdate(obj.(*v1.ConfigMap)) - }, + AddFunc: c.handleConfigMapAdd, + UpdateFunc: c.handleConfigMapUpdate, DeleteFunc: c.handleConfigMapDelete, }, ) @@ -80,22 +52,16 @@ func NewCertificateController( return c } -func (c *CertificateController) Run(stopCh <-chan struct{}) error { +func (c *CertificateController) Run(stopCh <-chan struct{}) { defer runtime.HandleCrash() defer c.queue.ShutDown() log.Info("starting certificate controller") defer log.Info("shutting down certificate controller") - if !cache.WaitForCacheSync(stopCh, c.podListerSynced, c.configMapListerSynced) { - return fmt.Errorf("timed out waiting for cache to sync") - } - log.Info("caches are synced") - go wait.Until(c.worker, time.Second, stopCh) <-stopCh - return nil } func (c *CertificateController) worker() { @@ -122,11 +88,11 @@ func (c *CertificateController) processNextWorkItem() bool { } func (c *CertificateController) syncNamespace(ns string) error { - conduitConfigMap, err := c.configMapLister.ConfigMaps(c.namespace). - Get(k8s.CertificateBundleName) + conduitConfigMap, err := c.k8sAPI.CM.Lister().ConfigMaps(c.namespace). + Get(pkgK8s.CertificateBundleName) if apierrors.IsNotFound(err) { log.Warnf("configmap [%s] not found in namespace [%s]", - k8s.CertificateBundleName, c.namespace) + pkgK8s.CertificateBundleName, c.namespace) return nil } if err != nil { @@ -134,28 +100,34 @@ func (c *CertificateController) syncNamespace(ns string) error { } configMap := &v1.ConfigMap{ - ObjectMeta: meta.ObjectMeta{Name: k8s.CertificateBundleName}, + ObjectMeta: meta.ObjectMeta{Name: pkgK8s.CertificateBundleName}, Data: conduitConfigMap.Data, } log.Debugf("adding configmap [%s] to namespace [%s]", - k8s.CertificateBundleName, ns) - _, err = c.client.CoreV1().ConfigMaps(ns).Create(configMap) + pkgK8s.CertificateBundleName, ns) + _, err = c.k8sAPI.Client.CoreV1().ConfigMaps(ns).Create(configMap) if apierrors.IsAlreadyExists(err) { - _, err = c.client.CoreV1().ConfigMaps(ns).Update(configMap) + _, err = c.k8sAPI.Client.CoreV1().ConfigMaps(ns).Update(configMap) } return err } -func (c *CertificateController) handlePodUpdate(pod *v1.Pod) { +func (c *CertificateController) handlePodAdd(obj interface{}) { + pod := obj.(*v1.Pod) if c.isInjectedPod(pod) && !c.filterNamespace(pod.Namespace) { c.queue.Add(pod.Namespace) } } -func (c *CertificateController) handleConfigMapUpdate(cm *v1.ConfigMap) { - if cm.Namespace == c.namespace && cm.Name == k8s.CertificateBundleName { +func (c *CertificateController) handlePodUpdate(oldObj, newObj interface{}) { + c.handlePodAdd(newObj) +} + +func (c *CertificateController) handleConfigMapAdd(obj interface{}) { + cm := obj.(*v1.ConfigMap) + if cm.Namespace == c.namespace && cm.Name == pkgK8s.CertificateBundleName { namespaces, err := c.getInjectedNamespaces() if err != nil { log.Errorf("error getting namespaces: %s", err) @@ -168,6 +140,10 @@ func (c *CertificateController) handleConfigMapUpdate(cm *v1.ConfigMap) { } } +func (c *CertificateController) handleConfigMapUpdate(oldObj, newObj interface{}) { + c.handleConfigMapAdd(newObj) +} + func (c *CertificateController) handleConfigMapDelete(obj interface{}) { configMap, ok := obj.(*v1.ConfigMap) if !ok { @@ -183,7 +159,7 @@ func (c *CertificateController) handleConfigMapDelete(obj interface{}) { } } - if configMap.Name == k8s.CertificateBundleName && configMap.Namespace != c.namespace { + if configMap.Name == pkgK8s.CertificateBundleName && configMap.Namespace != c.namespace { injected, err := c.isInjectedNamespace(configMap.Namespace) if err != nil { log.Errorf("error getting pods in namespace [%s]: %s", configMap.Namespace, err) @@ -191,14 +167,14 @@ func (c *CertificateController) handleConfigMapDelete(obj interface{}) { } if injected { log.Infof("configmap [%s] in namespace [%s] deleted; recreating it", - k8s.CertificateBundleName, configMap.Namespace) + pkgK8s.CertificateBundleName, configMap.Namespace) c.queue.Add(configMap.Namespace) } } } func (c *CertificateController) getInjectedNamespaces() ([]string, error) { - pods, err := c.podLister.List(labels.Everything()) + pods, err := c.k8sAPI.Pod.Lister().List(labels.Everything()) if err != nil { return nil, err } @@ -223,7 +199,7 @@ func (c *CertificateController) filterNamespace(ns string) bool { } func (c *CertificateController) isInjectedNamespace(ns string) (bool, error) { - pods, err := c.podLister.Pods(ns).List(labels.Everything()) + pods, err := c.k8sAPI.Pod.Lister().Pods(ns).List(labels.Everything()) if err != nil { return false, err } @@ -236,6 +212,6 @@ func (c *CertificateController) isInjectedNamespace(ns string) (bool, error) { } func (c *CertificateController) isInjectedPod(pod *v1.Pod) bool { - _, ok := pod.Annotations[k8s.CreatedByAnnotation] + _, ok := pod.Annotations[pkgK8s.CreatedByAnnotation] return ok } diff --git a/controller/ca/controller_test.go b/controller/ca/controller_test.go index b3510df06fcd9..703141f4a2d31 100644 --- a/controller/ca/controller_test.go +++ b/controller/ca/controller_test.go @@ -1,62 +1,76 @@ package ca import ( + "fmt" "testing" "time" - "github.com/runconduit/conduit/pkg/k8s" + "github.com/runconduit/conduit/controller/k8s" + pkgK8s "github.com/runconduit/conduit/pkg/k8s" "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" + testingK8s "k8s.io/client-go/testing" ) var ( - conduitNS = "conduitest" - - conduitNamespace = &v1.Namespace{ - ObjectMeta: meta.ObjectMeta{Name: conduitNS}, - } - - conduitConfigMap = &v1.ConfigMap{ - ObjectMeta: meta.ObjectMeta{ - Namespace: conduitNS, - Name: k8s.CertificateBundleName, - }, - } - + conduitNS = "conduitest" injectedNS = "injecttest" - injectedNamespace = &v1.Namespace{ - ObjectMeta: meta.ObjectMeta{Name: injectedNS}, + conduitConfigs = []string{ + fmt.Sprintf(` +apiVersion: v1 +kind: Namespace +metadata: + name: %s`, conduitNS), + fmt.Sprintf(` +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: %s + name: %s`, conduitNS, pkgK8s.CertificateBundleName), } - injectedConfigMap = &v1.ConfigMap{ - ObjectMeta: meta.ObjectMeta{ - Namespace: injectedNS, - Name: k8s.CertificateBundleName, - }, - } - - injectedPod = &v1.Pod{ - ObjectMeta: meta.ObjectMeta{ - Namespace: injectedNS, - Annotations: map[string]string{ - k8s.CreatedByAnnotation: "conduit", - }, - }, + injectedNSConfig = fmt.Sprintf(` +apiVersion: v1 +kind: Namespace +metadata: + name: %s`, injectedNS) + + injectedConfigs = []string{ + injectedNSConfig, + fmt.Sprintf(` +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: %s + name: %s`, injectedNS, pkgK8s.CertificateBundleName), + fmt.Sprintf(` +apiVersion: v1 +kind: Pod +metadata: + namespace: %s + annotations: + %s: conduit`, injectedNS, pkgK8s.CreatedByAnnotation), } ) func TestCertificateController(t *testing.T) { t.Run("creates new configmap in injected namespace on pod add", func(t *testing.T) { - controller, sharedInformers, client, synced := new(injectedNamespace) - stopCh := run(controller, sharedInformers) + controller, synced, stopCh, err := new(injectedNSConfig) + if err != nil { + t.Fatal(err.Error()) + } defer close(stopCh) - controller.handlePodUpdate(injectedPod) + controller.handlePodUpdate(nil, &v1.Pod{ + ObjectMeta: meta.ObjectMeta{ + Namespace: injectedNS, + Annotations: map[string]string{ + pkgK8s.CreatedByAnnotation: "conduit", + }, + }, + }) select { case <-synced: @@ -64,7 +78,7 @@ func TestCertificateController(t *testing.T) { t.Fatal("timed out waiting for sync") } - action := client.Actions()[len(client.Actions())-1] + action := getAction(controller) if !action.Matches("create", "configmaps") { t.Fatalf("expected action to be configmap create, got: %+v", action) @@ -77,12 +91,18 @@ func TestCertificateController(t *testing.T) { }) t.Run("updates configmap in injected namespace on configmap update", func(t *testing.T) { - controller, sharedInformers, client, synced := new( - injectedNamespace, injectedPod, injectedConfigMap) - stopCh := run(controller, sharedInformers) + controller, synced, stopCh, err := new(injectedConfigs...) + if err != nil { + t.Fatal(err.Error()) + } defer close(stopCh) - controller.handleConfigMapUpdate(conduitConfigMap) + controller.handleConfigMapUpdate(nil, &v1.ConfigMap{ + ObjectMeta: meta.ObjectMeta{ + Namespace: conduitNS, + Name: pkgK8s.CertificateBundleName, + }, + }) select { case <-synced: @@ -90,7 +110,7 @@ func TestCertificateController(t *testing.T) { t.Fatal("timed out waiting for sync") } - action := client.Actions()[len(client.Actions())-1] + action := getAction(controller) if !action.Matches("update", "configmaps") { t.Fatalf("expected action to be configmap update, got: %+v", action) @@ -103,12 +123,18 @@ func TestCertificateController(t *testing.T) { }) t.Run("re-adds configmap in injected namespace on configmap delete", func(t *testing.T) { - controller, sharedInformers, client, synced := new( - injectedNamespace, injectedPod, injectedConfigMap) - stopCh := run(controller, sharedInformers) + controller, synced, stopCh, err := new(injectedConfigs...) + if err != nil { + t.Fatal(err.Error()) + } defer close(stopCh) - controller.handleConfigMapDelete(injectedConfigMap) + controller.handleConfigMapDelete(&v1.ConfigMap{ + ObjectMeta: meta.ObjectMeta{ + Namespace: injectedNS, + Name: pkgK8s.CertificateBundleName, + }, + }) select { case <-synced: @@ -116,7 +142,7 @@ func TestCertificateController(t *testing.T) { t.Fatal("timed out waiting for sync") } - action := client.Actions()[len(client.Actions())-1] + action := getAction(controller) // we expect an update instead of a create here because we didn't actually // delete the config map; we just triggered a delete event @@ -131,16 +157,14 @@ func TestCertificateController(t *testing.T) { }) } -func new(fixtures ...runtime.Object) (*CertificateController, informers.SharedInformerFactory, *fake.Clientset, chan bool) { - withConduit := append([]runtime.Object{conduitNamespace, conduitConfigMap}, fixtures...) - clientSet := fake.NewSimpleClientset(withConduit...) - sharedInformers := informers.NewSharedInformerFactory(clientSet, 10*time.Minute) - controller := NewCertificateController( - clientSet, - conduitNS, - sharedInformers.Core().V1().Pods(), - sharedInformers.Core().V1().ConfigMaps(), - ) +func new(fixtures ...string) (*CertificateController, chan bool, chan struct{}, error) { + withConduit := append(conduitConfigs, fixtures...) + k8sAPI, err := k8s.NewFakeAPI(withConduit...) + if err != nil { + return nil, nil, nil, fmt.Errorf("NewFakeAPI returned an error: %s", err) + } + + controller := NewCertificateController(conduitNS, k8sAPI) synced := make(chan bool, 1) controller.syncHandler = func(key string) error { @@ -149,13 +173,17 @@ func new(fixtures ...runtime.Object) (*CertificateController, informers.SharedIn return err } - return controller, sharedInformers, clientSet, synced -} + if err := controller.k8sAPI.Sync(); err != nil { + return nil, nil, nil, fmt.Errorf("k8sAPI.Sync() returned an error: %s", err) + } -func run(c *CertificateController, i informers.SharedInformerFactory) chan struct{} { stopCh := make(chan struct{}) - i.Start(stopCh) - go c.Run(stopCh) - cache.WaitForCacheSync(stopCh, c.podListerSynced, c.configMapListerSynced) - return stopCh + go controller.Run(stopCh) + + return controller, synced, stopCh, nil +} + +func getAction(controller *CertificateController) testingK8s.Action { + client := controller.k8sAPI.Client.(*fake.Clientset) + return client.Actions()[len(client.Actions())-1] } diff --git a/controller/cmd/ca-distributor/main.go b/controller/cmd/ca-distributor/main.go index d5e60cd3d7167..d3a661f23186e 100644 --- a/controller/cmd/ca-distributor/main.go +++ b/controller/cmd/ca-distributor/main.go @@ -5,13 +5,11 @@ import ( "os" "os/signal" "syscall" - "time" "github.com/runconduit/conduit/controller/ca" "github.com/runconduit/conduit/controller/k8s" "github.com/runconduit/conduit/pkg/version" log "github.com/sirupsen/logrus" - "k8s.io/client-go/informers" ) const configMapName = "conduit-ca-bundle" @@ -35,22 +33,27 @@ func main() { stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt, syscall.SIGTERM) - clientSet, err := k8s.NewClientSet(*kubeConfigPath) + k8sClient, err := k8s.NewClientSet(*kubeConfigPath) if err != nil { - log.Fatalf(err.Error()) + log.Fatal(err.Error()) } + k8sAPI := k8s.NewAPI(k8sClient) + + controller := ca.NewCertificateController(*controllerNamespace, k8sAPI) + + go func() { + err := k8sAPI.Sync() + if err != nil { + log.Fatal(err.Error()) + } + }() - sharedInformers := informers.NewSharedInformerFactory(clientSet, 10*time.Minute) - controller := ca.NewCertificateController( - clientSet, - *controllerNamespace, - sharedInformers.Core().V1().Pods(), - sharedInformers.Core().V1().ConfigMaps(), - ) stopCh := make(chan struct{}) - sharedInformers.Start(stopCh) - go controller.Run(stopCh) + go func() { + log.Info("starting distributor") + controller.Run(stopCh) + }() <-stop diff --git a/proxy-init/Dockerfile b/proxy-init/Dockerfile index 040fff86238cc..44118dd8e43e0 100644 --- a/proxy-init/Dockerfile +++ b/proxy-init/Dockerfile @@ -1,5 +1,5 @@ ## compile proxy-init utility -FROM gcr.io/runconduit/go-deps:07e4dc9b as golang +FROM gcr.io/runconduit/go-deps:bca359b1 as golang WORKDIR /go/src/github.com/runconduit/conduit COPY ./proxy-init ./proxy-init RUN CGO_ENABLED=0 GOOS=linux go install -v -installsuffix cgo ./proxy-init/ diff --git a/web/Dockerfile b/web/Dockerfile index 0918421c53f87..92b971dc27823 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -24,7 +24,7 @@ ENV NODE_ENV production RUN $ROOT/bin/web build ## compile go server -FROM gcr.io/runconduit/go-deps:07e4dc9b as golang +FROM gcr.io/runconduit/go-deps:bca359b1 as golang WORKDIR /go/src/github.com/runconduit/conduit COPY web web COPY controller controller From 82f023006cee44c679c3e1ee78fec445b7826ffa Mon Sep 17 00:00:00 2001 From: Kevin Lingerfelt Date: Wed, 13 Jun 2018 14:31:53 -0700 Subject: [PATCH 3/6] Only install CA distributor when --enable-tls flag is set Signed-off-by: Kevin Lingerfelt --- cli/cmd/install.go | 10 + cli/cmd/testdata/install_default.golden | 129 ------------ cli/cmd/testdata/install_output.golden | 262 ++++++++++++------------ cli/install/template.go | 150 +++++++------- test/stat/stat_test.go | 2 +- 5 files changed, 220 insertions(+), 333 deletions(-) diff --git a/cli/cmd/install.go b/cli/cmd/install.go index bff288dd64154..bbd665f478b60 100644 --- a/cli/cmd/install.go +++ b/cli/cmd/install.go @@ -120,6 +120,16 @@ func render(config installConfig, w io.Writer, options *installOptions) error { if err != nil { return err } + if config.EnableTLS { + tlsTemplate, err := template.New("conduit").Parse(install.TlsTemplate) + if err != nil { + return err + } + err = tlsTemplate.Execute(buf, config) + if err != nil { + return err + } + } injectOptions := newInjectOptions() injectOptions.proxyConfigOptions = options.proxyConfigOptions return InjectYAML(buf, w, injectOptions) diff --git a/cli/cmd/testdata/install_default.golden b/cli/cmd/testdata/install_default.golden index 1ea17181efaef..439006e8a7799 100644 --- a/cli/cmd/testdata/install_default.golden +++ b/cli/cmd/testdata/install_default.golden @@ -40,49 +40,6 @@ subjects: name: conduit-controller namespace: conduit -### Service Account CA ### ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: conduit-ca - namespace: conduit - -### CA RBAC ### ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: conduit-ca -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["create"] -- apiGroups: [""] - resources: ["configmaps"] - resourceNames: [conduit-ca-bundle] - verbs: ["update"] -- apiGroups: ["extensions", "apps"] - resources: ["deployments", "replicasets"] - verbs: ["list", "get", "watch"] -- apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] - verbs: ["list", "get", "watch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: conduit-ca -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: conduit-ca -subjects: -- kind: ServiceAccount - name: conduit-ca - namespace: conduit - ### Service Account Prometheus ### --- kind: ServiceAccount @@ -289,92 +246,6 @@ spec: serviceAccount: conduit-controller status: {} --- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - annotations: - conduit.io/created-by: conduit/cli undefined - creationTimestamp: null - labels: - conduit.io/control-plane-component: ca-bundle-distributor - name: ca-bundle-distributor - namespace: conduit -spec: - replicas: 1 - strategy: {} - template: - metadata: - annotations: - conduit.io/created-by: conduit/cli undefined - conduit.io/proxy-version: undefined - creationTimestamp: null - labels: - conduit.io/control-plane-component: ca-bundle-distributor - conduit.io/control-plane-ns: conduit - conduit.io/proxy-deployment: ca-bundle-distributor - spec: - containers: - - args: - - ca-distributor - - -controller-namespace=conduit - - -log-level=info - - -logtostderr=true - image: gcr.io/runconduit/controller:undefined - imagePullPolicy: IfNotPresent - name: ca-distributor - resources: {} - - env: - - name: CONDUIT_PROXY_LOG - value: warn,conduit_proxy=info - - name: CONDUIT_PROXY_BIND_TIMEOUT - value: 10s - - name: CONDUIT_PROXY_CONTROL_URL - value: tcp://proxy-api.conduit.svc.cluster.local:8086 - - name: CONDUIT_PROXY_CONTROL_LISTENER - value: tcp://0.0.0.0:4190 - - name: CONDUIT_PROXY_METRICS_LISTENER - value: tcp://0.0.0.0:4191 - - name: CONDUIT_PROXY_PRIVATE_LISTENER - value: tcp://127.0.0.1:4140 - - name: CONDUIT_PROXY_PUBLIC_LISTENER - value: tcp://0.0.0.0:4143 - - name: CONDUIT_PROXY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: gcr.io/runconduit/proxy:undefined - imagePullPolicy: IfNotPresent - name: conduit-proxy - ports: - - containerPort: 4143 - name: conduit-proxy - - containerPort: 4191 - name: conduit-metrics - resources: {} - securityContext: - runAsUser: 2102 - initContainers: - - args: - - --incoming-proxy-port - - "4143" - - --outgoing-proxy-port - - "4140" - - --proxy-uid - - "2102" - - --inbound-ports-to-ignore - - 4190,4191 - image: gcr.io/runconduit/proxy-init:undefined - imagePullPolicy: IfNotPresent - name: conduit-init - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: false - serviceAccount: conduit-ca -status: {} ---- kind: Service apiVersion: v1 metadata: diff --git a/cli/cmd/testdata/install_output.golden b/cli/cmd/testdata/install_output.golden index 53e87f6af1ad5..0e36b7da2ebc6 100644 --- a/cli/cmd/testdata/install_output.golden +++ b/cli/cmd/testdata/install_output.golden @@ -40,49 +40,6 @@ subjects: name: conduit-controller namespace: Namespace -### Service Account CA ### ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: conduit-ca - namespace: Namespace - -### CA RBAC ### ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: conduit-ca -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["create"] -- apiGroups: [""] - resources: ["configmaps"] - resourceNames: [CertificateBundleName] - verbs: ["update"] -- apiGroups: ["extensions", "apps"] - resources: ["deployments", "replicasets"] - verbs: ["list", "get", "watch"] -- apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] - verbs: ["list", "get", "watch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: conduit-ca -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: conduit-ca -subjects: -- kind: ServiceAccount - name: conduit-ca - namespace: Namespace - ### Service Account Prometheus ### --- kind: ServiceAccount @@ -290,93 +247,6 @@ spec: serviceAccount: conduit-controller status: {} --- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - annotations: - CreatedByAnnotation: CliVersion - creationTimestamp: null - labels: - ControllerComponentLabel: ca-bundle-distributor - name: ca-bundle-distributor - namespace: Namespace -spec: - replicas: 1 - strategy: {} - template: - metadata: - annotations: - CreatedByAnnotation: CliVersion - conduit.io/created-by: conduit/cli undefined - conduit.io/proxy-version: undefined - creationTimestamp: null - labels: - ControllerComponentLabel: ca-bundle-distributor - conduit.io/control-plane-ns: Namespace - conduit.io/proxy-deployment: ca-bundle-distributor - spec: - containers: - - args: - - ca-distributor - - -controller-namespace=Namespace - - -log-level=ControllerLogLevel - - -logtostderr=true - image: ControllerImage - imagePullPolicy: ImagePullPolicy - name: ca-distributor - resources: {} - - env: - - name: CONDUIT_PROXY_LOG - value: warn,conduit_proxy=info - - name: CONDUIT_PROXY_BIND_TIMEOUT - value: 10s - - name: CONDUIT_PROXY_CONTROL_URL - value: tcp://proxy-api.Namespace.svc.cluster.local:8086 - - name: CONDUIT_PROXY_CONTROL_LISTENER - value: tcp://0.0.0.0:4190 - - name: CONDUIT_PROXY_METRICS_LISTENER - value: tcp://0.0.0.0:4191 - - name: CONDUIT_PROXY_PRIVATE_LISTENER - value: tcp://127.0.0.1:4140 - - name: CONDUIT_PROXY_PUBLIC_LISTENER - value: tcp://0.0.0.0:4143 - - name: CONDUIT_PROXY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: gcr.io/runconduit/proxy:undefined - imagePullPolicy: IfNotPresent - name: conduit-proxy - ports: - - containerPort: 4143 - name: conduit-proxy - - containerPort: 4191 - name: conduit-metrics - resources: {} - securityContext: - runAsUser: 2102 - initContainers: - - args: - - --incoming-proxy-port - - "4143" - - --outgoing-proxy-port - - "4140" - - --proxy-uid - - "2102" - - --inbound-ports-to-ignore - - 4190,4191 - image: gcr.io/runconduit/proxy-init:undefined - imagePullPolicy: IfNotPresent - name: conduit-init - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: false - serviceAccount: conduit-ca -status: {} ---- kind: Service apiVersion: v1 metadata: @@ -917,4 +787,136 @@ data: options: path: /var/lib/grafana/dashboards homeDashboardId: conduit-top-line + +### Service Account CA ### +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: conduit-ca + namespace: Namespace + +### CA RBAC ### +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: conduit-ca +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: [CertificateBundleName] + verbs: ["update"] +- apiGroups: ["extensions", "apps"] + resources: ["deployments", "replicasets"] + verbs: ["list", "get", "watch"] +- apiGroups: [""] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] + verbs: ["list", "get", "watch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: conduit-ca +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: conduit-ca +subjects: +- kind: ServiceAccount + name: conduit-ca + namespace: Namespace + +### CA Distributor ### +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + CreatedByAnnotation: CliVersion + creationTimestamp: null + labels: + ControllerComponentLabel: ca-bundle-distributor + name: ca-bundle-distributor + namespace: Namespace +spec: + replicas: 1 + strategy: {} + template: + metadata: + annotations: + CreatedByAnnotation: CliVersion + conduit.io/created-by: conduit/cli undefined + conduit.io/proxy-version: undefined + creationTimestamp: null + labels: + ControllerComponentLabel: ca-bundle-distributor + conduit.io/control-plane-ns: Namespace + conduit.io/proxy-deployment: ca-bundle-distributor + spec: + containers: + - args: + - ca-distributor + - -controller-namespace=Namespace + - -log-level=ControllerLogLevel + - -logtostderr=true + image: ControllerImage + imagePullPolicy: ImagePullPolicy + name: ca-distributor + resources: {} + - env: + - name: CONDUIT_PROXY_LOG + value: warn,conduit_proxy=info + - name: CONDUIT_PROXY_BIND_TIMEOUT + value: 10s + - name: CONDUIT_PROXY_CONTROL_URL + value: tcp://proxy-api.Namespace.svc.cluster.local:8086 + - name: CONDUIT_PROXY_CONTROL_LISTENER + value: tcp://0.0.0.0:4190 + - name: CONDUIT_PROXY_METRICS_LISTENER + value: tcp://0.0.0.0:4191 + - name: CONDUIT_PROXY_PRIVATE_LISTENER + value: tcp://127.0.0.1:4140 + - name: CONDUIT_PROXY_PUBLIC_LISTENER + value: tcp://0.0.0.0:4143 + - name: CONDUIT_PROXY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/runconduit/proxy:undefined + imagePullPolicy: IfNotPresent + name: conduit-proxy + ports: + - containerPort: 4143 + name: conduit-proxy + - containerPort: 4191 + name: conduit-metrics + resources: {} + securityContext: + runAsUser: 2102 + initContainers: + - args: + - --incoming-proxy-port + - "4143" + - --outgoing-proxy-port + - "4140" + - --proxy-uid + - "2102" + - --inbound-ports-to-ignore + - 4190,4191 + image: gcr.io/runconduit/proxy-init:undefined + imagePullPolicy: IfNotPresent + name: conduit-init + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + privileged: false + serviceAccount: conduit-ca +status: {} --- diff --git a/cli/install/template.go b/cli/install/template.go index 05f4be90b2aa4..f3c78d8cc086e 100644 --- a/cli/install/template.go +++ b/cli/install/template.go @@ -43,49 +43,6 @@ subjects: name: conduit-controller namespace: {{.Namespace}} -### Service Account CA ### ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: conduit-ca - namespace: {{.Namespace}} - -### CA RBAC ### ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: conduit-ca -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["create"] -- apiGroups: [""] - resources: ["configmaps"] - resourceNames: [{{.CertificateBundleName}}] - verbs: ["update"] -- apiGroups: ["extensions", "apps"] - resources: ["deployments", "replicasets"] - verbs: ["list", "get", "watch"] -- apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] - verbs: ["list", "get", "watch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: conduit-ca -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: conduit-ca -subjects: -- kind: ServiceAccount - name: conduit-ca - namespace: {{.Namespace}} - ### Service Account Prometheus ### --- kind: ServiceAccount @@ -232,36 +189,6 @@ spec: - "-log-level={{.ControllerLogLevel}}" - "-logtostderr=true" ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - name: ca-bundle-distributor - namespace: {{.Namespace}} - labels: - {{.ControllerComponentLabel}}: ca-bundle-distributor - annotations: - {{.CreatedByAnnotation}}: {{.CliVersion}} -spec: - replicas: {{.ControllerReplicas}} - template: - metadata: - labels: - {{.ControllerComponentLabel}}: ca-bundle-distributor - annotations: - {{.CreatedByAnnotation}}: {{.CliVersion}} - spec: - serviceAccount: conduit-ca - containers: - - name: ca-distributor - image: {{.ControllerImage}} - imagePullPolicy: {{.ImagePullPolicy}} - args: - - "ca-distributor" - - "-controller-namespace={{.Namespace}}" - - "-log-level={{.ControllerLogLevel}}" - - "-logtostderr=true" - ### Web ### --- kind: Service @@ -635,3 +562,80 @@ data: path: /var/lib/grafana/dashboards homeDashboardId: conduit-top-line ` + +const TlsTemplate = ` +### Service Account CA ### +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: conduit-ca + namespace: {{.Namespace}} + +### CA RBAC ### +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: conduit-ca +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: [{{.CertificateBundleName}}] + verbs: ["update"] +- apiGroups: ["extensions", "apps"] + resources: ["deployments", "replicasets"] + verbs: ["list", "get", "watch"] +- apiGroups: [""] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] + verbs: ["list", "get", "watch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: conduit-ca +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: conduit-ca +subjects: +- kind: ServiceAccount + name: conduit-ca + namespace: {{.Namespace}} + +### CA Distributor ### +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: ca-bundle-distributor + namespace: {{.Namespace}} + labels: + {{.ControllerComponentLabel}}: ca-bundle-distributor + annotations: + {{.CreatedByAnnotation}}: {{.CliVersion}} +spec: + replicas: {{.ControllerReplicas}} + template: + metadata: + labels: + {{.ControllerComponentLabel}}: ca-bundle-distributor + annotations: + {{.CreatedByAnnotation}}: {{.CliVersion}} + spec: + serviceAccount: conduit-ca + containers: + - name: ca-distributor + image: {{.ControllerImage}} + imagePullPolicy: {{.ImagePullPolicy}} + args: + - "ca-distributor" + - "-controller-namespace={{.Namespace}}" + - "-log-level={{.ControllerLogLevel}}" + - "-logtostderr=true" + +` diff --git a/test/stat/stat_test.go b/test/stat/stat_test.go index d0407efad60cf..9afe79431d92d 100644 --- a/test/stat/stat_test.go +++ b/test/stat/stat_test.go @@ -75,7 +75,7 @@ func parseRows(out string) (map[string]*rowStat, error) { rows := strings.Split(out, "\n") rows = rows[1 : len(rows)-1] // strip header and trailing newline - expectedRowCount := 5 + expectedRowCount := 4 if len(rows) != expectedRowCount { return nil, fmt.Errorf( "Expected [%d] rows in stat output, got [%d]; full output:\n%s", From f36e61ae81b795ef9f7935b4c1fc61200ab63bd1 Mon Sep 17 00:00:00 2001 From: Kevin Lingerfelt Date: Wed, 13 Jun 2018 14:52:27 -0700 Subject: [PATCH 4/6] Only copy CA bundle into namespaces where inject pods have the same controller Signed-off-by: Kevin Lingerfelt --- controller/ca/controller.go | 3 +-- controller/ca/controller_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/controller/ca/controller.go b/controller/ca/controller.go index 50923403fcb66..c0fd54b7c4fbd 100644 --- a/controller/ca/controller.go +++ b/controller/ca/controller.go @@ -212,6 +212,5 @@ func (c *CertificateController) isInjectedNamespace(ns string) (bool, error) { } func (c *CertificateController) isInjectedPod(pod *v1.Pod) bool { - _, ok := pod.Annotations[pkgK8s.CreatedByAnnotation] - return ok + return pkgK8s.GetControllerNs(pod.ObjectMeta) == c.namespace } diff --git a/controller/ca/controller_test.go b/controller/ca/controller_test.go index 703141f4a2d31..fd831ddd6b3c0 100644 --- a/controller/ca/controller_test.go +++ b/controller/ca/controller_test.go @@ -50,8 +50,8 @@ apiVersion: v1 kind: Pod metadata: namespace: %s - annotations: - %s: conduit`, injectedNS, pkgK8s.CreatedByAnnotation), + labels: + %s: %s`, injectedNS, pkgK8s.ControllerNSLabel, conduitNS), } ) @@ -66,8 +66,8 @@ func TestCertificateController(t *testing.T) { controller.handlePodUpdate(nil, &v1.Pod{ ObjectMeta: meta.ObjectMeta{ Namespace: injectedNS, - Annotations: map[string]string{ - pkgK8s.CreatedByAnnotation: "conduit", + Labels: map[string]string{ + pkgK8s.ControllerNSLabel: conduitNS, }, }, }) From 85dc6dc899638f2e20fb48dbe558f02cd455661f Mon Sep 17 00:00:00 2001 From: Kevin Lingerfelt Date: Thu, 14 Jun 2018 11:29:26 -0700 Subject: [PATCH 5/6] Update API config to only watch pods and configmaps Signed-off-by: Kevin Lingerfelt --- cli/cmd/testdata/install_default.golden | 2 +- cli/cmd/testdata/install_output.golden | 7 ++----- cli/install/template.go | 7 ++----- controller/ca/controller.go | 10 +++++----- controller/cmd/ca-distributor/main.go | 6 +++++- 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/cli/cmd/testdata/install_default.golden b/cli/cmd/testdata/install_default.golden index 439006e8a7799..58c6cf820570e 100644 --- a/cli/cmd/testdata/install_default.golden +++ b/cli/cmd/testdata/install_default.golden @@ -23,7 +23,7 @@ rules: resources: ["deployments", "replicasets"] verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers"] verbs: ["list", "get", "watch"] --- diff --git a/cli/cmd/testdata/install_output.golden b/cli/cmd/testdata/install_output.golden index 0e36b7da2ebc6..0cce16a32e055 100644 --- a/cli/cmd/testdata/install_output.golden +++ b/cli/cmd/testdata/install_output.golden @@ -23,7 +23,7 @@ rules: resources: ["deployments", "replicasets"] verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers"] verbs: ["list", "get", "watch"] --- @@ -810,11 +810,8 @@ rules: resources: ["configmaps"] resourceNames: [CertificateBundleName] verbs: ["update"] -- apiGroups: ["extensions", "apps"] - resources: ["deployments", "replicasets"] - verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] + resources: ["pods", "configmaps"] verbs: ["list", "get", "watch"] --- diff --git a/cli/install/template.go b/cli/install/template.go index f3c78d8cc086e..6fa2968aae0bd 100644 --- a/cli/install/template.go +++ b/cli/install/template.go @@ -26,7 +26,7 @@ rules: resources: ["deployments", "replicasets"] verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] + resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers"] verbs: ["list", "get", "watch"] --- @@ -586,11 +586,8 @@ rules: resources: ["configmaps"] resourceNames: [{{.CertificateBundleName}}] verbs: ["update"] -- apiGroups: ["extensions", "apps"] - resources: ["deployments", "replicasets"] - verbs: ["list", "get", "watch"] - apiGroups: [""] - resources: ["pods", "endpoints", "services", "namespaces", "replicationcontrollers", "configmaps"] + resources: ["pods", "configmaps"] verbs: ["list", "get", "watch"] --- diff --git a/controller/ca/controller.go b/controller/ca/controller.go index c0fd54b7c4fbd..04427b61a59ba 100644 --- a/controller/ca/controller.go +++ b/controller/ca/controller.go @@ -32,14 +32,14 @@ func NewCertificateController(conduitNamespace string, k8sAPI *k8s.API) *Certifi workqueue.DefaultControllerRateLimiter(), "certificates"), } - k8sAPI.Pod.Informer().AddEventHandler( + k8sAPI.Pod().Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ AddFunc: c.handlePodAdd, UpdateFunc: c.handlePodUpdate, }, ) - k8sAPI.CM.Informer().AddEventHandler( + k8sAPI.CM().Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ AddFunc: c.handleConfigMapAdd, UpdateFunc: c.handleConfigMapUpdate, @@ -88,7 +88,7 @@ func (c *CertificateController) processNextWorkItem() bool { } func (c *CertificateController) syncNamespace(ns string) error { - conduitConfigMap, err := c.k8sAPI.CM.Lister().ConfigMaps(c.namespace). + conduitConfigMap, err := c.k8sAPI.CM().Lister().ConfigMaps(c.namespace). Get(pkgK8s.CertificateBundleName) if apierrors.IsNotFound(err) { log.Warnf("configmap [%s] not found in namespace [%s]", @@ -174,7 +174,7 @@ func (c *CertificateController) handleConfigMapDelete(obj interface{}) { } func (c *CertificateController) getInjectedNamespaces() ([]string, error) { - pods, err := c.k8sAPI.Pod.Lister().List(labels.Everything()) + pods, err := c.k8sAPI.Pod().Lister().List(labels.Everything()) if err != nil { return nil, err } @@ -199,7 +199,7 @@ func (c *CertificateController) filterNamespace(ns string) bool { } func (c *CertificateController) isInjectedNamespace(ns string) (bool, error) { - pods, err := c.k8sAPI.Pod.Lister().Pods(ns).List(labels.Everything()) + pods, err := c.k8sAPI.Pod().Lister().Pods(ns).List(labels.Everything()) if err != nil { return false, err } diff --git a/controller/cmd/ca-distributor/main.go b/controller/cmd/ca-distributor/main.go index d3a661f23186e..640113cd79653 100644 --- a/controller/cmd/ca-distributor/main.go +++ b/controller/cmd/ca-distributor/main.go @@ -37,7 +37,11 @@ func main() { if err != nil { log.Fatal(err.Error()) } - k8sAPI := k8s.NewAPI(k8sClient) + k8sAPI := k8s.NewAPI( + k8sClient, + k8s.CM, + k8s.Pod, + ) controller := ca.NewCertificateController(*controllerNamespace, k8sAPI) From 7fcd7a21519c32825488c2944ea08b1741ff0b16 Mon Sep 17 00:00:00 2001 From: Kevin Lingerfelt Date: Thu, 14 Jun 2018 15:07:06 -0700 Subject: [PATCH 6/6] Address review feedback Signed-off-by: Kevin Lingerfelt --- cli/install/template.go | 1 - controller/cmd/ca-distributor/main.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/cli/install/template.go b/cli/install/template.go index 6fa2968aae0bd..f2c2c1b60c28f 100644 --- a/cli/install/template.go +++ b/cli/install/template.go @@ -634,5 +634,4 @@ spec: - "-controller-namespace={{.Namespace}}" - "-log-level={{.ControllerLogLevel}}" - "-logtostderr=true" - ` diff --git a/controller/cmd/ca-distributor/main.go b/controller/cmd/ca-distributor/main.go index 640113cd79653..163d0bf7ce8d4 100644 --- a/controller/cmd/ca-distributor/main.go +++ b/controller/cmd/ca-distributor/main.go @@ -12,8 +12,6 @@ import ( log "github.com/sirupsen/logrus" ) -const configMapName = "conduit-ca-bundle" - func main() { controllerNamespace := flag.String("controller-namespace", "conduit", "namespace in which Conduit is installed") kubeConfigPath := flag.String("kubeconfig", "", "path to kube config")