From 7b1edea5524f01fe46e717c08615ccd483346a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Nussbaumer?= Date: Tue, 6 Feb 2024 16:59:42 +0100 Subject: [PATCH 1/4] feat: use controller-runtime's client with integrated caching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit permits to get rid of kubediscovery package and to simplify the codebase linked to: https://github.com/postfinance/kubenurse/issues/55 Signed-off-by: Clément Nussbaumer --- go.mod | 22 ++-- go.sum | 61 ++++++++--- internal/kubediscovery/kubediscovery.go | 106 ------------------- internal/kubediscovery/kubediscovery_test.go | 55 ---------- internal/kubediscovery/nodewatcher.go | 82 -------------- internal/kubediscovery/nodewatcher_test.go | 45 -------- internal/kubenurse/handler.go | 5 +- internal/kubenurse/handler_test.go | 6 +- internal/kubenurse/k8s-auth.go | 7 ++ internal/kubenurse/server.go | 12 +-- internal/kubenurse/server_test.go | 5 +- internal/servicecheck/neighbours.go | 83 +++++++++++++++ internal/servicecheck/servicecheck.go | 30 +----- internal/servicecheck/servicecheck_test.go | 16 ++- internal/servicecheck/types.go | 17 +-- main.go | 48 +++++++-- 16 files changed, 219 insertions(+), 381 deletions(-) delete mode 100644 internal/kubediscovery/kubediscovery.go delete mode 100644 internal/kubediscovery/kubediscovery_test.go delete mode 100644 internal/kubediscovery/nodewatcher.go delete mode 100644 internal/kubediscovery/nodewatcher_test.go create mode 100644 internal/kubenurse/k8s-auth.go create mode 100644 internal/servicecheck/neighbours.go diff --git a/go.mod b/go.mod index 3abcf266..49394839 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( k8s.io/api v0.29.0 k8s.io/apimachinery v0.29.0 k8s.io/client-go v0.29.0 + sigs.k8s.io/controller-runtime v0.17.0 ) require ( @@ -18,16 +19,20 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -40,21 +45,26 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - golang.org/x/net v0.17.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.29.0 // indirect + k8s.io/component-base v0.29.0 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index f8270350..e7b4c990 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,15 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= +github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -22,6 +29,8 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -39,6 +48,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -63,10 +74,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= +github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -94,9 +105,17 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -104,8 +123,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -114,27 +133,29 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= @@ -154,19 +175,25 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= +k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= +k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= +k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= +k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= +sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/kubediscovery/kubediscovery.go b/internal/kubediscovery/kubediscovery.go deleted file mode 100644 index c36af220..00000000 --- a/internal/kubediscovery/kubediscovery.go +++ /dev/null @@ -1,106 +0,0 @@ -// Package kubediscovery implements a discovery mechanism to find other k8s resources. -package kubediscovery - -import ( - "context" - "fmt" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/client-go/kubernetes" -) - -// Client provides the kubediscovery client methods. -type Client struct { - k8s kubernetes.Interface - nodeCache *nodeCache - allowUnschedulable bool -} - -// NodeSchedulability determines if the kubernetes node is in schedulable mode -// or not. -type NodeSchedulability string - -// Constants to define the NodeSchedulability of kubernetes Nodes -const ( - NodeSchedulabilityUnknown NodeSchedulability = "Unknown" - NodeSchedulable NodeSchedulability = "Schedulable" - NodeUnschedulable NodeSchedulability = "Unschedulable" -) - -// Neighbour represents a kubenurse which should be reachable -type Neighbour struct { - PodName string - PodIP string - HostIP string - NodeName string - NodeSchedulable NodeSchedulability - Phase v1.PodPhase - Terminating bool -} - -// New creates a new kubediscovery client. The context is used to stop the k8s watchers/informers. -// When allowUnschedulable is true, no node watcher is created and kubenurses -// on unschedulable nodes are considered as neighbours. -func New(ctx context.Context, cliset kubernetes.Interface, allowUnschedulable bool) (*Client, error) { - var ( - nc *nodeCache - err error - ) - - // Watch nodes only if we do not consider kubenurses on unschedulable nodes - if !allowUnschedulable { - nc, err = watchNodes(ctx, cliset) - if err != nil { - return nil, fmt.Errorf("starting node watcher: %w", err) - } - } - - return &Client{ - k8s: cliset, - nodeCache: nc, - allowUnschedulable: allowUnschedulable, - }, nil -} - -// GetNeighbours returns a slice of neighbour kubenurses for the given namespace and labelSelector. -func (c *Client) GetNeighbours(ctx context.Context, namespace, labelSelector string) ([]Neighbour, error) { - // Get all pods - pods, err := c.k8s.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: labelSelector, - }) - if err != nil { - return nil, fmt.Errorf("list pods: %w", err) - } - - var neighbours = make([]Neighbour, len(pods.Items)) - - // process pods - for idx := range pods.Items { - pod := pods.Items[idx] - - // If we allow unschedulable kubenurses, we set the schedulability - // to unknown in order not to have to set up a node watcher. - sched := NodeSchedulabilityUnknown - if !c.allowUnschedulable { - sched = NodeUnschedulable - if c.nodeCache.isSchedulable(pod.Spec.NodeName) { - sched = NodeSchedulable - } - } - - n := Neighbour{ - PodName: pod.Name, - PodIP: pod.Status.PodIP, - HostIP: pod.Status.HostIP, - Phase: pod.Status.Phase, - NodeName: pod.Spec.NodeName, - Terminating: pod.DeletionTimestamp != nil, - NodeSchedulable: sched, - } - neighbours[idx] = n - } - - return neighbours, nil -} diff --git a/internal/kubediscovery/kubediscovery_test.go b/internal/kubediscovery/kubediscovery_test.go deleted file mode 100644 index 64a91b90..00000000 --- a/internal/kubediscovery/kubediscovery_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package kubediscovery - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/fake" -) - -var ( - kubenursePod = v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubenurse-dummy", - Labels: map[string]string{ - "app": "kubenurse", - }, - }, - } - differentPod = v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "different", - Labels: map[string]string{ - "app": "different", - }, - }, - } -) - -func TestGetNeighbours(t *testing.T) { - r := require.New(t) - fakeClient := fake.NewSimpleClientset() - - createFakePods(fakeClient) - - client, err := New(context.Background(), fakeClient, false) - r.NoError(err) - - neighbours, err := client.GetNeighbours(context.Background(), "kube-system", "app=kubenurse") - r.NoError(err) - r.Len(neighbours, 1) - r.Equal(kubenursePod.ObjectMeta.Name, neighbours[0].PodName) -} - -func createFakePods(k8s kubernetes.Interface) { - for _, pod := range []v1.Pod{kubenursePod, differentPod} { - _, err := k8s.CoreV1().Pods("kube-system").Create(context.Background(), &pod, metav1.CreateOptions{}) - if err != nil { - panic(err) - } - } -} diff --git a/internal/kubediscovery/nodewatcher.go b/internal/kubediscovery/nodewatcher.go deleted file mode 100644 index 7280558a..00000000 --- a/internal/kubediscovery/nodewatcher.go +++ /dev/null @@ -1,82 +0,0 @@ -package kubediscovery - -import ( - "context" - "fmt" - "sync" - "time" - - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" -) - -const ( - resyncPeriod = time.Hour * 1 -) - -type nodeCache struct { - nodes map[string]bool - mu *sync.RWMutex -} - -// watchNodes starts an informer to watch v1.Node resource, the context can be used to stop the informer -func watchNodes(ctx context.Context, client kubernetes.Interface) (*nodeCache, error) { - nc := nodeCache{ - nodes: make(map[string]bool), - mu: new(sync.RWMutex), - } - - informer := informers.NewSharedInformerFactory(client, resyncPeriod).Core().V1().Nodes().Informer() - - _, err := informer.AddEventHandler( - cache.ResourceEventHandlerFuncs{ - AddFunc: nc.add, - UpdateFunc: nc.update, - DeleteFunc: nc.delete, - }, - ) - if err != nil { - return nil, fmt.Errorf("cannot add event handler for watching nodes: %w", err) - } - - go informer.Run(ctx.Done()) - - if ok := cache.WaitForCacheSync(ctx.Done(), informer.HasSynced); !ok { - return nil, fmt.Errorf("watching nodes: initial cache sync not successful") - } - - return &nc, nil -} - -func (nc *nodeCache) add(obj interface{}) { - node := obj.(*corev1.Node) - - nc.mu.Lock() - nc.nodes[node.Name] = node.Spec.Unschedulable - nc.mu.Unlock() -} - -func (nc *nodeCache) delete(obj interface{}) { - node := obj.(*corev1.Node) - - nc.mu.Lock() - delete(nc.nodes, node.Name) - nc.mu.Unlock() -} - -func (nc *nodeCache) update(_, obj interface{}) { - node := obj.(*corev1.Node) - - nc.mu.Lock() - nc.nodes[node.Name] = node.Spec.Unschedulable - nc.mu.Unlock() -} - -func (nc *nodeCache) isSchedulable(node string) bool { - nc.mu.RLock() - defer nc.mu.RUnlock() - - return !nc.nodes[node] -} diff --git a/internal/kubediscovery/nodewatcher_test.go b/internal/kubediscovery/nodewatcher_test.go deleted file mode 100644 index 58a8053e..00000000 --- a/internal/kubediscovery/nodewatcher_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package kubediscovery - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" -) - -func TestNodeWatcher(t *testing.T) { - node := &corev1.Node{} - node.Name = "testnode" - node.Spec.Unschedulable = false - - r := require.New(t) - fakeClient := fake.NewSimpleClientset() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // start the informer - nc, err := watchNodes(ctx, fakeClient) - r.NoError(err) - r.NotNil(nc) - - _, err = fakeClient.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) - r.NoError(err) - r.True(nc.isSchedulable(node.Name), "node is schedulable") - - node.Spec.Unschedulable = true - - _, err = fakeClient.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) - r.NoError(err) - time.Sleep(100 * time.Millisecond) // the informer needs some time... - r.False(nc.isSchedulable(node.Name), "node is not schedulable") - - err = fakeClient.CoreV1().Nodes().Delete(ctx, node.Name, metav1.DeleteOptions{}) - r.NoError(err) - - r.True(nc.isSchedulable("unknown"), "node not in cache") -} diff --git a/internal/kubenurse/handler.go b/internal/kubenurse/handler.go index 9a1ef0d9..0b95ba70 100644 --- a/internal/kubenurse/handler.go +++ b/internal/kubenurse/handler.go @@ -5,7 +5,6 @@ import ( "net/http" "os" - "github.com/postfinance/kubenurse/internal/kubediscovery" "github.com/postfinance/kubenurse/internal/servicecheck" ) @@ -35,8 +34,8 @@ func (s *Server) aliveHandler() func(w http.ResponseWriter, r *http.Request) { servicecheck.Result // kubediscovery - NeighbourhoodState string `json:"neighbourhood_state"` - Neighbourhood []kubediscovery.Neighbour `json:"neighbourhood"` + NeighbourhoodState string `json:"neighbourhood_state"` + Neighbourhood []servicecheck.Neighbour `json:"neighbourhood"` } // Run checks now diff --git a/internal/kubenurse/handler_test.go b/internal/kubenurse/handler_test.go index 5bf65092..c459b0cc 100644 --- a/internal/kubenurse/handler_test.go +++ b/internal/kubenurse/handler_test.go @@ -7,15 +7,15 @@ import ( "testing" "github.com/stretchr/testify/require" - "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestServerHandler(t *testing.T) { r := require.New(t) - fakeClient := fake.NewSimpleClientset() - + fakeClient := fake.NewFakeClient() kubenurse, err := New(context.Background(), fakeClient) + r.NoError(err) r.NotNil(kubenurse) diff --git a/internal/kubenurse/k8s-auth.go b/internal/kubenurse/k8s-auth.go new file mode 100644 index 00000000..c51b51cb --- /dev/null +++ b/internal/kubenurse/k8s-auth.go @@ -0,0 +1,7 @@ +//go:build debug + +package kubenurse + +import ( + _ "k8s.io/client-go/plugin/pkg/client/auth" // permits to use all authentication providers +) diff --git a/internal/kubenurse/server.go b/internal/kubenurse/server.go index 516a0ca6..f85a927a 100644 --- a/internal/kubenurse/server.go +++ b/internal/kubenurse/server.go @@ -12,12 +12,11 @@ import ( "sync" "time" - "github.com/postfinance/kubenurse/internal/kubediscovery" "github.com/postfinance/kubenurse/internal/servicecheck" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" - "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" ) const defaultCheckInterval = 5 * time.Second @@ -56,7 +55,7 @@ type Server struct { // * KUBENURSE_CHECK_ME_SERVICE // * KUBENURSE_CHECK_NEIGHBOURHOOD // * KUBENURSE_CHECK_INTERVAL -func New(ctx context.Context, k8s kubernetes.Interface) (*Server, error) { //nolint:funlen // TODO: use a flag parsing library (e.g. ff) to reduce complexity +func New(ctx context.Context, c client.Client) (*Server, error) { //nolint:funlen // TODO: use a flag parsing library (e.g. ff) to reduce complexity mux := http.NewServeMux() checkInterval := defaultCheckInterval @@ -100,11 +99,6 @@ func New(ctx context.Context, k8s kubernetes.Interface) (*Server, error) { //nol collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), ) - discovery, err := kubediscovery.New(ctx, k8s, server.allowUnschedulable) - if err != nil { - return nil, fmt.Errorf("create k8s discovery client: %w", err) - } - var histogramBuckets []float64 if bucketsString := os.Getenv("KUBENURSE_HISTOGRAM_BUCKETS"); bucketsString != "" { @@ -124,7 +118,7 @@ func New(ctx context.Context, k8s kubernetes.Interface) (*Server, error) { //nol } // setup checker - chk, err := servicecheck.New(ctx, discovery, promRegistry, server.allowUnschedulable, 3*time.Second, histogramBuckets) + chk, err := servicecheck.New(ctx, c, promRegistry, server.allowUnschedulable, 3*time.Second, histogramBuckets) if err != nil { return nil, err } diff --git a/internal/kubenurse/server_test.go b/internal/kubenurse/server_test.go index b45d47c7..5e38c1a0 100644 --- a/internal/kubenurse/server_test.go +++ b/internal/kubenurse/server_test.go @@ -5,14 +5,13 @@ import ( "testing" "github.com/stretchr/testify/require" - "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestCombined(t *testing.T) { r := require.New(t) - fakeClient := fake.NewSimpleClientset() - + fakeClient := fake.NewFakeClient() kubenurse, err := New(context.Background(), fakeClient) r.NoError(err) r.NotNil(kubenurse) diff --git a/internal/servicecheck/neighbours.go b/internal/servicecheck/neighbours.go new file mode 100644 index 00000000..4d24fd98 --- /dev/null +++ b/internal/servicecheck/neighbours.go @@ -0,0 +1,83 @@ +package servicecheck + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Neighbour represents a kubenurse which should be reachable +type Neighbour struct { + PodName string + PodIP string + HostIP string + NodeName string +} + +// GetNeighbours returns a slice of neighbour kubenurses for the given namespace and labelSelector. +func (c *Checker) GetNeighbours(ctx context.Context, namespace, labelSelector string) ([]Neighbour, error) { + // Get all pods + pods := v1.PodList{} + selector, _ := labels.Parse(labelSelector) + err := c.client.List(ctx, &pods, &client.ListOptions{ + LabelSelector: selector, + Namespace: namespace, + }) + + if err != nil { + return nil, fmt.Errorf("list pods: %w", err) + } + + var neighbours = make([]Neighbour, 0, len(pods.Items)) + + // process pods + for idx := range pods.Items { + pod := pods.Items[idx] + + if !c.allowUnschedulable { // if we disallow unschedulable nodes, we have to check their status + n := v1.Node{} + if err := c.client.Get(ctx, types.NamespacedName{Name: pod.Spec.NodeName}, &n); err == nil { + if n.Spec.Unschedulable { // node unschedulable, we do not include this pod in the neighbour list + continue + } + } + } + + if pod.Status.Phase != v1.PodRunning || // only query running pods (excludes pending ones) + pod.DeletionTimestamp != nil { // exclude terminating pods + continue + } + + n := Neighbour{ + PodName: pod.Name, + PodIP: pod.Status.PodIP, + HostIP: pod.Status.HostIP, + NodeName: pod.Spec.NodeName, + } + neighbours = append(neighbours, n) + } + + return neighbours, nil +} + +// checkNeighbours checks the /alwayshappy endpoint from every discovered kubenurse neighbour. Neighbour pods on nodes +// which are not schedulable are excluded from this check to avoid possible false errors. +func (c *Checker) checkNeighbours(nh []Neighbour) { + for _, neighbour := range nh { + neighbour := neighbour // pin + + check := func(ctx context.Context) (string, error) { + if c.UseTLS { + return c.doRequest(ctx, "https://"+neighbour.PodIP+":8443/alwayshappy") + } + + return c.doRequest(ctx, "http://"+neighbour.PodIP+":8080/alwayshappy") + } + + _, _ = c.measure(check, "path_"+neighbour.NodeName) + } +} diff --git a/internal/servicecheck/servicecheck.go b/internal/servicecheck/servicecheck.go index f5698e72..7178513f 100644 --- a/internal/servicecheck/servicecheck.go +++ b/internal/servicecheck/servicecheck.go @@ -11,9 +11,8 @@ import ( "os" "time" - "github.com/postfinance/kubenurse/internal/kubediscovery" "github.com/prometheus/client_golang/prometheus" - v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -25,7 +24,7 @@ const ( // New configures the checker with a httpClient and a cache timeout for check // results. Other parameters of the Checker struct need to be configured separately. -func New(_ context.Context, discovery *kubediscovery.Client, promRegistry *prometheus.Registry, +func New(_ context.Context, cl client.Client, promRegistry *prometheus.Registry, allowUnschedulable bool, cacheTTL time.Duration, durationHistogramBuckets []float64) (*Checker, error) { errorCounter := prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -80,7 +79,7 @@ func New(_ context.Context, discovery *kubediscovery.Client, promRegistry *prome return &Checker{ allowUnschedulable: allowUnschedulable, - discovery: discovery, + client: cl, httpClient: httpClient, cacheTTL: cacheTTL, errorCounter: errorCounter, @@ -121,7 +120,7 @@ func (c *Checker) Run() (Result, bool) { if c.SkipCheckNeighbourhood { res.NeighbourhoodState = skippedStr } else { - res.Neighbourhood, err = c.discovery.GetNeighbours(context.TODO(), c.KubenurseNamespace, c.NeighbourFilter) + res.Neighbourhood, err = c.GetNeighbours(context.TODO(), c.KubenurseNamespace, c.NeighbourFilter) haserr = haserr || (err != nil) // Neighbourhood special error treating @@ -202,27 +201,6 @@ func (c *Checker) MeService(ctx context.Context) (string, error) { return c.doRequest(ctx, c.KubenurseServiceURL+"/alwayshappy") } -// checkNeighbours checks the /alwayshappy endpoint from every discovered kubenurse neighbour. Neighbour pods on nodes -// which are not schedulable are excluded from this check to avoid possible false errors. -func (c *Checker) checkNeighbours(nh []kubediscovery.Neighbour) { - for _, neighbour := range nh { - neighbour := neighbour // pin - if neighbour.Phase == v1.PodRunning && // only query running pods (excludes pending ones) - !neighbour.Terminating && // exclude terminating pods - (c.allowUnschedulable || neighbour.NodeSchedulable == kubediscovery.NodeSchedulable) { - check := func(ctx context.Context) (string, error) { - if c.UseTLS { - return c.doRequest(ctx, "https://"+neighbour.PodIP+":8443/alwayshappy") - } - - return c.doRequest(ctx, "http://"+neighbour.PodIP+":8080/alwayshappy") - } - - _, _ = c.measure(check, "path_"+neighbour.NodeName) - } - } -} - // measure implements metric collections for the check func (c *Checker) measure(check Check, label string) (string, error) { start := time.Now() diff --git a/internal/servicecheck/servicecheck_test.go b/internal/servicecheck/servicecheck_test.go index 07cf4db8..6fba11ee 100644 --- a/internal/servicecheck/servicecheck_test.go +++ b/internal/servicecheck/servicecheck_test.go @@ -5,17 +5,17 @@ import ( "testing" "time" - "github.com/postfinance/kubenurse/internal/kubediscovery" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) var fakeNeighbourPod = v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "kubenurse-dummy", + Name: "kubenurse-dummy", + Namespace: "kube-system", Labels: map[string]string{ "app": "kubenurse", }, @@ -26,6 +26,7 @@ var fakeNeighbourPod = v1.Pod{ Status: v1.PodStatus{ HostIP: "127.0.0.1", PodIP: "127.0.0.1", + Phase: v1.PodRunning, }, } @@ -33,14 +34,9 @@ func TestCombined(t *testing.T) { r := require.New(t) // fake client, with a dummy neighbour pod - fakeClient := fake.NewSimpleClientset() - _, err := fakeClient.CoreV1().Pods("kube-system").Create(context.Background(), &fakeNeighbourPod, metav1.CreateOptions{}) - r.NoError(err) - - discovery, err := kubediscovery.New(context.Background(), fakeClient, false) - r.NoError(err) + fakeClient := fake.NewFakeClient(&fakeNeighbourPod) - checker, err := New(context.Background(), discovery, prometheus.NewRegistry(), false, 3*time.Second, prometheus.DefBuckets) + checker, err := New(context.Background(), fakeClient, prometheus.NewRegistry(), false, 3*time.Second, prometheus.DefBuckets) r.NoError(err) r.NotNil(checker) diff --git a/internal/servicecheck/types.go b/internal/servicecheck/types.go index 47c9e758..c1d72e5e 100644 --- a/internal/servicecheck/types.go +++ b/internal/servicecheck/types.go @@ -5,8 +5,8 @@ import ( "net/http" "time" - "github.com/postfinance/kubenurse/internal/kubediscovery" "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" ) // Checker implements the kubenurse checker @@ -35,7 +35,8 @@ type Checker struct { // TLS UseTLS bool - discovery *kubediscovery.Client + // Controller runtime cached client + client client.Client // metrics errorCounter *prometheus.CounterVec @@ -56,12 +57,12 @@ type Checker struct { // Result contains the result of a performed check run type Result struct { - APIServerDirect string `json:"api_server_direct"` - APIServerDNS string `json:"api_server_dns"` - MeIngress string `json:"me_ingress"` - MeService string `json:"me_service"` - NeighbourhoodState string `json:"neighbourhood_state"` - Neighbourhood []kubediscovery.Neighbour `json:"neighbourhood"` + APIServerDirect string `json:"api_server_direct"` + APIServerDNS string `json:"api_server_dns"` + MeIngress string `json:"me_ingress"` + MeService string `json:"me_service"` + NeighbourhoodState string `json:"neighbourhood_state"` + Neighbourhood []Neighbour `json:"neighbourhood"` } // Check is the signature used by all checks that the checker can execute. diff --git a/main.go b/main.go index 441ee388..f1b5055c 100644 --- a/main.go +++ b/main.go @@ -3,15 +3,19 @@ package main import ( "context" + "errors" "fmt" "log" + "os" "os/signal" "syscall" "time" "github.com/postfinance/kubenurse/internal/kubenurse" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" + corev1 "k8s.io/api/core/v1" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -22,20 +26,48 @@ func main() { ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() - // create in-cluster config - config, err := rest.InClusterConfig() + restConf, err := controllerruntime.GetConfig() if err != nil { - log.Printf("creating in-cluster configuration: %s", err) + log.Println(err) return } - cliset, err := kubernetes.NewForConfig(config) + kubenurseNs := os.Getenv("KUBENURSE_NAMESPACE") + + ca, err := cache.New(restConf, cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Namespaces: map[string]cache.Config{ + kubenurseNs: {}, + }}, + &corev1.Node{}: {}, + }, + }) + + if err != nil { + log.Printf("error during cache creation: %s", err) + return + } + + go func() { + if err = ca.Start(ctx); !errors.Is(err, context.Canceled) { + log.Printf("client cache error: %s", err) + cancel() + } + }() + + opts := client.Options{ + Cache: &client.CacheOptions{ + Reader: ca, + }, + } + + c, err := client.New(restConf, opts) if err != nil { - log.Printf("creating clientset: %s", err) + log.Printf("error while starting controller-runtime client: %s", err) return } - server, err := kubenurse.New(ctx, cliset) + server, err := kubenurse.New(ctx, c) if err != nil { log.Printf("%s", err) return From 92b4922d1814a8b65375a0bafc59d92465d59a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Nussbaumer?= Date: Tue, 6 Feb 2024 18:17:32 +0100 Subject: [PATCH 2/4] chore: remove "caching" of results and simplify code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Clément Nussbaumer --- internal/kubenurse/handler.go | 8 +++---- .../kubenurse/{k8s-auth.go => k8s_auth.go} | 0 internal/kubenurse/server.go | 2 +- internal/servicecheck/cache.go | 22 ------------------- internal/servicecheck/servicecheck.go | 10 ++------- internal/servicecheck/types.go | 10 ++------- 6 files changed, 9 insertions(+), 43 deletions(-) rename internal/kubenurse/{k8s-auth.go => k8s_auth.go} (100%) delete mode 100644 internal/servicecheck/cache.go diff --git a/internal/kubenurse/handler.go b/internal/kubenurse/handler.go index 0b95ba70..e71dcb97 100644 --- a/internal/kubenurse/handler.go +++ b/internal/kubenurse/handler.go @@ -38,15 +38,15 @@ func (s *Server) aliveHandler() func(w http.ResponseWriter, r *http.Request) { Neighbourhood []servicecheck.Neighbour `json:"neighbourhood"` } - // Run checks now - res, haserr := s.checker.Run() - if haserr { + res := s.checker.LastCheckResult + if res == nil { w.WriteHeader(http.StatusInternalServerError) + return } // Add additional data out := Output{ - Result: res, + Result: *res, Headers: r.Header, UserAgent: r.UserAgent(), RequestURI: r.RequestURI, diff --git a/internal/kubenurse/k8s-auth.go b/internal/kubenurse/k8s_auth.go similarity index 100% rename from internal/kubenurse/k8s-auth.go rename to internal/kubenurse/k8s_auth.go diff --git a/internal/kubenurse/server.go b/internal/kubenurse/server.go index f85a927a..1059ab05 100644 --- a/internal/kubenurse/server.go +++ b/internal/kubenurse/server.go @@ -118,7 +118,7 @@ func New(ctx context.Context, c client.Client) (*Server, error) { //nolint:funle } // setup checker - chk, err := servicecheck.New(ctx, c, promRegistry, server.allowUnschedulable, 3*time.Second, histogramBuckets) + chk, err := servicecheck.New(ctx, c, promRegistry, server.allowUnschedulable, 1*time.Second, histogramBuckets) if err != nil { return nil, err } diff --git a/internal/servicecheck/cache.go b/internal/servicecheck/cache.go deleted file mode 100644 index 72fbf22c..00000000 --- a/internal/servicecheck/cache.go +++ /dev/null @@ -1,22 +0,0 @@ -package servicecheck - -import "time" - -// retrieveResultFromCache returns the latest check result from cache, if any. -// If the result is expired or none is available, this function will return nil. -func (c *Checker) retrieveResultFromCache() *Result { - if c.cachedResult != nil && c.cachedResult.expiration.After(time.Now()) { - return c.cachedResult.result - } - - return nil -} - -// cacheResult sets a check result to the cache and expires it after the -// Checker.cacheTTL is exceeded. -func (c *Checker) cacheResult(result *Result) { - c.cachedResult = &CachedResult{ - result: result, - expiration: time.Now().Add(c.cacheTTL), - } -} diff --git a/internal/servicecheck/servicecheck.go b/internal/servicecheck/servicecheck.go index 7178513f..eee65e37 100644 --- a/internal/servicecheck/servicecheck.go +++ b/internal/servicecheck/servicecheck.go @@ -96,12 +96,6 @@ func (c *Checker) Run() (Result, bool) { err error ) - // Check if a result is cached and return it - cacheRes := c.retrieveResultFromCache() - if cacheRes != nil { - return *cacheRes, false - } - // Run Checks res := Result{} @@ -134,8 +128,8 @@ func (c *Checker) Run() (Result, bool) { } } - // Cache result - c.cacheResult(&res) + // Cache result (used for /alive handler) + c.LastCheckResult = &res return res, haserr } diff --git a/internal/servicecheck/types.go b/internal/servicecheck/types.go index c1d72e5e..fede2c3c 100644 --- a/internal/servicecheck/types.go +++ b/internal/servicecheck/types.go @@ -45,8 +45,8 @@ type Checker struct { // Http Client for https requests httpClient *http.Client - // cachedResult represents a cached check result - cachedResult *CachedResult + // LastCheckResult represents a cached check result + LastCheckResult *Result // cacheTTL defines the TTL of how long a cached result is valid cacheTTL time.Duration @@ -67,9 +67,3 @@ type Result struct { // Check is the signature used by all checks that the checker can execute. type Check func(ctx context.Context) (string, error) - -// CachedResult represents a cached check result that is valid until the expiration. -type CachedResult struct { - result *Result - expiration time.Time -} From 8d891b6a209654a579d495da2754564abfcd6373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Nussbaumer?= Date: Tue, 6 Feb 2024 18:21:52 +0100 Subject: [PATCH 3/4] fix: don't log nil error returned when the cache terminates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Clément Nussbaumer --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index f1b5055c..a35c2cc4 100644 --- a/main.go +++ b/main.go @@ -49,7 +49,7 @@ func main() { } go func() { - if err = ca.Start(ctx); !errors.Is(err, context.Canceled) { + if err = ca.Start(ctx); err != nil && !errors.Is(err, context.Canceled) { log.Printf("client cache error: %s", err) cancel() } From 62e737c9d4cb73df8ea6ea9f0a32c2551d020ac7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Nussbaumer?= Date: Mon, 19 Feb 2024 14:35:08 +0100 Subject: [PATCH 4/4] fix(neighbours): only check other kubenurse pods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #110 Signed-off-by: Clément Nussbaumer --- internal/servicecheck/neighbours.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/servicecheck/neighbours.go b/internal/servicecheck/neighbours.go index 4d24fd98..c09c75c3 100644 --- a/internal/servicecheck/neighbours.go +++ b/internal/servicecheck/neighbours.go @@ -3,6 +3,7 @@ package servicecheck import ( "context" "fmt" + "os" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" @@ -34,6 +35,8 @@ func (c *Checker) GetNeighbours(ctx context.Context, namespace, labelSelector st var neighbours = make([]Neighbour, 0, len(pods.Items)) + var hostname, _ = os.Hostname() + // process pods for idx := range pods.Items { pod := pods.Items[idx] @@ -52,6 +55,10 @@ func (c *Checker) GetNeighbours(ctx context.Context, namespace, labelSelector st continue } + if pod.Name == hostname { // only quey other pods, not the currently running pod + continue + } + n := Neighbour{ PodName: pod.Name, PodIP: pod.Status.PodIP,