Skip to content

Commit

Permalink
Integrate kuberesolver into Buildbarn binaries
Browse files Browse the repository at this point in the history
When running Buildbarn on bare metal Kubernetes clusters, it may not
always be useful to set up things like kube-proxy, a CNI, or coredns.
Just some plain kubelets that only run pods with 'hostNetwork: true'
will do.

Though very light-weight, it does make it a bit harder to establish
network connections between individual components. It requires
applications to do a 'kubectl get endpoints' against the API server to
figure out the addresses of services are.

This change does exactly that. It allows users to register a custom gRPC
connection schema like kubernetes://${service}.${namespace}:${port} that
automatically expands to the right set of systems.
  • Loading branch information
EdSchouten committed Feb 27, 2024
1 parent f52022c commit a4267fc
Show file tree
Hide file tree
Showing 8 changed files with 321 additions and 116 deletions.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ require (
github.com/prometheus/client_golang v1.17.0
github.com/prometheus/client_model v0.5.0
github.com/prometheus/common v0.45.0
github.com/sercand/kuberesolver/v5 v5.1.1
github.com/stretchr/testify v1.8.4
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1
go.opentelemetry.io/contrib/propagators/b3 v1.21.1
Expand Down Expand Up @@ -72,6 +73,7 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/fxtlabs/primes v0.0.0-20150821004651-dad82d10a449 h1:HOYnhuVrhAVGKdg3rZapII640so7QfXQmkLkefUN/uM=
github.com/fxtlabs/primes v0.0.0-20150821004651-dad82d10a449/go.mod h1:i+vbdOOivRRh2j+WwBkjZXloGN/+KAqfKDwNfUJeugc=
github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
Expand Down Expand Up @@ -164,6 +166,8 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne
github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
github.com/sercand/kuberesolver/v5 v5.1.1 h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aeprPTHb6yY=
github.com/sercand/kuberesolver/v5 v5.1.1/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand Down Expand Up @@ -277,6 +281,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down
12 changes: 12 additions & 0 deletions go_dependencies.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,12 @@ def go_dependencies():
sum = "h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=",
version = "v1.14.4",
)
go_repository(
name = "com_github_fsnotify_fsnotify",
importpath = "github.com/fsnotify/fsnotify",
sum = "h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=",
version = "v1.6.0",
)
go_repository(
name = "com_github_fxtlabs_primes",
importpath = "github.com/fxtlabs/primes",
Expand Down Expand Up @@ -566,6 +572,12 @@ def go_dependencies():
sum = "h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=",
version = "v1.10.0",
)
go_repository(
name = "com_github_sercand_kuberesolver_v5",
importpath = "github.com/sercand/kuberesolver/v5",
sum = "h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aeprPTHb6yY=",
version = "v5.1.1",
)
go_repository(
name = "com_github_sergi_go_diff",
importpath = "github.com/sergi/go-diff",
Expand Down
3 changes: 3 additions & 0 deletions pkg/global/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ go_library(
"resource_limits_linux.go",
"resource_limits_nonunix.go",
"resource_limits_unix.go",
"simple_k8s_client.go",
"umask_nonunix.go",
"umask_unix.go",
],
Expand All @@ -28,6 +29,7 @@ go_library(
"@com_github_prometheus_client_golang//prometheus",
"@com_github_prometheus_client_golang//prometheus/promhttp",
"@com_github_prometheus_client_golang//prometheus/push",
"@com_github_sercand_kuberesolver_v5//:go_default_library",
"@io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc//:otelgrpc",
"@io_opentelemetry_go_contrib_propagators_b3//:b3",
"@io_opentelemetry_go_otel//:otel",
Expand All @@ -40,6 +42,7 @@ go_library(
"@io_opentelemetry_go_otel_trace//:trace",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//resolver",
"@org_golang_google_grpc//status",
] + select({
"@io_bazel_rules_go//go/platform:android": [
Expand Down
22 changes: 22 additions & 0 deletions pkg/global/apply_configuration.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,11 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/client_golang/prometheus/push"
"github.com/sercand/kuberesolver/v5"

"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/resolver"
"google.golang.org/grpc/status"

"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
Expand Down Expand Up @@ -109,6 +111,26 @@ func ApplyConfiguration(configuration *pb.Configuration) (*LifecycleState, bb_gr
}
log.SetOutput(io.MultiWriter(logWriters...))

// gRPC resolvers for connecting to Kubernetes service endpoints
// without using cluster internal DNS.
for schema, resolverConfiguration := range configuration.GetGrpcKubernetesResolvers() {
roundTripper, err := bb_http.NewRoundTripperFromConfiguration(resolverConfiguration.ApiServerHttpClient)
if err != nil {
return nil, nil, util.StatusWrapf(err, "Failed to create HTTP client for gRPC Kubernetes resolver for schema %#v", schema)
}
resolver.Register(
kuberesolver.NewBuilder(
newSimpleK8sClient(
&http.Client{
Transport: bb_http.NewMetricsRoundTripper(roundTripper, "GRPCKubernetesResolver"),
},
resolverConfiguration.ApiServerUrl,
),
schema,
),
)
}

grpcClientDialer := bb_grpc.NewLazyClientDialer(bb_grpc.BaseClientDialer)
var grpcUnaryInterceptors []grpc.UnaryClientInterceptor
var grpcStreamInterceptors []grpc.StreamClientInterceptor
Expand Down
34 changes: 34 additions & 0 deletions pkg/global/simple_k8s_client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package global

import (
"net/http"

"github.com/sercand/kuberesolver/v5"
)

type simpleK8sClient struct {
httpClient *http.Client
url string
}

// newSimpleK8sClient creates a Kubernetes API server client for use
// with kuberesolver. The implementation that ships with kuberesolver
// makes strong assumptions about pathnames and environment variables.
func newSimpleK8sClient(httpClient *http.Client, url string) kuberesolver.K8sClient {
return &simpleK8sClient{
httpClient: httpClient,
url: url,
}
}

func (kc *simpleK8sClient) GetRequest(url string) (*http.Request, error) {
return http.NewRequest(http.MethodGet, url, nil)
}

func (kc *simpleK8sClient) Do(req *http.Request) (*http.Response, error) {
return kc.httpClient.Do(req)
}

func (kc *simpleK8sClient) Host() string {
return kc.url
}
Loading

0 comments on commit a4267fc

Please sign in to comment.