Skip to content

Commit

Permalink
Create Health Checks Provider to interact with Google Cloud
Browse files Browse the repository at this point in the history
This separates business logic and handling health checks resources
  • Loading branch information
panslava committed Sep 9, 2022
1 parent 6b146e8 commit 5767597
Show file tree
Hide file tree
Showing 13 changed files with 476 additions and 106 deletions.
4 changes: 2 additions & 2 deletions cmd/glbc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
flag "github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/ingress-gce/pkg/frontendconfig"
"k8s.io/ingress-gce/pkg/healthchecks"
"k8s.io/ingress-gce/pkg/healthchecksl4"
"k8s.io/ingress-gce/pkg/ingparams"
"k8s.io/ingress-gce/pkg/l4lb"
"k8s.io/ingress-gce/pkg/psc"
Expand Down Expand Up @@ -279,7 +279,7 @@ func runControllers(ctx *ingctx.ControllerContext) {

fwc := firewalls.NewFirewallController(ctx, flags.F.NodePortRanges.Values())

healthchecks.InitializeL4(ctx.Cloud, ctx)
healthchecksl4.Initialize(ctx.Cloud, ctx)

if flags.F.RunL4Controller {
l4Controller := l4lb.NewILBController(ctx, stopCh)
Expand Down
22 changes: 2 additions & 20 deletions pkg/healthchecks/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,13 @@ limitations under the License.
package healthchecks

import (
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
computealpha "google.golang.org/api/compute/v0.alpha"
computebeta "google.golang.org/api/compute/v0.beta"
compute "google.golang.org/api/compute/v1"
"google.golang.org/api/compute/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/ingress-gce/pkg/translator"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"

"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
)

// HealthCheckProvider is an interface to manage a single GCE health check.
Expand Down Expand Up @@ -58,19 +56,3 @@ type HealthChecker interface {
Delete(name string, scope meta.KeyType) error
Get(name string, version meta.Version, scope meta.KeyType) (*translator.HealthCheck, error)
}

// L4HealthChecks defines methods for creating and deleting health checks (and their firewall rules) for l4 services
type L4HealthChecks interface {
// EnsureL4HealthCheck creates health check (and firewall rule) for l4 service
EnsureL4HealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult
// DeleteHealthCheck deletes health check (and firewall rule) for l4 service
DeleteHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
}

type EnsureL4HealthCheckResult struct {
HCName string
HCLink string
HCFirewallRuleName string
GceResourceInError string
Err error
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package healthchecks
package healthchecksl4

import (
"fmt"
"strconv"
"sync"

cloudprovider "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
Expand All @@ -30,6 +29,7 @@ import (
"k8s.io/ingress-gce/pkg/composite"
"k8s.io/ingress-gce/pkg/events"
"k8s.io/ingress-gce/pkg/firewalls"
"k8s.io/ingress-gce/pkg/healthchecksprovider"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"
"k8s.io/klog/v2"
Expand All @@ -49,20 +49,21 @@ const (
var (
// instanceLock to prevent duplicate initialization.
instanceLock = &sync.Mutex{}
// instance is a singleton instance, created by InitializeL4
// instance is a singleton instance, created by Initialize
instance *l4HealthChecks
)

type l4HealthChecks struct {
// sharedResourceLock serializes operations on the healthcheck and firewall
// resources shared across multiple Services.
sharedResourcesLock sync.Mutex
hcProvider healthChecksProvider
cloud *gce.Cloud
recorderFactory events.RecorderProducer
}

// InitializeL4 creates singleton instance, must be run before L4() func
func InitializeL4(cloud *gce.Cloud, recorderFactory events.RecorderProducer) {
// Initialize creates singleton instance, must be run before GetInstance() func
func Initialize(cloud *gce.Cloud, recorderFactory events.RecorderProducer) {
instanceLock.Lock()
defer instanceLock.Unlock()

Expand All @@ -74,25 +75,27 @@ func InitializeL4(cloud *gce.Cloud, recorderFactory events.RecorderProducer) {
instance = &l4HealthChecks{
cloud: cloud,
recorderFactory: recorderFactory,
hcProvider: healthchecksprovider.NewHealthChecks(cloud, meta.VersionGA),
}
klog.V(3).Infof("Initialized L4 Healthchecks")
}

// FakeL4 creates instance of l4HealthChecks. Use for test only.
func FakeL4(cloud *gce.Cloud, recorderFactory events.RecorderProducer) *l4HealthChecks {
// Fake creates instance of l4HealthChecks. Use for test only.
func Fake(cloud *gce.Cloud, recorderFactory events.RecorderProducer) *l4HealthChecks {
instance = &l4HealthChecks{
cloud: cloud,
recorderFactory: recorderFactory,
hcProvider: healthchecksprovider.NewHealthChecks(cloud, meta.VersionGA),
}
return instance
}

// L4 returns singleton instance, must be run after InitializeL4
func L4() *l4HealthChecks {
// GetInstance returns singleton instance, must be run after Initialize
func GetInstance() *l4HealthChecks {
return instance
}

// EnsureL4HealthCheck and firewall rules exist for the L4
// EnsureHealthCheck and firewall rules exist for the L4
// LoadBalancer Service.
//
// The healthcheck and firewall will be shared between different K8s
Expand All @@ -102,8 +105,7 @@ func L4() *l4HealthChecks {
// Firewall rules are always created at in the Global scope (vs
// Regional). This means that one Firewall rule is created for
// Services of different scope (Global vs Regional).

func (l4hc *l4HealthChecks) EnsureL4HealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult {
func (l4hc *l4HealthChecks) EnsureHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult {
namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}

hcName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC)
Expand Down Expand Up @@ -154,7 +156,7 @@ func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L
defer l4hc.sharedResourcesLock.Unlock()
}

err := utils.IgnoreHTTPNotFound(l4hc.deleteHealthCheck(hcName, scope))
err := l4hc.hcProvider.Delete(hcName, scope)
if err != nil {
// Ignore deletion error due to health check in use by another resource.
if !utils.IsInUsedByError(err) {
Expand All @@ -169,17 +171,11 @@ func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L
}

func (l4hc *l4HealthChecks) ensureL4HealthCheckInternal(hcName string, svcName types.NamespacedName, shared bool, path string, port int32, scope meta.KeyType, l4Type utils.L4LBType) (*composite.HealthCheck, string, error) {
selfLink := ""
key, err := composite.CreateKey(l4hc.cloud, hcName, scope)
if err != nil {
return nil, selfLink, fmt.Errorf("Failed to create key for healthcheck with name %s for service %s", hcName, svcName.String())
}
hc, err := composite.GetHealthCheck(l4hc.cloud, key, meta.VersionGA)
hc, err := l4hc.hcProvider.Get(hcName, scope)
if err != nil {
if !utils.IsNotFoundError(err) {
return nil, selfLink, err
}
return nil, "", err
}

var region string
if scope == meta.Regional {
region = l4hc.cloud.Region()
Expand All @@ -189,22 +185,25 @@ func (l4hc *l4HealthChecks) ensureL4HealthCheckInternal(hcName string, svcName t
if hc == nil {
// Create the healthcheck
klog.V(2).Infof("Creating healthcheck %s for service %s, shared = %v. Expected healthcheck: %v", hcName, svcName, shared, expectedHC)
err = composite.CreateHealthCheck(l4hc.cloud, key, expectedHC)
err = l4hc.hcProvider.Create(expectedHC)
if err != nil {
return nil, "", err
}
selfLink, err := l4hc.hcProvider.SelfLink(expectedHC.Name, scope)
if err != nil {
return nil, selfLink, err
return nil, "", err
}
selfLink = cloudprovider.SelfLink(meta.VersionGA, l4hc.cloud.ProjectID(), "healthChecks", key)
return expectedHC, selfLink, nil
}
selfLink = hc.SelfLink
selfLink := hc.SelfLink
if !needToUpdateHealthChecks(hc, expectedHC) {
// nothing to do
klog.V(3).Infof("Healthcheck %v already exists", hcName)
return hc, selfLink, nil
}
mergeHealthChecks(hc, expectedHC)
klog.V(2).Infof("Updating healthcheck %s for service %s, updated healthcheck: %v", hcName, svcName, expectedHC)
err = composite.UpdateHealthCheck(l4hc.cloud, key, expectedHC)
err = l4hc.hcProvider.Update(expectedHC.Name, scope, expectedHC)
if err != nil {
return nil, selfLink, err
}
Expand All @@ -227,14 +226,6 @@ func (l4hc *l4HealthChecks) ensureFirewall(svc *corev1.Service, hcFwName string,
return firewalls.EnsureL4LBFirewallForHc(svc, sharedHC, &hcFWRParams, l4hc.cloud, l4hc.recorderFactory.Recorder(svc.Namespace))
}

func (l4hc *l4HealthChecks) deleteHealthCheck(name string, scope meta.KeyType) error {
key, err := composite.CreateKey(l4hc.cloud, name, scope)
if err != nil {
return fmt.Errorf("Failed to create composite key for healthcheck %s - %w", name, err)
}
return composite.DeleteHealthCheck(l4hc.cloud, key, meta.VersionGA)
}

func (l4hc *l4HealthChecks) deleteHealthCheckFirewall(svc *corev1.Service, hcName, hcFwName string, sharedHC bool, l4Type utils.L4LBType) (string, error) {
namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}

Expand Down Expand Up @@ -266,12 +257,12 @@ func (l4hc *l4HealthChecks) healthCheckFirewallSafeToDelete(hcName string, share
if l4Type == utils.XLB {
scopeToCheck = meta.Global
}
key, err := composite.CreateKey(l4hc.cloud, hcName, scopeToCheck)

hc, err := l4hc.hcProvider.Get(hcName, scopeToCheck)
if err != nil {
return false, fmt.Errorf("Failed to create composite key for healthcheck %s - %w", hcName, err)
return false, fmt.Errorf("l4hc.hcProvider.Get(%s, %s) returned error %w, want nil", hcName, scopeToCheck, err)
}
_, err = composite.GetHealthCheck(l4hc.cloud, key, meta.VersionGA)
return utils.IsNotFoundError(err), nil
return hc == nil, nil
}

func (l4hc *l4HealthChecks) deleteFirewall(name string, svc *corev1.Service) error {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package healthchecks
package healthchecksl4

import (
"testing"
Expand Down Expand Up @@ -109,7 +109,7 @@ func TestCompareHealthChecks(t *testing.T) {
}
}

func TestCreateHealthCheck(t *testing.T) {
func TestNewHealthCheck(t *testing.T) {
t.Parallel()
namespaceName := types.NamespacedName{Name: "svc", Namespace: "default"}

Expand Down
33 changes: 33 additions & 0 deletions pkg/healthchecksl4/interfaces.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package healthchecksl4

import (
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
v1 "k8s.io/api/core/v1"
"k8s.io/ingress-gce/pkg/composite"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"
)

// L4HealthChecks defines methods for creating and deleting health checks (and their firewall rules) for l4 services
type L4HealthChecks interface {
// EnsureHealthCheck creates health check (and firewall rule) for l4 service
EnsureHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult
// DeleteHealthCheck deletes health check (and firewall rule) for l4 service
DeleteHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
}

type EnsureL4HealthCheckResult struct {
HCName string
HCLink string
HCFirewallRuleName string
GceResourceInError string
Err error
}

type healthChecksProvider interface {
Get(name string, scope meta.KeyType) (*composite.HealthCheck, error)
Create(healthCheck *composite.HealthCheck) error
Update(name string, scope meta.KeyType, updatedHealthCheck *composite.HealthCheck) error
Delete(name string, scope meta.KeyType) error
SelfLink(name string, scope meta.KeyType) (string, error)
}
86 changes: 86 additions & 0 deletions pkg/healthchecksprovider/healthchecksprovider.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package healthchecksprovider

import (
"fmt"

cloudprovider "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
"k8s.io/ingress-gce/pkg/composite"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/legacy-cloud-providers/gce"
)

type HealthChecks struct {
cloud *gce.Cloud
version meta.Version
}

func NewHealthChecks(cloud *gce.Cloud, version meta.Version) *HealthChecks {
return &HealthChecks{
cloud: cloud,
version: version,
}
}

func (hc *HealthChecks) Get(name string, scope meta.KeyType) (*composite.HealthCheck, error) {
key, err := hc.createKey(name, scope)
if err != nil {
return nil, fmt.Errorf("hc.createKey(%s, %s) returned error %w, want nil", name, scope, err)
}
healthCheck, err := composite.GetHealthCheck(hc.cloud, key, hc.version)
if err != nil {
if utils.IsNotFoundError(err) {
return nil, nil
}
return nil, fmt.Errorf("composite.GetHealthCheck(_, %v, %v) returned error %w, want nil", key, meta.VersionGA, err)
}
return healthCheck, nil
}

func (hc *HealthChecks) Create(healthCheck *composite.HealthCheck) error {
key, err := hc.createKey(healthCheck.Name, healthCheck.Scope)
if err != nil {
return fmt.Errorf("hc.createKey(%s, %s) returned error: %w, want nil", healthCheck.Name, healthCheck.Scope, err)
}

err = composite.CreateHealthCheck(hc.cloud, key, healthCheck)
if err != nil {
return fmt.Errorf("composite.CreateHealthCheck(_, %s, %v) returned error %w, want nil", key, healthCheck, err)
}
return nil
}

func (hc *HealthChecks) Update(name string, scope meta.KeyType, updatedHealthCheck *composite.HealthCheck) error {
key, err := hc.createKey(name, scope)
if err != nil {
return fmt.Errorf("hc.createKey(%s, %s) returned error: %w, want nil", name, scope, err)
}

err = composite.UpdateHealthCheck(hc.cloud, key, updatedHealthCheck)
if err != nil {
return fmt.Errorf("composite.UpdateHealthCheck(_, %s, %v) returned error %w, want nil", key, updatedHealthCheck, err)
}
return nil
}

func (hc *HealthChecks) Delete(name string, scope meta.KeyType) error {
key, err := hc.createKey(name, scope)
if err != nil {
return fmt.Errorf("hc.createKey(%s, %s) returned error %w, want nil", name, scope, err)
}

return utils.IgnoreHTTPNotFound(composite.DeleteHealthCheck(hc.cloud, key, hc.version))
}

func (hc *HealthChecks) SelfLink(name string, scope meta.KeyType) (string, error) {
key, err := hc.createKey(name, scope)
if err != nil {
return "", fmt.Errorf("hc.createKey(%s, %s) returned error %w, want nil", name, scope, err)
}

return cloudprovider.SelfLink(meta.VersionGA, hc.cloud.ProjectID(), "healthChecks", key), nil
}

func (hc *HealthChecks) createKey(name string, scope meta.KeyType) (*meta.Key, error) {
return composite.CreateKey(hc.cloud, name, scope)
}
Loading

0 comments on commit 5767597

Please sign in to comment.