Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create Health Checks Provider to interact with Google Cloud #1763

Merged
merged 1 commit into from
Sep 9, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/glbc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
flag "github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/ingress-gce/pkg/frontendconfig"
"k8s.io/ingress-gce/pkg/healthchecks"
"k8s.io/ingress-gce/pkg/healthchecksl4"
"k8s.io/ingress-gce/pkg/ingparams"
"k8s.io/ingress-gce/pkg/l4lb"
"k8s.io/ingress-gce/pkg/psc"
Expand Down Expand Up @@ -279,7 +279,7 @@ func runControllers(ctx *ingctx.ControllerContext) {

fwc := firewalls.NewFirewallController(ctx, flags.F.NodePortRanges.Values())

healthchecks.InitializeL4(ctx.Cloud, ctx)
healthchecksl4.Initialize(ctx.Cloud, ctx)

if flags.F.RunL4Controller {
l4Controller := l4lb.NewILBController(ctx, stopCh)
Expand Down
22 changes: 2 additions & 20 deletions pkg/healthchecks/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,13 @@ limitations under the License.
package healthchecks

import (
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
computealpha "google.golang.org/api/compute/v0.alpha"
computebeta "google.golang.org/api/compute/v0.beta"
compute "google.golang.org/api/compute/v1"
"google.golang.org/api/compute/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/ingress-gce/pkg/translator"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"

"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
)

// HealthCheckProvider is an interface to manage a single GCE health check.
Expand Down Expand Up @@ -58,19 +56,3 @@ type HealthChecker interface {
Delete(name string, scope meta.KeyType) error
Get(name string, version meta.Version, scope meta.KeyType) (*translator.HealthCheck, error)
}

// L4HealthChecks defines methods for creating and deleting health checks (and their firewall rules) for l4 services
type L4HealthChecks interface {
// EnsureL4HealthCheck creates health check (and firewall rule) for l4 service
EnsureL4HealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult
// DeleteHealthCheck deletes health check (and firewall rule) for l4 service
DeleteHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
}

type EnsureL4HealthCheckResult struct {
HCName string
HCLink string
HCFirewallRuleName string
GceResourceInError string
Err error
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package healthchecks
package healthchecksl4
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe it's better to name the package l4healthchecks?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WDYT?


import (
"fmt"
"strconv"
"sync"

cloudprovider "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
Expand All @@ -30,6 +29,7 @@ import (
"k8s.io/ingress-gce/pkg/composite"
"k8s.io/ingress-gce/pkg/events"
"k8s.io/ingress-gce/pkg/firewalls"
"k8s.io/ingress-gce/pkg/healthchecksprovider"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"
"k8s.io/klog/v2"
Expand All @@ -49,20 +49,21 @@ const (
var (
// instanceLock to prevent duplicate initialization.
instanceLock = &sync.Mutex{}
// instance is a singleton instance, created by InitializeL4
// instance is a singleton instance, created by Initialize
instance *l4HealthChecks
)

type l4HealthChecks struct {
// sharedResourceLock serializes operations on the healthcheck and firewall
// resources shared across multiple Services.
sharedResourcesLock sync.Mutex
hcProvider healthChecksProvider
cloud *gce.Cloud
recorderFactory events.RecorderProducer
}

// InitializeL4 creates singleton instance, must be run before L4() func
func InitializeL4(cloud *gce.Cloud, recorderFactory events.RecorderProducer) {
// Initialize creates singleton instance, must be run before GetInstance() func
func Initialize(cloud *gce.Cloud, recorderFactory events.RecorderProducer) {
instanceLock.Lock()
defer instanceLock.Unlock()

Expand All @@ -74,25 +75,27 @@ func InitializeL4(cloud *gce.Cloud, recorderFactory events.RecorderProducer) {
instance = &l4HealthChecks{
cloud: cloud,
recorderFactory: recorderFactory,
hcProvider: healthchecksprovider.NewHealthChecks(cloud, meta.VersionGA),
}
klog.V(3).Infof("Initialized L4 Healthchecks")
}

// FakeL4 creates instance of l4HealthChecks. Use for test only.
func FakeL4(cloud *gce.Cloud, recorderFactory events.RecorderProducer) *l4HealthChecks {
// Fake creates instance of l4HealthChecks. Use for test only.
func Fake(cloud *gce.Cloud, recorderFactory events.RecorderProducer) *l4HealthChecks {
instance = &l4HealthChecks{
cloud: cloud,
recorderFactory: recorderFactory,
hcProvider: healthchecksprovider.NewHealthChecks(cloud, meta.VersionGA),
}
return instance
}

// L4 returns singleton instance, must be run after InitializeL4
func L4() *l4HealthChecks {
// GetInstance returns singleton instance, must be run after Initialize
func GetInstance() *l4HealthChecks {
return instance
}

// EnsureL4HealthCheck and firewall rules exist for the L4
// EnsureHealthCheck and firewall rules exist for the L4
// LoadBalancer Service.
//
// The healthcheck and firewall will be shared between different K8s
Expand All @@ -102,8 +105,7 @@ func L4() *l4HealthChecks {
// Firewall rules are always created at in the Global scope (vs
// Regional). This means that one Firewall rule is created for
// Services of different scope (Global vs Regional).

func (l4hc *l4HealthChecks) EnsureL4HealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult {
func (l4hc *l4HealthChecks) EnsureHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult {
namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}

hcName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC)
Expand Down Expand Up @@ -154,7 +156,7 @@ func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L
defer l4hc.sharedResourcesLock.Unlock()
}

err := utils.IgnoreHTTPNotFound(l4hc.deleteHealthCheck(hcName, scope))
err := l4hc.hcProvider.Delete(hcName, scope)
if err != nil {
// Ignore deletion error due to health check in use by another resource.
if !utils.IsInUsedByError(err) {
Expand All @@ -169,17 +171,11 @@ func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L
}

func (l4hc *l4HealthChecks) ensureL4HealthCheckInternal(hcName string, svcName types.NamespacedName, shared bool, path string, port int32, scope meta.KeyType, l4Type utils.L4LBType) (*composite.HealthCheck, string, error) {
selfLink := ""
key, err := composite.CreateKey(l4hc.cloud, hcName, scope)
if err != nil {
return nil, selfLink, fmt.Errorf("Failed to create key for healthcheck with name %s for service %s", hcName, svcName.String())
}
hc, err := composite.GetHealthCheck(l4hc.cloud, key, meta.VersionGA)
hc, err := l4hc.hcProvider.Get(hcName, scope)
if err != nil {
if !utils.IsNotFoundError(err) {
return nil, selfLink, err
}
return nil, "", err
}

var region string
if scope == meta.Regional {
region = l4hc.cloud.Region()
Expand All @@ -189,22 +185,25 @@ func (l4hc *l4HealthChecks) ensureL4HealthCheckInternal(hcName string, svcName t
if hc == nil {
// Create the healthcheck
klog.V(2).Infof("Creating healthcheck %s for service %s, shared = %v. Expected healthcheck: %v", hcName, svcName, shared, expectedHC)
err = composite.CreateHealthCheck(l4hc.cloud, key, expectedHC)
err = l4hc.hcProvider.Create(expectedHC)
if err != nil {
return nil, "", err
}
selfLink, err := l4hc.hcProvider.SelfLink(expectedHC.Name, scope)
if err != nil {
return nil, selfLink, err
return nil, "", err
}
selfLink = cloudprovider.SelfLink(meta.VersionGA, l4hc.cloud.ProjectID(), "healthChecks", key)
return expectedHC, selfLink, nil
}
selfLink = hc.SelfLink
selfLink := hc.SelfLink
if !needToUpdateHealthChecks(hc, expectedHC) {
// nothing to do
klog.V(3).Infof("Healthcheck %v already exists", hcName)
return hc, selfLink, nil
}
mergeHealthChecks(hc, expectedHC)
klog.V(2).Infof("Updating healthcheck %s for service %s, updated healthcheck: %v", hcName, svcName, expectedHC)
err = composite.UpdateHealthCheck(l4hc.cloud, key, expectedHC)
err = l4hc.hcProvider.Update(expectedHC.Name, scope, expectedHC)
if err != nil {
return nil, selfLink, err
}
Expand All @@ -227,14 +226,6 @@ func (l4hc *l4HealthChecks) ensureFirewall(svc *corev1.Service, hcFwName string,
return firewalls.EnsureL4LBFirewallForHc(svc, sharedHC, &hcFWRParams, l4hc.cloud, l4hc.recorderFactory.Recorder(svc.Namespace))
}

func (l4hc *l4HealthChecks) deleteHealthCheck(name string, scope meta.KeyType) error {
key, err := composite.CreateKey(l4hc.cloud, name, scope)
if err != nil {
return fmt.Errorf("Failed to create composite key for healthcheck %s - %w", name, err)
}
return composite.DeleteHealthCheck(l4hc.cloud, key, meta.VersionGA)
}

func (l4hc *l4HealthChecks) deleteHealthCheckFirewall(svc *corev1.Service, hcName, hcFwName string, sharedHC bool, l4Type utils.L4LBType) (string, error) {
namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}

Expand Down Expand Up @@ -266,12 +257,12 @@ func (l4hc *l4HealthChecks) healthCheckFirewallSafeToDelete(hcName string, share
if l4Type == utils.XLB {
scopeToCheck = meta.Global
}
key, err := composite.CreateKey(l4hc.cloud, hcName, scopeToCheck)

hc, err := l4hc.hcProvider.Get(hcName, scopeToCheck)
if err != nil {
return false, fmt.Errorf("Failed to create composite key for healthcheck %s - %w", hcName, err)
return false, fmt.Errorf("l4hc.hcProvider.Get(%s, %s) returned error %w, want nil", hcName, scopeToCheck, err)
}
_, err = composite.GetHealthCheck(l4hc.cloud, key, meta.VersionGA)
return utils.IsNotFoundError(err), nil
return hc == nil, nil
}

func (l4hc *l4HealthChecks) deleteFirewall(name string, svc *corev1.Service) error {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package healthchecks
package healthchecksl4

import (
"testing"
Expand Down Expand Up @@ -109,7 +109,7 @@ func TestCompareHealthChecks(t *testing.T) {
}
}

func TestCreateHealthCheck(t *testing.T) {
func TestNewHealthCheck(t *testing.T) {
t.Parallel()
namespaceName := types.NamespacedName{Name: "svc", Namespace: "default"}

Expand Down
33 changes: 33 additions & 0 deletions pkg/healthchecksl4/interfaces.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package healthchecksl4

import (
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
v1 "k8s.io/api/core/v1"
"k8s.io/ingress-gce/pkg/composite"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"
)

// L4HealthChecks defines methods for creating and deleting health checks (and their firewall rules) for l4 services
type L4HealthChecks interface {
// EnsureHealthCheck creates health check (and firewall rule) for l4 service
EnsureHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult
// DeleteHealthCheck deletes health check (and firewall rule) for l4 service
DeleteHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
}

type EnsureL4HealthCheckResult struct {
HCName string
HCLink string
HCFirewallRuleName string
GceResourceInError string
Err error
}

type healthChecksProvider interface {
Get(name string, scope meta.KeyType) (*composite.HealthCheck, error)
Create(healthCheck *composite.HealthCheck) error
Update(name string, scope meta.KeyType, updatedHealthCheck *composite.HealthCheck) error
Delete(name string, scope meta.KeyType) error
SelfLink(name string, scope meta.KeyType) (string, error)
}
86 changes: 86 additions & 0 deletions pkg/healthchecksprovider/healthchecksprovider.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package healthchecksprovider

import (
"fmt"

cloudprovider "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
"k8s.io/ingress-gce/pkg/composite"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/legacy-cloud-providers/gce"
)

type HealthChecks struct {
cloud *gce.Cloud
version meta.Version
}

func NewHealthChecks(cloud *gce.Cloud, version meta.Version) *HealthChecks {
return &HealthChecks{
cloud: cloud,
version: version,
}
}

func (hc *HealthChecks) Get(name string, scope meta.KeyType) (*composite.HealthCheck, error) {
key, err := hc.createKey(name, scope)
if err != nil {
return nil, fmt.Errorf("hc.createKey(%s, %s) returned error %w, want nil", name, scope, err)
}
healthCheck, err := composite.GetHealthCheck(hc.cloud, key, hc.version)
if err != nil {
if utils.IsNotFoundError(err) {
return nil, nil
}
return nil, fmt.Errorf("composite.GetHealthCheck(_, %v, %v) returned error %w, want nil", key, meta.VersionGA, err)
}
return healthCheck, nil
}

func (hc *HealthChecks) Create(healthCheck *composite.HealthCheck) error {
key, err := hc.createKey(healthCheck.Name, healthCheck.Scope)
if err != nil {
return fmt.Errorf("hc.createKey(%s, %s) returned error: %w, want nil", healthCheck.Name, healthCheck.Scope, err)
}

err = composite.CreateHealthCheck(hc.cloud, key, healthCheck)
if err != nil {
return fmt.Errorf("composite.CreateHealthCheck(_, %s, %v) returned error %w, want nil", key, healthCheck, err)
}
return nil
}

func (hc *HealthChecks) Update(name string, scope meta.KeyType, updatedHealthCheck *composite.HealthCheck) error {
key, err := hc.createKey(name, scope)
if err != nil {
return fmt.Errorf("hc.createKey(%s, %s) returned error: %w, want nil", name, scope, err)
}

err = composite.UpdateHealthCheck(hc.cloud, key, updatedHealthCheck)
if err != nil {
return fmt.Errorf("composite.UpdateHealthCheck(_, %s, %v) returned error %w, want nil", key, updatedHealthCheck, err)
}
return nil
}

func (hc *HealthChecks) Delete(name string, scope meta.KeyType) error {
key, err := hc.createKey(name, scope)
if err != nil {
return fmt.Errorf("hc.createKey(%s, %s) returned error %w, want nil", name, scope, err)
}

return utils.IgnoreHTTPNotFound(composite.DeleteHealthCheck(hc.cloud, key, hc.version))
}

func (hc *HealthChecks) SelfLink(name string, scope meta.KeyType) (string, error) {
key, err := hc.createKey(name, scope)
if err != nil {
return "", fmt.Errorf("hc.createKey(%s, %s) returned error %w, want nil", name, scope, err)
}

return cloudprovider.SelfLink(meta.VersionGA, hc.cloud.ProjectID(), "healthChecks", key), nil
}

func (hc *HealthChecks) createKey(name string, scope meta.KeyType) (*meta.Key, error) {
return composite.CreateKey(hc.cloud, name, scope)
}
Loading