Skip to content

Commit

Permalink
Merge pull request #1763 from panslava/create-hc-provider
Browse files Browse the repository at this point in the history
Create Health Checks Provider to interact with Google Cloud
  • Loading branch information
k8s-ci-robot authored Sep 9, 2022
2 parents 6b146e8 + 5767597 commit 4312cb9
Show file tree
Hide file tree
Showing 13 changed files with 476 additions and 106 deletions.
4 changes: 2 additions & 2 deletions cmd/glbc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
flag "github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/ingress-gce/pkg/frontendconfig"
"k8s.io/ingress-gce/pkg/healthchecks"
"k8s.io/ingress-gce/pkg/healthchecksl4"
"k8s.io/ingress-gce/pkg/ingparams"
"k8s.io/ingress-gce/pkg/l4lb"
"k8s.io/ingress-gce/pkg/psc"
Expand Down Expand Up @@ -279,7 +279,7 @@ func runControllers(ctx *ingctx.ControllerContext) {

fwc := firewalls.NewFirewallController(ctx, flags.F.NodePortRanges.Values())

healthchecks.InitializeL4(ctx.Cloud, ctx)
healthchecksl4.Initialize(ctx.Cloud, ctx)

if flags.F.RunL4Controller {
l4Controller := l4lb.NewILBController(ctx, stopCh)
Expand Down
22 changes: 2 additions & 20 deletions pkg/healthchecks/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,13 @@ limitations under the License.
package healthchecks

import (
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
computealpha "google.golang.org/api/compute/v0.alpha"
computebeta "google.golang.org/api/compute/v0.beta"
compute "google.golang.org/api/compute/v1"
"google.golang.org/api/compute/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/ingress-gce/pkg/translator"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"

"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
)

// HealthCheckProvider is an interface to manage a single GCE health check.
Expand Down Expand Up @@ -58,19 +56,3 @@ type HealthChecker interface {
Delete(name string, scope meta.KeyType) error
Get(name string, version meta.Version, scope meta.KeyType) (*translator.HealthCheck, error)
}

// L4HealthChecks defines methods for creating and deleting health checks (and their firewall rules) for l4 services
type L4HealthChecks interface {
// EnsureL4HealthCheck creates health check (and firewall rule) for l4 service
EnsureL4HealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult
// DeleteHealthCheck deletes health check (and firewall rule) for l4 service
DeleteHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
}

type EnsureL4HealthCheckResult struct {
HCName string
HCLink string
HCFirewallRuleName string
GceResourceInError string
Err error
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package healthchecks
package healthchecksl4

import (
"fmt"
"strconv"
"sync"

cloudprovider "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
Expand All @@ -30,6 +29,7 @@ import (
"k8s.io/ingress-gce/pkg/composite"
"k8s.io/ingress-gce/pkg/events"
"k8s.io/ingress-gce/pkg/firewalls"
"k8s.io/ingress-gce/pkg/healthchecksprovider"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"
"k8s.io/klog/v2"
Expand All @@ -49,20 +49,21 @@ const (
var (
// instanceLock to prevent duplicate initialization.
instanceLock = &sync.Mutex{}
// instance is a singleton instance, created by InitializeL4
// instance is a singleton instance, created by Initialize
instance *l4HealthChecks
)

type l4HealthChecks struct {
// sharedResourceLock serializes operations on the healthcheck and firewall
// resources shared across multiple Services.
sharedResourcesLock sync.Mutex
hcProvider healthChecksProvider
cloud *gce.Cloud
recorderFactory events.RecorderProducer
}

// InitializeL4 creates singleton instance, must be run before L4() func
func InitializeL4(cloud *gce.Cloud, recorderFactory events.RecorderProducer) {
// Initialize creates singleton instance, must be run before GetInstance() func
func Initialize(cloud *gce.Cloud, recorderFactory events.RecorderProducer) {
instanceLock.Lock()
defer instanceLock.Unlock()

Expand All @@ -74,25 +75,27 @@ func InitializeL4(cloud *gce.Cloud, recorderFactory events.RecorderProducer) {
instance = &l4HealthChecks{
cloud: cloud,
recorderFactory: recorderFactory,
hcProvider: healthchecksprovider.NewHealthChecks(cloud, meta.VersionGA),
}
klog.V(3).Infof("Initialized L4 Healthchecks")
}

// FakeL4 creates instance of l4HealthChecks. Use for test only.
func FakeL4(cloud *gce.Cloud, recorderFactory events.RecorderProducer) *l4HealthChecks {
// Fake creates instance of l4HealthChecks. Use for test only.
func Fake(cloud *gce.Cloud, recorderFactory events.RecorderProducer) *l4HealthChecks {
instance = &l4HealthChecks{
cloud: cloud,
recorderFactory: recorderFactory,
hcProvider: healthchecksprovider.NewHealthChecks(cloud, meta.VersionGA),
}
return instance
}

// L4 returns singleton instance, must be run after InitializeL4
func L4() *l4HealthChecks {
// GetInstance returns singleton instance, must be run after Initialize
func GetInstance() *l4HealthChecks {
return instance
}

// EnsureL4HealthCheck and firewall rules exist for the L4
// EnsureHealthCheck and firewall rules exist for the L4
// LoadBalancer Service.
//
// The healthcheck and firewall will be shared between different K8s
Expand All @@ -102,8 +105,7 @@ func L4() *l4HealthChecks {
// Firewall rules are always created at in the Global scope (vs
// Regional). This means that one Firewall rule is created for
// Services of different scope (Global vs Regional).

func (l4hc *l4HealthChecks) EnsureL4HealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult {
func (l4hc *l4HealthChecks) EnsureHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult {
namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}

hcName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC)
Expand Down Expand Up @@ -154,7 +156,7 @@ func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L
defer l4hc.sharedResourcesLock.Unlock()
}

err := utils.IgnoreHTTPNotFound(l4hc.deleteHealthCheck(hcName, scope))
err := l4hc.hcProvider.Delete(hcName, scope)
if err != nil {
// Ignore deletion error due to health check in use by another resource.
if !utils.IsInUsedByError(err) {
Expand All @@ -169,17 +171,11 @@ func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L
}

func (l4hc *l4HealthChecks) ensureL4HealthCheckInternal(hcName string, svcName types.NamespacedName, shared bool, path string, port int32, scope meta.KeyType, l4Type utils.L4LBType) (*composite.HealthCheck, string, error) {
selfLink := ""
key, err := composite.CreateKey(l4hc.cloud, hcName, scope)
if err != nil {
return nil, selfLink, fmt.Errorf("Failed to create key for healthcheck with name %s for service %s", hcName, svcName.String())
}
hc, err := composite.GetHealthCheck(l4hc.cloud, key, meta.VersionGA)
hc, err := l4hc.hcProvider.Get(hcName, scope)
if err != nil {
if !utils.IsNotFoundError(err) {
return nil, selfLink, err
}
return nil, "", err
}

var region string
if scope == meta.Regional {
region = l4hc.cloud.Region()
Expand All @@ -189,22 +185,25 @@ func (l4hc *l4HealthChecks) ensureL4HealthCheckInternal(hcName string, svcName t
if hc == nil {
// Create the healthcheck
klog.V(2).Infof("Creating healthcheck %s for service %s, shared = %v. Expected healthcheck: %v", hcName, svcName, shared, expectedHC)
err = composite.CreateHealthCheck(l4hc.cloud, key, expectedHC)
err = l4hc.hcProvider.Create(expectedHC)
if err != nil {
return nil, "", err
}
selfLink, err := l4hc.hcProvider.SelfLink(expectedHC.Name, scope)
if err != nil {
return nil, selfLink, err
return nil, "", err
}
selfLink = cloudprovider.SelfLink(meta.VersionGA, l4hc.cloud.ProjectID(), "healthChecks", key)
return expectedHC, selfLink, nil
}
selfLink = hc.SelfLink
selfLink := hc.SelfLink
if !needToUpdateHealthChecks(hc, expectedHC) {
// nothing to do
klog.V(3).Infof("Healthcheck %v already exists", hcName)
return hc, selfLink, nil
}
mergeHealthChecks(hc, expectedHC)
klog.V(2).Infof("Updating healthcheck %s for service %s, updated healthcheck: %v", hcName, svcName, expectedHC)
err = composite.UpdateHealthCheck(l4hc.cloud, key, expectedHC)
err = l4hc.hcProvider.Update(expectedHC.Name, scope, expectedHC)
if err != nil {
return nil, selfLink, err
}
Expand All @@ -227,14 +226,6 @@ func (l4hc *l4HealthChecks) ensureFirewall(svc *corev1.Service, hcFwName string,
return firewalls.EnsureL4LBFirewallForHc(svc, sharedHC, &hcFWRParams, l4hc.cloud, l4hc.recorderFactory.Recorder(svc.Namespace))
}

func (l4hc *l4HealthChecks) deleteHealthCheck(name string, scope meta.KeyType) error {
key, err := composite.CreateKey(l4hc.cloud, name, scope)
if err != nil {
return fmt.Errorf("Failed to create composite key for healthcheck %s - %w", name, err)
}
return composite.DeleteHealthCheck(l4hc.cloud, key, meta.VersionGA)
}

func (l4hc *l4HealthChecks) deleteHealthCheckFirewall(svc *corev1.Service, hcName, hcFwName string, sharedHC bool, l4Type utils.L4LBType) (string, error) {
namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}

Expand Down Expand Up @@ -266,12 +257,12 @@ func (l4hc *l4HealthChecks) healthCheckFirewallSafeToDelete(hcName string, share
if l4Type == utils.XLB {
scopeToCheck = meta.Global
}
key, err := composite.CreateKey(l4hc.cloud, hcName, scopeToCheck)

hc, err := l4hc.hcProvider.Get(hcName, scopeToCheck)
if err != nil {
return false, fmt.Errorf("Failed to create composite key for healthcheck %s - %w", hcName, err)
return false, fmt.Errorf("l4hc.hcProvider.Get(%s, %s) returned error %w, want nil", hcName, scopeToCheck, err)
}
_, err = composite.GetHealthCheck(l4hc.cloud, key, meta.VersionGA)
return utils.IsNotFoundError(err), nil
return hc == nil, nil
}

func (l4hc *l4HealthChecks) deleteFirewall(name string, svc *corev1.Service) error {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package healthchecks
package healthchecksl4

import (
"testing"
Expand Down Expand Up @@ -109,7 +109,7 @@ func TestCompareHealthChecks(t *testing.T) {
}
}

func TestCreateHealthCheck(t *testing.T) {
func TestNewHealthCheck(t *testing.T) {
t.Parallel()
namespaceName := types.NamespacedName{Name: "svc", Namespace: "default"}

Expand Down
33 changes: 33 additions & 0 deletions pkg/healthchecksl4/interfaces.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package healthchecksl4

import (
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
v1 "k8s.io/api/core/v1"
"k8s.io/ingress-gce/pkg/composite"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"
)

// L4HealthChecks defines methods for creating and deleting health checks (and their firewall rules) for l4 services
type L4HealthChecks interface {
// EnsureHealthCheck creates health check (and firewall rule) for l4 service
EnsureHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult
// DeleteHealthCheck deletes health check (and firewall rule) for l4 service
DeleteHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
}

type EnsureL4HealthCheckResult struct {
HCName string
HCLink string
HCFirewallRuleName string
GceResourceInError string
Err error
}

type healthChecksProvider interface {
Get(name string, scope meta.KeyType) (*composite.HealthCheck, error)
Create(healthCheck *composite.HealthCheck) error
Update(name string, scope meta.KeyType, updatedHealthCheck *composite.HealthCheck) error
Delete(name string, scope meta.KeyType) error
SelfLink(name string, scope meta.KeyType) (string, error)
}
86 changes: 86 additions & 0 deletions pkg/healthchecksprovider/healthchecksprovider.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package healthchecksprovider

import (
"fmt"

cloudprovider "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
"k8s.io/ingress-gce/pkg/composite"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/legacy-cloud-providers/gce"
)

type HealthChecks struct {
cloud *gce.Cloud
version meta.Version
}

func NewHealthChecks(cloud *gce.Cloud, version meta.Version) *HealthChecks {
return &HealthChecks{
cloud: cloud,
version: version,
}
}

func (hc *HealthChecks) Get(name string, scope meta.KeyType) (*composite.HealthCheck, error) {
key, err := hc.createKey(name, scope)
if err != nil {
return nil, fmt.Errorf("hc.createKey(%s, %s) returned error %w, want nil", name, scope, err)
}
healthCheck, err := composite.GetHealthCheck(hc.cloud, key, hc.version)
if err != nil {
if utils.IsNotFoundError(err) {
return nil, nil
}
return nil, fmt.Errorf("composite.GetHealthCheck(_, %v, %v) returned error %w, want nil", key, meta.VersionGA, err)
}
return healthCheck, nil
}

func (hc *HealthChecks) Create(healthCheck *composite.HealthCheck) error {
key, err := hc.createKey(healthCheck.Name, healthCheck.Scope)
if err != nil {
return fmt.Errorf("hc.createKey(%s, %s) returned error: %w, want nil", healthCheck.Name, healthCheck.Scope, err)
}

err = composite.CreateHealthCheck(hc.cloud, key, healthCheck)
if err != nil {
return fmt.Errorf("composite.CreateHealthCheck(_, %s, %v) returned error %w, want nil", key, healthCheck, err)
}
return nil
}

func (hc *HealthChecks) Update(name string, scope meta.KeyType, updatedHealthCheck *composite.HealthCheck) error {
key, err := hc.createKey(name, scope)
if err != nil {
return fmt.Errorf("hc.createKey(%s, %s) returned error: %w, want nil", name, scope, err)
}

err = composite.UpdateHealthCheck(hc.cloud, key, updatedHealthCheck)
if err != nil {
return fmt.Errorf("composite.UpdateHealthCheck(_, %s, %v) returned error %w, want nil", key, updatedHealthCheck, err)
}
return nil
}

func (hc *HealthChecks) Delete(name string, scope meta.KeyType) error {
key, err := hc.createKey(name, scope)
if err != nil {
return fmt.Errorf("hc.createKey(%s, %s) returned error %w, want nil", name, scope, err)
}

return utils.IgnoreHTTPNotFound(composite.DeleteHealthCheck(hc.cloud, key, hc.version))
}

func (hc *HealthChecks) SelfLink(name string, scope meta.KeyType) (string, error) {
key, err := hc.createKey(name, scope)
if err != nil {
return "", fmt.Errorf("hc.createKey(%s, %s) returned error %w, want nil", name, scope, err)
}

return cloudprovider.SelfLink(meta.VersionGA, hc.cloud.ProjectID(), "healthChecks", key), nil
}

func (hc *HealthChecks) createKey(name string, scope meta.KeyType) (*meta.Key, error) {
return composite.CreateKey(hc.cloud, name, scope)
}
Loading

0 comments on commit 4312cb9

Please sign in to comment.