ci: disable china price gen (#4763)

Co-authored-by: Jonathan Innis <jonathan.innis.ji@gmail.com>
Co-authored-by: Jonathan Innis <joinnis@amazon.com>

hack/code/prices_gen.go

@@ -39,12 +39,50 @@ import (
 	"github.com/aws/karpenter/pkg/test"
 )
 
-func main() {
+func getAWSRegions(partition string) []string {
+	switch partition {
+	case "aws":
+		return []string{"us-east-1"}
+	case "aws-us-gov":
+		return []string{"us-gov-east-1", "us-gov-west-1"}
+	case "aws-cn":
+		return []string{"cn-north-1"}
+	default:
+		panic("invalid partition")
+	}
+}
+
+func getPartitionSuffix(partition string) string {
+	switch partition {
+	case "aws":
+		return "AWS"
+	case "aws-us-gov":
+		return "USGov"
+	case "aws-cn":
+		return "CN"
+	default:
+		panic("invalid partition")
+	}
+}
+
+type Options struct {
+	partition string
+	output    string
+}
+
+func NewOptions() *Options {
+	o := &Options{}
+	flag.StringVar(&o.partition, "partition", "aws", "The partition to generate prices for. Valid options are \"aws\", \"aws-us-gov\", and \"aws-cn\".")
+	flag.StringVar(&o.output, "output", "pkg/providers/pricing/zz_generated.pricing_aws.go", "The destination for the generated go file.")
 	flag.Parse()
-	if flag.NArg() != 1 {
-		log.Fatalf("Usage: %s pkg/providers/pricing/zz_generated.pricing.go", os.Args[0])
+	if !lo.Contains([]string{"aws", "aws-us-gov", "aws-cn"}, o.partition) {
+		log.Fatal("invalid partition: must be \"aws\", \"aws-us-gov\", or \"aws-cn\"")
 	}
+	return o
+}
 
+func main() {
+	opts := NewOptions()
 	f, err := os.Create("pricing.heapprofile")
 	if err != nil {
 		log.Fatal("could not create memory profile: ", err)
@@ -58,34 +96,23 @@ func main() {
 	ctx = settings.ToContext(ctx, test.Settings())
 	sess := session.Must(session.NewSession())
 	ec2 := ec22.New(sess)
-	updateStarted := time.Now()
 	src := &bytes.Buffer{}
 	fmt.Fprintln(src, "//go:build !ignore_autogenerated")
 	license := lo.Must(os.ReadFile("hack/boilerplate.go.txt"))
 	fmt.Fprintln(src, string(license))
 	fmt.Fprintln(src, "package pricing")
-	fmt.Fprintln(src, `import "time"`)
 	now := time.Now().UTC().Format(time.RFC3339)
 	fmt.Fprintf(src, "// generated at %s for %s\n\n\n", now, region)
-	fmt.Fprintf(src, "var initialPriceUpdate, _ = time.Parse(time.RFC3339, \"%s\")\n", now)
-	fmt.Fprintln(src, "var initialOnDemandPrices = map[string]map[string]float64{}")
-	fmt.Fprintln(src, "func init() {")
+	fmt.Fprintf(src, "var InitialOnDemandPrices%s = map[string]map[string]float64{\n", getPartitionSuffix(opts.partition))
 	// record prices for each region we are interested in
-	for _, region := range []string{"us-east-1", "us-gov-west-1", "us-gov-east-1", "cn-north-1"} {
+	for _, region := range getAWSRegions(opts.partition) {
 		log.Println("fetching for", region)
 		pricingProvider := pricing.NewProvider(ctx, pricing.NewAPI(sess, region), ec2, region)
 		controller := pricing.NewController(pricingProvider)
 		_, err := controller.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{}})
 		if err != nil {
 			log.Fatalf("failed to initialize pricing provider %s", err)
 		}
-		for {
-			if pricingProvider.OnDemandLastUpdated().After(updateStarted) && pricingProvider.SpotLastUpdated().After(updateStarted) {
-				break
-			}
-			log.Println("waiting on pricing update...")
-			time.Sleep(1 * time.Second)
-		}
 		instanceTypes := pricingProvider.InstanceTypes()
 		sort.Strings(instanceTypes)
 
@@ -94,13 +121,13 @@ func main() {
 	fmt.Fprintln(src, "}")
 	formatted, err := format.Source(src.Bytes())
 	if err != nil {
-		if err := os.WriteFile(flag.Arg(0), src.Bytes(), 0644); err != nil {
+		if err := os.WriteFile(opts.output, src.Bytes(), 0644); err != nil {
 			log.Fatalf("writing output, %s", err)
 		}
 		log.Fatalf("formatting generated source, %s", err)
 	}
 
-	if err := os.WriteFile(flag.Arg(0), formatted, 0644); err != nil {
+	if err := os.WriteFile(opts.output, formatted, 0644); err != nil {
 		log.Fatalf("writing output, %s", err)
 	}
 	runtime.GC()
@@ -111,7 +138,7 @@ func main() {
 
 func writePricing(src *bytes.Buffer, instanceNames []string, region string, getPrice func(instanceType string) (float64, bool)) {
 	fmt.Fprintf(src, "// %s\n", region)
-	fmt.Fprintf(src, "initialOnDemandPrices[%q] = map[string]float64{\n", region)
+	fmt.Fprintf(src, "%q: {\n", region)
 	lineLen := 0
 	sort.Strings(instanceNames)
 	previousFamily := ""
@@ -144,7 +171,7 @@ func writePricing(src *bytes.Buffer, instanceNames []string, region string, getP
 			fmt.Fprintln(src)
 		}
 	}
-	fmt.Fprintln(src, "\n}")
+	fmt.Fprintln(src, "\n},")
 	fmt.Fprintln(src)
 }
 

hack/codegen.sh

@@ -19,14 +19,22 @@ bandwidth() {
 }
 
 pricing() {
-  GENERATED_FILE="pkg/providers/pricing/zz_generated.pricing.go"
-  NO_UPDATE=$' pkg/providers/pricing/zz_generated.pricing.go | 4 ++--\n 1 file changed, 2 insertions(+), 2 deletions(-)'
-  SUBJECT="Pricing"
-
-  go run hack/code/prices_gen.go -- "${GENERATED_FILE}"
-
-  GIT_DIFF=$(git diff --stat "${GENERATED_FILE}")
-  checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT} beside timestamps since last update" "${GENERATED_FILE}"
+  declare -a PARTITIONS=(
+    "aws"
+    "aws-us-gov"
+    # "aws-cn"
+  )
+
+  for partition in "${PARTITIONS[@]}"; do
+    GENERATED_FILE="pkg/providers/pricing/zz_generated.pricing_${partition//-/_}.go"
+    NO_UPDATE=" ${GENERATED_FILE} "$'| 4 ++--\n 1 file changed, 2 insertions(+), 2 deletions(-)'
+    SUBJECT="Pricing"
+
+    go run hack/code/prices_gen.go --partition "$partition" --output "$GENERATED_FILE"
+
+    GIT_DIFF=$(git diff --stat "${GENERATED_FILE}")
+    checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT} beside timestamps since last update" "${GENERATED_FILE}"
+  done
 }
 
 vpcLimits() {

pkg/providers/instancetype/nodeclass_test.go

@@ -1369,7 +1369,6 @@ var _ = Describe("NodeClass/InstanceTypes", func() {
 				},
 			})
 			Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed())
-			Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue())
 
 			nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{
 				{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot}},
@@ -1396,7 +1395,6 @@ var _ = Describe("NodeClass/InstanceTypes", func() {
 				},
 			})
 			Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed())
-			Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue())
 
 			// not restricting to the zone so we can get any zone
 			nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{

pkg/providers/instancetype/nodetemplate_test.go

@@ -1400,7 +1400,6 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() {
 				},
 			})
 			Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed())
-			Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue())
 
 			provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{
 				{Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}},
@@ -1427,7 +1426,6 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() {
 				},
 			})
 			Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed())
-			Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue())
 
 			// not restricting to the zone so we can get any zone
 			provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{

pkg/providers/pricing/pricing.go

@@ -18,7 +18,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -39,6 +39,8 @@ import (
 	"github.com/aws/karpenter-core/pkg/utils/pretty"
 )
 
+var initialOnDemandPrices = lo.Assign(InitialOnDemandPricesAWS, InitialOnDemandPricesUSGov, InitialOnDemandPricesCN)
+
 // Provider provides actual pricing data to the AWS cloud provider to allow it to make more informed decisions
 // regarding which instances to launch.  This is initialized at startup with a periodically updated static price list to
 // support running in locations where pricing data is unavailable.  In those cases the static pricing data provides a
@@ -52,10 +54,9 @@ type Provider struct {
 	cm      *pretty.ChangeMonitor
 
 	mu                 sync.RWMutex
-	onDemandUpdateTime time.Time
 	onDemandPrices     map[string]float64
-	spotUpdateTime     time.Time
 	spotPrices         map[string]zonal
+	spotPricingUpdated bool
 }
 
 // zonalPricing is used to capture the per-zone price
@@ -67,11 +68,6 @@ type zonal struct {
 	prices       map[string]float64
 }
 
-type Err struct {
-	error
-	lastUpdateTime time.Time
-}
-
 func newZonalPricing(defaultPrice float64) zonal {
 	z := zonal{
 		prices: map[string]float64{},
@@ -117,20 +113,6 @@ func (p *Provider) InstanceTypes() []string {
 	return lo.Union(lo.Keys(p.onDemandPrices), lo.Keys(p.spotPrices))
 }
 
-// OnDemandLastUpdated returns the time that the on-demand pricing was last updated
-func (p *Provider) OnDemandLastUpdated() time.Time {
-	p.mu.RLock()
-	defer p.mu.RUnlock()
-	return p.onDemandUpdateTime
-}
-
-// SpotLastUpdated returns the time that the spot pricing was last updated
-func (p *Provider) SpotLastUpdated() time.Time {
-	p.mu.RLock()
-	defer p.mu.RUnlock()
-	return p.spotUpdateTime
-}
-
 // OnDemandPrice returns the last known on-demand price for a given instance type, returning an error if there is no
 // known on-demand pricing for the instance type.
 func (p *Provider) OnDemandPrice(instanceType string) (float64, bool) {
@@ -149,7 +131,7 @@ func (p *Provider) SpotPrice(instanceType string, zone string) (float64, bool) {
 	p.mu.RLock()
 	defer p.mu.RUnlock()
 	if val, ok := p.spotPrices[instanceType]; ok {
-		if p.spotUpdateTime.Equal(initialPriceUpdate) {
+		if !p.spotPricingUpdated {
 			return val.defaultPrice, true
 		}
 		if price, ok := p.spotPrices[instanceType].prices[zone]; ok {
@@ -205,15 +187,14 @@ func (p *Provider) UpdateOnDemandPricing(ctx context.Context) error {
 	defer p.mu.Unlock()
 	err := multierr.Append(onDemandErr, onDemandMetalErr)
 	if err != nil {
-		return &Err{error: err, lastUpdateTime: p.onDemandUpdateTime}
+		return fmt.Errorf("retreiving on-demand pricing data, %w", err)
 	}
 
 	if len(onDemandPrices) == 0 || len(onDemandMetalPrices) == 0 {
-		return &Err{error: errors.New("no on-demand pricing found"), lastUpdateTime: p.onDemandUpdateTime}
+		return fmt.Errorf("no on-demand pricing found")
 	}
 
 	p.onDemandPrices = lo.Assign(onDemandPrices, onDemandMetalPrices)
-	p.onDemandUpdateTime = time.Now()
 	for instanceType, price := range p.onDemandPrices {
 		InstancePriceEstimate.With(prometheus.Labels{
 			InstanceTypeLabel: instanceType,
@@ -364,10 +345,10 @@ func (p *Provider) UpdateSpotPricing(ctx context.Context) error {
 	defer p.mu.Unlock()
 
 	if err != nil {
-		return &Err{error: err, lastUpdateTime: p.spotUpdateTime}
+		return fmt.Errorf("retrieving spot pricing data, %w", err)
 	}
 	if len(prices) == 0 {
-		return &Err{error: errors.New("no spot pricing found"), lastUpdateTime: p.spotUpdateTime}
+		return fmt.Errorf("no spot pricing found")
 	}
 	for it, zoneData := range prices {
 		if _, ok := p.spotPrices[it]; !ok {
@@ -379,7 +360,7 @@ func (p *Provider) UpdateSpotPricing(ctx context.Context) error {
 		totalOfferings += len(zoneData)
 	}
 
-	p.spotUpdateTime = time.Now()
+	p.spotPricingUpdated = true
 	if p.cm.HasChanged("spot-prices", p.spotPrices) {
 		logging.FromContext(ctx).With(
 			"instance-type-count", len(p.onDemandPrices),
@@ -415,6 +396,5 @@ func (p *Provider) Reset() {
 	p.onDemandPrices = staticPricing
 	// default our spot pricing to the same as the on-demand pricing until a price update
 	p.spotPrices = populateInitialSpotPricing(staticPricing)
-	p.onDemandUpdateTime = initialPriceUpdate
-	p.spotUpdateTime = initialPriceUpdate
+	p.spotPricingUpdated = false
 }

pkg/providers/pricing/suite_test.go

@@ -83,6 +83,22 @@ var _ = AfterEach(func() {
 })
 
 var _ = Describe("Pricing", func() {
+	DescribeTable(
+		"should return correct static data for all partitions",
+		func(staticPricing map[string]map[string]float64) {
+			for region, prices := range staticPricing {
+				provider := pricing.NewProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, region)
+				for instance, price := range prices {
+					val, ok := provider.OnDemandPrice(instance)
+					Expect(ok).To(BeTrue())
+					Expect(val).To(Equal(price))
+				}
+			}
+		},
+		Entry("aws", pricing.InitialOnDemandPricesAWS),
+		Entry("aws-us-gov", pricing.InitialOnDemandPricesUSGov),
+		Entry("aws-cn", pricing.InitialOnDemandPricesCN),
+	)
 	It("should return static on-demand data if pricing API fails", func() {
 		awsEnv.PricingAPI.NextError.Set(fmt.Errorf("failed"))
 		ExpectReconcileFailed(ctx, controller, types.NamespacedName{})
@@ -106,9 +122,7 @@ var _ = Describe("Pricing", func() {
 				fake.NewOnDemandPrice("c99.large", 1.23),
 			},
 		})
-		updateStart := time.Now()
 		ExpectReconcileFailed(ctx, controller, types.NamespacedName{})
-		Eventually(func() bool { return awsEnv.PricingProvider.OnDemandLastUpdated().After(updateStart) }).Should(BeTrue())
 
 		price, ok := awsEnv.PricingProvider.OnDemandPrice("c98.large")
 		Expect(ok).To(BeTrue())
@@ -156,9 +170,7 @@ var _ = Describe("Pricing", func() {
 				fake.NewOnDemandPrice("c99.large", 1.23),
 			},
 		})
-		updateStart := time.Now()
 		ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{})
-		Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(updateStart) }).Should(BeTrue())
 
 		price, ok := awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1b")
 		Expect(ok).To(BeTrue())
@@ -194,9 +206,7 @@ var _ = Describe("Pricing", func() {
 				fake.NewOnDemandPrice("c99.large", 1.23),
 			},
 		})
-		updateStart := time.Now()
 		ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{})
-		Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(updateStart) }).Should(BeTrue())
 
 		price, ok := awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1a")
 		Expect(ok).To(BeTrue())
@@ -224,9 +234,7 @@ var _ = Describe("Pricing", func() {
 				fake.NewOnDemandPrice("c99.large", 1.23),
 			},
 		})
-		updateStart := time.Now()
 		ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{})
-		Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(updateStart) }).Should(BeTrue())
 
 		_, ok := awsEnv.PricingProvider.SpotPrice("c99.large", "test-zone-1b")
 		Expect(ok).To(BeFalse())
@@ -254,7 +262,6 @@ var _ = Describe("Pricing", func() {
 			},
 		})
 		ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{})
-		Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(updateStart) }, 5*time.Second).Should(BeTrue())
 		inp := awsEnv.EC2API.DescribeSpotPriceHistoryInput.Clone()
 		Expect(lo.Map(inp.ProductDescriptions, func(x *string, _ int) string { return *x })).
 			To(ContainElements("Linux/UNIX", "Linux/UNIX (Amazon VPC)"))