Allow separate service ID and alias (#211)

Fixes aws-controllers-k8s/community#994

Description of changes:
This pull request supports including an optional `--model-name` command line argument for any `ack-generate` generator verb. The generator will use this argument to override the service name when looking up the API files in `aws-sdk-go/models/apis`. 

Currently we reference the `metadata.serviceId` field from the `api-2.json` file as the `ServiceIDClean` variable. `ServiceIDClean` is used in all code generator templates as the import path for `aws-sdk-go` and when referencing the controller name (eg. `{{ .ServiceIDClean}}-controller`). This pull request will redirect `ServiceIDClean` to use the service alias provided when calling `ack-generate`, instead. Therefore, all subsequent ACK repositories should be named according to the AWS SDK Go package name, rather than the API file's definition of `serviceId`. 

`ServiceID` has been removed from the template variables, instead replaced by `AWSSDKModelServiceID` if it is needed. `ServiceIDClean` now refers to the name of the AWS SDK Go package name.

For services that need to use the `--model-name` command line argument, such as `opensearchservice` and `elbv2`, developers should use the `ACK_GENERATE_MODEL_NAME` environment variable when calling `make build-controller`. Changes to the CI/CD system will need to be made to accommodate this customisation.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

cmd/ack-generate/command/apis.go

@@ -37,9 +37,8 @@ const (
 )
 
 var (
-	optGenVersion    string
-	optAPIsInputPath string
-	apisVersionPath  string
+	optGenVersion   string
+	apisVersionPath string
 )
 
 // apiCmd is the command that generates service API types
@@ -97,10 +96,13 @@ func generateAPIs(cmd *cobra.Command, args []string) error {
 	if err := ensureSDKRepo(ctx, optCacheDir, optRefreshCache); err != nil {
 		return err
 	}
+	if optModelName == "" {
+		optModelName = svcAlias
+	}
 	sdkHelper := ackmodel.NewSDKHelper(sdkDir)
-	sdkAPI, err := sdkHelper.API(svcAlias)
+	sdkAPI, err := sdkHelper.API(optModelName)
 	if err != nil {
-		newSvcAlias, err := FallBackFindServiceID(sdkDir, svcAlias)
+		newSvcAlias, err := FallBackFindServiceID(sdkDir, optModelName)
 		if err != nil {
 			return err
 		}
@@ -110,7 +112,7 @@ func generateAPIs(cmd *cobra.Command, args []string) error {
 		}
 	}
 	model, err := ackmodel.New(
-		sdkAPI, optGenVersion, optGeneratorConfigPath, ackgenerate.DefaultConfig,
+		sdkAPI, svcAlias, optGenVersion, optGeneratorConfigPath, ackgenerate.DefaultConfig,
 	)
 	if err != nil {
 		return err

cmd/ack-generate/command/controller.go

@@ -62,10 +62,13 @@ func generateController(cmd *cobra.Command, args []string) error {
 	if err := ensureSDKRepo(ctx, optCacheDir, optRefreshCache); err != nil {
 		return err
 	}
+	if optModelName == "" {
+		optModelName = svcAlias
+	}
 	sdkHelper := ackmodel.NewSDKHelper(sdkDir)
-	sdkAPI, err := sdkHelper.API(svcAlias)
+	sdkAPI, err := sdkHelper.API(optModelName)
 	if err != nil {
-		newSvcAlias, err := FallBackFindServiceID(sdkDir, svcAlias)
+		newSvcAlias, err := FallBackFindServiceID(sdkDir, optModelName)
 		if err != nil {
 			return err
 		}
@@ -79,7 +82,7 @@ func generateController(cmd *cobra.Command, args []string) error {
 		return err
 	}
 	m, err := ackmodel.New(
-		sdkAPI, latestAPIVersion, optGeneratorConfigPath, ackgenerate.DefaultConfig,
+		sdkAPI, svcAlias, latestAPIVersion, optGeneratorConfigPath, ackgenerate.DefaultConfig,
 	)
 	if err != nil {
 		return err

cmd/ack-generate/command/crossplane.go

@@ -26,7 +26,6 @@ import (
 	"github.com/spf13/cobra"
 
 	cpgenerate "github.com/aws-controllers-k8s/code-generator/pkg/generate/crossplane"
-	"github.com/aws-controllers-k8s/code-generator/pkg/model"
 	ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
 )
 
@@ -56,17 +55,20 @@ func generateCrossplane(_ *cobra.Command, args []string) error {
 		return err
 	}
 	svcAlias := strings.ToLower(args[0])
-	sdkHelper := model.NewSDKHelper(sdkDir)
+	if optModelName == "" {
+		optModelName = svcAlias
+	}
+	sdkHelper := ackmodel.NewSDKHelper(sdkDir)
 	sdkHelper.APIGroupSuffix = "aws.crossplane.io"
-	sdkAPI, err := sdkHelper.API(svcAlias)
+	sdkAPI, err := sdkHelper.API(optModelName)
 	if err != nil {
-		newSvcAlias, err := FallBackFindServiceID(sdkDir, svcAlias)
+		newSvcAlias, err := FallBackFindServiceID(sdkDir, optModelName)
 		if err != nil {
 			return err
 		}
 		sdkAPI, err = sdkHelper.API(newSvcAlias) // retry with serviceID
 		if err != nil {
-			return fmt.Errorf("cannot get the API model for service %s", svcAlias)
+			return fmt.Errorf("service %s not found", svcAlias)
 		}
 	}
 	cfgPath := filepath.Join(providerDir, "apis", svcAlias, optGenVersion, "generator-config.yaml")
@@ -78,7 +80,7 @@ func generateCrossplane(_ *cobra.Command, args []string) error {
 		cfgPath = ""
 	}
 	m, err := ackmodel.New(
-		sdkAPI, optGenVersion, cfgPath, cpgenerate.DefaultConfig,
+		sdkAPI, svcAlias, optGenVersion, cfgPath, cpgenerate.DefaultConfig,
 	)
 	if err != nil {
 		return err

cmd/ack-generate/command/olm.go

@@ -86,10 +86,13 @@ func generateOLMAssets(cmd *cobra.Command, args []string) error {
 	if err := ensureSDKRepo(ctx, optCacheDir, optRefreshCache); err != nil {
 		return err
 	}
+	if optModelName == "" {
+		optModelName = svcAlias
+	}
 	sdkHelper := ackmodel.NewSDKHelper(sdkDir)
-	sdkAPI, err := sdkHelper.API(svcAlias)
+	sdkAPI, err := sdkHelper.API(optModelName)
 	if err != nil {
-		newSvcAlias, err := FallBackFindServiceID(sdkDir, svcAlias)
+		newSvcAlias, err := FallBackFindServiceID(sdkDir, optModelName)
 		if err != nil {
 			return err
 		}
@@ -104,7 +107,7 @@ func generateOLMAssets(cmd *cobra.Command, args []string) error {
 		return err
 	}
 	m, err := ackmodel.New(
-		sdkAPI, latestAPIVersion, optGeneratorConfigPath, ackgenerate.DefaultConfig,
+		sdkAPI, svcAlias, latestAPIVersion, optGeneratorConfigPath, ackgenerate.DefaultConfig,
 	)
 	if err != nil {
 		return err

cmd/ack-generate/command/release.go

@@ -74,10 +74,13 @@ func generateRelease(cmd *cobra.Command, args []string) error {
 	if err := ensureSDKRepo(ctx, optCacheDir, optRefreshCache); err != nil {
 		return err
 	}
+	if optModelName == "" {
+		optModelName = svcAlias
+	}
 	sdkHelper := ackmodel.NewSDKHelper(sdkDir)
-	sdkAPI, err := sdkHelper.API(svcAlias)
+	sdkAPI, err := sdkHelper.API(optModelName)
 	if err != nil {
-		newSvcAlias, err := FallBackFindServiceID(sdkDir, svcAlias)
+		newSvcAlias, err := FallBackFindServiceID(sdkDir, optModelName)
 		if err != nil {
 			return err
 		}
@@ -87,7 +90,7 @@ func generateRelease(cmd *cobra.Command, args []string) error {
 		}
 	}
 	m, err := ackmodel.New(
-		sdkAPI, "", optGeneratorConfigPath, ackgenerate.DefaultConfig,
+		sdkAPI, svcAlias, "", optGeneratorConfigPath, ackgenerate.DefaultConfig,
 	)
 	if err != nil {
 		return err

cmd/ack-generate/command/root.go

@@ -34,6 +34,7 @@ var (
 	optCacheDir            string
 	optRefreshCache        bool
 	optAWSSDKGoVersion     string
+	optModelName           string
 	defaultTemplateDirs    []string
 	optTemplateDirs        []string
 	defaultServicesDir     string
@@ -121,6 +122,9 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(
 		&optAWSSDKGoVersion, "aws-sdk-go-version", "", "Version of github.com/aws/aws-sdk-go used to generate apis and controllers files",
 	)
+	rootCmd.PersistentFlags().StringVar(
+		&optModelName, "model-name", "", "the name of the service model package",
+	)
 }
 
 // Execute adds all child commands to the root command and sets flags

pkg/generate/templateset/vars.go

@@ -21,9 +21,6 @@ type MetaVars struct {
 	// alias does not match the ServiceID. e.g. The AWS Step Functions API has
 	// a ServiceID of "SFN" and a service alias of "states"...
 	ServiceAlias string
-	// ServiceID is the exact string that appears in the AWS service API's
-	// api-2.json descriptor file under `metadata.serviceId`
-	ServiceID string
 	// ServiceIDClean is the ServiceID lowercased and stripped of any
 	// non-alphanumeric characters
 	ServiceIDClean string
@@ -34,6 +31,9 @@ type MetaVars struct {
 	// for custom resources, e.g. "sns.services.k8s.aws" or
 	// "sfn.services.k8s.aws"
 	APIGroup string
+	// AWSSDKModelServiceID is the exact string that appears in the AWS service API's
+	// api-2.json descriptor file under `metadata.serviceId`
+	AWSSDKModelServiceID string
 	// SDKAPIInterfaceTypeName is the name of the interface type used by the
 	// aws-sdk-go services/$SERVICE/api.go file
 	SDKAPIInterfaceTypeName string

pkg/model/model.go

@@ -34,13 +34,14 @@ var (
 // Model contains the ACK model for the generator to process and apply
 // templates against.
 type Model struct {
-	SDKAPI       *SDKAPI
-	serviceAlias string
-	apiVersion   string
-	crds         []*CRD
-	typeDefs     []*TypeDef
-	typeImports  map[string]string
-	typeRenames  map[string]string
+	SDKAPI         *SDKAPI
+	servicePackage string
+	serviceAlias   string
+	apiVersion     string
+	crds           []*CRD
+	typeDefs       []*TypeDef
+	typeImports    map[string]string
+	typeRenames    map[string]string
 	// Instructions to the code generator how to handle the API and its
 	// resources
 	cfg *ackgenconfig.Config
@@ -51,10 +52,10 @@ type Model struct {
 func (m *Model) MetaVars() templateset.MetaVars {
 	return templateset.MetaVars{
 		ServiceAlias:            m.serviceAlias,
-		ServiceID:               m.SDKAPI.ServiceID(),
-		ServiceIDClean:          m.SDKAPI.ServiceIDClean(),
-		APIGroup:                m.SDKAPI.APIGroup(),
+		ServiceIDClean:          m.ServiceIDClean(),
+		APIGroup:                m.APIGroup(),
 		APIVersion:              m.apiVersion,
+		AWSSDKModelServiceID:    m.SDKAPI.AWSSDKModelServiceID(),
 		SDKAPIInterfaceTypeName: m.SDKAPI.SDKAPIInterfaceTypeName(),
 		CRDNames:                m.crdNames(),
 	}
@@ -691,11 +692,29 @@ func (m *Model) GetConfig() *ackgenconfig.Config {
 	return m.cfg
 }
 
+// APIGroup returns the normalized Kubernetes APIGroup for the AWS service API,
+// e.g. "sns.services.k8s.aws"
+func (m *Model) APIGroup() string {
+	serviceID := m.servicePackage
+	suffix := "services.k8s.aws"
+	if m.SDKAPI.apiGroupSuffix != "" {
+		suffix = m.SDKAPI.apiGroupSuffix
+	}
+	return fmt.Sprintf("%s.%s", serviceID, suffix)
+}
+
+// ServiceIDClean returns a lowercased, whitespace-stripped ServiceID
+func (m *Model) ServiceIDClean() string {
+	serviceID := strings.ToLower(m.servicePackage)
+	return strings.Replace(serviceID, " ", "", -1)
+}
+
 // New returns a new Model struct for a supplied API model.
 // Optionally, pass a file path to a generator config file that can be used to
 // instruct the code generator how to handle the API properly
 func New(
 	SDKAPI *SDKAPI,
+	servicePackage string,
 	apiVersion string,
 	configPath string,
 	defaultConfig ackgenconfig.Config,
@@ -705,12 +724,11 @@ func New(
 		return nil, err
 	}
 	m := &Model{
-		SDKAPI: SDKAPI,
-		// TODO(jaypipes): Handle cases where service alias and service ID
-		// don't match (Step Functions)
-		serviceAlias: SDKAPI.ServiceID(),
-		apiVersion:   apiVersion,
-		cfg:          &cfg,
+		SDKAPI:         SDKAPI,
+		servicePackage: servicePackage,
+		serviceAlias:   SDKAPI.AWSSDKModelServiceID(),
+		apiVersion:     apiVersion,
+		cfg:            &cfg,
 	}
 	m.ApplyShapeIgnoreRules()
 	return m, nil

pkg/model/multiversion/manager.go

@@ -51,6 +51,7 @@ func NewAPIVersionManager(
 	sdkCacheDir string,
 	metadataPath string,
 	serviceAlias string,
+	serviceModelName string,
 	hubVersion string,
 	apisInfo map[string]ackmetadata.APIInfo,
 	defaultConfig ackgenconfig.Config,
@@ -94,13 +95,14 @@ func NewAPIVersionManager(
 			return nil, err
 		}
 
-		SDKAPI, err := SDKAPIHelper.API(serviceAlias)
+		SDKAPI, err := SDKAPIHelper.API(serviceModelName)
 		if err != nil {
 			return nil, err
 		}
 
 		i, err := ackmodel.New(
 			SDKAPI,
+			serviceAlias,
 			version.APIVersion,
 			apiInfo.GeneratorConfigPath,
 			defaultConfig,

pkg/model/sdk_helper.go

@@ -246,8 +246,8 @@ func (a *SDKAPI) GetOutputShapeRef(
 }
 
 // getMemberByPath returns a ShapeRef given a root Shape and a dot-notation
-// object search path. Given the explicit type check for list type members 
-// both ".." and "." notations work currently. 
+// object search path. Given the explicit type check for list type members
+// both ".." and "." notations work currently.
 // TODO: Add support for other types such as map.
 func getMemberByPath(
 	shape *awssdkmodel.Shape,
@@ -353,39 +353,24 @@ func (a *SDKAPI) HasConflictingTypeName(typeName string, cfg *ackgenconfig.Confi
 		util.InStrings(cleanTypeName, crdListResourceNames)
 }
 
-// ServiceID returns the exact `metadata.serviceId` attribute for the AWS
-// service APi's api-2.json file
-func (a *SDKAPI) ServiceID() string {
+// AWSSDKModelServiceID returns the exact `metadata.serviceId` attribute for the AWS
+// service APi's api-2.json file.
+// This MAY NOT MATCH the AWS SDK Go package used by the service. For example:
+// AWS SDK Go uses `opensearchservice` whereas the service ID is `opensearch`
+func (a *SDKAPI) AWSSDKModelServiceID() string {
 	if a == nil || a.API == nil {
 		return ""
 	}
 	return awssdkmodel.ServiceID(a.API)
 }
 
-// ServiceIDClean returns a lowercased, whitespace-stripped ServiceID
-func (a *SDKAPI) ServiceIDClean() string {
-	serviceID := strings.ToLower(a.ServiceID())
-	return strings.Replace(serviceID, " ", "", -1)
-}
-
 func (a *SDKAPI) GetServiceFullName() string {
 	if a == nil || a.API == nil {
 		return ""
 	}
 	return a.API.Metadata.ServiceFullName
 }
 
-// APIGroup returns the normalized Kubernetes APIGroup for the AWS service API,
-// e.g. "sns.services.k8s.aws"
-func (a *SDKAPI) APIGroup() string {
-	serviceID := a.ServiceIDClean()
-	suffix := "services.k8s.aws"
-	if a.apiGroupSuffix != "" {
-		suffix = a.apiGroupSuffix
-	}
-	return fmt.Sprintf("%s.%s", serviceID, suffix)
-}
-
 // SDKAPIInterfaceTypeName returns the name of the aws-sdk-go primary API
 // interface type name.
 func (a *SDKAPI) SDKAPIInterfaceTypeName() string {