Add support for external clients

pkg/generator/steps/openapi/clients_diff.go

@@ -51,7 +51,10 @@ func calcClientsDiff(ctx *gencontext.GenContext, openAPIGenerator *OpenAPIGenera
 			continue
 		}
 
-		schemaDirPath := ctx.GetWorkspace().GetApiSchemaDirRelPath(cl)
+		schemaDirPath, err := ctx.GetWorkspace().GetUniversalApiSchemaDirRelPath(cl)
+		if err != nil {
+			return clientsDiff{}, wrapError(err)
+		}
 		needGenerate, err := openAPIGenerator.NeedGenerateClient(ctx, schemaDirPath)
 		if err != nil {
 			return clientsDiff{}, wrapError(err)

pkg/generator/steps/openapi/openapi.go

@@ -173,7 +173,7 @@ func (g *OpenAPIGenerator) GenerateServer(ctx *gencontext.GenContext) error {
 		return fmt.Errorf("failed to generate server: %w", err)
 	}
 
-	err = updateGenerationTime(ctx, ctx.GetServiceName(), filepath.Dir(schemaPath))
+	err = updateGenerationTime(ctx, ctx.GetServiceName(), schemaDir, filepath.Dir(schemaPath))
 	if err != nil {
 		return err
 	}
@@ -186,7 +186,10 @@ func (g *OpenAPIGenerator) GenerateClient(ctx *gencontext.GenContext, clientName
 	if len(g.clientAssetsPath) == 0 {
 		return fmt.Errorf("failed to generate client: no generator available for language: %s", g.language)
 	}
-	inputSchemaPath := ctx.GetWorkspace().GetApiSchemaAbsPath(clientName, "api.yaml")
+	inputSchemaPath, err := ctx.GetWorkspace().GetUniversalApiSchemaAbsPath(clientName, "api.yaml")
+	if err != nil {
+		return fmt.Errorf("failed to generate client: %w", err)
+	}
 	schemaPath, err := g.makeClientEnrichedSchema(ctx, inputSchemaPath)
 	if err != nil {
 		return fmt.Errorf("failed to generate client: %w", err)
@@ -196,8 +199,12 @@ func (g *OpenAPIGenerator) GenerateClient(ctx *gencontext.GenContext, clientName
 	if err != nil {
 		return fmt.Errorf("failed to generate client: %w", err)
 	}
+	schemaDir, err := ctx.GetWorkspace().GetUniversalApiSchemaDirRelPath(clientName)
+	if err != nil {
+		return fmt.Errorf("failed to generate client: %w", err)
+	}
 
-	err = updateGenerationTime(ctx, clientName, filepath.Dir(schemaPath))
+	err = updateGenerationTime(ctx, clientName, schemaDir, filepath.Dir(schemaPath))
 	if err != nil {
 		return err
 	}
@@ -309,11 +316,10 @@ func makeFileUpdateMap(ctx *gencontext.GenContext, schemaDir string, tmpSchemaDi
 	return fileMap, nil
 }
 
-func updateGenerationTime(ctx *gencontext.GenContext, targetServiceName string, tmpSchemaDir string) error {
-	schemaDir := ctx.GetWorkspace().GetApiSchemaDirRelPath(targetServiceName)
-	ctx.Logger.Infof("updating generation time in: %s", schemaDir)
+func updateGenerationTime(ctx *gencontext.GenContext, targetServiceName string, svcSchemaDir string, tmpSchemaDir string) error {
+	ctx.Logger.Infof("updating generation time in: %s", svcSchemaDir)
 
-	fileMap, err := makeFileUpdateMap(ctx, schemaDir, tmpSchemaDir)
+	fileMap, err := makeFileUpdateMap(ctx, svcSchemaDir, tmpSchemaDir)
 	if err != nil {
 		return fmt.Errorf("failed to write file update times: %w", err)
 	}

pkg/generator/steps/openapi/openapi_client.go

@@ -10,6 +10,7 @@ import (
 	gencontext "github.com/mify-io/mify/pkg/generator/gen-context"
 	"github.com/mify-io/mify/pkg/generator/lib/endpoints"
 	"github.com/mify-io/mify/pkg/generator/steps/openapi/processors"
+	"github.com/mify-io/mify/pkg/mifyconfig"
 )
 
 func (g *OpenAPIGenerator) makeClientEnrichedSchema(ctx *gencontext.GenContext, schemaPath string) (string, error) {
@@ -41,7 +42,7 @@ func (g *OpenAPIGenerator) makeClientEnrichedSchema(ctx *gencontext.GenContext,
 	return g.saveEnrichedSchema(ctx, doc, schemaPath, CACHE_CLIENT_SUBDIR)
 }
 
-func (g *OpenAPIGenerator) getServiceBasePath(ctx *gencontext.GenContext, schemaPath string) (string, error) {
+func (g *OpenAPIGenerator) getServiceBasePath(ctx *gencontext.GenContext, schemaPath string, isExternal bool) (string, error) {
 	doc, err := g.readSchema(ctx, schemaPath)
 	if err != nil {
 		return "", fmt.Errorf("failed to read schema: %s: %w", schemaPath, err)
@@ -56,20 +57,41 @@ func (g *OpenAPIGenerator) getServiceBasePath(ctx *gencontext.GenContext, schema
 	}
 	srv := servers[0].(map[interface{}]interface{})
 	ctx.Logger.Infof("processing server: %s", srv["url"])
-	u, err := url.Parse("//" + srv["url"].(string))
+	urlStr := "//" + srv["url"].(string)
+	if isExternal {
+		urlStr = srv["url"].(string)
+		return urlStr, nil
+	}
+	u, err := url.Parse(urlStr)
 	if err != nil {
 		return "", fmt.Errorf("failed to parse server url %s: %w", srv["url"], err)
 	}
 	ctx.Logger.Infof("got server path: %s", u.Path)
 	return u.Path, nil
 }
 
-func (g *OpenAPIGenerator) doGenerateClient(
-	ctx *gencontext.GenContext, clientName string, schemaPath string) error {
+func (g *OpenAPIGenerator) makeServiceEndpoint(
+	ctx *gencontext.GenContext, clientName string, schemaPath string) (string, error) {
+	cfg, err := mifyconfig.ReadServiceConfig(ctx.GetWorkspace().BasePath, clientName)
+	if err != nil {
+		return "", err
+	}
+	basePath, err := g.getServiceBasePath(ctx, schemaPath, cfg.IsExternal)
+	if err != nil {
+		return "", err
+	}
+	if cfg.IsExternal {
+		return basePath, err
+	}
 	endpoints, err := ctx.EndpointsResolver.ResolveEndpoints(clientName)
 	if err != nil {
-		return err
+		return "", err
 	}
+	return endpoints.Api + basePath, err
+}
+
+func (g *OpenAPIGenerator) doGenerateClient(
+	ctx *gencontext.GenContext, clientName string, schemaPath string) error {
 
 	postProcessor, err := processors.NewPostProcessor(g.language)
 	if err != nil {
@@ -81,13 +103,13 @@ func (g *OpenAPIGenerator) doGenerateClient(
 		return err
 	}
 
-	basePath, err := g.getServiceBasePath(ctx, schemaPath)
+	serviceEndpoint, err := g.makeServiceEndpoint(ctx, clientName, schemaPath)
 	if err != nil {
 		return err
 	}
 
 	err = runOpenapiGenerator(ctx, g.basePath, schemaPath, g.clientAssetsPath,
-		generatorConf.TargetPath, generatorConf.PackageName, clientName, endpoints.Api+basePath, g.info)
+		generatorConf.TargetPath, generatorConf.PackageName, clientName, serviceEndpoint, g.info)
 	if err != nil {
 		return fmt.Errorf("failed to run openapi-generator: %w", err)
 	}

pkg/generator/steps/schema/impl.go

@@ -11,6 +11,7 @@ import (
 	gencontext "github.com/mify-io/mify/pkg/generator/gen-context"
 	"github.com/mify-io/mify/pkg/generator/steps/schema/context"
 	"github.com/mify-io/mify/pkg/mifyconfig"
+	"github.com/mify-io/mify/pkg/workspace"
 )
 
 func execute(ctx *gencontext.GenContext) (*context.SchemaContext, error) {
@@ -32,8 +33,11 @@ func collectSchemas(ctx *gencontext.GenContext) (context.AllSchemas, error) {
 
 	for _, f := range files {
 		serviceName := f.Name()
+		if f.Name() == workspace.ExternalSchemasDir {
+			continue
+		}
 
-		openapiSchemas, err := extractOpenapiSchemas(ctx, serviceName)
+		openapiSchemas, err := extractOpenapiSchemas(ctx, serviceName, false)
 		if err != nil {
 			return nil, fmt.Errorf("can't collect openapi schemas for service '%s': %w", serviceName, err)
 		}
@@ -46,11 +50,37 @@ func collectSchemas(ctx *gencontext.GenContext) (context.AllSchemas, error) {
 		schemas[serviceName] = context.NewServiceSchemas(openapiSchemas, mifySchema)
 	}
 
+	if _, err := os.Stat(ctx.GetWorkspace().GetExternalSchemasRootAbsPath()); os.IsNotExist(err) {
+		return schemas, nil
+	}
+
+	externalFiles, err := os.ReadDir(ctx.GetWorkspace().GetExternalSchemasRootAbsPath())
+	if err != nil {
+		return nil, fmt.Errorf("can't iterate external schemas directory: %w", err)
+	}
+	for _, f := range externalFiles {
+		serviceName := f.Name()
+		openapiSchemas, err := extractOpenapiSchemas(ctx, serviceName, true)
+		if err != nil {
+			return nil, fmt.Errorf("can't collect openapi schemas for external service '%s': %w", serviceName, err)
+		}
+
+		mifySchema, err := makeExternalServiceMifySchema(ctx, serviceName)
+		if err != nil {
+			return nil, fmt.Errorf("can't create mify schema for external service '%s': %w", serviceName, err)
+		}
+
+		schemas[serviceName] = context.NewServiceSchemas(openapiSchemas, mifySchema)
+	}
+
 	return schemas, nil
 }
 
-func extractOpenapiSchemas(ctx *gencontext.GenContext, forService string) (context.OpenapiServiceSchemas, error) {
+func extractOpenapiSchemas(ctx *gencontext.GenContext, forService string, isExternal bool) (context.OpenapiServiceSchemas, error) {
 	openapiSchemasDir := ctx.GetWorkspace().GetApiSchemaDirAbsPath(forService)
+	if isExternal {
+		openapiSchemasDir = ctx.GetWorkspace().GetExternalApiSchemaDirAbsPath(forService)
+	}
 
 	files, err := os.ReadDir(openapiSchemasDir)
 	if err != nil {
@@ -98,3 +128,10 @@ func extractMifySchema(ctx *gencontext.GenContext, forService string) (*mifyconf
 
 	return config, nil
 }
+
+func makeExternalServiceMifySchema(ctx *gencontext.GenContext, forService string) (*mifyconfig.ServiceConfig, error) {
+	return &mifyconfig.ServiceConfig{
+		ServiceName: forService,
+		IsExternal: true,
+	}, nil
+}

pkg/generator/steps/schema/validation.go

@@ -12,6 +12,9 @@ func validateCtx(ctx *gencontext.GenContext) error {
 	if ctx.GetServiceName() == workspace.DevRunnerName {
 		return nil // Dev runner shouldn't have any scheme
 	}
+	if ctx.MustGetMifySchema().IsExternal {
+		return nil // Dev runner shouldn't have any scheme
+	}
 
 	schemas := ctx.GetSchemaCtx().GetAllSchemas()
 	serviceSchemas, ok := schemas[ctx.GetServiceName()]

pkg/mifyconfig/service_config.go

@@ -1,6 +1,7 @@
 package mifyconfig
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path"
@@ -28,6 +29,10 @@ var LanguagesList = []ServiceLanguage{
 	ServiceLanguageJs,
 }
 
+var (
+	ErrNoSuchService = errors.New("no such service")
+)
+
 type ServiceOpenAPIClientConfig struct{}
 
 type ServiceOpenAPIConfig struct {
@@ -53,13 +58,19 @@ type ServiceConfig struct {
 
 	OpenAPI  ServiceOpenAPIConfig `yaml:"openapi,omitempty"`
 	Postgres PostgresConfig       `yaml:"postgres,omitempty"`
+
+	IsExternal bool `yaml:"-"`
 }
 
 func ReadServiceCfg(path string) (*ServiceConfig, error) {
 	wrapErr := func(err error) error {
 		return fmt.Errorf("failed to read service config: %w", err)
 	}
 
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil, wrapErr(ErrNoSuchService)
+	}
+
 	rawData, err := os.ReadFile(path)
 	if err != nil {
 		return nil, wrapErr(err)
@@ -74,12 +85,33 @@ func ReadServiceCfg(path string) (*ServiceConfig, error) {
 	return &data, nil
 }
 
+func tryReadExternalService(workspaceDir, serviceName string) (*ServiceConfig, error) {
+	wrapErr := func(err error) error {
+		return fmt.Errorf("failed to read external service config: %w", err)
+	}
+	externalSchemaPath := path.Join(workspaceDir, "schemas", "mify-external", serviceName, "api", "api.yaml")
+	if _, err := os.Stat(externalSchemaPath); os.IsNotExist(err) {
+		return nil, wrapErr(ErrNoSuchService)
+	}
+	return &ServiceConfig{
+		ServiceName: serviceName,
+		IsExternal:  true,
+	}, nil
+}
+
 // TODO: remove (use ReadServiceCfg)
 func ReadServiceConfig(workspaceDir string, serviceName string) (ServiceConfig, error) {
 	schemaDir := path.Join(workspaceDir, "schemas", serviceName)
 	path := filepath.Join(schemaDir, ServiceConfigName)
 
 	cfg, err := ReadServiceCfg(path)
+	if err == nil {
+		return *cfg, nil
+	}
+	if err != nil && !errors.Is(err, ErrNoSuchService) {
+		return ServiceConfig{}, err
+	}
+	cfg, err = tryReadExternalService(workspaceDir, serviceName)
 	if err != nil {
 		return ServiceConfig{}, err
 	}

pkg/workspace/description.go

@@ -14,13 +14,14 @@ import (
 )
 
 const (
-	ApiGatewayName    = "api-gateway"
-	MainApiSchemaName = "api.yaml"
-	MifySchemaName    = "service.mify.yaml"
-	CloudSchemaName   = "cloud.mify.yaml"
-	GoServicesDirName = "go-services"
-	DevRunnerName     = "dev-runner"
-	TmpSubdir         = "services"
+	ApiGatewayName     = "api-gateway"
+	MainApiSchemaName  = "api.yaml"
+	MifySchemaName     = "service.mify.yaml"
+	CloudSchemaName    = "cloud.mify.yaml"
+	GoServicesDirName  = "go-services"
+	DevRunnerName      = "dev-runner"
+	TmpSubdir          = "services"
+	ExternalSchemasDir = "mify-external"
 )
 
 var (
@@ -82,6 +83,9 @@ func (c Description) GetApiServices() []string {
 		if !f.IsDir() {
 			continue
 		}
+		if f.Name() == ExternalSchemasDir {
+			continue
+		}
 		services = append(services, f.Name())
 	}
 	return services
@@ -98,6 +102,9 @@ func (c Description) GetFrontendServices() ([]string, error) {
 		if !f.IsDir() {
 			continue
 		}
+		if f.Name() == ExternalSchemasDir {
+			continue
+		}
 
 		cfgPath := c.GetMifySchemaAbsPath(f.Name())
 		cfg, err := mifyconfig.ReadServiceCfg(cfgPath)
@@ -192,6 +199,58 @@ func (c Description) GetApiSchemaGenAbsPath(serviceName string) string {
 	return path.Join(c.BasePath, "schemas", serviceName, "api/api_generated.yaml")
 }
 
+func (c Description) GetExternalSchemasRootRelPath() string {
+	return path.Join(c.GetSchemasRootRelPath(), ExternalSchemasDir)
+}
+
+func (c Description) GetExternalSchemasRootAbsPath() string {
+	return path.Join(c.BasePath, c.GetExternalSchemasRootRelPath())
+}
+
+func (c Description) GetExternalSchemasRelPath(serviceName string) string {
+	return path.Join(c.GetExternalSchemasRootRelPath(), serviceName)
+}
+
+func (c Description) GetExternalSchemasAbsPath(serviceName string) string {
+	return path.Join(c.BasePath, c.GetSchemasRelPath(serviceName))
+}
+
+func (c Description) GetExternalApiSchemaDirRelPath(serviceName string) string {
+	return path.Join(c.GetExternalSchemasRootRelPath(), serviceName, "api")
+}
+
+func (c Description) GetExternalApiSchemaDirAbsPath(serviceName string) string {
+	return path.Join(c.BasePath, c.GetExternalApiSchemaDirRelPath(serviceName))
+}
+
+func (c Description) GetExternalApiSchemaAbsPath(serviceName string, schemaName string) string {
+	return path.Join(c.BasePath, c.GetExternalApiSchemaDirRelPath(serviceName), schemaName)
+}
+
+func (c Description) GetUniversalApiSchemaAbsPath(serviceName string, schemaName string) (string, error) {
+	svcSchema := c.GetApiSchemaAbsPath(serviceName, schemaName)
+	if _, err := os.Stat(svcSchema); err == nil {
+		return svcSchema, nil
+	}
+	externalSvcSchema := c.GetExternalApiSchemaAbsPath(serviceName, schemaName)
+	if _, err := os.Stat(externalSvcSchema); err == nil {
+		return externalSvcSchema, nil
+	}
+	return "", ErrNoSuchService
+}
+
+func (c Description) GetUniversalApiSchemaDirRelPath(serviceName string) (string, error) {
+	svcSchemaDir := c.GetApiSchemaDirAbsPath(serviceName)
+	if _, err := os.Stat(svcSchemaDir); err == nil {
+		return c.GetApiSchemaDirRelPath(serviceName), nil
+	}
+	externalSvcSchemaDir := c.GetExternalApiSchemaDirAbsPath(serviceName)
+	if _, err := os.Stat(externalSvcSchemaDir); err == nil {
+		return c.GetExternalApiSchemaDirRelPath(serviceName), nil
+	}
+	return "", ErrNoSuchService
+}
+
 func (c *Description) GetRepository() string {
 	return fmt.Sprintf("%s/%s/%s",
 		c.Config.GitHost,