Support backlink attributes via the importer

We're introducing backlink attributes, which allow you to say 'find all the features that this team owns' or similar, based on another catalog type. This is challenging for the importer as this means attributes in an importer config can reference each other. e.g. if I'm creating both Teams and Features in my catalog, I need to wait until Features exists before creating the backlink attribute in Teams. To resolve this, we split the UpdateTypeSchema process into two parts: 1. Create/Update all the types and their schemas 2. Create any new backlinks that don't exist yet (as the other attribute must now be present). Note that to make the code sane, I've pulled the 'if dryRun' up one level as dryRun doesn't have any of these concerns.
incident-io · Mar 25, 2024 · 3c25f84 · 3c25f84
1 parent b0ca478
commit 3c25f84
Show file tree

Hide file tree

Showing 2 changed files with 127 additions and 33 deletions.
diff --git a/cmd/catalog-importer/cmd/sync.go b/cmd/catalog-importer/cmd/sync.go
@@ -240,33 +240,73 @@ func (opt *SyncOptions) Run(ctx context.Context, logger kitlog.Logger, cfg *conf
 		}
 	}
 
-	// Update type schemas to match config (excluding backlinks)
 	OUT("\n↻ Syncing catalog type schemas...")
-	for _, outputType := range cfg.Outputs() {
-		baseModel, enumModels := output.MarshalType(outputType)
-		for _, model := range append(enumModels, baseModel) {
-			catalogType := catalogTypesByOutput[model.TypeName]
-
-			var updatedCatalogType client.CatalogTypeV2
-			if opt.DryRun {
-				logger.Log("msg", "dry-run active, which means we fake a response")
-				updatedCatalogType = *catalogType // they start the same
-
-				// Then we pretend like we've already updated the schema, which means we rebuild the
-				// attributes.
-				updatedCatalogType.Schema = client.CatalogTypeSchemaV2{
-					Version:    updatedCatalogType.Schema.Version,
-					Attributes: []client.CatalogTypeAttributeV2{},
+	if opt.DryRun {
+		for _, outputType := range cfg.Outputs() {
+			baseModel, enumModels := output.MarshalType(outputType)
+			for _, model := range append(enumModels, baseModel) {
+				catalogType := catalogTypesByOutput[model.TypeName]
+
+				var updatedCatalogType client.CatalogTypeV2
+				if opt.DryRun {
+					logger.Log("msg", "dry-run active, which means we fake a response")
+					updatedCatalogType = *catalogType // they start the same
+
+					// Then we pretend like we've already updated the schema, which means we rebuild the
+					// attributes.
+					updatedCatalogType.Schema = client.CatalogTypeSchemaV2{
+						Version:    updatedCatalogType.Schema.Version,
+						Attributes: []client.CatalogTypeAttributeV2{},
+					}
+					for _, attr := range model.Attributes {
+						updatedCatalogType.Schema.Attributes = append(updatedCatalogType.Schema.Attributes, client.CatalogTypeAttributeV2{
+							Id:                *attr.Id,
+							Name:              attr.Name,
+							Type:              attr.Type,
+							Array:             attr.Array,
+							Mode:              client.CatalogTypeAttributeV2Mode(*attr.Mode),
+							BacklinkAttribute: attr.BacklinkAttribute,
+						})
+					}
 				}
+				OUT("  ✔ %s (id=%s)", model.TypeName, catalogType.Id)
+				DIFF("  ", *catalogType, updatedCatalogType)
+			}
+		}
+	} else {
+		// Update all the type schemas except for new backlinks, which could reference
+		// attributes that don't exist yet.
+		catalogTypeVersions := map[string]int64{}
+		for _, outputType := range cfg.Outputs() {
+			baseModel, enumModels := output.MarshalType(outputType)
+			for _, model := range append(enumModels, baseModel) {
+				catalogType := catalogTypesByOutput[model.TypeName]
+
+				attributesWithoutNewBacklinks := []client.CatalogTypeAttributePayloadV2{}
 				for _, attr := range model.Attributes {
-					updatedCatalogType.Schema.Attributes = append(updatedCatalogType.Schema.Attributes, client.CatalogTypeAttributeV2{
-						Id:    *attr.Id,
-						Name:  attr.Name,
-						Type:  attr.Type,
-						Array: attr.Array,
-					})
+					if attr.Mode != nil && *attr.Mode == client.CatalogTypeAttributePayloadV2ModeBacklink {
+						inCurrentSchema := false
+						// Does it exist in the current schema?
+						if attr.Id == nil {
+							logger.Log("msg", "attribute ID is empty, which shouldn't be possible")
+							continue
+						}
+
+						for _, existingAttr := range catalogType.Schema.Attributes {
+							if existingAttr.Id == *attr.Id {
+								inCurrentSchema = true
+								break
+							}
+						}
+
+						if !inCurrentSchema {
+							// We can't add new backlinks yet in case the attribute it refers to doesn't exist yet.
+							continue
+						}
+					}
+					attributesWithoutNewBacklinks = append(attributesWithoutNewBacklinks, attr)
 				}
-			} else {
+
 				logger.Log("msg", "updating catalog type", "catalog_type_id", catalogType.Id)
 				result, err := cl.CatalogV2UpdateTypeWithResponse(ctx, catalogType.Id, client.CatalogV2UpdateTypeJSONRequestBody{
 					Name:          model.Name,
@@ -281,19 +321,65 @@ func (opt *SyncOptions) Run(ctx context.Context, logger kitlog.Logger, cfg *conf
 
 				version := result.JSON200.CatalogType.Schema.Version
 				logger.Log("msg", "updating catalog type schema", "catalog_type_id", catalogType.Id, "version", version)
-				schemaResult, err := cl.CatalogV2UpdateTypeSchemaWithResponse(ctx, catalogType.Id, client.CatalogV2UpdateTypeSchemaJSONRequestBody{
+				schema, err := cl.CatalogV2UpdateTypeSchemaWithResponse(ctx, catalogType.Id, client.CatalogV2UpdateTypeSchemaJSONRequestBody{
 					Version:    version,
-					Attributes: model.Attributes,
+					Attributes: attributesWithoutNewBacklinks,
 				})
 				if err != nil {
 					return errors.Wrap(err, "updating catalog type schema")
 				}
 
-				updatedCatalogType = schemaResult.JSON200.CatalogType
+				catalogTypeVersions[catalogType.Id] = schema.JSON200.CatalogType.Schema.Version
+
+				OUT("  ✔ %s (id=%s)", model.TypeName, catalogType.Id)
 			}
-			OUT("  ✔ %s (id=%s)", model.TypeName, catalogType.Id)
-			if opt.DryRun {
-				DIFF("  ", *catalogType, updatedCatalogType)
+		}
+
+		// Then go through again and create any types that do have new backlinks
+		OUT("\n↻ Syncing backlink attributes...")
+		for _, outputType := range cfg.Outputs() {
+			baseModel, enumModels := output.MarshalType(outputType)
+			for _, model := range append(enumModels, baseModel) {
+				catalogType := catalogTypesByOutput[model.TypeName]
+
+				hasNewBacklinks := false
+				for _, attr := range model.Attributes {
+					if attr.Mode != nil && attr.BacklinkAttribute != nil {
+						inCurrentSchema := false
+						// Does it exist in the current schema?
+						if attr.Id == nil {
+							logger.Log("msg", "attribute ID is empty, which shouldn't be possible")
+							continue
+						}
+
+						for _, existingAttr := range catalogType.Schema.Attributes {
+							if existingAttr.Id == *attr.Id {
+								inCurrentSchema = true
+								break
+							}
+						}
+
+						if !inCurrentSchema {
+							hasNewBacklinks = true
+						}
+					}
+				}
+
+				if !hasNewBacklinks {
+					continue
+				}
+				version := catalogTypeVersions[catalogType.Id]
+				logger.Log("msg", "updating catalog type schema: creating backlink attribute(s)", "catalog_type_id", catalogType.Id, "version", version)
+
+				_, err = cl.CatalogV2UpdateTypeSchemaWithResponse(ctx, catalogType.Id, client.CatalogV2UpdateTypeSchemaJSONRequestBody{
+					Version:    version,
+					Attributes: model.Attributes,
+				})
+				if err != nil {
+					return errors.Wrap(err, "updating catalog type schema")
+				}
+
+				OUT("  ✔ %s (id=%s)", model.TypeName, catalogType.Id)
 			}
 		}
 	}

diff --git a/output/marshal.go b/output/marshal.go
@@ -51,12 +51,19 @@ func MarshalType(output *Output) (base *CatalogTypeModel, enumTypes []*CatalogTy
 			attrType = attr.Type.String
 		}
 
+		mode := client.CatalogTypeAttributePayloadV2ModeManual
+		if attr.BacklinkAttribute.Valid {
+			mode = client.CatalogTypeAttributePayloadV2ModeBacklink
+		}
+
 		base.Attributes = append(
 			base.Attributes, client.CatalogTypeAttributePayloadV2{
-				Id:    lo.ToPtr(attr.ID),
-				Name:  attr.Name,
-				Type:  attrType,
-				Array: attr.Array,
+				Id:                lo.ToPtr(attr.ID),
+				Name:              attr.Name,
+				Type:              attrType,
+				Array:             attr.Array,
+				BacklinkAttribute: attr.BacklinkAttribute.Ptr(),
+				Mode:              lo.ToPtr(mode),
 			})
 
 		// The enums we generate should be returned as types too, as we'll need to sync them
@@ -72,6 +79,7 @@ func MarshalType(output *Output) (base *CatalogTypeModel, enumTypes []*CatalogTy
 						Id:   lo.ToPtr("description"),
 						Name: "Description",
 						Type: "String",
+						Mode: lo.ToPtr(client.CatalogTypeAttributePayloadV2ModeManual),
 					},
 				},
 				SourceAttribute: lo.ToPtr(*attr),