feat!(compose): add ability to pull and compose import component defs (…

…#406)

* feat: enhance component definition handling with import and merge capabilities

Refactor and rename unit tests for clarity and consistency

Add new unit test for imported component definitions

* feat(composition): added ability to handle multiple component definitions in a remote file

* chore: add comments to ComposeComponentDefinitions

* feat(validate): add component definition composition to validate on comp def method

* feat(tools/compose): add component definition composition to tools compose cmd

* refactor!(validation-store): rm fetchFromRemoteLink, AddFromLink, GetHrefIds, SetHrefIds as they were unused and create AddLulaValidation func
fix(oscal/component): update BackMatterTopMap to return an instantiated map if the backmatter is nil
refactor(validate): update ValidateOnCompDef to use the new simplified validationstore
tests: update related tests

src/cmd/tools/compose.go

@@ -80,7 +80,7 @@ func Compose(inputFile, outputFile string) error {
 		return err
 	}
 
-	err = composition.ComposeComponentValidations(model.ComponentDefinition)
+	err = composition.ComposeComponentDefinitions(model.ComponentDefinition)
 	if err != nil {
 		return err
 	}

src/cmd/tools/compose_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 var (
-	validInputFile   = "../../test/unit/common/compilation/component-definition-local-and-remote.yaml"
+	validInputFile   = "../../test/unit/common/composition/component-definition-import-compdefs.yaml"
 	invalidInputFile = "../../test/unit/common/valid-api-spec.yaml"
 )
 

src/cmd/validate/validate.go

@@ -11,6 +11,7 @@ import (
 	"github.com/defenseunicorns/go-oscal/src/pkg/uuid"
 	oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2"
 	"github.com/defenseunicorns/lula/src/pkg/common"
+	"github.com/defenseunicorns/lula/src/pkg/common/composition"
 	"github.com/defenseunicorns/lula/src/pkg/common/oscal"
 	validationstore "github.com/defenseunicorns/lula/src/pkg/common/validation-store"
 	"github.com/defenseunicorns/lula/src/pkg/message"
@@ -137,16 +138,16 @@ func ValidateOnPath(path string) (findingMap map[string]oscalTypes_1_1_2.Finding
 // ValidateOnCompDef takes a single ComponentDefinition object
 // It will perform a validation and add data to a referenced report object
 func ValidateOnCompDef(compDef *oscalTypes_1_1_2.ComponentDefinition) (map[string]oscalTypes_1_1_2.Finding, []oscalTypes_1_1_2.Observation, error) {
-	// Create a validation store from the back-matter if it exists
-	var validationStore *validationstore.ValidationStore
-	if compDef.BackMatter != nil {
-		validationStore = validationstore.NewValidationStoreFromBackMatter(*compDef.BackMatter)
-	} else {
-		validationStore = validationstore.NewValidationStore()
+	err := composition.ComposeComponentDefinitions(compDef)
+	if err != nil {
+		return nil, nil, err
+
 	}
 
-	// Loops all the way down
+	// Create a validation store from the back-matter if it exists
+	validationStore := validationstore.NewValidationStoreFromBackMatter(*compDef.BackMatter)
 
+	// Loops all the way down
 	findings := make(map[string]oscalTypes_1_1_2.Finding)
 	observations := make([]oscalTypes_1_1_2.Observation, 0)
 
@@ -187,74 +188,57 @@ func ValidateOnCompDef(compDef *oscalTypes_1_1_2.ComponentDefinition) (map[strin
 					for _, link := range *implementedRequirement.Links {
 						// TODO: potentially use rel to determine the type of validation (Validation Types discussion)
 						if common.IsLulaLink(link) {
-							ids, err := validationStore.AddFromLink(link)
+							id := common.TrimIdPrefix(link.Href)
+							lulaValidation, err := validationStore.GetLulaValidation(id)
 							if err != nil {
-								message.Debugf("Error adding validation from link %s: %v", link.Href, err)
+								message.Debugf("Error getting lula validation %s: %v", id, err)
 								// Handle error as an output to observations
-								observation := createObservation("TEST", "[Failed Observation]: %s\n%s\n", implementedRequirement.ControlId, link.Text)
+								observation := createObservation("TEST", "[Failed Observation]: %s - %s\n%s\n", implementedRequirement.ControlId, id, link.Text)
 								observation.RelevantEvidence = &[]oscalTypes_1_1_2.RelevantEvidence{
 									{
 										Description: "Result: not-satistfied\n",
-										Remarks:     fmt.Sprintf("Error adding validation from link: %v\n", err),
+										Remarks:     fmt.Sprintf("Error getting lula validation: %v\n", err),
 									},
 								}
 								fail = 1
 								relatedObservations, tempObservations = appendObservations(relatedObservations, tempObservations, observation)
-							}
+							} else {
+								// Add the description of the validation now that we have the ID
+								observation := createObservation("TEST", "[TEST]: %s - %s\n%s\n", implementedRequirement.ControlId, id, link.Text)
 
-							for _, id := range ids {
-								lulaValidation, err := validationStore.GetLulaValidation(id)
+								err = lulaValidation.Validate()
 								if err != nil {
-									message.Debugf("Error getting lula validation %s: %v", id, err)
+									message.Debugf("Error getting validating yaml: %v", err)
 									// Handle error as an output to observations
-									observation := createObservation("TEST", "[Failed Observation]: %s - %s\n%s\n", implementedRequirement.ControlId, id, link.Text)
-									observation.RelevantEvidence = &[]oscalTypes_1_1_2.RelevantEvidence{
-										{
-											Description: "Result: not-satistfied\n",
-											Remarks:     fmt.Sprintf("Error getting lula validation: %v\n", err),
-										},
-									}
-									fail = 1
-									relatedObservations, tempObservations = appendObservations(relatedObservations, tempObservations, observation)
+									lulaValidation.Result.Failing = 1
+									lulaValidation.Result.Observations = map[string]string{"Validation Error": err.Error()}
+								}
+								// Individual result state
+								if lulaValidation.Result.Passing > 0 && lulaValidation.Result.Failing <= 0 {
+									lulaValidation.Result.State = "satisfied"
 								} else {
-									// Add the description of the validation now that we have the ID
-									observation := createObservation("TEST", "[TEST]: %s - %s\n%s\n", implementedRequirement.ControlId, id, link.Text)
-
-									err = lulaValidation.Validate()
-									if err != nil {
-										message.Debugf("Error getting validating yaml: %v", err)
-										// Handle error as an output to observations
-										lulaValidation.Result.Failing = 1
-										lulaValidation.Result.Observations = map[string]string{"Validation Error": err.Error()}
-									}
-									// Individual result state
-									if lulaValidation.Result.Passing > 0 && lulaValidation.Result.Failing <= 0 {
-										lulaValidation.Result.State = "satisfied"
-									} else {
-										lulaValidation.Result.State = "not-satisfied"
-									}
-
-									// Add remarks if Result has Observations
-									var remarks string
-									if len(lulaValidation.Result.Observations) > 0 {
-										for k, v := range lulaValidation.Result.Observations {
-											remarks += fmt.Sprintf("%s: %s\n", k, v)
-										}
-									}
+									lulaValidation.Result.State = "not-satisfied"
+								}
 
-									observation.RelevantEvidence = &[]oscalTypes_1_1_2.RelevantEvidence{
-										{
-											Description: fmt.Sprintf("Result: %s\n", lulaValidation.Result.State),
-											Remarks:     remarks,
-										},
+								// Add remarks if Result has Observations
+								var remarks string
+								if len(lulaValidation.Result.Observations) > 0 {
+									for k, v := range lulaValidation.Result.Observations {
+										remarks += fmt.Sprintf("%s: %s\n", k, v)
 									}
+								}
 
-									pass += lulaValidation.Result.Passing
-									fail += lulaValidation.Result.Failing
-
-									relatedObservations, tempObservations = appendObservations(relatedObservations, tempObservations, observation)
+								observation.RelevantEvidence = &[]oscalTypes_1_1_2.RelevantEvidence{
+									{
+										Description: fmt.Sprintf("Result: %s\n", lulaValidation.Result.State),
+										Remarks:     remarks,
+									},
 								}
 
+								pass += lulaValidation.Result.Passing
+								fail += lulaValidation.Result.Failing
+
+								relatedObservations, tempObservations = appendObservations(relatedObservations, tempObservations, observation)
 							}
 						}
 					}

src/pkg/common/composition/composition.go

@@ -1,13 +1,89 @@
 package composition
 
 import (
+	"bytes"
 	"fmt"
 
 	gooscalUtils "github.com/defenseunicorns/go-oscal/src/pkg/utils"
+	"github.com/defenseunicorns/go-oscal/src/pkg/validation"
 	oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2"
 	"github.com/defenseunicorns/lula/src/pkg/common"
+	"github.com/defenseunicorns/lula/src/pkg/common/network"
+	"github.com/defenseunicorns/lula/src/pkg/common/oscal"
 )
 
+func ComposeComponentDefinitions(compDef *oscalTypes_1_1_2.ComponentDefinition) error {
+	if compDef == nil {
+		return fmt.Errorf("component definition is nil")
+	}
+
+	// Compose the component validations
+	err := ComposeComponentValidations(compDef)
+	if err != nil {
+		return err
+	}
+
+	// If there are no components, create an empty array
+	// Components aren't required by oscal but are by merge?
+	// TODO: fix merge to match required OSCAL fields
+	if compDef.Components == nil {
+		compDef.Components = &[]oscalTypes_1_1_2.DefinedComponent{}
+	}
+
+	// Same as above
+	if compDef.BackMatter == nil {
+		compDef.BackMatter = &oscalTypes_1_1_2.BackMatter{}
+	}
+
+	if compDef.ImportComponentDefinitions != nil {
+		for _, importComponentDef := range *compDef.ImportComponentDefinitions {
+			// Fetch the response
+			response, err := network.Fetch(importComponentDef.Href)
+			if err != nil {
+				return err
+			}
+
+			// Handle multi-docs
+			split := bytes.Split(response, []byte(common.YAML_DELIMITER))
+			// Unmarshal the component definition
+			for _, file := range split {
+				importDef, err := oscal.NewOscalComponentDefinitionFromBytes(file)
+				if err != nil {
+					return err
+				}
+
+				// create a validator
+				validator, err := validation.NewValidator(file)
+				if err != nil {
+					return err
+				}
+				// Validate the component definition
+				err = validator.Validate()
+				if err != nil {
+					return err
+				}
+
+				// Recurse and compose the component definition
+				err = ComposeComponentDefinitions(&importDef)
+				if err != nil {
+					return err
+				}
+
+				// Merge the component definitions
+				compDef, err = oscal.MergeComponentDefinitions(compDef, &importDef)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	compDef.Metadata.LastModified = gooscalUtils.GetTimestamp()
+	compDef.ImportComponentDefinitions = nil
+
+	return nil
+}
+
 // ComposeComponentValidations compiles the component validations by adding the remote resources to the back matter and updating with back matter links.
 func ComposeComponentValidations(compDef *oscalTypes_1_1_2.ComponentDefinition) error {
 
@@ -17,8 +93,9 @@ func ComposeComponentValidations(compDef *oscalTypes_1_1_2.ComponentDefinition)
 
 	resourceMap := NewResourceStoreFromBackMatter(compDef.BackMatter)
 
-	if *compDef.Components == nil {
-		return fmt.Errorf("no components found in component definition")
+	// If there are no components, there is nothing to do
+	if compDef.Components == nil {
+		return nil
 	}
 
 	for componentIndex, component := range *compDef.Components {

src/pkg/common/composition/composition_test.go

@@ -12,11 +12,100 @@ import (
 )
 
 const (
-	allRemote      = "../../../test/e2e/scenarios/validation-composition/component-definition.yaml"
-	allLocal       = "../../../test/unit/common/compilation/component-definition-all-local.yaml"
-	localAndRemote = "../../../test/unit/common/compilation/component-definition-local-and-remote.yaml"
+	allRemote          = "../../../test/e2e/scenarios/validation-composition/component-definition.yaml"
+	allLocal           = "../../../test/unit/common/composition/component-definition-all-local.yaml"
+	localAndRemote     = "../../../test/unit/common/composition/component-definition-local-and-remote.yaml"
+	subComponentDef    = "../../../test/unit/common/composition/component-definition-import-compdefs.yaml"
+	compDefMultiImport = "../../../test/unit/common/composition/component-definition-import-multi-compdef.yaml"
 )
 
+func TestComposeComponentDefinitions(t *testing.T) {
+	t.Run("No imports, local validations", func(t *testing.T) {
+		og := getComponentDef(allLocal, t)
+		compDef := getComponentDef(allLocal, t)
+		reset, err := common.SetCwdToFileDir(allLocal)
+		defer reset()
+		if err != nil {
+			t.Fatalf("Error setting cwd to file dir: %v", err)
+		}
+		err = composition.ComposeComponentDefinitions(compDef)
+		if err != nil {
+			t.Fatalf("Error composing component definitions: %v", err)
+		}
+
+		// Only the last-modified timestamp should be different
+		if !reflect.DeepEqual(*og.BackMatter, *compDef.BackMatter) {
+			t.Error("expected the back matter to be unchanged")
+		}
+	})
+
+	t.Run("No imports, remote validations", func(t *testing.T) {
+		og := getComponentDef(allRemote, t)
+		compDef := getComponentDef(allRemote, t)
+		reset, err := common.SetCwdToFileDir(allRemote)
+		defer reset()
+		if err != nil {
+			t.Fatalf("Error setting cwd to file dir: %v", err)
+		}
+		err = composition.ComposeComponentDefinitions(compDef)
+		if err != nil {
+			t.Fatalf("Error composing component definitions: %v", err)
+		}
+
+		if reflect.DeepEqual(*og, *compDef) {
+			t.Errorf("expected component definition to have changed.")
+		}
+	})
+
+	t.Run("Imports, no components", func(t *testing.T) {
+		og := getComponentDef(subComponentDef, t)
+		compDef := getComponentDef(subComponentDef, t)
+		reset, err := common.SetCwdToFileDir(subComponentDef)
+		defer reset()
+		if err != nil {
+			t.Fatalf("Error setting cwd to file dir: %v", err)
+		}
+		err = composition.ComposeComponentDefinitions(compDef)
+		if err != nil {
+			t.Fatalf("Error composing component definitions: %v", err)
+		}
+
+		if compDef.Components == og.Components {
+			t.Error("expected there to be components")
+		}
+
+		if compDef.BackMatter == og.BackMatter {
+			t.Error("expected the back matter to be changed")
+		}
+	})
+
+	t.Run("imports, no components, multiple component definitions from import", func(t *testing.T) {
+		og := getComponentDef(compDefMultiImport, t)
+		compDef := getComponentDef(compDefMultiImport, t)
+		reset, err := common.SetCwdToFileDir(compDefMultiImport)
+		defer reset()
+		if err != nil {
+			t.Fatalf("Error setting cwd to file dir: %v", err)
+		}
+		err = composition.ComposeComponentDefinitions(compDef)
+		if err != nil {
+			t.Fatalf("Error composing component definitions: %v", err)
+		}
+		if compDef.Components == og.Components {
+			t.Error("expected there to be components")
+		}
+
+		if compDef.BackMatter == og.BackMatter {
+			t.Error("expected the back matter to be changed")
+		}
+
+		if len(*compDef.Components) != 1 {
+			t.Error("expected there to be 2 components")
+		}
+	})
+
+}
+
 func TestCompileComponentValidations(t *testing.T) {
 
 	t.Run("all local", func(t *testing.T) {

src/pkg/common/oscal/component.go

@@ -49,6 +49,16 @@ func NewOscalComponentDefinition(source string, data []byte) (componentDefinitio
 	return oscalModels.ComponentDefinition, nil
 }
 
+// NewOscalComponentDefinitionFromBytes consumes a byte array and returns a new single OscalComponentDefinitionModel object
+func NewOscalComponentDefinitionFromBytes(data []byte) (componentDefinition oscalTypes_1_1_2.ComponentDefinition, err error) {
+	var oscalModels oscalTypes_1_1_2.OscalModels
+	err = yaml.Unmarshal(data, &oscalModels)
+	if err != nil {
+		return componentDefinition, err
+	}
+	return *oscalModels.ComponentDefinition, nil
+}
+
 // This function should perform a merge of two component-definitions where maintaining the original component-definition is the primary concern.
 func MergeComponentDefinitions(original *oscalTypes_1_1_2.ComponentDefinition, latest *oscalTypes_1_1_2.ComponentDefinition) (*oscalTypes_1_1_2.ComponentDefinition, error) {
 
@@ -378,11 +388,11 @@ func ControlToImplementedRequirement(control oscalTypes_1_1_2.Control, targetRem
 
 // Returns a map of the uuid - description of the back-matter resources
 func BackMatterToMap(backMatter oscalTypes_1_1_2.BackMatter) (resourceMap map[string]string) {
+	resourceMap = make(map[string]string)
 	if backMatter.Resources == nil {
-		return nil
+		return resourceMap
 	}
 
-	resourceMap = make(map[string]string)
 	for _, resource := range *backMatter.Resources {
 		// perform a check to see if the key already exists (meaning duplicitive uuid use)
 		_, exists := resourceMap[resource.UUID]