feat!: #367 compiling external/remote validations (#384)

* refactor(common.go): ValidationFromString now uses `validationData.UnmarshalYaml`

* refactor(validation-store): extracted constants and helpers from validation-store to common as they are used in wider areas

* refactor(validate): replace lula link check with common.IsLulaLink

* feat(common): Validation now has ToResource method

* tests(common): add tests for Validation.ToResource()

* feat(compilation): Added tests and functionality for the validation compilation command TODO: create the actual command

* feat(compile): create compile command

* docs: create compile command documentation

* feat(tools): move the compile command under the tools cmd

* feat(compilation): now updates the timestamp for the component definition

* chore: renamecompile command and all associations to use the compose verbiage

docs/commands/tools/compose.md

@@ -0,0 +1,30 @@
+# Compose Command
+
+The `compose` command is used to compose an OSCAL component definition. It is used to compose remote validations within a component definition in order to resolve any references for portability.
+
+## Usage
+
+```bash
+lula tools compose -f <input-file> -o <output-file>
+```
+
+## Options
+
+- `-f, --input-file`: The path to the target OSCAL component definition.
+- `-o, --output-file`: The path to the output file. If not specified, the output file will be the original filename with `-composed` appended.
+
+## Examples
+
+To compose an OSCAL Model:
+```bash
+lula tools compose -f ./oscal-component.yaml
+```
+
+To indicate a specific output file:
+```bash
+lula tools compose -f ./oscal-component.yaml -o composed-oscal-component.yaml
+```
+
+## Notes
+
+If the input file does not exist, an error will be returned. The composed OSCAL Component Definition will be written to the specified output file. If no output file is specified, the composed definition will be written to a file with the original filename and `-composed` appended.

src/cmd/tools/compose.go

@@ -0,0 +1,110 @@
+package tools
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	gooscalUtils "github.com/defenseunicorns/go-oscal/src/pkg/utils"
+	"github.com/defenseunicorns/lula/src/pkg/common"
+	"github.com/defenseunicorns/lula/src/pkg/common/composition"
+	"github.com/defenseunicorns/lula/src/pkg/common/oscal"
+	"github.com/defenseunicorns/lula/src/pkg/message"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
+)
+
+type composeFlags struct {
+	InputFile  string // -f --input-file
+	OutputFile string // -o --output-file
+}
+
+var composeOpts = &composeFlags{}
+
+var composeHelp = `
+To compose an OSCAL Model:
+	lula tools compose -f ./oscal-component.yaml
+
+To indicate a specific output file:
+	lula tools compose -f ./oscal-component.yaml -o composed-oscal-component.yaml
+`
+
+func init() {
+	composeCmd := &cobra.Command{
+		Use:     "compose",
+		Short:   "compose an OSCAL component definition",
+		Long:    "Lula Composition of an OSCAL component definition. Used to compose remote validations within a component definition in order to resolve any references for portability.",
+		Example: composeHelp,
+		Run: func(cmd *cobra.Command, args []string) {
+			if composeOpts.InputFile == "" {
+				message.Fatal(errors.New("flag input-file is not set"),
+					"Please specify an input file with the -f flag")
+			}
+			err := Compose(composeOpts.InputFile, composeOpts.OutputFile)
+			if err != nil {
+				message.Fatalf(err, "Composition error: %s", err)
+			}
+		},
+	}
+
+	toolsCmd.AddCommand(composeCmd)
+
+	composeCmd.Flags().StringVarP(&composeOpts.InputFile, "input-file", "f", "", "the path to the target OSCAL component definition")
+	composeCmd.Flags().StringVarP(&composeOpts.OutputFile, "output-file", "o", "", "the path to the output file. If not specified, the output file will be the original filename with `-composed` appended")
+}
+
+func Compose(inputFile, outputFile string) error {
+	_, err := os.Stat(inputFile)
+	if os.IsNotExist(err) {
+		return fmt.Errorf("input file: %v does not exist - unable to compose document", inputFile)
+	}
+
+	data, err := os.ReadFile(inputFile)
+	if err != nil {
+		return err
+	}
+
+	// Change Cwd to the directory of the component definition
+	dirPath := filepath.Dir(inputFile)
+	message.Infof("changing cwd to %s", dirPath)
+	resetCwd, err := common.SetCwdToFileDir(dirPath)
+	if err != nil {
+		return err
+	}
+
+	model, err := oscal.NewOscalModel(data)
+	if err != nil {
+		return err
+	}
+
+	err = composition.ComposeComponentValidations(model.ComponentDefinition)
+	if err != nil {
+		return err
+	}
+
+	// Reset Cwd to original before outputting
+	resetCwd()
+
+	var b bytes.Buffer
+	// Format the output
+	yamlEncoder := yaml.NewEncoder(&b)
+	yamlEncoder.SetIndent(2)
+	yamlEncoder.Encode(model)
+
+	outputFileName := outputFile
+	if outputFileName == "" {
+		outputFileName = strings.TrimSuffix(inputFile, filepath.Ext(inputFile)) + "-composed" + filepath.Ext(inputFile)
+	}
+
+	message.Infof("Writing Composed OSCAL Component Definition to: %s", outputFileName)
+
+	err = gooscalUtils.WriteOutput(b.Bytes(), outputFileName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

src/cmd/tools/compose_test.go

@@ -0,0 +1,52 @@
+package tools_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/defenseunicorns/lula/src/cmd/tools"
+	"github.com/defenseunicorns/lula/src/pkg/common/oscal"
+)
+
+var (
+	validInputFile   = "../../test/unit/common/compilation/component-definition-local-and-remote.yaml"
+	invalidInputFile = "../../test/unit/common/valid-api-spec.yaml"
+)
+
+func TestComposeComponentDefinition(t *testing.T) {
+	t.Parallel()
+	tempDir := t.TempDir()
+	outputFile := filepath.Join(tempDir, "output.yaml")
+
+	t.Run("composes valid component definition", func(t *testing.T) {
+		err := tools.Compose(validInputFile, outputFile)
+		if err != nil {
+			t.Fatalf("error composing component definition: %s", err)
+		}
+
+		compiledBytes, err := os.ReadFile(outputFile)
+		if err != nil {
+			t.Fatalf("error reading composed component definition: %s", err)
+		}
+		compiledModel, err := oscal.NewOscalModel(compiledBytes)
+		if err != nil {
+			t.Fatalf("error creating oscal model from composed component definition: %s", err)
+		}
+
+		if compiledModel.ComponentDefinition.BackMatter.Resources == nil {
+			t.Fatal("composed component definition is nil")
+		}
+
+		if len(*compiledModel.ComponentDefinition.BackMatter.Resources) <= 1 {
+			t.Fatalf("expected 2 resources, got %d", len(*compiledModel.ComponentDefinition.BackMatter.Resources))
+		}
+	})
+
+	t.Run("invalid component definition throws error", func(t *testing.T) {
+		err := tools.Compose(invalidInputFile, outputFile)
+		if err == nil {
+			t.Fatal("expected error composing invalid component definition")
+		}
+	})
+}

src/cmd/validate/validate.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/defenseunicorns/go-oscal/src/pkg/uuid"
@@ -185,8 +184,7 @@ func ValidateOnCompDef(compDef oscalTypes_1_1_2.ComponentDefinition) (map[string
 				if implementedRequirement.Links != nil {
 					for _, link := range *implementedRequirement.Links {
 						// TODO: potentially use rel to determine the type of validation (Validation Types discussion)
-						rel := strings.Split(link.Rel, ".")
-						if link.Text == "Lula Validation" || rel[0] == "lula" {
+						if common.IsLulaLink(link) {
 							ids, err := validationStore.AddFromLink(link)
 							if err != nil {
 								return map[string]oscalTypes_1_1_2.Finding{}, []oscalTypes_1_1_2.Observation{}, err

src/pkg/common/common.go

@@ -5,18 +5,40 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
+	oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2"
 	"github.com/defenseunicorns/lula/src/pkg/domains/api"
 	kube "github.com/defenseunicorns/lula/src/pkg/domains/kubernetes"
 	"github.com/defenseunicorns/lula/src/pkg/message"
 	"github.com/defenseunicorns/lula/src/pkg/providers/kyverno"
 	"github.com/defenseunicorns/lula/src/pkg/providers/opa"
 	"github.com/defenseunicorns/lula/src/types"
 	goversion "github.com/hashicorp/go-version"
+)
 
-	"sigs.k8s.io/yaml"
+const (
+	UUID_PREFIX    = "#"
+	WILDCARD       = "*"
+	YAML_DELIMITER = "---"
 )
 
+// TrimIdPrefix trims the id prefix from the given id
+func TrimIdPrefix(id string) string {
+	return strings.TrimPrefix(id, UUID_PREFIX)
+}
+
+// AddIdPrefix adds the id prefix to the given id
+func AddIdPrefix(id string) string {
+	return fmt.Sprintf("%s%s", UUID_PREFIX, id)
+}
+
+// IsLulaLink checks if the link is a lula link
+func IsLulaLink(link oscalTypes_1_1_2.Link) bool {
+	rel := strings.Split(link.Rel, ".")
+	return link.Text == "Lula Validation" || rel[0] == "lula"
+}
+
 // ReadFileToBytes reads a file to bytes
 func ReadFileToBytes(path string) ([]byte, error) {
 	var data []byte
@@ -135,10 +157,8 @@ func ValidationFromString(raw string) (validation types.LulaValidation, err erro
 	}
 
 	var validationData Validation
-
-	err = yaml.Unmarshal([]byte(raw), &validationData)
+	err = validationData.UnmarshalYaml([]byte(raw))
 	if err != nil {
-		message.Fatalf(err, "error unmarshalling yaml: %s", err.Error())
 		return validation, err
 	}
 

src/pkg/common/common_test.go

@@ -273,3 +273,63 @@ func TestSwitchCwd(t *testing.T) {
 		})
 	}
 }
+
+func TestValidationToResource(t *testing.T) {
+	t.Parallel()
+	t.Run("It populates a resource from a validation", func(t *testing.T) {
+		t.Parallel()
+		validation := &common.Validation{
+			Metadata: common.Metadata{
+				UUID: "1234",
+				Name: "Test Validation",
+			},
+			Provider: common.Provider{
+				Type: "test",
+			},
+			Domain: common.Domain{
+				Type: "test",
+			},
+		}
+
+		resource, err := validation.ToResource()
+		if err != nil {
+			t.Errorf("ToResource() error = %v", err)
+		}
+
+		if resource.Title != validation.Metadata.Name {
+			t.Errorf("ToResource() title = %v, want %v", resource.Title, validation.Metadata.Name)
+		}
+
+		if resource.UUID != validation.Metadata.UUID {
+			t.Errorf("ToResource() UUID = %v, want %v", resource.UUID, validation.Metadata.UUID)
+		}
+
+		if resource.Description == "" {
+			t.Errorf("ToResource() description = %v, want %v", resource.Description, "")
+		}
+	})
+
+	t.Run("It adds a UUID if one does not exist", func(t *testing.T) {
+		t.Parallel()
+		validation := &common.Validation{
+			Metadata: common.Metadata{
+				Name: "Test Validation",
+			},
+			Provider: common.Provider{
+				Type: "test",
+			},
+			Domain: common.Domain{
+				Type: "test",
+			},
+		}
+
+		resource, err := validation.ToResource()
+		if err != nil {
+			t.Errorf("ToResource() error = %v", err)
+		}
+
+		if resource.UUID == validation.Metadata.UUID {
+			t.Errorf("ToResource() description = \"\", want a valid UUID")
+		}
+	})
+}