Merge branch 'main' into 542_deterministic_writes

.github/workflows/scan-codeql.yaml

@@ -42,7 +42,7 @@ jobs:
         uses: ./.github/actions/golang
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@2d790406f505036ef40ecba973cc774a50395aac # v3.25.13
+        uses: github/codeql-action/init@5cf07d8b700b67e235fbb65cbc84f69c0cf10464 # v3.25.14
         with:
           languages: ${{ matrix.language }}
           # config-file: ./.github/codeql.yaml #Uncomment once config file is needed.
@@ -52,7 +52,7 @@ jobs:
 
       - name: Perform CodeQL Analysis
         id: scan
-        uses: github/codeql-action/analyze@2d790406f505036ef40ecba973cc774a50395aac # v3.25.13
+        uses: github/codeql-action/analyze@5cf07d8b700b67e235fbb65cbc84f69c0cf10464 # v3.25.14
         with:
           category: "/language:${{matrix.language}}"
 

.github/workflows/scorecard.yaml

@@ -46,6 +46,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@2d790406f505036ef40ecba973cc774a50395aac # v3.25.13
+        uses: github/codeql-action/upload-sarif@5cf07d8b700b67e235fbb65cbc84f69c0cf10464 # v3.25.14
         with:
           sarif_file: results.sarif

docs/cli-commands/assessments/evaluate.md

@@ -2,6 +2,28 @@
 
 Evaluate serves as a method for verifying the compliance of a component/system against an established threshold to determine if it is more or less compliant than a previous assessment. 
 
+## Usage
+
+To evaluate two results (threshold and latest) in a single OSCAL file:
+```bash
+lula evaluate -f assessment-results.yaml
+```
+
+To evaluate the latest results in two assessment results files:
+```bash
+lula evaluate -f assessment-results-threshold.yaml -f assessment-results-new.yaml
+```
+
+To print a summary of the observation results:
+```bash
+lula evaluate -f assessment-results.yaml --summary
+```
+
+## Options
+
+- `-f, --file`: The path to the file(s) to be evaluated.
+- `-s, --summary`: [Optional] Prints a summary of the evaluation.
+
 ## Expected Process
 
 ### No Existing Data

go.mod

@@ -1,12 +1,12 @@
 module github.com/defenseunicorns/lula
 
-go 1.22.3
+go 1.22.5
 
 require (
 	github.com/defenseunicorns/go-oscal v0.5.0
 	github.com/hashicorp/go-version v1.7.0
 	github.com/kyverno/kyverno-json v0.0.3
-	github.com/open-policy-agent/opa v0.66.0
+	github.com/open-policy-agent/opa v0.67.0
 	github.com/pterm/pterm v0.12.79
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/sergi/go-diff v1.3.1
@@ -33,7 +33,7 @@ require (
 	github.com/aquilax/truncate v1.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/containerd/console v1.0.3 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -111,22 +111,22 @@ require (
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/yashtewari/glob-intersection v0.2.0 // indirect
 	github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea // indirect
-	go.opentelemetry.io/otel v1.23.1 // indirect
-	go.opentelemetry.io/otel/metric v1.23.1 // indirect
-	go.opentelemetry.io/otel/sdk v1.23.1 // indirect
-	go.opentelemetry.io/otel/trace v1.23.1 // indirect
+	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	go.starlark.net v0.0.0-20240123142251-f86470692795 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
-	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/oauth2 v0.17.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/term v0.21.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/term v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/evanphx/json-patch.v5 v5.9.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

src/cmd/evaluate/evaluate.go

@@ -2,11 +2,13 @@ package evaluate
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/defenseunicorns/go-oscal/src/pkg/files"
 	oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2"
 	"github.com/defenseunicorns/lula/src/pkg/common"
 	"github.com/defenseunicorns/lula/src/pkg/common/oscal"
+	"github.com/defenseunicorns/lula/src/pkg/common/result"
 	"github.com/defenseunicorns/lula/src/pkg/message"
 	"github.com/spf13/cobra"
 )
@@ -20,7 +22,8 @@ To evaluate two results (threshold and latest) in a single OSCAL file:
 `
 
 type flags struct {
-	files []string
+	files   []string
+	summary bool
 }
 
 var opts = &flags{}
@@ -39,18 +42,19 @@ var evaluateCmd = &cobra.Command{
 			message.Fatal(err, err.Error())
 		}
 
-		EvaluateAssessments(assessmentMap)
+		EvaluateAssessments(assessmentMap, opts.summary)
 	},
 }
 
 func EvaluateCommand() *cobra.Command {
 
 	evaluateCmd.Flags().StringArrayVarP(&opts.files, "file", "f", []string{}, "Path to the file to be evaluated")
+	evaluateCmd.Flags().BoolVarP(&opts.summary, "summary", "s", false, "Print a summary of the evaluation")
 	// insert flag options here
 	return evaluateCmd
 }
 
-func EvaluateAssessments(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResults) {
+func EvaluateAssessments(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResults, summary bool) {
 	// Identify the threshold & latest for comparison
 	resultMap, err := oscal.IdentifyResults(assessmentMap)
 	if err != nil {
@@ -69,22 +73,41 @@ func EvaluateAssessments(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentRe
 	}
 
 	if resultMap["threshold"] != nil && resultMap["latest"] != nil {
+		var findingsWithoutObservations []string
 		// Compare the assessment results
 		spinner := message.NewProgressSpinner("Evaluating Assessment Results %s against %s", resultMap["threshold"].UUID, resultMap["latest"].UUID)
 		defer spinner.Stop()
 
 		message.Debugf("threshold UUID: %s / latest UUID: %s", resultMap["threshold"].UUID, resultMap["latest"].UUID)
 
-		status, findings, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"])
+		status, resultComparison, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"])
 		if err != nil {
 			message.Fatal(err, err.Error())
 		}
 
+		// Print summary
+		if summary {
+			message.Info("Summary of All Observations:")
+			findingsWithoutObservations = result.Collapse(resultComparison).PrintObservationComparisonTable(false, true, false)
+			if len(findingsWithoutObservations) > 0 {
+				message.Warnf("%d Finding(s) Without Observations", len(findingsWithoutObservations))
+				message.Info(strings.Join(findingsWithoutObservations, ", "))
+			}
+		}
+
+		// Check 'status' - Result if evaluation is passing or failing
+		// Fails if anything went from satisfied -> not-satisfied OR if any old findings are removed (doesn't matter whether they were satisfied or not)
 		if status {
-			if len(findings["new-passing-findings"]) > 0 {
+			// Print new-passing-findings
+			newSatisfied := resultComparison["new-satisfied"]
+			nowSatisfied := resultComparison["now-satisfied"]
+			if len(newSatisfied) > 0 || len(nowSatisfied) > 0 {
 				message.Info("New passing finding Target-Ids:")
-				for _, finding := range findings["new-passing-findings"] {
-					message.Infof("%s", finding.Target.TargetId)
+				for id := range newSatisfied {
+					message.Infof("%s", id)
+				}
+				for id := range nowSatisfied {
+					message.Infof("%s", id)
 				}
 
 				message.Infof("New threshold identified - threshold will be updated to result %s", resultMap["latest"].UUID)
@@ -97,19 +120,33 @@ func EvaluateAssessments(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentRe
 				oscal.UpdateProps("threshold", "https://docs.lula.dev/ns", "true", resultMap["threshold"].Props)
 			}
 
-			if len(findings["new-failing-findings"]) > 0 {
+			// Print new-not-satisfied
+			newFailing := resultComparison["new-not-satisfied"]
+			if len(newFailing) > 0 {
 				message.Info("New failing finding Target-Ids:")
-				for _, finding := range findings["new-failing-findings"] {
-					message.Infof("%s", finding.Target.TargetId)
+				for id := range newFailing {
+					message.Infof("%s", id)
 				}
 			}
-			message.Info("Evaluation Passed Successfully")
 
+			message.Info("Evaluation Passed Successfully")
 		} else {
-			message.Warn("Evaluation Failed against the following findings:")
-			for _, finding := range findings["no-longer-satisfied"] {
-				message.Warnf("%s", finding.Target.TargetId)
+			// Print no-longer-satisfied
+			message.Warn("Evaluation Failed against the following:")
+
+			// Alternative printing in a single table
+			failedFindings := map[string]result.ResultComparisonMap{
+				"no-longer-satisfied":   resultComparison["no-longer-satisfied"],
+				"removed-satisfied":     resultComparison["removed-satisfied"],
+				"removed-not-satisfied": resultComparison["removed-not-satisfied"],
 			}
+			findingsWithoutObservations = result.Collapse(failedFindings).PrintObservationComparisonTable(true, false, true)
+			// handle controls that failed but didn't have observations
+			if len(findingsWithoutObservations) > 0 {
+				message.Warnf("%d Failed Finding(s) Without Observations", len(findingsWithoutObservations))
+				message.Info(strings.Join(findingsWithoutObservations, ", "))
+			}
+
 			message.Fatalf(fmt.Errorf("failed to meet established threshold"), "failed to meet established threshold")
 
 			// retain result as threshold

src/pkg/common/oscal/assessment-results.go

@@ -9,6 +9,7 @@ import (
 	"github.com/defenseunicorns/go-oscal/src/pkg/uuid"
 	oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2"
 	"github.com/defenseunicorns/lula/src/config"
+	"github.com/defenseunicorns/lula/src/pkg/common/result"
 	"gopkg.in/yaml.v3"
 )
 
@@ -116,14 +117,6 @@ func MergeAssessmentResults(original *oscalTypes_1_1_2.AssessmentResults, latest
 	return original, nil
 }
 
-func GenerateFindingsMap(findings []oscalTypes_1_1_2.Finding) map[string]oscalTypes_1_1_2.Finding {
-	findingsMap := make(map[string]oscalTypes_1_1_2.Finding)
-	for _, finding := range findings {
-		findingsMap[finding.Target.TargetId] = finding
-	}
-	return findingsMap
-}
-
 // IdentifyResults produces a map containing the threshold result and a result used for comparison
 func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResults) (map[string]*oscalTypes_1_1_2.Result, error) {
 	resultMap := make(map[string]*oscalTypes_1_1_2.Result)
@@ -178,58 +171,83 @@ func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResult
 	}
 }
 
-func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalTypes_1_1_2.Result) (bool, map[string][]oscalTypes_1_1_2.Finding, error) {
+func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalTypes_1_1_2.Result) (bool, map[string]result.ResultComparisonMap, error) {
+	var status bool = true
+
 	if thresholdResult.Findings == nil || newResult.Findings == nil {
 		return false, nil, fmt.Errorf("results must contain findings to evaluate")
 	}
 
-	// Store unique findings for review here
-	findings := make(map[string][]oscalTypes_1_1_2.Finding, 0)
-	result := true
-
-	findingMapThreshold := GenerateFindingsMap(*thresholdResult.Findings)
-	findingMapNew := GenerateFindingsMap(*newResult.Findings)
-
-	// For a given oldResult - we need to prove that the newResult implements all of the oldResult findings/controls
-	// We are explicitly iterating through the findings in order to collect a delta to display
-
-	for targetId, finding := range findingMapThreshold {
-		if _, ok := findingMapNew[targetId]; !ok {
-			// If the new result does not contain the finding of the old result
-			// set result to fail, add finding to the findings map and continue
-			result = false
-			findings[targetId] = append(findings["no-longer-satisfied"], finding)
-		} else {
-			// If the finding is present in each map - we need to check if the state has changed from "not-satisfied" to "satisfied"
-			if finding.Target.Status.State == "satisfied" {
-				// Was previously satisfied - compare state
-				if findingMapNew[targetId].Target.Status.State == "not-satisfied" {
-					// If the new finding is now not-satisfied - set result to false and add to findings
-					result = false
-					findings["no-longer-satisfied"] = append(findings["no-longer-satisfied"], finding)
-				}
-			} else {
-				// was previously not-satisfied but now is satisfied
-				if findingMapNew[targetId].Target.Status.State == "satisfied" {
-					// If the new finding is now satisfied - add to new-passing-findings
-					findings["new-passing-findings"] = append(findings["new-passing-findings"], finding)
-				}
-			}
-			delete(findingMapNew, targetId)
-		}
+	// Compare threshold result to new result and vice versa
+	comparedToThreshold := result.NewResultComparisonMap(*newResult, *thresholdResult)
+
+	// Group by categories
+	categories := []struct {
+		name        string
+		stateChange result.StateChange
+		satisfied   bool
+		status      bool
+	}{
+		{
+			name:        "new-satisfied",
+			stateChange: result.NEW,
+			satisfied:   true,
+			status:      true,
+		},
+		{
+			name:        "new-not-satisfied",
+			stateChange: result.NEW,
+			satisfied:   false,
+			status:      true,
+		},
+		{
+			name:        "no-longer-satisfied",
+			stateChange: result.SATISFIED_TO_NOT_SATISFIED,
+			satisfied:   false,
+			status:      false,
+		},
+		{
+			name:        "now-satisfied",
+			stateChange: result.NOT_SATISFIED_TO_SATISFIED,
+			satisfied:   true,
+			status:      true,
+		},
+		{
+			name:        "unchanged-not-satisfied",
+			stateChange: result.UNCHANGED,
+			satisfied:   false,
+			status:      true,
+		},
+		{
+			name:        "unchanged-satisfied",
+			stateChange: result.UNCHANGED,
+			satisfied:   true,
+			status:      true,
+		},
+		{
+			name:        "removed-not-satisfied",
+			stateChange: result.REMOVED,
+			satisfied:   false,
+			status:      false,
+		},
+		{
+			name:        "removed-satisfied",
+			stateChange: result.REMOVED,
+			satisfied:   true,
+			status:      false,
+		},
 	}
 
-	// All remaining findings in the new map are new findings
-	for _, finding := range findingMapNew {
-		if finding.Target.Status.State == "satisfied" {
-			findings["new-passing-findings"] = append(findings["new-passing-findings"], finding)
-		} else {
-			findings["new-failing-findings"] = append(findings["new-failing-findings"], finding)
+	categorizedResultComparisons := make(map[string]result.ResultComparisonMap)
+	for _, c := range categories {
+		results := result.GetResultComparisonMap(comparedToThreshold, c.stateChange, c.satisfied)
+		categorizedResultComparisons[c.name] = results
+		if len(results) > 0 && !c.status {
+			status = false
 		}
-
 	}
 
-	return result, findings, nil
+	return status, categorizedResultComparisons, nil
 }
 
 func MakeAssessmentResultsDeterministic(assessment *oscalTypes_1_1_2.AssessmentResults) {