From b5aff8b9421ef3ed98b3d52f59373ecdea067df4 Mon Sep 17 00:00:00 2001 From: mistahj67 <26472282+mistahj67@users.noreply.github.com> Date: Tue, 20 Aug 2024 17:29:14 -0700 Subject: [PATCH] Stage/v5.14.0 (#804) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: correctly stream process json data normalization to utf8 for bot… (#764) * fix: correctly stream process json data normalization to utf8 for both metatag validation and writing to disk * test: update expected errors in negative tests * chore: restore go mod files for reverted package * chore: correct module name * chore: remove duplicate import * doc: format inline comment Co-authored-by: Ulises Rangel --------- Co-authored-by: Ulises Rangel * Bed-4168: fix: Inaccurate asset group counts attempt 2(#794) * revert BED-4662 PR 767 (#797) * updated table (#798) * chore: prep-for-codereview --------- Co-authored-by: Dillon Lees Co-authored-by: Ulises Rangel Co-authored-by: Michael Lipka Co-authored-by: stephanieslamb --- cmd/api/src/api/v2/agi.go | 8 +- cmd/api/src/api/v2/agi_internal_test.go | 4 +- cmd/api/src/api/v2/agi_test.go | 20 +- .../api/v2/file_uploads_integration_test.go | 13 +- cmd/api/src/daemons/datapipe/analysis.go | 3 +- cmd/api/src/database/saved_queries.go | 2 +- cmd/api/src/go.mod | 4 +- cmd/api/src/go.sum | 10 +- cmd/api/src/model/filter.go | 24 +- cmd/api/src/model/filter_test.go | 27 +- cmd/api/src/model/saved_queries.go | 2 +- cmd/api/src/model/saved_queries_test.go | 7 +- cmd/api/src/queries/graph.go | 37 +-- cmd/api/src/queries/graph_integration_test.go | 6 +- cmd/api/src/queries/mocks/graph.go | 8 +- cmd/api/src/services/agi/agi.go | 55 ++-- .../src/services/fileupload/file_upload.go | 18 +- .../services/fileupload/file_upload_test.go | 119 ++++++--- cmd/ui/src/ducks/auth/authSlice.ts | 2 +- go.work | 3 +- packages/go/analysis/go.mod | 2 +- packages/go/analysis/go.sum | 2 +- packages/go/bomenc/encodings.go | 198 ++++++++++++++ packages/go/bomenc/encodings_test.go | 233 ++++++++++++++++ packages/go/bomenc/go.mod | 33 +++ packages/go/bomenc/go.sum | 15 ++ packages/go/bomenc/normalize.go | 85 ++++++ packages/go/bomenc/normalize_test.go | 250 ++++++++++++++++++ packages/go/cache/go.mod | 10 +- packages/go/cache/go.sum | 2 +- packages/go/crypto/go.mod | 2 +- packages/go/crypto/go.sum | 2 +- packages/go/cypher/go.mod | 4 +- packages/go/cypher/go.sum | 5 +- packages/go/dawgs/go.mod | 6 +- packages/go/dawgs/go.sum | 7 +- packages/go/errors/go.mod | 2 +- packages/go/errors/go.sum | 3 +- packages/go/lab/go.mod | 2 +- packages/go/lab/go.sum | 3 +- packages/go/schemagen/go.mod | 4 +- packages/go/schemagen/go.sum | 6 +- packages/go/slicesext/go.mod | 2 +- packages/go/slicesext/go.sum | 3 +- packages/go/stbernard/go.mod | 2 +- packages/go/stbernard/go.sum | 3 +- .../javascript/bh-shared-ui/src/constants.ts | 21 +- .../bh-shared-ui/src/test-utils.jsx | 2 +- 48 files changed, 1048 insertions(+), 233 deletions(-) create mode 100644 packages/go/bomenc/encodings.go create mode 100644 packages/go/bomenc/encodings_test.go create mode 100644 packages/go/bomenc/go.mod create mode 100644 packages/go/bomenc/go.sum create mode 100644 packages/go/bomenc/normalize.go create mode 100644 packages/go/bomenc/normalize_test.go diff --git a/cmd/api/src/api/v2/agi.go b/cmd/api/src/api/v2/agi.go index 1c1f8c6621..1003c62756 100644 --- a/cmd/api/src/api/v2/agi.go +++ b/cmd/api/src/api/v2/agi.go @@ -431,7 +431,7 @@ func (s Resources) getAssetGroupMembers(response http.ResponseWriter, request *h } else if assetGroup, err := s.DB.GetAssetGroup(request.Context(), int32(assetGroupID)); err != nil { api.HandleDatabaseError(request, response, err) return agMembers, err - } else if assetGroupNodes, err := s.GraphQuery.GetAssetGroupNodes(request.Context(), assetGroup.Tag); err != nil { + } else if assetGroupNodes, err := s.GraphQuery.GetAssetGroupNodes(request.Context(), assetGroup.Tag, assetGroup.SystemGroup); err != nil { api.WriteErrorResponse(request.Context(), api.BuildErrorResponse(http.StatusInternalServerError, fmt.Sprintf("Graph error fetching nodes for asset group ID %v: %v", assetGroup.ID, err), request), response) return agMembers, err } else if agMembers, err = parseAGMembersFromNodes(assetGroupNodes, assetGroup.Selectors, int(assetGroup.ID)).SortBy(sortByColumns); err != nil { @@ -520,16 +520,14 @@ func parseAGMembersFromNodes(nodes graph.NodeSet, selectors model.AssetGroupSele if node.Kinds.ContainsOneOf(azure.Entity) { if tenantID, err := node.Properties.Get(azure.TenantID.String()).String(); err != nil { - log.Warnf("%s is missing for node %d, skipping AG Membership...", azure.TenantID.String(), node.ID) - continue + log.Warnf("%s is missing for node %d", azure.TenantID.String(), node.ID) } else { agMember.EnvironmentKind = azure.Tenant.String() agMember.EnvironmentID = tenantID } } else if node.Kinds.ContainsOneOf(ad.Entity) { if domainSID, err := node.Properties.Get(ad.DomainSID.String()).String(); err != nil { - log.Warnf("%s is missing for node %d, skipping AG Membership...", ad.DomainSID.String(), node.ID) - continue + log.Warnf("%s is missing for node %d", ad.DomainSID.String(), node.ID) } else { agMember.EnvironmentKind = ad.Domain.String() agMember.EnvironmentID = domainSID diff --git a/cmd/api/src/api/v2/agi_internal_test.go b/cmd/api/src/api/v2/agi_internal_test.go index 6bb86bbd2e..734faf841c 100644 --- a/cmd/api/src/api/v2/agi_internal_test.go +++ b/cmd/api/src/api/v2/agi_internal_test.go @@ -100,7 +100,7 @@ func TestParseAGMembersFromNodes_(t *testing.T) { func TestParseAGMembersFromNodes_MissingNodeProperties(t *testing.T) { nodes := graph.NodeSet{ - // the parse fn should handle nodes with missing name and missing properties with warnings and no output + // the parse fn should handle nodes with missing name and missing properties with warnings 1: &graph.Node{ ID: 1, Kinds: graph.Kinds{ad.Entity, ad.Domain}, @@ -125,5 +125,5 @@ func TestParseAGMembersFromNodes_MissingNodeProperties(t *testing.T) { SystemSelector: false, }}, 1) - require.Equal(t, 0, len(members)) + require.Equal(t, 2, len(members)) } diff --git a/cmd/api/src/api/v2/agi_test.go b/cmd/api/src/api/v2/agi_test.go index 66a6f30961..e94e6faead 100644 --- a/cmd/api/src/api/v2/agi_test.go +++ b/cmd/api/src/api/v2/agi_test.go @@ -1211,7 +1211,7 @@ func TestResources_ListAssetGroupMembers(t *testing.T) { GetAssetGroup(gomock.Any(), gomock.Any()). Return(assetGroup, nil) mockGraph.EXPECT(). - GetAssetGroupNodes(gomock.Any(), gomock.Any()). + GetAssetGroupNodes(gomock.Any(), gomock.Any(), gomock.Any()). Return(graph.NodeSet{}, fmt.Errorf("GetAssetGroupNodes fail")) }, Test: func(output apitest.Output) { @@ -1228,7 +1228,7 @@ func TestResources_ListAssetGroupMembers(t *testing.T) { GetAssetGroup(gomock.Any(), gomock.Any()). Return(assetGroup, nil) mockGraph.EXPECT(). - GetAssetGroupNodes(gomock.Any(), gomock.Any()). + GetAssetGroupNodes(gomock.Any(), gomock.Any(), gomock.Any()). Return(graph.NodeSet{ 1: &graph.Node{ ID: 1, @@ -1278,7 +1278,7 @@ func TestResources_ListAssetGroupMembers(t *testing.T) { GetAssetGroup(gomock.Any(), gomock.Any()). Return(assetGroup, nil) mockGraph.EXPECT(). - GetAssetGroupNodes(gomock.Any(), gomock.Any()). + GetAssetGroupNodes(gomock.Any(), gomock.Any(), gomock.Any()). Return(graph.NodeSet{ 1: &graph.Node{ ID: 1, @@ -1314,7 +1314,7 @@ func TestResources_ListAssetGroupMembers(t *testing.T) { GetAssetGroup(gomock.Any(), gomock.Any()). Return(assetGroup, nil) mockGraph.EXPECT(). - GetAssetGroupNodes(gomock.Any(), gomock.Any()). + GetAssetGroupNodes(gomock.Any(), gomock.Any(), gomock.Any()). Return(graph.NodeSet{ 1: &graph.Node{ ID: 1, @@ -1355,7 +1355,7 @@ func TestResources_ListAssetGroupMembers(t *testing.T) { GetAssetGroup(gomock.Any(), gomock.Any()). Return(assetGroup, nil) mockGraph.EXPECT(). - GetAssetGroupNodes(gomock.Any(), gomock.Any()). + GetAssetGroupNodes(gomock.Any(), gomock.Any(), gomock.Any()). Return(graph.NodeSet{ 1: &graph.Node{ ID: 1, @@ -1398,7 +1398,7 @@ func TestResources_ListAssetGroupMembers(t *testing.T) { GetAssetGroup(gomock.Any(), gomock.Any()). Return(assetGroup, nil) mockGraph.EXPECT(). - GetAssetGroupNodes(gomock.Any(), gomock.Any()). + GetAssetGroupNodes(gomock.Any(), gomock.Any(), gomock.Any()). Return(graph.NodeSet{ 1: &graph.Node{ ID: 1, @@ -1433,7 +1433,7 @@ func TestResources_ListAssetGroupMembers(t *testing.T) { GetAssetGroup(gomock.Any(), gomock.Any()). Return(assetGroup, nil) mockGraph.EXPECT(). - GetAssetGroupNodes(gomock.Any(), gomock.Any()). + GetAssetGroupNodes(gomock.Any(), gomock.Any(), gomock.Any()). Return(graph.NodeSet{ 1: &graph.Node{ ID: 1, @@ -1574,7 +1574,7 @@ func TestResources_ListAssetGroupMembersCount(t *testing.T) { GetAssetGroup(gomock.Any(), gomock.Any()). Return(assetGroup, nil) mockGraph.EXPECT(). - GetAssetGroupNodes(gomock.Any(), gomock.Any()). + GetAssetGroupNodes(gomock.Any(), gomock.Any(), gomock.Any()). Return(graph.NodeSet{}, fmt.Errorf("GetAssetGroupNodes fail")) }, Test: func(output apitest.Output) { @@ -1591,7 +1591,7 @@ func TestResources_ListAssetGroupMembersCount(t *testing.T) { GetAssetGroup(gomock.Any(), gomock.Any()). Return(assetGroup, nil) mockGraph.EXPECT(). - GetAssetGroupNodes(gomock.Any(), gomock.Any()). + GetAssetGroupNodes(gomock.Any(), gomock.Any(), gomock.Any()). Return(graph.NodeSet{ 1: &graph.Node{ ID: 1, @@ -1627,7 +1627,7 @@ func TestResources_ListAssetGroupMembersCount(t *testing.T) { GetAssetGroup(gomock.Any(), gomock.Any()). Return(assetGroup, nil) mockGraph.EXPECT(). - GetAssetGroupNodes(gomock.Any(), gomock.Any()). + GetAssetGroupNodes(gomock.Any(), gomock.Any(), gomock.Any()). Return(graph.NodeSet{ 1: &graph.Node{ ID: 1, diff --git a/cmd/api/src/api/v2/file_uploads_integration_test.go b/cmd/api/src/api/v2/file_uploads_integration_test.go index f6a15e04d6..64565a33cc 100644 --- a/cmd/api/src/api/v2/file_uploads_integration_test.go +++ b/cmd/api/src/api/v2/file_uploads_integration_test.go @@ -27,11 +27,10 @@ import ( "net/http" "testing" - "github.com/specterops/bloodhound/mediatypes" - "github.com/specterops/bloodhound/src/services/fileupload" - "github.com/specterops/bloodhound/headers" + "github.com/specterops/bloodhound/mediatypes" "github.com/specterops/bloodhound/src/api/v2/integration" + "github.com/specterops/bloodhound/src/services/fileupload" "github.com/specterops/bloodhound/src/test/fixtures/fixtures" "github.com/stretchr/testify/assert" ) @@ -170,7 +169,7 @@ func Test_FileUploadWorkFlowVersion5(t *testing.T) { "v5/ingest/sessions.json", }) - //Assert that we created stuff we expected + // Assert that we created stuff we expected testCtx.AssertIngest(fixtures.IngestAssertions) } @@ -189,7 +188,7 @@ func Test_FileUploadWorkFlowVersion6(t *testing.T) { "v6/ingest/sessions.json", }) - //Assert that we created stuff we expected + // Assert that we created stuff we expected testCtx.AssertIngest(fixtures.IngestAssertions) testCtx.AssertIngest(fixtures.IngestAssertionsv6) testCtx.AssertIngest(fixtures.PropertyAssertions) @@ -240,7 +239,7 @@ func Test_CompressedFileUploadWorkFlowVersion5(t *testing.T) { "v5/ingest/sessions.json", }) - //Assert that we created stuff we expected + // Assert that we created stuff we expected testCtx.AssertIngest(fixtures.IngestAssertions) testCtx.AssertIngest(fixtures.PropertyAssertions) } @@ -260,7 +259,7 @@ func Test_CompressedFileUploadWorkFlowVersion6(t *testing.T) { "v6/ingest/sessions.json", }) - //Assert that we created stuff we expected + // Assert that we created stuff we expected testCtx.AssertIngest(fixtures.IngestAssertions) testCtx.AssertIngest(fixtures.IngestAssertionsv6) testCtx.AssertIngest(fixtures.PropertyAssertions) diff --git a/cmd/api/src/daemons/datapipe/analysis.go b/cmd/api/src/daemons/datapipe/analysis.go index 00d7ef92c2..b76f928c93 100644 --- a/cmd/api/src/daemons/datapipe/analysis.go +++ b/cmd/api/src/daemons/datapipe/analysis.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" - "github.com/specterops/bloodhound/analysis" adAnalysis "github.com/specterops/bloodhound/analysis/ad" "github.com/specterops/bloodhound/dawgs/graph" "github.com/specterops/bloodhound/log" @@ -92,7 +91,7 @@ func RunAnalysisOperations(ctx context.Context, db database.Database, graphDB gr stats.LogStats() } - if err := agi.RunAssetGroupIsolationCollections(ctx, db, graphDB, analysis.GetNodeKindDisplayLabel); err != nil { + if err := agi.RunAssetGroupIsolationCollections(ctx, db, graphDB); err != nil { collectedErrors = append(collectedErrors, fmt.Errorf("asset group isolation collection failed: %w", err)) agiFailed = true } diff --git a/cmd/api/src/database/saved_queries.go b/cmd/api/src/database/saved_queries.go index 1ba0bf0ca9..e171a7c2f1 100644 --- a/cmd/api/src/database/saved_queries.go +++ b/cmd/api/src/database/saved_queries.go @@ -113,7 +113,7 @@ func (s *BloodhoundDB) GetSharedSavedQueries(ctx context.Context, userID uuid.UU func (s *BloodhoundDB) GetPublicSavedQueries(ctx context.Context) (model.SavedQueries, error) { savedQueries := model.SavedQueries{} - result := s.db.WithContext(ctx).Select("sqp.*").Joins("JOIN saved_queries_permissions sqp ON sqp.query_id = saved_queries.id").Where("sqp.public = true").Find(&savedQueries) + result := s.db.WithContext(ctx).Select("saved_queries.*").Joins("JOIN saved_queries_permissions sqp ON sqp.query_id = saved_queries.id").Where("sqp.public = true").Find(&savedQueries) return savedQueries, CheckError(result) } diff --git a/cmd/api/src/go.mod b/cmd/api/src/go.mod index 7d13bf4e76..f9e7d14cb0 100644 --- a/cmd/api/src/go.mod +++ b/cmd/api/src/go.mod @@ -39,7 +39,7 @@ require ( github.com/pquerna/otp v1.4.0 github.com/prometheus/client_golang v1.16.0 github.com/russellhaering/goxmldsig v1.4.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/unrolled/secure v1.13.0 github.com/zenazn/goji v1.0.1 go.uber.org/mock v0.2.0 @@ -79,7 +79,7 @@ require ( github.com/prometheus/procfs v0.11.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/cmd/api/src/go.sum b/cmd/api/src/go.sum index f464503cdd..edf408c38b 100644 --- a/cmd/api/src/go.sum +++ b/cmd/api/src/go.sum @@ -61,8 +61,6 @@ github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/schema v1.2.0 h1:YufUaxZYCKGFuAq3c96BOhjgd5nmXiOY9NGzF247Tsc= -github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E= github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= @@ -211,8 +209,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/unrolled/secure v1.13.0 h1:sdr3Phw2+f8Px8HE5sd1EHdj1aV3yUwed/uZXChLFsk= github.com/unrolled/secure v1.13.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -253,7 +250,7 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -277,8 +274,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/cmd/api/src/model/filter.go b/cmd/api/src/model/filter.go index cc31f297a7..53d8c61228 100644 --- a/cmd/api/src/model/filter.go +++ b/cmd/api/src/model/filter.go @@ -39,7 +39,6 @@ const ( LessThanOrEquals FilterOperator = "lte" Equals FilterOperator = "eq" NotEquals FilterOperator = "neq" - Contains FilterOperator = "in" GreaterThanSymbol string = ">" GreaterThanOrEqualsSymbol string = ">=" @@ -47,7 +46,6 @@ const ( LessThanOrEqualsSymbol string = "<=" EqualsSymbol string = "=" NotEqualsSymbol string = "<>" - ContainsSymbol string = "like" TrueString = "true" FalseString = "false" @@ -81,9 +79,6 @@ func ParseFilterOperator(raw string) (FilterOperator, error) { case NotEquals: return NotEquals, nil - case Contains: - return Contains, nil - default: return "", fmt.Errorf("unknown query parameter filter predicate: %s", raw) } @@ -170,25 +165,14 @@ func (s QueryParameterFilterMap) BuildSQLFilter() (SQLFilter, error) { predicate = EqualsSymbol case NotEquals: predicate = NotEqualsSymbol - case Contains: - predicate = ContainsSymbol default: return SQLFilter{}, fmt.Errorf("invalid filter predicate specified") } - switch predicate { - case ContainsSymbol: - result.WriteString(filter.Name) - result.WriteString(" ") - result.WriteString(predicate) - filter.Value = fmt.Sprintf("%%%s%%", filter.Value) - result.WriteString(" lower(?)") - default: - result.WriteString(filter.Name) - result.WriteString(" ") - result.WriteString(predicate) - result.WriteString(" ?") - } + result.WriteString(filter.Name) + result.WriteString(" ") + result.WriteString(predicate) + result.WriteString(" ?") params = append(params, filter.Value) firstFilter = false diff --git a/cmd/api/src/model/filter_test.go b/cmd/api/src/model/filter_test.go index a9393925fa..370669007f 100644 --- a/cmd/api/src/model/filter_test.go +++ b/cmd/api/src/model/filter_test.go @@ -76,28 +76,19 @@ func TestModel_BuildSQLFilter_Success(t *testing.T) { IsStringData: false, } - stringContains := model.QueryParameterFilter{ - Name: "filtercolumn6", - Operator: model.Contains, - Value: "something", - IsStringData: true, - } - expectedResults := map[string]model.SQLFilter{ - "numericMin": {SQLString: fmt.Sprintf("%s > ?", numericMin.Name), Params: []any{numericMin.Value}}, - "numericMax": {SQLString: fmt.Sprintf("%s < ?", numericMax.Name), Params: []any{numericMax.Value}}, - "stringValue": {SQLString: fmt.Sprintf("%s = ?", stringValue.Name), Params: []any{stringValue.Value}}, - "boolEquals": {SQLString: fmt.Sprintf("%s = ?", boolEquals.Name), Params: []any{boolEquals.Value}}, - "boolNotEquals": {SQLString: fmt.Sprintf("%s <> ?", boolNotEquals.Name), Params: []any{boolNotEquals.Value}}, - "stringContains": {SQLString: fmt.Sprintf("%s like lower(?)", stringContains.Name), Params: []any{stringContains.Value}}, + "numericMin": {SQLString: fmt.Sprintf("%s > ?", numericMin.Name), Params: []any{numericMin.Value}}, + "numericMax": {SQLString: fmt.Sprintf("%s < ?", numericMax.Name), Params: []any{numericMax.Value}}, + "stringValue": {SQLString: fmt.Sprintf("%s = ?", stringValue.Name), Params: []any{stringValue.Value}}, + "boolEquals": {SQLString: fmt.Sprintf("%s = ?", boolEquals.Name), Params: []any{boolEquals.Value}}, + "boolNotEquals": {SQLString: fmt.Sprintf("%s <> ?", boolNotEquals.Name), Params: []any{boolNotEquals.Value}}, } queryParameterFilterMap := model.QueryParameterFilterMap{ - numericMax.Name: model.QueryParameterFilters{numericMin, numericMax}, - stringValue.Name: model.QueryParameterFilters{stringValue}, - boolEquals.Name: model.QueryParameterFilters{boolEquals}, - boolNotEquals.Name: model.QueryParameterFilters{boolNotEquals}, - stringContains.Name: model.QueryParameterFilters{stringContains}, + numericMax.Name: model.QueryParameterFilters{numericMin, numericMax}, + stringValue.Name: model.QueryParameterFilters{stringValue}, + boolEquals.Name: model.QueryParameterFilters{boolEquals}, + boolNotEquals.Name: model.QueryParameterFilters{boolNotEquals}, } result, err := queryParameterFilterMap.BuildSQLFilter() diff --git a/cmd/api/src/model/saved_queries.go b/cmd/api/src/model/saved_queries.go index 7dd4000cfc..076b781604 100644 --- a/cmd/api/src/model/saved_queries.go +++ b/cmd/api/src/model/saved_queries.go @@ -57,7 +57,7 @@ func (s SavedQueries) ValidFilters() map[string][]FilterOperator { "user_id": {Equals, NotEquals}, "name": {Equals, NotEquals}, "query": {Equals, NotEquals}, - "description": {Equals, NotEquals, Contains}, + "description": {Equals, NotEquals}, } } diff --git a/cmd/api/src/model/saved_queries_test.go b/cmd/api/src/model/saved_queries_test.go index 9b485ff3c2..a045509173 100644 --- a/cmd/api/src/model/saved_queries_test.go +++ b/cmd/api/src/model/saved_queries_test.go @@ -41,12 +41,7 @@ func TestSavedQueries_ValidFilters(t *testing.T) { for _, column := range []string{"user_id", "name", "query", "description"} { operators, ok := validFilters[column] require.True(t, ok) - switch column { - case "description": - require.Equal(t, 3, len(operators)) - default: - require.Equal(t, 2, len(operators)) - } + require.Equal(t, 2, len(operators)) } } diff --git a/cmd/api/src/queries/graph.go b/cmd/api/src/queries/graph.go index f26b2a8634..fc8b94cd43 100644 --- a/cmd/api/src/queries/graph.go +++ b/cmd/api/src/queries/graph.go @@ -24,7 +24,6 @@ import ( "fmt" "net/http" "net/url" - "slices" "sort" "strconv" "strings" @@ -132,7 +131,7 @@ func BuildEntityQueryParams(request *http.Request, queryName string, pathDelegat type Graph interface { GetAssetGroupComboNode(ctx context.Context, owningObjectID string, assetGroupTag string) (map[string]any, error) - GetAssetGroupNodes(ctx context.Context, assetGroupTag string) (graph.NodeSet, error) + GetAssetGroupNodes(ctx context.Context, assetGroupTag string, isSystemGroup bool) (graph.NodeSet, error) GetAllShortestPaths(ctx context.Context, startNodeID string, endNodeID string, filter graph.Criteria) (graph.PathSet, error) SearchNodesByName(ctx context.Context, nodeKinds graph.Kinds, nameQuery string, skip int, limit int) ([]model.SearchResult, error) SearchByNameOrObjectID(ctx context.Context, searchValue string, searchType string) (graph.NodeSet, error) @@ -223,42 +222,20 @@ func (s *GraphQuery) GetAssetGroupComboNode(ctx context.Context, owningObjectID }) } -func (s *GraphQuery) GetAssetGroupNodes(ctx context.Context, assetGroupTag string) (graph.NodeSet, error) { +func (s *GraphQuery) GetAssetGroupNodes(ctx context.Context, assetGroupTag string, isSystemGroup bool) (graph.NodeSet, error) { var ( assetGroupNodes graph.NodeSet err error ) - return assetGroupNodes, s.Graph.ReadTransaction(ctx, func(tx graph.Transaction) error { - if assetGroupNodes, err = ops.FetchNodeSet(tx.Nodes().Filterf(func() graph.Criteria { - filters := []graph.Criteria{ - query.KindIn(query.Node(), azure.Entity, ad.Entity), - query.Or( - query.StringContains(query.NodeProperty(common.SystemTags.String()), assetGroupTag), - query.StringContains(query.NodeProperty(common.UserTags.String()), assetGroupTag), - ), - } - return query.And(filters...) - })); err != nil { + err = s.Graph.ReadTransaction(ctx, func(tx graph.Transaction) error { + if assetGroupNodes, err = agi.FetchAssetGroupNodes(tx, assetGroupTag, isSystemGroup); err != nil { return err - } else { - for _, node := range assetGroupNodes { - // We need to filter out nodes that do not contain an exact tag match - var ( - systemTags, _ = node.Properties.Get(common.SystemTags.String()).String() - userTags, _ = node.Properties.Get(common.UserTags.String()).String() - allTags = append(strings.Split(systemTags, " "), strings.Split(userTags, " ")...) - ) - - if !slices.Contains(allTags, assetGroupTag) { - assetGroupNodes.Remove(node.ID) - } else { - node.Properties.Set("type", analysis.GetNodeKindDisplayLabel(node)) - } - } - return nil } + return nil }) + + return assetGroupNodes, err } func (s *GraphQuery) GetAllShortestPaths(ctx context.Context, startNodeID string, endNodeID string, filter graph.Criteria) (graph.PathSet, error) { diff --git a/cmd/api/src/queries/graph_integration_test.go b/cmd/api/src/queries/graph_integration_test.go index 02813468d9..f169f502d8 100644 --- a/cmd/api/src/queries/graph_integration_test.go +++ b/cmd/api/src/queries/graph_integration_test.go @@ -295,13 +295,13 @@ func TestGetAssetGroupNodes(t *testing.T) { }, func(harness integration.HarnessDetails, db graph.Database) { graphQuery := queries.NewGraphQuery(db, cache.Cache{}, config.Configuration{}) - tierZeroNodes, err := graphQuery.GetAssetGroupNodes(context.Background(), harness.AssetGroupNodesHarness.TierZeroTag) + tierZeroNodes, err := graphQuery.GetAssetGroupNodes(context.Background(), harness.AssetGroupNodesHarness.TierZeroTag, true) require.Nil(t, err) - customGroup1Nodes, err := graphQuery.GetAssetGroupNodes(context.Background(), harness.AssetGroupNodesHarness.CustomTag1) + customGroup1Nodes, err := graphQuery.GetAssetGroupNodes(context.Background(), harness.AssetGroupNodesHarness.CustomTag1, false) require.Nil(t, err) - customGroup2Nodes, err := graphQuery.GetAssetGroupNodes(context.Background(), harness.AssetGroupNodesHarness.CustomTag2) + customGroup2Nodes, err := graphQuery.GetAssetGroupNodes(context.Background(), harness.AssetGroupNodesHarness.CustomTag2, false) require.Nil(t, err) require.True(t, tierZeroNodes.Contains(harness.AssetGroupNodesHarness.GroupB)) diff --git a/cmd/api/src/queries/mocks/graph.go b/cmd/api/src/queries/mocks/graph.go index c02f45b1fd..7a6368472c 100644 --- a/cmd/api/src/queries/mocks/graph.go +++ b/cmd/api/src/queries/mocks/graph.go @@ -135,18 +135,18 @@ func (mr *MockGraphMockRecorder) GetAssetGroupComboNode(arg0, arg1, arg2 interfa } // GetAssetGroupNodes mocks base method. -func (m *MockGraph) GetAssetGroupNodes(arg0 context.Context, arg1 string) (graph.NodeSet, error) { +func (m *MockGraph) GetAssetGroupNodes(arg0 context.Context, arg1 string, arg2 bool) (graph.NodeSet, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAssetGroupNodes", arg0, arg1) + ret := m.ctrl.Call(m, "GetAssetGroupNodes", arg0, arg1, arg2) ret0, _ := ret[0].(graph.NodeSet) ret1, _ := ret[1].(error) return ret0, ret1 } // GetAssetGroupNodes indicates an expected call of GetAssetGroupNodes. -func (mr *MockGraphMockRecorder) GetAssetGroupNodes(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockGraphMockRecorder) GetAssetGroupNodes(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAssetGroupNodes", reflect.TypeOf((*MockGraph)(nil).GetAssetGroupNodes), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAssetGroupNodes", reflect.TypeOf((*MockGraph)(nil).GetAssetGroupNodes), arg0, arg1, arg2) } // GetEntityByObjectId mocks base method. diff --git a/cmd/api/src/services/agi/agi.go b/cmd/api/src/services/agi/agi.go index 083556f800..cbb72cf5bb 100644 --- a/cmd/api/src/services/agi/agi.go +++ b/cmd/api/src/services/agi/agi.go @@ -19,7 +19,10 @@ package agi import ( "context" + "slices" + "strings" + "github.com/specterops/bloodhound/analysis" "github.com/specterops/bloodhound/dawgs/graph" "github.com/specterops/bloodhound/dawgs/ops" "github.com/specterops/bloodhound/dawgs/query" @@ -36,7 +39,38 @@ type AgiData interface { CreateAssetGroupCollection(ctx context.Context, collection model.AssetGroupCollection, entries model.AssetGroupCollectionEntries) error } -func RunAssetGroupIsolationCollections(ctx context.Context, db AgiData, graphDB graph.Database, kindGetter func(*graph.Node) string) error { +func FetchAssetGroupNodes(tx graph.Transaction, assetGroupTag string, isSystemGroup bool) (graph.NodeSet, error) { + var ( + assetGroupNodes graph.NodeSet + tagPropertyStr = common.SystemTags.String() + err error + ) + + if !isSystemGroup { + tagPropertyStr = common.UserTags.String() + } + + if assetGroupNodes, err = ops.FetchNodeSet(tx.Nodes().Filterf(func() graph.Criteria { + return query.And( + query.KindIn(query.Node(), ad.Entity, azure.Entity), + query.StringContains(query.NodeProperty(tagPropertyStr), assetGroupTag), + ) + })); err != nil { + return graph.NodeSet{}, err + } else { + // tags are space seperated, so we have to loop and remove any that are not exact matches + for _, node := range assetGroupNodes { + tags, _ := node.Properties.Get(tagPropertyStr).String() + if !slices.Contains(strings.Split(tags, " "), assetGroupTag) { + assetGroupNodes.Remove(node.ID) + } + } + } + + return assetGroupNodes, err +} + +func RunAssetGroupIsolationCollections(ctx context.Context, db AgiData, graphDB graph.Database) error { defer log.Measure(log.LevelInfo, "Asset Group Isolation Collections")() if assetGroups, err := db.GetAllAssetGroups(ctx, "", model.SQLFilter{}); err != nil { @@ -44,18 +78,7 @@ func RunAssetGroupIsolationCollections(ctx context.Context, db AgiData, graphDB } else { return graphDB.WriteTransaction(ctx, func(tx graph.Transaction) error { for _, assetGroup := range assetGroups { - if assetGroupNodes, err := ops.FetchNodes(tx.Nodes().Filterf(func() graph.Criteria { - tagPropertyStr := common.SystemTags.String() - - if !assetGroup.SystemGroup { - tagPropertyStr = common.UserTags.String() - } - - return query.And( - query.KindIn(query.Node(), ad.Entity, azure.Entity), - query.StringContains(query.NodeProperty(tagPropertyStr), assetGroup.Tag), - ) - })); err != nil { + if assetGroupNodes, err := FetchAssetGroupNodes(tx, assetGroup.Tag, assetGroup.SystemGroup); err != nil { return err } else { var ( @@ -65,16 +88,18 @@ func RunAssetGroupIsolationCollections(ctx context.Context, db AgiData, graphDB } ) - for idx, node := range assetGroupNodes { + idx := 0 + for _, node := range assetGroupNodes { if objectID, err := node.Properties.Get(common.ObjectID.String()).String(); err != nil { log.Errorf("Node %d that does not have valid %s property", node.ID, common.ObjectID) } else { entries[idx] = model.AssetGroupCollectionEntry{ ObjectID: objectID, - NodeLabel: kindGetter(node), + NodeLabel: analysis.GetNodeKindDisplayLabel(node), Properties: node.Properties.Map, } } + idx++ } // Enter a collection, even if it's empty to signal that we did do a tagging/collection run diff --git a/cmd/api/src/services/fileupload/file_upload.go b/cmd/api/src/services/fileupload/file_upload.go index ca5b211b12..87f6caf9fc 100644 --- a/cmd/api/src/services/fileupload/file_upload.go +++ b/cmd/api/src/services/fileupload/file_upload.go @@ -18,7 +18,6 @@ package fileupload import ( - "bufio" "context" "errors" "fmt" @@ -27,6 +26,7 @@ import ( "os" "time" + "github.com/specterops/bloodhound/bomenc" "github.com/specterops/bloodhound/headers" "github.com/specterops/bloodhound/mediatypes" "github.com/specterops/bloodhound/src/model/ingest" @@ -120,18 +120,12 @@ func WriteAndValidateZip(src io.Reader, dst io.Writer) error { } func WriteAndValidateJSON(src io.Reader, dst io.Writer) error { - tr := io.TeeReader(src, dst) - bufReader := bufio.NewReader(tr) - if b, err := bufReader.Peek(3); err != nil { + normalizedContent, err := bomenc.NormalizeToUTF8(src) + if err != nil { return err - } else { - if b[0] == UTF8BOM1 && b[1] == UTF8BOM2 && b[2] == UTF8BMO3 { - if _, err := bufReader.Discard(3); err != nil { - return err - } - } } - _, err := ValidateMetaTag(bufReader, true) + tr := io.TeeReader(normalizedContent, dst) + _, err = ValidateMetaTag(tr, true) return err } @@ -147,7 +141,7 @@ func SaveIngestFile(location string, request *http.Request) (string, model.FileT } else if utils.HeaderMatches(request.Header, headers.ContentType.String(), ingest.AllowedZipFileUploadTypes...) { return tempFile.Name(), model.FileTypeZip, WriteAndValidateFile(fileData, tempFile, WriteAndValidateZip) } else { - //We should never get here since this is checked a level above + // We should never get here since this is checked a level above return "", model.FileTypeJson, fmt.Errorf("invalid content type for ingest file") } } diff --git a/cmd/api/src/services/fileupload/file_upload_test.go b/cmd/api/src/services/fileupload/file_upload_test.go index 59943adf18..9d81504440 100644 --- a/cmd/api/src/services/fileupload/file_upload_test.go +++ b/cmd/api/src/services/fileupload/file_upload_test.go @@ -18,6 +18,7 @@ package fileupload import ( "bytes" + "errors" "io" "os" "strings" @@ -27,42 +28,9 @@ import ( "github.com/stretchr/testify/assert" ) -func TestWriteAndValidateJSON(t *testing.T) { - t.Run("trigger invalid json on bad json", func(t *testing.T) { - var ( - writer = bytes.Buffer{} - badJSON = strings.NewReader("{[]}") - ) - err := WriteAndValidateJSON(badJSON, &writer) - assert.ErrorIs(t, err, ErrInvalidJSON) - }) - - t.Run("succeed on good json", func(t *testing.T) { - var ( - writer = bytes.Buffer{} - goodJSON = strings.NewReader(`{"meta": {"methods": 0, "type": "sessions", "count": 0, "version": 5}, "data": []}`) - ) - err := WriteAndValidateJSON(goodJSON, &writer) - assert.Nil(t, err) - }) - - t.Run("succeed on utf-8 BOM json", func(t *testing.T) { - var ( - writer = bytes.Buffer{} - ) - - file, err := os.Open("../../test/fixtures/fixtures/utf8bomjson.json") - assert.Nil(t, err) - err = WriteAndValidateJSON(io.Reader(file), &writer) - assert.Nil(t, err) - }) -} - func TestWriteAndValidateZip(t *testing.T) { t.Run("valid zip file is ok", func(t *testing.T) { - var ( - writer = bytes.Buffer{} - ) + writer := bytes.Buffer{} file, err := os.Open("../../test/fixtures/fixtures/goodzip.zip") assert.Nil(t, err) @@ -81,3 +49,86 @@ func TestWriteAndValidateZip(t *testing.T) { assert.Equal(t, err, ingest.ErrInvalidZipFile) }) } + +func TestWriteAndValidateJSON(t *testing.T) { + tests := []struct { + name string + input []byte + expectedOutput []byte + expectedError error + }{ + { + name: "UTF-8 without BOM", + input: []byte(`{"meta": {"type": "domains", "version": 4, "count": 1}, "data": [{"domain": "example.com"}]}`), + expectedOutput: []byte(`{"meta": {"type": "domains", "version": 4, "count": 1}, "data": [{"domain": "example.com"}]}`), + expectedError: nil, + }, + { + name: "UTF-8 with BOM", + input: append([]byte{0xEF, 0xBB, 0xBF}, []byte(`{"meta": {"type": "domains", "version": 4, "count": 1}, "data": [{"domain": "example.com"}]}`)...), + expectedOutput: []byte(`{"meta": {"type": "domains", "version": 4, "count": 1}, "data": [{"domain": "example.com"}]}`), + expectedError: nil, + }, + { + name: "UTF-16BE with BOM", + input: []byte{0xFE, 0xFF, 0x00, 0x7B, 0x00, 0x22, 0x00, 0x6D, 0x00, 0x65, 0x00, 0x74, 0x00, 0x61, 0x00, 0x22, 0x00, 0x3A, 0x00, 0x20, 0x00, 0x7B, 0x00, 0x22, 0x00, 0x74, 0x00, 0x79, 0x00, 0x70, 0x00, 0x65, 0x00, 0x22, 0x00, 0x3A, 0x00, 0x20, 0x00, 0x22, 0x00, 0x64, 0x00, 0x6F, 0x00, 0x6D, 0x00, 0x61, 0x00, 0x69, 0x00, 0x6E, 0x00, 0x73, 0x00, 0x22, 0x00, 0x2C, 0x00, 0x20, 0x00, 0x22, 0x00, 0x76, 0x00, 0x65, 0x00, 0x72, 0x00, 0x73, 0x00, 0x69, 0x00, 0x6F, 0x00, 0x6E, 0x00, 0x22, 0x00, 0x3A, 0x00, 0x20, 0x00, 0x34, 0x00, 0x2C, 0x00, 0x20, 0x00, 0x22, 0x00, 0x63, 0x00, 0x6F, 0x00, 0x75, 0x00, 0x6E, 0x00, 0x74, 0x00, 0x22, 0x00, 0x3A, 0x00, 0x20, 0x00, 0x31, 0x00, 0x7D, 0x00, 0x2C, 0x00, 0x20, 0x00, 0x22, 0x00, 0x64, 0x00, 0x61, 0x00, 0x74, 0x00, 0x61, 0x00, 0x22, 0x00, 0x3A, 0x00, 0x20, 0x00, 0x5B, 0x00, 0x7B, 0x00, 0x22, 0x00, 0x64, 0x00, 0x6F, 0x00, 0x6D, 0x00, 0x61, 0x00, 0x69, 0x00, 0x6E, 0x00, 0x22, 0x00, 0x3A, 0x00, 0x20, 0x00, 0x22, 0x00, 0x65, 0x00, 0x78, 0x00, 0x61, 0x00, 0x6D, 0x00, 0x70, 0x00, 0x6C, 0x00, 0x65, 0x00, 0x2E, 0x00, 0x63, 0x00, 0x6F, 0x00, 0x6D, 0x00, 0x22, 0x00, 0x7D, 0x00, 0x5D, 0x00, 0x7D}, + expectedOutput: []byte{0x7b, 0x22, 0x6d, 0x65, 0x74, 0x61, 0x22, 0x3a, 0x20, 0x7b, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3a, 0x20, 0x22, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x22, 0x2c, 0x20, 0x22, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x3a, 0x20, 0x34, 0x2c, 0x20, 0x22, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x3a, 0x20, 0x31, 0x7d, 0x2c, 0x20, 0x22, 0x64, 0x61, 0x74, 0x61, 0x22, 0x3a, 0x20, 0x5b, 0x7b, 0x22, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0x3a, 0x20, 0x22, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x22, 0x7d, 0x5d, 0x7d}, + expectedError: nil, + }, + { + name: "Missing meta tag", + input: []byte(`{"data": [{"domain": "example.com"}]}`), + expectedOutput: []byte(`{"data": [{"domain": "example.com"}]}`), + expectedError: ingest.ErrMetaTagNotFound, + }, + { + name: "Missing data tag", + input: []byte(`{"meta": {"type": "domains", "version": 4, "count": 1}}`), + expectedOutput: []byte(`{"meta": {"type": "domains", "version": 4, "count": 1}}`), + expectedError: ingest.ErrDataTagNotFound, + }, + // NOTE: this test discovers a bug where invalid JSON files are not being invalidated due to the current + // implemenation of ValidateMetaTag of decoding each token. + // { + // name: "Invalid JSON", + // input: []byte(`{"meta": {"type": "domains", "version": 4, "count": 1}, "data": [{"domain": "example.com"`), + // expectedOutput: []byte(`{"meta": {"type": "domains", "version": 4, "count": 1}, "data": [{"domain": "example.com"`), + // expectedError: ErrInvalidJSON, + // }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := bytes.NewReader(tt.input) + dst := &bytes.Buffer{} + + err := WriteAndValidateJSON(src, dst) + if tt.expectedError != nil { + assert.Error(t, err) + assert.ErrorIs(t, err, tt.expectedError) + } else { + assert.NoError(t, err) + } + assert.Equal(t, tt.expectedOutput, dst.Bytes()) + }) + } +} + +func TestWriteAndValidateJSON_NormalizationError(t *testing.T) { + src := &ErrorReader{err: errors.New("read error")} + dst := &bytes.Buffer{} + + err := WriteAndValidateJSON(src, dst) + + assert.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidJSON) +} + +// ErrorReader is a mock reader that always returns an error +type ErrorReader struct { + err error +} + +func (er *ErrorReader) Read(p []byte) (n int, err error) { + return 0, er.err +} diff --git a/cmd/ui/src/ducks/auth/authSlice.ts b/cmd/ui/src/ducks/auth/authSlice.ts index 17ef5f7cad..b2ad7619fe 100644 --- a/cmd/ui/src/ducks/auth/authSlice.ts +++ b/cmd/ui/src/ducks/auth/authSlice.ts @@ -80,7 +80,7 @@ export const login = createAsyncThunk( ); export const logout = createAsyncThunk('auth/logout', async () => { - return await apiClient.logout().catch(() => { }); + return await apiClient.logout().catch(() => {}); }); export const initialize = createAsyncThunk< diff --git a/go.work b/go.work index c5077a96c0..21c94c9931 100644 --- a/go.work +++ b/go.work @@ -14,11 +14,12 @@ // // SPDX-License-Identifier: Apache-2.0 -go 1.21 +go 1.21.3 use ( ./cmd/api/src ./packages/go/analysis + ./packages/go/bomenc ./packages/go/cache ./packages/go/conftool ./packages/go/crypto diff --git a/packages/go/analysis/go.mod b/packages/go/analysis/go.mod index 07f6010adc..b56c4506d8 100644 --- a/packages/go/analysis/go.mod +++ b/packages/go/analysis/go.mod @@ -21,7 +21,7 @@ go 1.21 require ( github.com/RoaringBitmap/roaring v1.3.0 github.com/bloodhoundad/azurehound/v2 v2.0.1 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 go.uber.org/mock v0.2.0 ) diff --git a/packages/go/analysis/go.sum b/packages/go/analysis/go.sum index 825742456a..5f1ca3c69e 100644 --- a/packages/go/analysis/go.sum +++ b/packages/go/analysis/go.sum @@ -7,7 +7,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/packages/go/bomenc/encodings.go b/packages/go/bomenc/encodings.go new file mode 100644 index 0000000000..a3add1c4bf --- /dev/null +++ b/packages/go/bomenc/encodings.go @@ -0,0 +1,198 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package bomenc + +import ( + "encoding/binary" + "io" +) + +// Encoding interface defines the methods that all encoding types must implement. +// This interface provides a unified way to handle different encodings throughout the package, +// allowing us to treat all encodings uniformly. This design facilitates easy extension +// and manipulation of different encoding types without altering the core logic. +type Encoding interface { + // Sequence returns the byte sequence that represents the Byte Order Mark (BOM) for this encoding. + // This method is crucial for identifying the specific byte sequence that indicates + // this encoding at the start of a file. + Sequence() []byte + + // String returns a human-readable string representation of the encoding. + // This is particularly useful for logging and user interfaces, providing + // a user-friendly name for the encoding. + String() string + + // HasSequence checks if the given data starts with this encoding's BOM sequence. + // This method allows for efficient checking of whether a given byte slice + // begins with this encoding's BOM, which is essential for encoding detection. + HasSequence(data Peeker) bool +} + +// Peeker interface defines a single method for introspecing the first n number of bytes in the underlying structure without modifying its read state or advancing its cursor. +type Peeker interface { + Peek(n int) ([]byte, error) +} + +// bomEncoding is the concrete implementation of the Encoding interface. +// It encapsulates all necessary information and behavior for a specific encoding, +// providing a consistent structure for handling different encodings. This approach +// allows us to create instances for each supported encoding while maintaining +// a uniform interface for interaction. +type bomEncoding struct { + encodingType string // A human-readable name for the encoding + sequence []byte // The BOM sequence for this encoding + hasSequenceFunc func(data Peeker) bool // Function to check if data starts with this encoding's BOM +} + +// String returns the human-readable name of the encoding. +// This method fulfills the Encoding interface and provides a simple way +// to get a string representation of the encoding. +func (s bomEncoding) String() string { + return s.encodingType +} + +// Sequence returns the BOM sequence for this encoding. +// This method fulfills the Encoding interface and provides access to the BOM sequence, +// which is essential for encoding detection and writing files with proper BOMs. +func (s bomEncoding) Sequence() []byte { + return s.sequence +} + +// HasSequence checks if the given data starts with this encoding's BOM sequence. +// This method fulfills the Encoding interface and provides a way to check for +// the presence of this encoding's BOM, which is crucial for encoding detection. +func (s bomEncoding) HasSequence(data Peeker) bool { + return s.hasSequenceFunc(data) +} + +// The following functions are used to check for specific encoding BOMs. +// By defining these as separate functions, we can easily reuse them +// and potentially extend them if more complex checking is needed in the future. +// This approach also keeps the bomEncoding struct clean and simple. + +func isUTF32BE(data Peeker) bool { + if buf, err := data.Peek(4); err != nil { + return false + } else { + return buf[0] == 0x00 && buf[1] == 0x00 && buf[2] == 0xFE && buf[3] == 0xFF + } +} + +func isUTF32LE(data Peeker) bool { + if buf, err := data.Peek(4); err != nil { + return false + } else if buf[0] != 0xFF || buf[1] != 0xFE || buf[2] != 0x00 || buf[3] != 0x00 { + return false + } else if buf, err := data.Peek(64); err != nil && err != io.EOF { // BOM + sample code points to check for valid sequences + return false + } else if err != nil && err == io.EOF && len(buf)%4 != 0 { + return false + } else { + // Check for valid UTF-32LE sequences + for i := 4; i+3 < len(buf); i += 4 { + codePoint := binary.LittleEndian.Uint32(buf[i : i+4]) + if codePoint > 0x10FFFF { + return false + } + } + // NOTE: There is an edge case where data may may include the UTF16LE BOM and a NULL code point + // followed by sampled code points that, when calculated, all fall within the unicode range. In this + // case, distinguishing between UTF32LE and UTF16LE encoded data is impossible from just the BOM + data. + // With the probability of occurence being low, we're opting to return true + return true + } +} + +func isUTF8(data Peeker) bool { + if buf, err := data.Peek(3); err != nil { + return false + } else { + return buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF + } +} + +func isUTF16BE(data Peeker) bool { + if buf, err := data.Peek(2); err != nil { + return false + } else { + return buf[0] == 0xFE && buf[1] == 0xFF + } +} + +func isUTF16LE(data Peeker) bool { + if buf, err := data.Peek(2); err != nil { + return false + } else { + if buf[0] == 0xFF && buf[1] == 0xFE { + if buf, err := data.Peek(4); err != nil { + return err == io.EOF && len(buf) == 2 // true: has UTF16LE BOM w/ no data, false: is invalid UTF16LE encoding + } else { + return !isUTF32LE(data) // true: is UTF16LE data, false: is UTF32LE data + } + } + return false + } +} + +// The following variables define the supported encodings. +// By defining these as package-level variables, we allow easy reference +// throughout the package and by users of the package. This design also +// facilitates potential future extension by simply adding new encoding variables. + +// Unknown represents an unknown or unrecognized encoding. +// Having an Unknown encoding allows us to handle cases where +// the encoding cannot be determined, providing a fallback option. +var Unknown Encoding = bomEncoding{ + encodingType: "Unknown", + sequence: nil, // Unknown encoding has no BOM sequence + hasSequenceFunc: func(data Peeker) bool { return false }, +} + +// UTF8 represents the UTF-8 encoding. +var UTF8 Encoding = bomEncoding{ + encodingType: "UTF-8", + sequence: []byte{0xEF, 0xBB, 0xBF}, // UTF-8 BOM sequence + hasSequenceFunc: isUTF8, +} + +// UTF16BE represents the UTF-16 Big Endian encoding. +var UTF16BE Encoding = bomEncoding{ + encodingType: "UTF-16 BE", + sequence: []byte{0xFE, 0xFF}, // UTF-16 BE BOM sequence + hasSequenceFunc: isUTF16BE, +} + +// UTF16LE represents the UTF-16 Little Endian encoding. +var UTF16LE Encoding = bomEncoding{ + encodingType: "UTF-16 LE", + sequence: []byte{0xFF, 0xFE}, // UTF-16 LE BOM sequence + hasSequenceFunc: isUTF16LE, +} + +// UTF32BE represents the UTF-32 Big Endian encoding. +var UTF32BE Encoding = bomEncoding{ + encodingType: "UTF-32 BE", + sequence: []byte{0x00, 0x00, 0xFE, 0xFF}, // UTF-32 BE BOM sequence + hasSequenceFunc: isUTF32BE, +} + +// UTF32LE represents the UTF-32 Little Endian encoding. +var UTF32LE Encoding = bomEncoding{ + encodingType: "UTF-32 LE", + sequence: []byte{0xFF, 0xFE, 0x00, 0x00}, // UTF-32 LE BOM sequence + hasSequenceFunc: isUTF32LE, +} diff --git a/packages/go/bomenc/encodings_test.go b/packages/go/bomenc/encodings_test.go new file mode 100644 index 0000000000..73a617a7f0 --- /dev/null +++ b/packages/go/bomenc/encodings_test.go @@ -0,0 +1,233 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package bomenc + +import ( + "bufio" + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEncodingInterface(t *testing.T) { + encodings := []struct { + name string + encoding Encoding + }{ + {name: "Unknown", encoding: Unknown}, + {name: "UTF8", encoding: UTF8}, + {name: "UTF16BE", encoding: UTF16BE}, + {name: "UTF16LE", encoding: UTF16LE}, + {name: "UTF32BE", encoding: UTF32BE}, + {name: "UTF32LE", encoding: UTF32LE}, + } + + for _, tt := range encodings { + t.Run(tt.name, func(t *testing.T) { + assert.NotEmpty(t, tt.encoding.String(), "Encoding String() should not be empty") + if tt.encoding.String() != Unknown.String() { + assert.NotEmpty(t, tt.encoding.Sequence(), "Encoding Sequence() should not be empty for non-Unknown encodings") + } + // Test HasSequence method + if tt.encoding.String() != Unknown.String() { + reader := bufio.NewReader(bytes.NewReader(tt.encoding.Sequence())) + assert.True(t, tt.encoding.HasSequence(reader), "HasSequence() should return true for its own sequence") + } + }) + } +} + +func TestEncodingValues(t *testing.T) { + tests := []struct { + name string + encoding Encoding + expectedType string + expectedSeq []byte + }{ + { + name: "Unknown", + encoding: Unknown, + expectedType: "Unknown", + expectedSeq: nil, + }, + { + name: "UTF-8", + encoding: UTF8, + expectedType: "UTF-8", + expectedSeq: []byte{0xEF, 0xBB, 0xBF}, + }, + { + name: "UTF-16 BE", + encoding: UTF16BE, + expectedType: "UTF-16 BE", + expectedSeq: []byte{0xFE, 0xFF}, + }, + { + name: "UTF-16 LE", + encoding: UTF16LE, + expectedType: "UTF-16 LE", + expectedSeq: []byte{0xFF, 0xFE}, + }, + { + name: "UTF-32 BE", + encoding: UTF32BE, + expectedType: "UTF-32 BE", + expectedSeq: []byte{0x00, 0x00, 0xFE, 0xFF}, + }, + { + name: "UTF-32 LE", + encoding: UTF32LE, + expectedType: "UTF-32 LE", + expectedSeq: []byte{0xFF, 0xFE, 0x00, 0x00}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedType, tt.encoding.String(), "Encoding type should match") + assert.Equal(t, tt.expectedSeq, tt.encoding.Sequence(), "Encoding sequence should match") + if tt.encoding.String() != Unknown.String() { + assert.True(t, tt.encoding.HasSequence(bufio.NewReader(bytes.NewReader(tt.expectedSeq))), "HasSequence() should return true for the expected sequence") + } + }) + } +} + +func TestBOMEncoding(t *testing.T) { + testCases := []struct { + name string + encoding bomEncoding + expectedString string + expectedSeq []byte + testData []byte + hasSequence bool + }{ + { + name: "Custom encoding", + encoding: bomEncoding{ + encodingType: "Custom", + sequence: []byte{0x01, 0x02, 0x03}, + hasSequenceFunc: func(input Peeker) bool { + if data, err := input.Peek(3); err == nil { + return len(data) >= 3 && data[0] == 0x01 && data[1] == 0x02 && data[2] == 0x03 + } + return false + }, + }, + expectedString: "Custom", + expectedSeq: []byte{0x01, 0x02, 0x03}, + testData: []byte{0x01, 0x02, 0x03, 0x04}, + hasSequence: true, + }, + { + name: "Empty encoding", + encoding: bomEncoding{ + encodingType: "", + sequence: []byte{}, + hasSequenceFunc: func(input Peeker) bool { + if data, err := input.Peek(0); err == io.EOF { + return len(data) == 0 + } + return false + }, + }, + expectedString: "", + expectedSeq: []byte{}, + testData: []byte{0x01, 0x02, 0x03}, + hasSequence: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expectedString, tc.encoding.String(), "bomEncoding String() should return correct value") + assert.Equal(t, tc.expectedSeq, tc.encoding.Sequence(), "bomEncoding Sequence() should return correct value") + assert.Equal(t, tc.hasSequence, tc.encoding.HasSequence(bufio.NewReader(bytes.NewReader(tc.testData))), "bomEncoding HasSequence() should return correct value") + }) + } +} + +func TestEncodingEquality(t *testing.T) { + testCases := []struct { + name string + enc1 Encoding + enc2 Encoding + expected bool + }{ + { + name: "Same encoding", + enc1: UTF8, + enc2: UTF8, + expected: true, + }, + { + name: "Different encodings", + enc1: UTF8, + enc2: UTF16BE, + expected: false, + }, + { + name: "Unknown and other encoding", + enc1: Unknown, + enc2: UTF8, + expected: false, + }, + { + name: "Both Unknown", + enc1: Unknown, + enc2: Unknown, + expected: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, tc.enc1.String() == tc.enc2.String(), "Encoding equality check should be correct") + }) + } +} + +func TestHasSequence(t *testing.T) { + testCases := []struct { + name string + encoding Encoding + input []byte + expected bool + }{ + {"UTF-8 with correct BOM", UTF8, []byte{0xEF, 0xBB, 0xBF, 0x68, 0x65, 0x6C, 0x6C, 0x6F}, true}, + {"UTF-8 without BOM", UTF8, []byte{0x68, 0x65, 0x6C, 0x6C, 0x6F}, false}, + {"UTF-16BE with correct BOM", UTF16BE, []byte{0xFE, 0xFF, 0x00, 0x68, 0x00, 0x65}, true}, + {"UTF-16BE without BOM", UTF16BE, []byte{0x00, 0x68, 0x00, 0x65}, false}, + {"UTF-16LE with correct BOM", UTF16LE, []byte{0xFF, 0xFE, 0x68, 0x00, 0x65, 0x00}, true}, + {"UTF-16LE without BOM", UTF16LE, []byte{0x68, 0x00, 0x65, 0x00}, false}, + {"UTF-32BE with correct BOM", UTF32BE, []byte{0x00, 0x00, 0xFE, 0xFF, 0x00, 0x00, 0x00, 0x68}, true}, + {"UTF-32BE without BOM", UTF32BE, []byte{0x00, 0x00, 0x00, 0x68}, false}, + {"UTF-32LE with correct BOM", UTF32LE, []byte{0xFF, 0xFE, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00}, true}, + {"UTF-32LE without BOM", UTF32LE, []byte{0x68, 0x00, 0x00, 0x00}, false}, + {"Unknown encoding", Unknown, []byte{0x68, 0x65, 0x6C, 0x6C, 0x6F}, false}, + {"Empty input", UTF8, []byte{}, false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := tc.encoding.HasSequence(bufio.NewReader(bytes.NewReader(tc.input))) + assert.Equal(t, tc.expected, result, "HasSequence() should correctly identify BOM presence") + }) + } +} diff --git a/packages/go/bomenc/go.mod b/packages/go/bomenc/go.mod new file mode 100644 index 0000000000..580d23fe3e --- /dev/null +++ b/packages/go/bomenc/go.mod @@ -0,0 +1,33 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +module github.com/specterops/bloodhound/bomenc + +go 1.21.3 + +require ( + github.com/stretchr/testify v1.9.0 + golang.org/x/text v0.17.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/packages/go/bomenc/go.sum b/packages/go/bomenc/go.sum new file mode 100644 index 0000000000..df75b7807b --- /dev/null +++ b/packages/go/bomenc/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/go/bomenc/normalize.go b/packages/go/bomenc/normalize.go new file mode 100644 index 0000000000..39dc0e99ad --- /dev/null +++ b/packages/go/bomenc/normalize.go @@ -0,0 +1,85 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package bomenc + +import ( + "bufio" + "errors" + "io" + + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/encoding/unicode/utf32" +) + +// DetectBOMEncoding detects the byte order mark in the given bytes and returns the corresponding Encoding. +// This function is crucial for determining the encoding of incoming data based on its BOM. +func DetectBOMEncoding(data Peeker) Encoding { + if UTF8.HasSequence(data) { + return UTF8 + } else if UTF16BE.HasSequence(data) { + return UTF16BE + } else if UTF16LE.HasSequence(data) { // NOTE: The byte sequence for UTF16LE matches the first two bytes for UTF32LE; this check's implementation ensures that the BOM does not accidently match UTF32LE + return UTF16LE + } else if UTF32LE.HasSequence(data) { + return UTF32LE + } else if UTF32BE.HasSequence(data) { + return UTF32BE + } else { + return Unknown + } +} + +// NormalizeToUTF8 converts the input to UTF-8, removing any BOM. +// This function is the main entry point for normalizing data from an io.Reader. +// It's useful when working with streams of data, such as file input. +func NormalizeToUTF8(input io.Reader) (io.Reader, error) { + buf := bufio.NewReader(input) + switch DetectBOMEncoding(buf).String() { + case UTF8.String(): + if _, err := buf.Discard(3); err != nil { + return nil, err + } else { + return buf, nil + } + case UTF16LE.String(): + return transformUTF16toUTF8(unicode.LittleEndian, buf) + case UTF16BE.String(): + return transformUTF16toUTF8(unicode.BigEndian, buf) + case UTF32LE.String(): + return transformUTF32toUTF8(utf32.LittleEndian, buf) + case UTF32BE.String(): + return transformUTF32toUTF8(utf32.BigEndian, buf) + default: + // Either Unknown or no BOM + return unicode.UTF8.NewDecoder().Reader(buf), nil + } +} + +// transformUTF16toUTF8 aids NormalizeToUTF8 in converting UTF16 data with a given endienness into UTF8 +func transformUTF16toUTF8(endianness unicode.Endianness, data io.Reader) (io.Reader, error) { + utf16leDecoder := unicode.UTF16(endianness, unicode.UseBOM).NewDecoder() + return utf16leDecoder.Reader(data), nil +} + +// transformUTF32toUTF8 aids NormalizeToUTF8 in converting UTF32 data with a given endienness into UTF8 +func transformUTF32toUTF8(endianness utf32.Endianness, data io.Reader) (io.Reader, error) { + utf16leDecoder := utf32.UTF32(endianness, utf32.UseBOM).NewDecoder() + return utf16leDecoder.Reader(data), nil +} + +// ErrUnknownEncodingInvalidUTF8 ... +var ErrUnknownEncodingInvalidUTF8 = errors.New("unknown encoding and not a valid UTF-8") diff --git a/packages/go/bomenc/normalize_test.go b/packages/go/bomenc/normalize_test.go new file mode 100644 index 0000000000..6544d92333 --- /dev/null +++ b/packages/go/bomenc/normalize_test.go @@ -0,0 +1,250 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package bomenc + +import ( + "bufio" + "bytes" + "errors" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/encoding/unicode" +) + +func TestDetectBOMEncoding(t *testing.T) { + tests := []struct { + name string + input []byte + expected Encoding + }{ + { + name: "UTF-8 BOM", + input: []byte{0xEF, 0xBB, 0xBF, 0x68, 0x65, 0x6C, 0x6C, 0x6F}, + expected: UTF8, + }, + { + name: "UTF-16BE BOM", + input: []byte{0xFE, 0xFF, 0x00, 0x68, 0x00, 0x65, 0x00, 0x6C, 0x00, 0x6C, 0x00, 0x6F}, + expected: UTF16BE, + }, + { + name: "UTF-16LE BOM", + input: []byte{0xFF, 0xFE, 0x68, 0x00, 0x65, 0x00, 0x6C, 0x00, 0x6C, 0x00, 0x6F, 0x00}, + expected: UTF16LE, + }, + { + name: "UTF-32BE BOM", + input: []byte{0x00, 0x00, 0xFE, 0xFF, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x65}, + expected: UTF32BE, + }, + { + name: "UTF-32LE BOM", + input: []byte{0xFF, 0xFE, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00}, + expected: UTF32LE, + }, + { + name: "No BOM", + input: []byte{0x68, 0x65, 0x6C, 0x6C, 0x6F}, + expected: Unknown, + }, + { + name: "Empty input", + input: []byte{}, + expected: Unknown, + }, + { + name: "Incomplete UTF-16LE BOM (should not be detected as UTF-16LE)", + input: []byte{0xFF, 0xFE, 0x68}, + expected: Unknown, + }, + { + name: "Incomplete UTF-32LE BOM (should not be detected as UTF-32LE)", + input: []byte{0xFF, 0xFE, 0x00}, + expected: Unknown, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reader := bufio.NewReader(bytes.NewReader(tt.input)) + result := DetectBOMEncoding(reader) + assert.Equal(t, tt.expected.String(), result.String(), "DetectBOMEncoding() should return the correct encoding") + }) + } +} + +func TestNormalizeToUTF8(t *testing.T) { + tests := []struct { + name string + input []byte + expected []byte + encFrom Encoding + wantErr bool + }{ + { + name: "UTF-8 BOM", + input: []byte{0xEF, 0xBB, 0xBF, 0x68, 0x65, 0x6C, 0x6C, 0x6F}, + expected: []byte("hello"), + encFrom: UTF8, + wantErr: false, + }, + { + name: "UTF-16BE BOM", + input: []byte{0xFE, 0xFF, 0x00, 0x68, 0x00, 0x65, 0x00, 0x6C, 0x00, 0x6C, 0x00, 0x6F}, + expected: []byte("hello"), + encFrom: UTF16BE, + wantErr: false, + }, + { + name: "UTF-16LE BOM", + input: []byte{0xFF, 0xFE, 0x68, 0x00, 0x65, 0x00, 0x6C, 0x00, 0x6C, 0x00, 0x6F, 0x00}, + expected: []byte("hello"), + encFrom: UTF16LE, + wantErr: false, + }, + { + name: "UTF-32BE BOM", + input: []byte{0x00, 0x00, 0xFE, 0xFF, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00, 0x6F}, + expected: []byte("hello"), + encFrom: UTF32BE, + wantErr: false, + }, + { + name: "UTF-32LE BOM", + input: []byte{0xFF, 0xFE, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00, 0x6F, 0x00, 0x00, 0x00}, + expected: []byte("hello"), + encFrom: UTF32LE, + wantErr: false, + }, + { + name: "No BOM (valid UTF-8)", + input: []byte("hello"), + expected: []byte("hello"), + encFrom: Unknown, + wantErr: false, + }, + { + name: "No BOM (invalid UTF-8)", + input: []byte{0xFF, 0xFE, 0xFD}, + expected: []byte{0xEF, 0xBF, 0xBD, 0xEF, 0xBF, 0xBD, 0xEF, 0xBF, 0xBD}, + encFrom: Unknown, + wantErr: false, + }, + { + name: "Empty input", + input: []byte{}, + expected: []byte{}, + encFrom: Unknown, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reader := bufio.NewReader(bytes.NewReader(tt.input)) + detectedEnc := DetectBOMEncoding(reader) + result, err := NormalizeToUTF8(reader) + assert.NoError(t, err) + actual, err := io.ReadAll(result) + + if tt.wantErr { + assert.Error(t, err, "NormalizeToUTF8() should return an error for invalid input") + return + } + + require.NoError(t, err, "NormalizeToUTF8() should not return an error for valid input") + assert.Equal(t, tt.expected, actual, "NormalizedContent() should return the correct normalized content") + assert.Equal(t, tt.encFrom.String(), detectedEnc.String(), "NormalizedFrom() should return the correct original encoding") + }) + } +} + +// Mock reader for testing error cases +type errorReader struct{} + +func (er errorReader) Read(p []byte) (n int, err error) { + return 0, errors.New("mock read error") +} + +func TestNormalizeToUTF8_ReaderError(t *testing.T) { + reader, err := NormalizeToUTF8(errorReader{}) + assert.NoError(t, err) + _, err = io.ReadAll(reader) + assert.Error(t, err, "NormalizeToUTF8() should return an error when the reader fails") +} + +func TestNormalizeToUTF8_LargeInput(t *testing.T) { + type testCase struct { + name string + input []byte + expected []byte + encFrom Encoding + } + + // Generate a large data set with 1000 Unicode code points + // NOTE: We don't want to begin the payload with a NULL character because it would then be impossible to discern between UTF16LE and UTF32LE + var runes []rune + for i := 0; i <= 1000; i++ { + runes = append(runes, rune(i%0x10FFFF)) + } + + utf8Bytes := []byte(string(runes)) + utf16LEBytes, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes(utf8Bytes) + require.NoError(t, err) + + tests := []testCase{ + { + name: "Large UTF-16LE input", + input: utf16LEBytes, + expected: utf8Bytes, + encFrom: UTF16LE, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reader := bufio.NewReader(bytes.NewReader(tt.input)) + detectedEnc := DetectBOMEncoding(reader) + assert.Equal(t, tt.encFrom.String(), detectedEnc.String(), "NormalizedFrom() should return the correct original encoding") + + result, err := NormalizeToUTF8(reader) + if err != nil { + t.Errorf("NormalizeToUTF8() error = %v", err) + // Print the first few bytes of the input for debugging + t.Logf("First 20 bytes of input: %v", tt.input[:20]) + // Print the detected encoding + assert.NoError(t, err) + t.Logf("Detected encoding: %v", detectedEnc) + return + } + + actual, err := io.ReadAll(result) + assert.NoError(t, err) + + if !bytes.Equal(tt.expected, actual) { + t.Errorf("NormalizedContent() = %v, want %v", actual, tt.expected) + // Print the first few bytes of the result and expected for debugging + t.Logf("First 20 bytes of result: %v", actual[:20]) + t.Logf("First 20 bytes of expected: %v", tt.expected[:20]) + t.Logf("First 20 bytes of input: %v", tt.input[:20]) + } + }) + } +} diff --git a/packages/go/cache/go.mod b/packages/go/cache/go.mod index 18f5b42f51..d02a6fee36 100644 --- a/packages/go/cache/go.mod +++ b/packages/go/cache/go.mod @@ -1,17 +1,17 @@ // Copyright 2023 Specter Ops, Inc. -// +// // Licensed under the Apache License, Version 2.0 // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// +// // SPDX-License-Identifier: Apache-2.0 module github.com/specterops/bloodhound/cache @@ -20,7 +20,7 @@ go 1.21 require ( github.com/hashicorp/golang-lru v0.6.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 ) require ( diff --git a/packages/go/cache/go.sum b/packages/go/cache/go.sum index caaa23039d..faab3ea4f0 100644 --- a/packages/go/cache/go.sum +++ b/packages/go/cache/go.sum @@ -4,6 +4,6 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/packages/go/crypto/go.mod b/packages/go/crypto/go.mod index 13e9d72d4c..0a8dbe79bb 100644 --- a/packages/go/crypto/go.mod +++ b/packages/go/crypto/go.mod @@ -29,7 +29,7 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect - github.com/stretchr/testify v1.8.4 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect diff --git a/packages/go/crypto/go.sum b/packages/go/crypto/go.sum index cb70c46d47..a743df069d 100644 --- a/packages/go/crypto/go.sum +++ b/packages/go/crypto/go.sum @@ -5,7 +5,7 @@ github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0g github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= diff --git a/packages/go/cypher/go.mod b/packages/go/cypher/go.mod index 4dd1b14df6..62ae46041e 100644 --- a/packages/go/cypher/go.mod +++ b/packages/go/cypher/go.mod @@ -22,7 +22,7 @@ require ( cuelang.org/go v0.5.0 github.com/antlr4-go/antlr/v4 v4.13.0 github.com/jackc/pgtype v1.14.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 ) require ( @@ -42,7 +42,7 @@ require ( golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/net v0.26.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.17.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/packages/go/cypher/go.sum b/packages/go/cypher/go.sum index 915b6cad71..1fbe1a8dd9 100644 --- a/packages/go/cypher/go.sum +++ b/packages/go/cypher/go.sum @@ -120,8 +120,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -175,7 +174,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= diff --git a/packages/go/dawgs/go.mod b/packages/go/dawgs/go.mod index 0b1f5d6417..f4fcc66836 100644 --- a/packages/go/dawgs/go.mod +++ b/packages/go/dawgs/go.mod @@ -27,7 +27,7 @@ require ( github.com/neo4j/neo4j-go-driver/v5 v5.9.0 github.com/specterops/bloodhound/cypher v0.0.0-00010101000000-000000000000 github.com/specterops/bloodhound/log v0.0.0-00010101000000-000000000000 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 go.uber.org/mock v0.2.0 ) @@ -50,9 +50,9 @@ require ( github.com/rs/zerolog v1.29.1 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect - golang.org/x/sync v0.7.0 // indirect + golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.17.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/packages/go/dawgs/go.sum b/packages/go/dawgs/go.sum index 2c359f36ec..22a3a252bf 100644 --- a/packages/go/dawgs/go.sum +++ b/packages/go/dawgs/go.sum @@ -60,19 +60,18 @@ github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU= go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/go/errors/go.mod b/packages/go/errors/go.mod index ac8712ce00..42af88d396 100644 --- a/packages/go/errors/go.mod +++ b/packages/go/errors/go.mod @@ -18,7 +18,7 @@ module github.com/specterops/bloodhound/errors go 1.21 -require github.com/stretchr/testify v1.8.4 +require github.com/stretchr/testify v1.9.0 require ( github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/packages/go/errors/go.sum b/packages/go/errors/go.sum index 3ee4af40e7..982327c4a7 100644 --- a/packages/go/errors/go.sum +++ b/packages/go/errors/go.sum @@ -14,8 +14,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/packages/go/lab/go.mod b/packages/go/lab/go.mod index fc8df5ef12..f7ea99a45a 100644 --- a/packages/go/lab/go.mod +++ b/packages/go/lab/go.mod @@ -18,7 +18,7 @@ module github.com/specterops/bloodhound/lab go 1.21 -require github.com/stretchr/testify v1.8.4 +require github.com/stretchr/testify v1.9.0 require ( github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/packages/go/lab/go.sum b/packages/go/lab/go.sum index 3ee4af40e7..982327c4a7 100644 --- a/packages/go/lab/go.sum +++ b/packages/go/lab/go.sum @@ -14,8 +14,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/packages/go/schemagen/go.mod b/packages/go/schemagen/go.mod index 02db695238..4040737bc9 100644 --- a/packages/go/schemagen/go.mod +++ b/packages/go/schemagen/go.mod @@ -36,9 +36,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect - github.com/stretchr/testify v1.8.4 // indirect + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/net v0.26.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.17.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/packages/go/schemagen/go.sum b/packages/go/schemagen/go.sum index 27649b1ccb..306f65160e 100644 --- a/packages/go/schemagen/go.sum +++ b/packages/go/schemagen/go.sum @@ -41,12 +41,10 @@ github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b/go.mod h1 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/packages/go/slicesext/go.mod b/packages/go/slicesext/go.mod index 87271cbd8b..a3257cffb2 100644 --- a/packages/go/slicesext/go.mod +++ b/packages/go/slicesext/go.mod @@ -18,7 +18,7 @@ module github.com/specterops/bloodhound/slicesext go 1.21 -require github.com/stretchr/testify v1.8.4 +require github.com/stretchr/testify v1.9.0 require ( github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/packages/go/slicesext/go.sum b/packages/go/slicesext/go.sum index 3ee4af40e7..982327c4a7 100644 --- a/packages/go/slicesext/go.sum +++ b/packages/go/slicesext/go.sum @@ -14,8 +14,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/packages/go/stbernard/go.mod b/packages/go/stbernard/go.mod index 7acb2cfc56..7495f0cdd2 100644 --- a/packages/go/stbernard/go.mod +++ b/packages/go/stbernard/go.mod @@ -23,7 +23,7 @@ require ( github.com/gofrs/uuid v4.4.0+incompatible github.com/specterops/bloodhound/log v0.0.0-00010101000000-000000000000 github.com/specterops/bloodhound/slicesext v0.0.0-00010101000000-000000000000 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 golang.org/x/mod v0.17.0 ) diff --git a/packages/go/stbernard/go.sum b/packages/go/stbernard/go.sum index d0ee2a3ec4..b41d07097e 100644 --- a/packages/go/stbernard/go.sum +++ b/packages/go/stbernard/go.sum @@ -25,8 +25,7 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/packages/javascript/bh-shared-ui/src/constants.ts b/packages/javascript/bh-shared-ui/src/constants.ts index 8e3823570b..b82980e83d 100644 --- a/packages/javascript/bh-shared-ui/src/constants.ts +++ b/packages/javascript/bh-shared-ui/src/constants.ts @@ -133,14 +133,13 @@ export const typography: Partial = { }, }; - const defaultPortalContainer = { // Defaults all MUI components that leverage the Modal construct to portal to a child of the applicationContainer element. // If not for this, any tailwind based components in a portal and outside the applicationContainer will not respect the current theme. // Controlling doodle components: https://tailwindcss.com/docs/dark-mode#toggling-dark-mode-manually // Modal construct: https://mui.com/material-ui/api/modal/ - container: () => document.getElementById('app-root') // Callback so this is re-run on useLayoutEffect within MUI -} + container: () => document.getElementById('app-root'), // Callback so this is re-run on useLayoutEffect within MUI +}; export const components = (theme: Theme): Partial => ({ MuiButton: { @@ -202,7 +201,7 @@ export const components = (theme: Theme): Partial => ({ }, MuiDialog: { defaultProps: { - ...defaultPortalContainer + ...defaultPortalContainer, }, styleOverrides: { root: { @@ -215,17 +214,17 @@ export const components = (theme: Theme): Partial => ({ }, MuiMenu: { defaultProps: { - ...defaultPortalContainer - } + ...defaultPortalContainer, + }, }, MuiAutocomplete: { defaultProps: { componentsProps: { popper: { - ...defaultPortalContainer - } - } - } + ...defaultPortalContainer, + }, + }, + }, }, MuiDialogActions: { styleOverrides: { @@ -236,7 +235,7 @@ export const components = (theme: Theme): Partial => ({ }, MuiPopover: { defaultProps: { - ...defaultPortalContainer + ...defaultPortalContainer, }, styleOverrides: { root: { diff --git a/packages/javascript/bh-shared-ui/src/test-utils.jsx b/packages/javascript/bh-shared-ui/src/test-utils.jsx index 392e468429..d9c12cac93 100644 --- a/packages/javascript/bh-shared-ui/src/test-utils.jsx +++ b/packages/javascript/bh-shared-ui/src/test-utils.jsx @@ -63,4 +63,4 @@ const customRender = ( // re-export everything export * from '@testing-library/react'; // override render method -export { customRender as render}; +export { customRender as render };