From 0c6a8ce933028fdf2e887b737f6bd9a03d052e1a Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 28 Mar 2024 01:30:32 -0400 Subject: [PATCH 1/5] MergeClone --- firstpartydata/extmerger.go | 60 -- firstpartydata/extmerger_test.go | 109 ---- firstpartydata/first_party_data.go | 169 +---- firstpartydata/first_party_data_test.go | 825 +++--------------------- go.mod | 2 +- ortb/clone.go | 145 ----- ortb/clone_test.go | 413 ------------ util/jsonutil/jsonutil.go | 2 +- util/jsonutil/merge.go | 185 ++++++ util/jsonutil/merge_test.go | 443 +++++++++++++ util/maputil/maputil.go | 3 +- util/reflectutil/slice.go | 49 ++ util/reflectutil/slice_test.go | 41 ++ util/sliceutil/clone.go | 2 +- 14 files changed, 863 insertions(+), 1585 deletions(-) delete mode 100644 firstpartydata/extmerger.go delete mode 100644 firstpartydata/extmerger_test.go create mode 100644 util/jsonutil/merge.go create mode 100644 util/jsonutil/merge_test.go create mode 100644 util/reflectutil/slice.go create mode 100644 util/reflectutil/slice_test.go diff --git a/firstpartydata/extmerger.go b/firstpartydata/extmerger.go deleted file mode 100644 index f3196bea996..00000000000 --- a/firstpartydata/extmerger.go +++ /dev/null @@ -1,60 +0,0 @@ -package firstpartydata - -import ( - "encoding/json" - "errors" - "fmt" - - "github.com/prebid/prebid-server/v2/util/sliceutil" - jsonpatch "gopkg.in/evanphx/json-patch.v4" -) - -var ( - ErrBadRequest = fmt.Errorf("invalid request ext") - ErrBadFPD = fmt.Errorf("invalid first party data ext") -) - -// extMerger tracks a JSON `ext` field within an OpenRTB request. The value of the -// `ext` field is expected to be modified when calling unmarshal on the same object -// and will later be updated when invoking Merge. -type extMerger struct { - ext *json.RawMessage // Pointer to the JSON `ext` field. - snapshot json.RawMessage // Copy of the original state of the JSON `ext` field. -} - -// Track saves a copy of the JSON `ext` field and stores a reference to the extension -// object for comparison later in the Merge call. -func (e *extMerger) Track(ext *json.RawMessage) { - e.ext = ext - e.snapshot = sliceutil.Clone(*ext) -} - -// Merge applies a JSON merge of the stored extension snapshot on top of the current -// JSON of the tracked extension object. -func (e extMerger) Merge() error { - if e.ext == nil { - return nil - } - - if len(e.snapshot) == 0 { - return nil - } - - if len(*e.ext) == 0 { - *e.ext = e.snapshot - return nil - } - - merged, err := jsonpatch.MergePatch(e.snapshot, *e.ext) - if err != nil { - if errors.Is(err, jsonpatch.ErrBadJSONDoc) { - return ErrBadRequest - } else if errors.Is(err, jsonpatch.ErrBadJSONPatch) { - return ErrBadFPD - } - return err - } - - *e.ext = merged - return nil -} diff --git a/firstpartydata/extmerger_test.go b/firstpartydata/extmerger_test.go deleted file mode 100644 index 4107b0d1144..00000000000 --- a/firstpartydata/extmerger_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package firstpartydata - -import ( - "encoding/json" - "testing" - - "github.com/prebid/prebid-server/v2/util/sliceutil" - "github.com/stretchr/testify/assert" -) - -func TestExtMerger(t *testing.T) { - t.Run("nil", func(t *testing.T) { - merger := extMerger{ext: nil, snapshot: json.RawMessage(`{"a":1}`)} - assert.NoError(t, merger.Merge()) - assert.Nil(t, merger.ext) - }) - - testCases := []struct { - name string - givenOriginal json.RawMessage - givenFPD json.RawMessage - expectedExt json.RawMessage - expectedErr string - }{ - { - name: "both-populated", - givenOriginal: json.RawMessage(`{"a":1,"b":2}`), - givenFPD: json.RawMessage(`{"b":200,"c":3}`), - expectedExt: json.RawMessage(`{"a":1,"b":200,"c":3}`), - }, - { - name: "both-nil", - givenFPD: nil, - givenOriginal: nil, - expectedExt: nil, - }, - { - name: "both-empty", - givenOriginal: json.RawMessage(`{}`), - givenFPD: json.RawMessage(`{}`), - expectedExt: json.RawMessage(`{}`), - }, - { - name: "ext-nil", - givenOriginal: json.RawMessage(`{"b":2}`), - givenFPD: nil, - expectedExt: json.RawMessage(`{"b":2}`), - }, - { - name: "ext-empty", - givenOriginal: json.RawMessage(`{"b":2}`), - givenFPD: json.RawMessage(`{}`), - expectedExt: json.RawMessage(`{"b":2}`), - }, - { - name: "ext-malformed", - givenOriginal: json.RawMessage(`{"b":2}`), - givenFPD: json.RawMessage(`malformed`), - expectedErr: "invalid first party data ext", - }, - { - name: "snapshot-nil", - givenOriginal: nil, - givenFPD: json.RawMessage(`{"a":1}`), - expectedExt: json.RawMessage(`{"a":1}`), - }, - { - name: "snapshot-empty", - givenOriginal: json.RawMessage(`{}`), - givenFPD: json.RawMessage(`{"a":1}`), - expectedExt: json.RawMessage(`{"a":1}`), - }, - { - name: "snapshot-malformed", - givenOriginal: json.RawMessage(`malformed`), - givenFPD: json.RawMessage(`{"a":1}`), - expectedErr: "invalid request ext", - }, - } - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - // Initialize A Ext Raw Message For Testing - simulatedExt := json.RawMessage(sliceutil.Clone(test.givenOriginal)) - - // Begin Tracking - var merger extMerger - merger.Track(&simulatedExt) - - // Unmarshal - simulatedExt.UnmarshalJSON(test.givenFPD) - - // Merge - actualErr := merger.Merge() - - if test.expectedErr == "" { - assert.NoError(t, actualErr, "error") - - if test.expectedExt == nil { - assert.Nil(t, simulatedExt, "json") - } else { - assert.JSONEq(t, string(test.expectedExt), string(simulatedExt), "json") - } - } else { - assert.EqualError(t, actualErr, test.expectedErr, "error") - } - }) - } -} diff --git a/firstpartydata/first_party_data.go b/firstpartydata/first_party_data.go index 8c77f61a3d6..e547b2991f4 100644 --- a/firstpartydata/first_party_data.go +++ b/firstpartydata/first_party_data.go @@ -2,18 +2,24 @@ package firstpartydata import ( "encoding/json" + "errors" "fmt" + "strings" "github.com/prebid/openrtb/v20/openrtb2" jsonpatch "gopkg.in/evanphx/json-patch.v4" "github.com/prebid/prebid-server/v2/errortypes" "github.com/prebid/prebid-server/v2/openrtb_ext" - "github.com/prebid/prebid-server/v2/ortb" "github.com/prebid/prebid-server/v2/util/jsonutil" "github.com/prebid/prebid-server/v2/util/ptrutil" ) +var ( + ErrBadRequest = fmt.Errorf("invalid request ext") + ErrBadFPD = fmt.Errorf("invalid first party data ext") +) + const ( siteKey = "site" appKey = "app" @@ -185,7 +191,7 @@ func resolveUser(fpdConfig *openrtb_ext.ORTB2, bidRequestUser *openrtb2.User, gl var err error newUser.Ext, err = jsonpatch.MergePatch(newUser.Ext, extData) if err != nil { - return nil, err + return nil, formatMergePatchError(err) } } else { newUser.Ext = extData @@ -195,42 +201,14 @@ func resolveUser(fpdConfig *openrtb_ext.ORTB2, bidRequestUser *openrtb2.User, gl newUser.Data = openRtbGlobalFPD[userDataKey] } if fpdConfigUser != nil { - if err := mergeUser(newUser, fpdConfigUser); err != nil { - return nil, err + if err := jsonutil.MergeClone(newUser, fpdConfigUser); err != nil { + return nil, formatMergeCloneError(err) } } return newUser, nil } -func mergeUser(v *openrtb2.User, overrideJSON json.RawMessage) error { - *v = *ortb.CloneUser(v) - - // Track EXTs - // It's not necessary to track `ext` fields in array items because the array - // items will be replaced entirely with the override JSON, so no merge is required. - var ext, extGeo extMerger - ext.Track(&v.Ext) - if v.Geo != nil { - extGeo.Track(&v.Geo.Ext) - } - - // Merge - if err := jsonutil.Unmarshal(overrideJSON, &v); err != nil { - return err - } - - // Merge EXTs - if err := ext.Merge(); err != nil { - return err - } - if err := extGeo.Merge(); err != nil { - return err - } - - return nil -} - func resolveSite(fpdConfig *openrtb_ext.ORTB2, bidRequestSite *openrtb2.Site, globalFPD map[string][]byte, openRtbGlobalFPD map[string][]openrtb2.Data, bidderName string) (*openrtb2.Site, error) { var fpdConfigSite json.RawMessage @@ -261,7 +239,7 @@ func resolveSite(fpdConfig *openrtb_ext.ORTB2, bidRequestSite *openrtb2.Site, gl var err error newSite.Ext, err = jsonpatch.MergePatch(newSite.Ext, extData) if err != nil { - return nil, err + return nil, formatMergePatchError(err) } } else { newSite.Ext = extData @@ -286,52 +264,8 @@ func resolveSite(fpdConfig *openrtb_ext.ORTB2, bidRequestSite *openrtb2.Site, gl } func mergeSite(v *openrtb2.Site, overrideJSON json.RawMessage, bidderName string) error { - *v = *ortb.CloneSite(v) - - // Track EXTs - // It's not necessary to track `ext` fields in array items because the array - // items will be replaced entirely with the override JSON, so no merge is required. - var ext, extPublisher, extContent, extContentProducer, extContentNetwork, extContentChannel extMerger - ext.Track(&v.Ext) - if v.Publisher != nil { - extPublisher.Track(&v.Publisher.Ext) - } - if v.Content != nil { - extContent.Track(&v.Content.Ext) - } - if v.Content != nil && v.Content.Producer != nil { - extContentProducer.Track(&v.Content.Producer.Ext) - } - if v.Content != nil && v.Content.Network != nil { - extContentNetwork.Track(&v.Content.Network.Ext) - } - if v.Content != nil && v.Content.Channel != nil { - extContentChannel.Track(&v.Content.Channel.Ext) - } - - // Merge - if err := jsonutil.Unmarshal(overrideJSON, &v); err != nil { - return err - } - - // Merge EXTs - if err := ext.Merge(); err != nil { - return err - } - if err := extPublisher.Merge(); err != nil { - return err - } - if err := extContent.Merge(); err != nil { - return err - } - if err := extContentProducer.Merge(); err != nil { - return err - } - if err := extContentNetwork.Merge(); err != nil { - return err - } - if err := extContentChannel.Merge(); err != nil { - return err + if err := jsonutil.MergeClone(v, overrideJSON); err != nil { + return formatMergeCloneError(err) } // Re-Validate Site @@ -344,6 +278,25 @@ func mergeSite(v *openrtb2.Site, overrideJSON json.RawMessage, bidderName string return nil } +func formatMergePatchError(err error) error { + if errors.Is(err, jsonpatch.ErrBadJSONDoc) { + return ErrBadRequest + } + + if errors.Is(err, jsonpatch.ErrBadJSONPatch) { + return ErrBadFPD + } + + return err +} + +func formatMergeCloneError(err error) error { + if strings.Contains(err.Error(), "invalid json on existing object") { + return ErrBadRequest + } + return ErrBadFPD +} + func resolveApp(fpdConfig *openrtb_ext.ORTB2, bidRequestApp *openrtb2.App, globalFPD map[string][]byte, openRtbGlobalFPD map[string][]openrtb2.Data, bidderName string) (*openrtb2.App, error) { var fpdConfigApp json.RawMessage @@ -375,7 +328,7 @@ func resolveApp(fpdConfig *openrtb_ext.ORTB2, bidRequestApp *openrtb2.App, globa var err error newApp.Ext, err = jsonpatch.MergePatch(newApp.Ext, extData) if err != nil { - return nil, err + return nil, formatMergePatchError(err) } } else { newApp.Ext = extData @@ -394,66 +347,14 @@ func resolveApp(fpdConfig *openrtb_ext.ORTB2, bidRequestApp *openrtb2.App, globa } if fpdConfigApp != nil { - if err := mergeApp(newApp, fpdConfigApp); err != nil { - return nil, err + if err := jsonutil.MergeClone(newApp, fpdConfigApp); err != nil { + return nil, formatMergeCloneError(err) } } return newApp, nil } -func mergeApp(v *openrtb2.App, overrideJSON json.RawMessage) error { - *v = *ortb.CloneApp(v) - - // Track EXTs - // It's not necessary to track `ext` fields in array items because the array - // items will be replaced entirely with the override JSON, so no merge is required. - var ext, extPublisher, extContent, extContentProducer, extContentNetwork, extContentChannel extMerger - ext.Track(&v.Ext) - if v.Publisher != nil { - extPublisher.Track(&v.Publisher.Ext) - } - if v.Content != nil { - extContent.Track(&v.Content.Ext) - } - if v.Content != nil && v.Content.Producer != nil { - extContentProducer.Track(&v.Content.Producer.Ext) - } - if v.Content != nil && v.Content.Network != nil { - extContentNetwork.Track(&v.Content.Network.Ext) - } - if v.Content != nil && v.Content.Channel != nil { - extContentChannel.Track(&v.Content.Channel.Ext) - } - - // Merge - if err := jsonutil.Unmarshal(overrideJSON, &v); err != nil { - return err - } - - // Merge EXTs - if err := ext.Merge(); err != nil { - return err - } - if err := extPublisher.Merge(); err != nil { - return err - } - if err := extContent.Merge(); err != nil { - return err - } - if err := extContentProducer.Merge(); err != nil { - return err - } - if err := extContentNetwork.Merge(); err != nil { - return err - } - if err := extContentChannel.Merge(); err != nil { - return err - } - - return nil -} - func buildExtData(data []byte) []byte { res := make([]byte, 0, len(data)+len(`"{"data":}"`)) res = append(res, []byte(`{"data":`)...) diff --git a/firstpartydata/first_party_data_test.go b/firstpartydata/first_party_data_test.go index f417a24d7e7..757a91ce6b7 100644 --- a/firstpartydata/first_party_data_test.go +++ b/firstpartydata/first_party_data_test.go @@ -4,14 +4,12 @@ import ( "encoding/json" "os" "path/filepath" - "reflect" "testing" "github.com/prebid/openrtb/v20/openrtb2" "github.com/prebid/prebid-server/v2/errortypes" "github.com/prebid/prebid-server/v2/openrtb_ext" "github.com/prebid/prebid-server/v2/util/jsonutil" - "github.com/prebid/prebid-server/v2/util/ptrutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -34,17 +32,17 @@ func TestExtractGlobalFPD(t *testing.T) { Publisher: &openrtb2.Publisher{ ID: "1", }, - Ext: json.RawMessage(`{"data": {"somesitefpd": "sitefpdDataTest"}}`), + Ext: json.RawMessage(`{"data":{"somesitefpd":"sitefpdDataTest"}}`), }, User: &openrtb2.User{ ID: "reqUserID", Yob: 1982, Gender: "M", - Ext: json.RawMessage(`{"data": {"someuserfpd": "userfpdDataTest"}}`), + Ext: json.RawMessage(`{"data":{"someuserfpd":"userfpdDataTest"}}`), }, App: &openrtb2.App{ ID: "appId", - Ext: json.RawMessage(`{"data": {"someappfpd": "appfpdDataTest"}}`), + Ext: json.RawMessage(`{"data":{"someappfpd":"appfpdDataTest"}}`), }, }, }, @@ -67,9 +65,9 @@ func TestExtractGlobalFPD(t *testing.T) { }, }}, expectedFpd: map[string][]byte{ - "site": []byte(`{"somesitefpd": "sitefpdDataTest"}`), - "user": []byte(`{"someuserfpd": "userfpdDataTest"}`), - "app": []byte(`{"someappfpd": "appfpdDataTest"}`), + "site": []byte(`{"somesitefpd":"sitefpdDataTest"}`), + "user": []byte(`{"someuserfpd":"userfpdDataTest"}`), + "app": []byte(`{"someappfpd":"appfpdDataTest"}`), }, }, { @@ -86,7 +84,7 @@ func TestExtractGlobalFPD(t *testing.T) { }, App: &openrtb2.App{ ID: "appId", - Ext: json.RawMessage(`{"data": {"someappfpd": "appfpdDataTest"}}`), + Ext: json.RawMessage(`{"data":{"someappfpd":"appfpdDataTest"}}`), }, }, }, @@ -106,7 +104,7 @@ func TestExtractGlobalFPD(t *testing.T) { }, }, expectedFpd: map[string][]byte{ - "app": []byte(`{"someappfpd": "appfpdDataTest"}`), + "app": []byte(`{"someappfpd":"appfpdDataTest"}`), "user": nil, "site": nil, }, @@ -127,7 +125,7 @@ func TestExtractGlobalFPD(t *testing.T) { ID: "reqUserID", Yob: 1982, Gender: "M", - Ext: json.RawMessage(`{"data": {"someuserfpd": "userfpdDataTest"}}`), + Ext: json.RawMessage(`{"data":{"someuserfpd":"userfpdDataTest"}}`), }, }, }, @@ -150,7 +148,7 @@ func TestExtractGlobalFPD(t *testing.T) { }, expectedFpd: map[string][]byte{ "app": nil, - "user": []byte(`{"someuserfpd": "userfpdDataTest"}`), + "user": []byte(`{"someuserfpd":"userfpdDataTest"}`), "site": nil, }, }, @@ -213,7 +211,7 @@ func TestExtractGlobalFPD(t *testing.T) { Publisher: &openrtb2.Publisher{ ID: "1", }, - Ext: json.RawMessage(`{"data": {"someappfpd": true}}`), + Ext: json.RawMessage(`{"data":{"someappfpd":true}}`), }, App: &openrtb2.App{ ID: "appId", @@ -238,7 +236,7 @@ func TestExtractGlobalFPD(t *testing.T) { expectedFpd: map[string][]byte{ "app": nil, "user": nil, - "site": []byte(`{"someappfpd": true}`), + "site": []byte(`{"someappfpd":true}`), }, }, } @@ -525,6 +523,7 @@ func TestExtractBidderConfigFPD(t *testing.T) { }) } } + func TestResolveFPD(t *testing.T) { testPath := "tests/resolvefpd" @@ -633,6 +632,7 @@ func TestResolveFPD(t *testing.T) { }) } } + func TestExtractFPDForBidders(t *testing.T) { if specFiles, err := os.ReadDir("./tests/extractfpdforbidders"); err == nil { for _, specFile := range specFiles { @@ -727,7 +727,7 @@ func TestResolveUser(t *testing.T) { globalFPD map[string][]byte openRtbGlobalFPD map[string][]openrtb2.Data expectedUser *openrtb2.User - expectError bool + expectError string }{ { description: "FPD config and bid request user are not specified", @@ -735,42 +735,42 @@ func TestResolveUser(t *testing.T) { }, { description: "FPD config user only is specified", - fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test"}`)}, + fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id":"test"}`)}, expectedUser: &openrtb2.User{ID: "test"}, }, { description: "FPD config and bid request user are specified", - fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id":"test1"}`)}, bidRequestUser: &openrtb2.User{ID: "test2"}, expectedUser: &openrtb2.User{ID: "test1"}, }, { description: "FPD config, bid request and global fpd user are specified, no input user ext", - fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id":"test1"}`)}, bidRequestUser: &openrtb2.User{ID: "test2"}, - globalFPD: map[string][]byte{userKey: []byte(`{"globalFPDUserData": "globalFPDUserDataValue"}`)}, + globalFPD: map[string][]byte{userKey: []byte(`{"globalFPDUserData":"globalFPDUserDataValue"}`)}, expectedUser: &openrtb2.User{ID: "test1", Ext: json.RawMessage(`{"data":{"globalFPDUserData":"globalFPDUserDataValue"}}`)}, }, { description: "FPD config, bid request user with ext and global fpd user are specified, no input user ext", - fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id":"test1"}`)}, bidRequestUser: &openrtb2.User{ID: "test2", Ext: json.RawMessage(`{"test":{"inputFPDUserData":"inputFPDUserDataValue"}}`)}, - globalFPD: map[string][]byte{userKey: []byte(`{"globalFPDUserData": "globalFPDUserDataValue"}`)}, + globalFPD: map[string][]byte{userKey: []byte(`{"globalFPDUserData":"globalFPDUserDataValue"}`)}, expectedUser: &openrtb2.User{ID: "test1", Ext: json.RawMessage(`{"data":{"globalFPDUserData":"globalFPDUserDataValue"},"test":{"inputFPDUserData":"inputFPDUserDataValue"}}`)}, }, { description: "FPD config, bid request and global fpd user are specified, with input user ext.data", fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1"}`)}, bidRequestUser: &openrtb2.User{ID: "test2", Ext: json.RawMessage(`{"data":{"inputFPDUserData":"inputFPDUserDataValue"}}`)}, - globalFPD: map[string][]byte{userKey: []byte(`{"globalFPDUserData": "globalFPDUserDataValue"}`)}, + globalFPD: map[string][]byte{userKey: []byte(`{"globalFPDUserData":"globalFPDUserDataValue"}`)}, expectedUser: &openrtb2.User{ID: "test1", Ext: json.RawMessage(`{"data":{"globalFPDUserData":"globalFPDUserDataValue","inputFPDUserData":"inputFPDUserDataValue"}}`)}, }, { description: "FPD config, bid request and global fpd user are specified, with input user ext.data malformed", - fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id":"test1"}`)}, bidRequestUser: &openrtb2.User{ID: "test2", Ext: json.RawMessage(`{"data":{"inputFPDUserData":"inputFPDUserDataValue"}}`)}, globalFPD: map[string][]byte{userKey: []byte(`malformed`)}, - expectError: true, + expectError: "invalid first party data ext", }, { description: "bid request and openrtb global fpd user are specified, no input user ext", @@ -786,7 +786,7 @@ func TestResolveUser(t *testing.T) { }, { description: "fpd config user, bid request and openrtb global fpd user are specified, no input user ext", - fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id":"test1"}`)}, bidRequestUser: &openrtb2.User{ID: "test2"}, openRtbGlobalFPD: map[string][]openrtb2.Data{userDataKey: { {ID: "DataId1", Name: "Name1"}, @@ -799,7 +799,7 @@ func TestResolveUser(t *testing.T) { }, { description: "fpd config user with ext, bid request and openrtb global fpd user are specified, no input user ext", - fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1", "ext":{"test":1}}`)}, + fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id":"test1","ext":{"test":1}}`)}, bidRequestUser: &openrtb2.User{ID: "test2"}, openRtbGlobalFPD: map[string][]openrtb2.Data{userDataKey: { {ID: "DataId1", Name: "Name1"}, @@ -813,8 +813,8 @@ func TestResolveUser(t *testing.T) { }, { description: "fpd config user with ext, bid requestuser with ext and openrtb global fpd user are specified, no input user ext", - fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1", "ext":{"test":1}}`)}, - bidRequestUser: &openrtb2.User{ID: "test2", Ext: json.RawMessage(`{"test":2, "key": "value"}`)}, + fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id":"test1","ext":{"test":1}}`)}, + bidRequestUser: &openrtb2.User{ID: "test2", Ext: json.RawMessage(`{"test":2,"key":"value"}`)}, openRtbGlobalFPD: map[string][]openrtb2.Data{userDataKey: { {ID: "DataId1", Name: "Name1"}, {ID: "DataId2", Name: "Name2"}, @@ -827,8 +827,8 @@ func TestResolveUser(t *testing.T) { }, { description: "fpd config user with malformed ext, bid requestuser with ext and openrtb global fpd user are specified, no input user ext", - fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1", "ext":{malformed}}`)}, - bidRequestUser: &openrtb2.User{ID: "test2", Ext: json.RawMessage(`{"test":2, "key": "value"}`)}, + fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1","ext":{malformed}}`)}, + bidRequestUser: &openrtb2.User{ID: "test2", Ext: json.RawMessage(`{"test":2,"key":"value"}`)}, openRtbGlobalFPD: map[string][]openrtb2.Data{userDataKey: { {ID: "DataId1", Name: "Name1"}, {ID: "DataId2", Name: "Name2"}, @@ -839,15 +839,15 @@ func TestResolveUser(t *testing.T) { }, Ext: json.RawMessage(`{"key":"value","test":1}`), }, - expectError: true, + expectError: "invalid first party data ext", }, } for _, test := range testCases { t.Run(test.description, func(t *testing.T) { resultUser, err := resolveUser(test.fpdConfig, test.bidRequestUser, test.globalFPD, test.openRtbGlobalFPD, "bidderA") - if test.expectError { - assert.Error(t, err, "expected error incorrect") + if len(test.expectError) > 0 { + assert.EqualError(t, err, test.expectError) } else { assert.NoError(t, err, "unexpected error returned") assert.Equal(t, test.expectedUser, resultUser, "Result user is incorrect") @@ -864,7 +864,7 @@ func TestResolveSite(t *testing.T) { globalFPD map[string][]byte openRtbGlobalFPD map[string][]openrtb2.Data expectedSite *openrtb2.Site - expectError bool + expectError string }{ { description: "FPD config and bid request site are not specified", @@ -872,42 +872,42 @@ func TestResolveSite(t *testing.T) { }, { description: "FPD config site only is specified", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test"}`)}, - expectError: true, + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id":"test"}`)}, + expectError: "incorrect First Party Data for bidder bidderA: Site object is not defined in request, but defined in FPD config", }, { description: "FPD config and bid request site are specified", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id":"test1"}`)}, bidRequestSite: &openrtb2.Site{ID: "test2"}, expectedSite: &openrtb2.Site{ID: "test1"}, }, { description: "FPD config, bid request and global fpd site are specified, no input site ext", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id":"test1"}`)}, bidRequestSite: &openrtb2.Site{ID: "test2"}, - globalFPD: map[string][]byte{siteKey: []byte(`{"globalFPDSiteData": "globalFPDSiteDataValue"}`)}, + globalFPD: map[string][]byte{siteKey: []byte(`{"globalFPDSiteData":"globalFPDSiteDataValue"}`)}, expectedSite: &openrtb2.Site{ID: "test1", Ext: json.RawMessage(`{"data":{"globalFPDSiteData":"globalFPDSiteDataValue"}}`)}, }, { description: "FPD config, bid request site with ext and global fpd site are specified, no input site ext", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id":"test1"}`)}, bidRequestSite: &openrtb2.Site{ID: "test2", Ext: json.RawMessage(`{"test":{"inputFPDSiteData":"inputFPDSiteDataValue"}}`)}, - globalFPD: map[string][]byte{siteKey: []byte(`{"globalFPDSiteData": "globalFPDSiteDataValue"}`)}, + globalFPD: map[string][]byte{siteKey: []byte(`{"globalFPDSiteData":"globalFPDSiteDataValue"}`)}, expectedSite: &openrtb2.Site{ID: "test1", Ext: json.RawMessage(`{"data":{"globalFPDSiteData":"globalFPDSiteDataValue"},"test":{"inputFPDSiteData":"inputFPDSiteDataValue"}}`)}, }, { description: "FPD config, bid request and global fpd site are specified, with input site ext.data", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id":"test1"}`)}, bidRequestSite: &openrtb2.Site{ID: "test2", Ext: json.RawMessage(`{"data":{"inputFPDSiteData":"inputFPDSiteDataValue"}}`)}, - globalFPD: map[string][]byte{siteKey: []byte(`{"globalFPDSiteData": "globalFPDSiteDataValue"}`)}, + globalFPD: map[string][]byte{siteKey: []byte(`{"globalFPDSiteData":"globalFPDSiteDataValue"}`)}, expectedSite: &openrtb2.Site{ID: "test1", Ext: json.RawMessage(`{"data":{"globalFPDSiteData":"globalFPDSiteDataValue","inputFPDSiteData":"inputFPDSiteDataValue"}}`)}, }, { description: "FPD config, bid request and global fpd site are specified, with input site ext.data malformed", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id":"test1"}`)}, bidRequestSite: &openrtb2.Site{ID: "test2", Ext: json.RawMessage(`{"data":{"inputFPDSiteData":"inputFPDSiteDataValue"}}`)}, globalFPD: map[string][]byte{siteKey: []byte(`malformed`)}, - expectError: true, + expectError: "invalid first party data ext", }, { description: "bid request and openrtb global fpd site are specified, no input site ext", @@ -946,7 +946,7 @@ func TestResolveSite(t *testing.T) { }, { description: "fpd config site, bid request and openrtb global fpd site are specified, no input site ext", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id":"test1"}`)}, bidRequestSite: &openrtb2.Site{ID: "test2"}, openRtbGlobalFPD: map[string][]openrtb2.Data{siteContentDataKey: { {ID: "DataId1", Name: "Name1"}, @@ -959,7 +959,7 @@ func TestResolveSite(t *testing.T) { }, { description: "fpd config site with ext, bid request and openrtb global fpd site are specified, no input site ext", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test1", "ext":{"test":1}}`)}, + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id":"test1","ext":{"test":1}}`)}, bidRequestSite: &openrtb2.Site{ID: "test2"}, openRtbGlobalFPD: map[string][]openrtb2.Data{siteContentDataKey: { {ID: "DataId1", Name: "Name1"}, @@ -973,8 +973,8 @@ func TestResolveSite(t *testing.T) { }, { description: "fpd config site with ext, bid request site with ext and openrtb global fpd site are specified, no input site ext", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test1", "ext":{"test":1}}`)}, - bidRequestSite: &openrtb2.Site{ID: "test2", Ext: json.RawMessage(`{"test":2, "key": "value"}`)}, + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id":"test1","ext":{"test":1}}`)}, + bidRequestSite: &openrtb2.Site{ID: "test2", Ext: json.RawMessage(`{"test":2,"key":"value"}`)}, openRtbGlobalFPD: map[string][]openrtb2.Data{siteContentDataKey: { {ID: "DataId1", Name: "Name1"}, {ID: "DataId2", Name: "Name2"}, @@ -987,8 +987,8 @@ func TestResolveSite(t *testing.T) { }, { description: "fpd config site with malformed ext, bid request site with ext and openrtb global fpd site are specified, no input site ext", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test1", "ext":{malformed}}`)}, - bidRequestSite: &openrtb2.Site{ID: "test2", Ext: json.RawMessage(`{"test":2, "key": "value"}`)}, + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id":"test1","ext":{malformed}}`)}, + bidRequestSite: &openrtb2.Site{ID: "test2", Ext: json.RawMessage(`{"test":2,"key":"value"}`)}, openRtbGlobalFPD: map[string][]openrtb2.Data{siteContentDataKey: { {ID: "DataId1", Name: "Name1"}, {ID: "DataId2", Name: "Name2"}, @@ -999,15 +999,15 @@ func TestResolveSite(t *testing.T) { }}, Ext: json.RawMessage(`{"key":"value","test":1}`), }, - expectError: true, + expectError: "invalid first party data ext", }, } for _, test := range testCases { t.Run(test.description, func(t *testing.T) { resultSite, err := resolveSite(test.fpdConfig, test.bidRequestSite, test.globalFPD, test.openRtbGlobalFPD, "bidderA") - if test.expectError { - assert.Error(t, err) + if len(test.expectError) > 0 { + assert.EqualError(t, err, test.expectError) } else { assert.NoError(t, err, "unexpected error returned") assert.Equal(t, test.expectedSite, resultSite, "Result site is incorrect") @@ -1024,7 +1024,7 @@ func TestResolveApp(t *testing.T) { globalFPD map[string][]byte openRtbGlobalFPD map[string][]openrtb2.Data expectedApp *openrtb2.App - expectError bool + expectError string }{ { description: "FPD config and bid request app are not specified", @@ -1032,42 +1032,42 @@ func TestResolveApp(t *testing.T) { }, { description: "FPD config app only is specified", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test"}`)}, - expectError: true, + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id":"test"}`)}, + expectError: "incorrect First Party Data for bidder bidderA: App object is not defined in request, but defined in FPD config", }, { description: "FPD config and bid request app are specified", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id":"test1"}`)}, bidRequestApp: &openrtb2.App{ID: "test2"}, expectedApp: &openrtb2.App{ID: "test1"}, }, { description: "FPD config, bid request and global fpd app are specified, no input app ext", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id":"test1"}`)}, bidRequestApp: &openrtb2.App{ID: "test2"}, - globalFPD: map[string][]byte{appKey: []byte(`{"globalFPDAppData": "globalFPDAppDataValue"}`)}, + globalFPD: map[string][]byte{appKey: []byte(`{"globalFPDAppData":"globalFPDAppDataValue"}`)}, expectedApp: &openrtb2.App{ID: "test1", Ext: json.RawMessage(`{"data":{"globalFPDAppData":"globalFPDAppDataValue"}}`)}, }, { description: "FPD config, bid request app with ext and global fpd app are specified, no input app ext", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id":"test1"}`)}, bidRequestApp: &openrtb2.App{ID: "test2", Ext: json.RawMessage(`{"test":{"inputFPDAppData":"inputFPDAppDataValue"}}`)}, - globalFPD: map[string][]byte{appKey: []byte(`{"globalFPDAppData": "globalFPDAppDataValue"}`)}, + globalFPD: map[string][]byte{appKey: []byte(`{"globalFPDAppData":"globalFPDAppDataValue"}`)}, expectedApp: &openrtb2.App{ID: "test1", Ext: json.RawMessage(`{"data":{"globalFPDAppData":"globalFPDAppDataValue"},"test":{"inputFPDAppData":"inputFPDAppDataValue"}}`)}, }, { description: "FPD config, bid request and global fpd app are specified, with input app ext.data", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id":"test1"}`)}, bidRequestApp: &openrtb2.App{ID: "test2", Ext: json.RawMessage(`{"data":{"inputFPDAppData":"inputFPDAppDataValue"}}`)}, - globalFPD: map[string][]byte{appKey: []byte(`{"globalFPDAppData": "globalFPDAppDataValue"}`)}, + globalFPD: map[string][]byte{appKey: []byte(`{"globalFPDAppData":"globalFPDAppDataValue"}`)}, expectedApp: &openrtb2.App{ID: "test1", Ext: json.RawMessage(`{"data":{"globalFPDAppData":"globalFPDAppDataValue","inputFPDAppData":"inputFPDAppDataValue"}}`)}, }, { description: "FPD config, bid request and global fpd app are specified, with input app ext.data malformed", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id":"test1"}`)}, bidRequestApp: &openrtb2.App{ID: "test2", Ext: json.RawMessage(`{"data":{"inputFPDAppData":"inputFPDAppDataValue"}}`)}, globalFPD: map[string][]byte{appKey: []byte(`malformed`)}, - expectError: true, + expectError: "invalid first party data ext", }, { description: "bid request and openrtb global fpd app are specified, no input app ext", @@ -1106,7 +1106,7 @@ func TestResolveApp(t *testing.T) { }, { description: "fpd config app, bid request and openrtb global fpd app are specified, no input app ext", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test1"}`)}, + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id":"test1"}`)}, bidRequestApp: &openrtb2.App{ID: "test2"}, openRtbGlobalFPD: map[string][]openrtb2.Data{appContentDataKey: { {ID: "DataId1", Name: "Name1"}, @@ -1119,7 +1119,7 @@ func TestResolveApp(t *testing.T) { }, { description: "fpd config app with ext, bid request and openrtb global fpd app are specified, no input app ext", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test1", "ext":{"test":1}}`)}, + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id":"test1","ext":{"test":1}}`)}, bidRequestApp: &openrtb2.App{ID: "test2"}, openRtbGlobalFPD: map[string][]openrtb2.Data{appContentDataKey: { {ID: "DataId1", Name: "Name1"}, @@ -1133,8 +1133,8 @@ func TestResolveApp(t *testing.T) { }, { description: "fpd config app with ext, bid request app with ext and openrtb global fpd app are specified, no input app ext", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test1", "ext":{"test":1}}`)}, - bidRequestApp: &openrtb2.App{ID: "test2", Ext: json.RawMessage(`{"test":2, "key": "value"}`)}, + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id":"test1","ext":{"test":1}}`)}, + bidRequestApp: &openrtb2.App{ID: "test2", Ext: json.RawMessage(`{"test":2,"key":"value"}`)}, openRtbGlobalFPD: map[string][]openrtb2.Data{appContentDataKey: { {ID: "DataId1", Name: "Name1"}, {ID: "DataId2", Name: "Name2"}, @@ -1147,8 +1147,8 @@ func TestResolveApp(t *testing.T) { }, { description: "fpd config app with malformed ext, bid request app with ext and openrtb global fpd app are specified, no input app ext", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test1", "ext":{malformed}}`)}, - bidRequestApp: &openrtb2.App{ID: "test2", Ext: json.RawMessage(`{"test":2, "key": "value"}`)}, + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id":"test1","ext":{malformed}}`)}, + bidRequestApp: &openrtb2.App{ID: "test2", Ext: json.RawMessage(`{"test":2,"key":"value"}`)}, openRtbGlobalFPD: map[string][]openrtb2.Data{appContentDataKey: { {ID: "DataId1", Name: "Name1"}, {ID: "DataId2", Name: "Name2"}, @@ -1159,15 +1159,15 @@ func TestResolveApp(t *testing.T) { }}, Ext: json.RawMessage(`{"key":"value","test":1}`), }, - expectError: true, + expectError: "invalid first party data ext", }, } for _, test := range testCases { t.Run(test.description, func(t *testing.T) { resultApp, err := resolveApp(test.fpdConfig, test.bidRequestApp, test.globalFPD, test.openRtbGlobalFPD, "bidderA") - if test.expectError { - assert.Error(t, err) + if len(test.expectError) > 0 { + assert.EqualError(t, err, test.expectError) } else { assert.NoError(t, err) assert.Equal(t, test.expectedApp, resultApp, "Result app is incorrect") @@ -1184,28 +1184,28 @@ func TestBuildExtData(t *testing.T) { }{ { description: "Input object with int value", - input: []byte(`{"someData": 123}`), - expectedRes: `{"data": {"someData": 123}}`, + input: []byte(`{"someData":123}`), + expectedRes: `{"data":{"someData":123}}`, }, { description: "Input object with bool value", - input: []byte(`{"someData": true}`), - expectedRes: `{"data": {"someData": true}}`, + input: []byte(`{"someData":true}`), + expectedRes: `{"data":{"someData":true}}`, }, { description: "Input object with string value", - input: []byte(`{"someData": "true"}`), - expectedRes: `{"data": {"someData": "true"}}`, + input: []byte(`{"someData":"true"}`), + expectedRes: `{"data":{"someData":"true"}}`, }, { description: "No input object", input: []byte(`{}`), - expectedRes: `{"data": {}}`, + expectedRes: `{"data":{}}`, }, { description: "Input object with object value", - input: []byte(`{"someData": {"moreFpdData": "fpddata"}}`), - expectedRes: `{"data": {"someData": {"moreFpdData": "fpddata"}}}`, + input: []byte(`{"someData":{"moreFpdData":"fpddata"}}`), + expectedRes: `{"data":{"someData":{"moreFpdData":"fpddata"}}}`, }, } @@ -1215,415 +1215,49 @@ func TestBuildExtData(t *testing.T) { } } -func TestMergeUser(t *testing.T) { - testCases := []struct { - name string - givenUser openrtb2.User - givenFPD json.RawMessage - expectedUser openrtb2.User - expectError bool - }{ - { - name: "empty", - givenUser: openrtb2.User{}, - givenFPD: []byte(`{}`), - expectedUser: openrtb2.User{}, - }, - { - name: "toplevel", - givenUser: openrtb2.User{ID: "1"}, - givenFPD: []byte(`{"id":"2"}`), - expectedUser: openrtb2.User{ID: "2"}, - }, - { - name: "toplevel-ext", - givenUser: openrtb2.User{Ext: []byte(`{"a":1,"b":2}`)}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}}`), - expectedUser: openrtb2.User{Ext: []byte(`{"a":1,"b":100,"c":3}`)}, - }, - { - name: "toplevel-ext-err", - givenUser: openrtb2.User{ID: "1", Ext: []byte(`malformed`)}, - givenFPD: []byte(`{"id":"2"}`), - expectError: true, - }, - { - name: "nested-geo", - givenUser: openrtb2.User{Geo: &openrtb2.Geo{Lat: ptrutil.ToPtr(1.0)}}, - givenFPD: []byte(`{"geo":{"lat": 2}}`), - expectedUser: openrtb2.User{Geo: &openrtb2.Geo{Lat: ptrutil.ToPtr(2.0)}}, - }, - { - name: "nested-geo-ext", - givenUser: openrtb2.User{Geo: &openrtb2.Geo{Ext: []byte(`{"a":1,"b":2}`)}}, - givenFPD: []byte(`{"geo":{"ext":{"b":100,"c":3}}}`), - expectedUser: openrtb2.User{Geo: &openrtb2.Geo{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}, - }, - { - name: "toplevel-ext-and-nested-geo-ext", - givenUser: openrtb2.User{Ext: []byte(`{"a":1,"b":2}`), Geo: &openrtb2.Geo{Ext: []byte(`{"a":10,"b":20}`)}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "geo":{"ext":{"b":100,"c":3}}}`), - expectedUser: openrtb2.User{Ext: []byte(`{"a":1,"b":100,"c":3}`), Geo: &openrtb2.Geo{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}, - }, - { - name: "nested-geo-ext-err", - givenUser: openrtb2.User{Geo: &openrtb2.Geo{Ext: []byte(`malformed`)}}, - givenFPD: []byte(`{"geo":{"ext":{"b":100,"c":3}}}`), - expectError: true, - }, - { - name: "fpd-err", - givenUser: openrtb2.User{ID: "1", Ext: []byte(`{"a":1}`)}, - givenFPD: []byte(`malformed`), - expectError: true, - }, - } - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - err := mergeUser(&test.givenUser, test.givenFPD) - - if test.expectError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, test.expectedUser, test.givenUser, "result user is incorrect") - } - }) - } -} - -func TestMergeApp(t *testing.T) { - testCases := []struct { - name string - givenApp openrtb2.App - givenFPD json.RawMessage - expectedApp openrtb2.App - expectError bool - }{ - { - name: "empty", - givenApp: openrtb2.App{}, - givenFPD: []byte(`{}`), - expectedApp: openrtb2.App{}, - }, - { - name: "toplevel", - givenApp: openrtb2.App{ID: "1"}, - givenFPD: []byte(`{"id":"2"}`), - expectedApp: openrtb2.App{ID: "2"}, - }, - { - name: "toplevel-ext", - givenApp: openrtb2.App{Ext: []byte(`{"a":1,"b":2}`)}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}}`), - expectedApp: openrtb2.App{Ext: []byte(`{"a":1,"b":100,"c":3}`)}, - }, - { - name: "toplevel-ext-err", - givenApp: openrtb2.App{ID: "1", Ext: []byte(`malformed`)}, - givenFPD: []byte(`{"id":"2"}`), - expectError: true, - }, - { - name: "nested-publisher", - givenApp: openrtb2.App{Publisher: &openrtb2.Publisher{Name: "pub1"}}, - givenFPD: []byte(`{"publisher":{"name": "pub2"}}`), - expectedApp: openrtb2.App{Publisher: &openrtb2.Publisher{Name: "pub2"}}, - }, - { - name: "nested-content", - givenApp: openrtb2.App{Content: &openrtb2.Content{Title: "content1"}}, - givenFPD: []byte(`{"content":{"title": "content2"}}`), - expectedApp: openrtb2.App{Content: &openrtb2.Content{Title: "content2"}}, - }, - { - name: "nested-content-producer", - givenApp: openrtb2.App{Content: &openrtb2.Content{Title: "content1", Producer: &openrtb2.Producer{Name: "producer1"}}}, - givenFPD: []byte(`{"content":{"title": "content2", "producer":{"name":"producer2"}}}`), - expectedApp: openrtb2.App{Content: &openrtb2.Content{Title: "content2", Producer: &openrtb2.Producer{Name: "producer2"}}}, - }, - { - name: "nested-content-network", - givenApp: openrtb2.App{Content: &openrtb2.Content{Title: "content1", Network: &openrtb2.Network{Name: "network1"}}}, - givenFPD: []byte(`{"content":{"title": "content2", "network":{"name":"network2"}}}`), - expectedApp: openrtb2.App{Content: &openrtb2.Content{Title: "content2", Network: &openrtb2.Network{Name: "network2"}}}, - }, - { - name: "nested-content-channel", - givenApp: openrtb2.App{Content: &openrtb2.Content{Title: "content1", Channel: &openrtb2.Channel{Name: "channel1"}}}, - givenFPD: []byte(`{"content":{"title": "content2", "channel":{"name":"channel2"}}}`), - expectedApp: openrtb2.App{Content: &openrtb2.Content{Title: "content2", Channel: &openrtb2.Channel{Name: "channel2"}}}, - }, - { - name: "nested-publisher-ext", - givenApp: openrtb2.App{Publisher: &openrtb2.Publisher{Ext: []byte(`{"a":1,"b":2}`)}}, - givenFPD: []byte(`{"publisher":{"ext":{"b":100,"c":3}}}`), - expectedApp: openrtb2.App{Publisher: &openrtb2.Publisher{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}, - }, - { - name: "nested-content-ext", - givenApp: openrtb2.App{Content: &openrtb2.Content{Ext: []byte(`{"a":1,"b":2}`)}}, - givenFPD: []byte(`{"content":{"ext":{"b":100,"c":3}}}`), - expectedApp: openrtb2.App{Content: &openrtb2.Content{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}, - }, - { - name: "nested-content-producer-ext", - givenApp: openrtb2.App{Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`{"a":1,"b":2}`)}}}, - givenFPD: []byte(`{"content":{"producer":{"ext":{"b":100,"c":3}}}}`), - expectedApp: openrtb2.App{Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}}, - }, - { - name: "nested-content-network-ext", - givenApp: openrtb2.App{Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`{"a":1,"b":2}`)}}}, - givenFPD: []byte(`{"content":{"network":{"ext":{"b":100,"c":3}}}}`), - expectedApp: openrtb2.App{Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}}, - }, - { - name: "nested-content-channel-ext", - givenApp: openrtb2.App{Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`{"a":1,"b":2}`)}}}, - givenFPD: []byte(`{"content":{"channel":{"ext":{"b":100,"c":3}}}}`), - expectedApp: openrtb2.App{Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}}, - }, - { - name: "toplevel-ext-and-nested-publisher-ext", - givenApp: openrtb2.App{Ext: []byte(`{"a":1,"b":2}`), Publisher: &openrtb2.Publisher{Ext: []byte(`{"a":10,"b":20}`)}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "publisher":{"ext":{"b":100,"c":3}}}`), - expectedApp: openrtb2.App{Ext: []byte(`{"a":1,"b":100,"c":3}`), Publisher: &openrtb2.Publisher{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}, - }, - { - name: "toplevel-ext-and-nested-content-ext", - givenApp: openrtb2.App{Ext: []byte(`{"a":1,"b":2}`), Content: &openrtb2.Content{Ext: []byte(`{"a":10,"b":20}`)}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "content":{"ext":{"b":100,"c":3}}}`), - expectedApp: openrtb2.App{Ext: []byte(`{"a":1,"b":100,"c":3}`), Content: &openrtb2.Content{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}, - }, - { - name: "toplevel-ext-and-nested-content-producer-ext", - givenApp: openrtb2.App{Ext: []byte(`{"a":1,"b":2}`), Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`{"a":10,"b":20}`)}}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "content":{"producer": {"ext":{"b":100,"c":3}}}}`), - expectedApp: openrtb2.App{Ext: []byte(`{"a":1,"b":100,"c":3}`), Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}}, - }, - { - name: "toplevel-ext-and-nested-content-network-ext", - givenApp: openrtb2.App{Ext: []byte(`{"a":1,"b":2}`), Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`{"a":10,"b":20}`)}}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "content":{"network": {"ext":{"b":100,"c":3}}}}`), - expectedApp: openrtb2.App{Ext: []byte(`{"a":1,"b":100,"c":3}`), Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}}, - }, - { - name: "toplevel-ext-and-nested-content-channel-ext", - givenApp: openrtb2.App{Ext: []byte(`{"a":1,"b":2}`), Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`{"a":10,"b":20}`)}}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "content":{"channel": {"ext":{"b":100,"c":3}}}}`), - expectedApp: openrtb2.App{Ext: []byte(`{"a":1,"b":100,"c":3}`), Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}}, - }, - { - name: "nested-publisher-ext-err", - givenApp: openrtb2.App{Publisher: &openrtb2.Publisher{Ext: []byte(`malformed`)}}, - givenFPD: []byte(`{"publisher":{"ext":{"b":100,"c":3}}}`), - expectError: true, - }, - { - name: "nested-content-ext-err", - givenApp: openrtb2.App{Content: &openrtb2.Content{Ext: []byte(`malformed`)}}, - givenFPD: []byte(`{"content":{"ext":{"b":100,"c":3}}}`), - expectError: true, - }, - { - name: "nested-content-producer-ext-err", - givenApp: openrtb2.App{Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`malformed`)}}}, - givenFPD: []byte(`{"content":{"producer": {"ext":{"b":100,"c":3}}}}`), - expectError: true, - }, - { - name: "nested-content-network-ext-err", - givenApp: openrtb2.App{Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`malformed`)}}}, - givenFPD: []byte(`{"content":{"network": {"ext":{"b":100,"c":3}}}}`), - expectError: true, - }, - { - name: "nested-content-channel-ext-err", - givenApp: openrtb2.App{Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`malformed`)}}}, - givenFPD: []byte(`{"content":{"channelx": {"ext":{"b":100,"c":3}}}}`), - expectError: true, - }, - { - name: "fpd-err", - givenApp: openrtb2.App{ID: "1", Ext: []byte(`{"a":1}`)}, - givenFPD: []byte(`malformed`), - expectError: true, - }, - } - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - err := mergeApp(&test.givenApp, test.givenFPD) - - if test.expectError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, test.expectedApp, test.givenApp, " result app is incorrect") - } - }) - } -} - func TestMergeSite(t *testing.T) { testCases := []struct { name string givenSite openrtb2.Site givenFPD json.RawMessage expectedSite openrtb2.Site - expectError bool + expectError string }{ { - name: "empty", - givenSite: openrtb2.Site{}, - givenFPD: []byte(`{}`), - expectError: true, - }, - { - name: "toplevel", + name: "valid-id", givenSite: openrtb2.Site{ID: "1"}, givenFPD: []byte(`{"id":"2"}`), expectedSite: openrtb2.Site{ID: "2"}, }, { - name: "toplevel-ext", - givenSite: openrtb2.Site{Page: "test.com/page", Ext: []byte(`{"a":1,"b":2}`)}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}}`), - expectedSite: openrtb2.Site{Page: "test.com/page", Ext: []byte(`{"a":1,"b":100,"c":3}`)}, - }, - { - name: "toplevel-ext-err", - givenSite: openrtb2.Site{ID: "1", Ext: []byte(`malformed`)}, - givenFPD: []byte(`{"id":"2"}`), - expectError: true, - }, - { - name: "nested-publisher", - givenSite: openrtb2.Site{Page: "test.com/page", Publisher: &openrtb2.Publisher{Name: "pub1"}}, - givenFPD: []byte(`{"publisher":{"name": "pub2"}}`), - expectedSite: openrtb2.Site{Page: "test.com/page", Publisher: &openrtb2.Publisher{Name: "pub2"}}, - }, - { - name: "nested-content", - givenSite: openrtb2.Site{Page: "test.com/page", Content: &openrtb2.Content{Title: "content1"}}, - givenFPD: []byte(`{"content":{"title": "content2"}}`), - expectedSite: openrtb2.Site{Page: "test.com/page", Content: &openrtb2.Content{Title: "content2"}}, + name: "valid-page", + givenSite: openrtb2.Site{Page: "1"}, + givenFPD: []byte(`{"page":"2"}`), + expectedSite: openrtb2.Site{Page: "2"}, }, { - name: "nested-content-producer", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Title: "content1", Producer: &openrtb2.Producer{Name: "producer1"}}}, - givenFPD: []byte(`{"content":{"title": "content2", "producer":{"name":"producer2"}}}`), - expectedSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Title: "content2", Producer: &openrtb2.Producer{Name: "producer2"}}}, + name: "invalid-id", + givenSite: openrtb2.Site{ID: "a"}, + givenFPD: []byte(`{"id":null}`), + expectError: "incorrect First Party Data for bidder BidderA: Site object cannot set empty page if req.site.id is empty", }, { - name: "nested-content-network", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Title: "content1", Network: &openrtb2.Network{Name: "network1"}}}, - givenFPD: []byte(`{"content":{"title": "content2", "network":{"name":"network2"}}}`), - expectedSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Title: "content2", Network: &openrtb2.Network{Name: "network2"}}}, + name: "invalid-page", + givenSite: openrtb2.Site{Page: "a"}, + givenFPD: []byte(`{"page":null}`), + expectError: "incorrect First Party Data for bidder BidderA: Site object cannot set empty page if req.site.id is empty", }, { - name: "nested-content-channel", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Title: "content1", Channel: &openrtb2.Channel{Name: "channel1"}}}, - givenFPD: []byte(`{"content":{"title": "content2", "channel":{"name":"channel2"}}}`), - expectedSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Title: "content2", Channel: &openrtb2.Channel{Name: "channel2"}}}, - }, - { - name: "nested-publisher-ext", - givenSite: openrtb2.Site{ID: "1", Publisher: &openrtb2.Publisher{Ext: []byte(`{"a":1,"b":2}`)}}, - givenFPD: []byte(`{"publisher":{"ext":{"b":100,"c":3}}}`), - expectedSite: openrtb2.Site{ID: "1", Publisher: &openrtb2.Publisher{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}, - }, - { - name: "nested-content-ext", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Ext: []byte(`{"a":1,"b":2}`)}}, - givenFPD: []byte(`{"content":{"ext":{"b":100,"c":3}}}`), - expectedSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}, - }, - { - name: "nested-content-producer-ext", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`{"a":1,"b":2}`)}}}, - givenFPD: []byte(`{"content":{"producer":{"ext":{"b":100,"c":3}}}}`), - expectedSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}}, - }, - { - name: "nested-content-network-ext", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`{"a":1,"b":2}`)}}}, - givenFPD: []byte(`{"content":{"network":{"ext":{"b":100,"c":3}}}}`), - expectedSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}}, - }, - { - name: "nested-content-channel-ext", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`{"a":1,"b":2}`)}}}, - givenFPD: []byte(`{"content":{"channel":{"ext":{"b":100,"c":3}}}}`), - expectedSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`{"a":1,"b":100,"c":3}`)}}}, - }, - { - name: "toplevel-ext-and-nested-publisher-ext", - givenSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1,"b":2}`), Publisher: &openrtb2.Publisher{Ext: []byte(`{"a":10,"b":20}`)}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "publisher":{"ext":{"b":100,"c":3}}}`), - expectedSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1,"b":100,"c":3}`), Publisher: &openrtb2.Publisher{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}, - }, - { - name: "toplevel-ext-and-nested-content-ext", - givenSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1,"b":2}`), Content: &openrtb2.Content{Ext: []byte(`{"a":10,"b":20}`)}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "content":{"ext":{"b":100,"c":3}}}`), - expectedSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1,"b":100,"c":3}`), Content: &openrtb2.Content{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}, - }, - { - name: "toplevel-ext-and-nested-content-producer-ext", - givenSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1,"b":2}`), Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`{"a":10,"b":20}`)}}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "content":{"producer": {"ext":{"b":100,"c":3}}}}`), - expectedSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1,"b":100,"c":3}`), Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}}, - }, - { - name: "toplevel-ext-and-nested-content-network-ext", - givenSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1,"b":2}`), Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`{"a":10,"b":20}`)}}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "content":{"network": {"ext":{"b":100,"c":3}}}}`), - expectedSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1,"b":100,"c":3}`), Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}}, - }, - { - name: "toplevel-ext-and-nested-content-channel-ext", - givenSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1,"b":2}`), Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`{"a":10,"b":20}`)}}}, - givenFPD: []byte(`{"ext":{"b":100,"c":3}, "content":{"channel": {"ext":{"b":100,"c":3}}}}`), - expectedSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1,"b":100,"c":3}`), Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`{"a":10,"b":100,"c":3}`)}}}, - }, - { - name: "nested-publisher-ext-err", - givenSite: openrtb2.Site{ID: "1", Publisher: &openrtb2.Publisher{Ext: []byte(`malformed`)}}, - givenFPD: []byte(`{"publisher":{"ext":{"b":100,"c":3}}}`), - expectError: true, - }, - { - name: "nested-content-ext-err", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Ext: []byte(`malformed`)}}, - givenFPD: []byte(`{"content":{"ext":{"b":100,"c":3}}}`), - expectError: true, - }, - { - name: "nested-content-producer-ext-err", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`malformed`)}}}, - givenFPD: []byte(`{"content":{"producer": {"ext":{"b":100,"c":3}}}}`), - expectError: true, - }, - { - name: "nested-content-network-ext-err", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`malformed`)}}}, - givenFPD: []byte(`{"content":{"network": {"ext":{"b":100,"c":3}}}}`), - expectError: true, - }, - { - name: "nested-content-channel-ext-err", - givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`malformed`)}}}, - givenFPD: []byte(`{"content":{"channelx": {"ext":{"b":100,"c":3}}}}`), - expectError: true, + name: "existing-err", + givenSite: openrtb2.Site{ID: "1", Ext: []byte(`malformed`)}, + givenFPD: []byte(`{"ext":{"a":1}}`), + expectError: "invalid request ext", }, { name: "fpd-err", givenSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1}`)}, givenFPD: []byte(`malformed`), - expectError: true, + expectError: "invalid first party data ext", }, } @@ -1631,8 +1265,8 @@ func TestMergeSite(t *testing.T) { t.Run(test.name, func(t *testing.T) { err := mergeSite(&test.givenSite, test.givenFPD, "BidderA") - if test.expectError { - assert.Error(t, err) + if len(test.expectError) > 0 { + assert.EqualError(t, err, test.expectError) } else { assert.NoError(t, err) assert.Equal(t, test.expectedSite, test.givenSite, " result Site is incorrect") @@ -1641,255 +1275,6 @@ func TestMergeSite(t *testing.T) { } } -// TestMergeObjectStructure detects when new nested objects are added to First Party Data supported -// fields, as these will invalidate the mergeSite, mergeApp, and mergeUser methods. If this test fails, -// fix the merge methods to add support and update this test to set a new baseline. -func TestMergeObjectStructure(t *testing.T) { - testCases := []struct { - name string - kind any - knownStructs []string - }{ - { - name: "Site", - kind: openrtb2.Site{}, - knownStructs: []string{ - "Publisher", - "Content", - "Content.Producer", - "Content.Network", - "Content.Channel", - }, - }, - { - name: "App", - kind: openrtb2.App{}, - knownStructs: []string{ - "Publisher", - "Content", - "Content.Producer", - "Content.Network", - "Content.Channel", - }, - }, - { - name: "User", - kind: openrtb2.User{}, - knownStructs: []string{ - "Geo", - }, - }, - } - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - nestedStructs := []string{} - - var discover func(parent string, t reflect.Type) - discover = func(parent string, t reflect.Type) { - fields := reflect.VisibleFields(t) - for _, field := range fields { - if field.Type.Kind() == reflect.Pointer && field.Type.Elem().Kind() == reflect.Struct { - nestedStructs = append(nestedStructs, parent+field.Name) - discover(parent+field.Name+".", field.Type.Elem()) - } - } - } - discover("", reflect.TypeOf(test.kind)) - - assert.ElementsMatch(t, test.knownStructs, nestedStructs) - }) - } -} - -// user memory protect test -func TestMergeUserMemoryProtection(t *testing.T) { - inputGeo := &openrtb2.Geo{ - Ext: json.RawMessage(`{"a":1,"b":2}`), - } - input := openrtb2.User{ - ID: "1", - Geo: inputGeo, - } - - err := mergeUser(&input, userFPD) - assert.NoError(t, err) - - // Input user object is expected to be a copy. Changes are ok. - assert.Equal(t, "2", input.ID, "user-id-copied") - - // Nested objects must be copied before changes. - assert.JSONEq(t, `{"a":1,"b":2}`, string(inputGeo.Ext), "geo-input") - assert.JSONEq(t, `{"a":1,"b":100,"c":3}`, string(input.Geo.Ext), "geo-copied") -} - -// app memory protect test -func TestMergeAppMemoryProtection(t *testing.T) { - inputPublisher := &openrtb2.Publisher{ - ID: "InPubId", - Ext: json.RawMessage(`{"a": "inputPubExt", "b": 1}`), - } - inputContent := &openrtb2.Content{ - ID: "InContentId", - Ext: json.RawMessage(`{"a": "inputContentExt", "b": 1}`), - Producer: &openrtb2.Producer{ - ID: "InProducerId", - Ext: json.RawMessage(`{"a": "inputProducerExt", "b": 1}`), - }, - Network: &openrtb2.Network{ - ID: "InNetworkId", - Ext: json.RawMessage(`{"a": "inputNetworkExt", "b": 1}`), - }, - Channel: &openrtb2.Channel{ - ID: "InChannelId", - Ext: json.RawMessage(`{"a": "inputChannelExt", "b": 1}`), - }, - } - input := openrtb2.App{ - ID: "InAppID", - Publisher: inputPublisher, - Content: inputContent, - Ext: json.RawMessage(`{"a": "inputAppExt", "b": 1}`), - } - - err := mergeApp(&input, fpdWithPublisherAndContent) - assert.NoError(t, err) - - // Input app object is expected to be a copy. Changes are ok. - assert.Equal(t, "FPDID", input.ID, "app-id-copied") - assert.JSONEq(t, `{"a": "FPDExt", "b": 2}`, string(input.Ext), "app-ext-copied") - - // Nested objects must be copied before changes. - assert.Equal(t, "InPubId", inputPublisher.ID, "app-pub-id-input") - assert.Equal(t, "FPDPubId", input.Publisher.ID, "app-pub-id-copied") - assert.JSONEq(t, `{"a": "inputPubExt", "b": 1}`, string(inputPublisher.Ext), "app-pub-ext-input") - assert.JSONEq(t, `{"a": "FPDPubExt", "b": 2}`, string(input.Publisher.Ext), "app-pub-ext-copied") - - assert.Equal(t, "InContentId", inputContent.ID, "app-content-id-input") - assert.Equal(t, "FPDContentId", input.Content.ID, "app-content-id-copied") - assert.JSONEq(t, `{"a": "inputContentExt", "b": 1}`, string(inputContent.Ext), "app-content-ext-input") - assert.JSONEq(t, `{"a": "FPDContentExt", "b": 2}`, string(input.Content.Ext), "app-content-ext-copied") - - assert.Equal(t, "InProducerId", inputContent.Producer.ID, "app-content-producer-id-input") - assert.Equal(t, "FPDProducerId", input.Content.Producer.ID, "app-content-producer-id-copied") - assert.JSONEq(t, `{"a": "inputProducerExt", "b": 1}`, string(inputContent.Producer.Ext), "app-content-producer-ext-input") - assert.JSONEq(t, `{"a": "FPDProducerExt", "b": 2}`, string(input.Content.Producer.Ext), "app-content-producer-ext-copied") - - assert.Equal(t, "InNetworkId", inputContent.Network.ID, "app-content-network-id-input") - assert.Equal(t, "FPDNetworkId", input.Content.Network.ID, "app-content-network-id-copied") - assert.JSONEq(t, `{"a": "inputNetworkExt", "b": 1}`, string(inputContent.Network.Ext), "app-content-network-ext-input") - assert.JSONEq(t, `{"a": "FPDNetworkExt", "b": 2}`, string(input.Content.Network.Ext), "app-content-network-ext-copied") - - assert.Equal(t, "InChannelId", inputContent.Channel.ID, "app-content-channel-id-input") - assert.Equal(t, "FPDChannelId", input.Content.Channel.ID, "app-content-channel-id-copied") - assert.JSONEq(t, `{"a": "inputChannelExt", "b": 1}`, string(inputContent.Channel.Ext), "app-content-channel-ext-input") - assert.JSONEq(t, `{"a": "FPDChannelExt", "b": 2}`, string(input.Content.Channel.Ext), "app-content-channel-ext-copied") -} - -// site memory protect test -func TestMergeSiteMemoryProtection(t *testing.T) { - inputPublisher := &openrtb2.Publisher{ - ID: "InPubId", - Ext: json.RawMessage(`{"a": "inputPubExt", "b": 1}`), - } - inputContent := &openrtb2.Content{ - ID: "InContentId", - Ext: json.RawMessage(`{"a": "inputContentExt", "b": 1}`), - Producer: &openrtb2.Producer{ - ID: "InProducerId", - Ext: json.RawMessage(`{"a": "inputProducerExt", "b": 1}`), - }, - Network: &openrtb2.Network{ - ID: "InNetworkId", - Ext: json.RawMessage(`{"a": "inputNetworkExt", "b": 1}`), - }, - Channel: &openrtb2.Channel{ - ID: "InChannelId", - Ext: json.RawMessage(`{"a": "inputChannelExt", "b": 1}`), - }, - } - input := openrtb2.Site{ - ID: "InSiteID", - Publisher: inputPublisher, - Content: inputContent, - Ext: json.RawMessage(`{"a": "inputSiteExt", "b": 1}`), - } - - err := mergeSite(&input, fpdWithPublisherAndContent, "BidderA") - assert.NoError(t, err) - - // Input app object is expected to be a copy. Changes are ok. - assert.Equal(t, "FPDID", input.ID, "site-id-copied") - assert.JSONEq(t, `{"a": "FPDExt", "b": 2}`, string(input.Ext), "site-ext-copied") - - // Nested objects must be copied before changes. - assert.Equal(t, "InPubId", inputPublisher.ID, "site-pub-id-input") - assert.Equal(t, "FPDPubId", input.Publisher.ID, "site-pub-id-copied") - assert.JSONEq(t, `{"a": "inputPubExt", "b": 1}`, string(inputPublisher.Ext), "site-pub-ext-input") - assert.JSONEq(t, `{"a": "FPDPubExt", "b": 2}`, string(input.Publisher.Ext), "site-pub-ext-copied") - - assert.Equal(t, "InContentId", inputContent.ID, "site-content-id-input") - assert.Equal(t, "FPDContentId", input.Content.ID, "site-content-id-copied") - assert.JSONEq(t, `{"a": "inputContentExt", "b": 1}`, string(inputContent.Ext), "site-content-ext-input") - assert.JSONEq(t, `{"a": "FPDContentExt", "b": 2}`, string(input.Content.Ext), "site-content-ext-copied") - - assert.Equal(t, "InProducerId", inputContent.Producer.ID, "site-content-producer-id-input") - assert.Equal(t, "FPDProducerId", input.Content.Producer.ID, "site-content-producer-id-copied") - assert.JSONEq(t, `{"a": "inputProducerExt", "b": 1}`, string(inputContent.Producer.Ext), "site-content-producer-ext-input") - assert.JSONEq(t, `{"a": "FPDProducerExt", "b": 2}`, string(input.Content.Producer.Ext), "site-content-producer-ext-copied") - - assert.Equal(t, "InNetworkId", inputContent.Network.ID, "site-content-network-id-input") - assert.Equal(t, "FPDNetworkId", input.Content.Network.ID, "site-content-network-id-copied") - assert.JSONEq(t, `{"a": "inputNetworkExt", "b": 1}`, string(inputContent.Network.Ext), "site-content-network-ext-input") - assert.JSONEq(t, `{"a": "FPDNetworkExt", "b": 2}`, string(input.Content.Network.Ext), "site-content-network-ext-copied") - - assert.Equal(t, "InChannelId", inputContent.Channel.ID, "site-content-channel-id-input") - assert.Equal(t, "FPDChannelId", input.Content.Channel.ID, "site-content-channel-id-copied") - assert.JSONEq(t, `{"a": "inputChannelExt", "b": 1}`, string(inputContent.Channel.Ext), "site-content-channel-ext-input") - assert.JSONEq(t, `{"a": "FPDChannelExt", "b": 2}`, string(input.Content.Channel.Ext), "site-content-channel-ext-copied") -} - -var ( - userFPD = []byte(` -{ - "id": "2", - "geo": { - "ext": { - "b": 100, - "c": 3 - } - } -} -`) - - fpdWithPublisherAndContent = []byte(` -{ - "id": "FPDID", - "ext": {"a": "FPDExt", "b": 2}, - "publisher": { - "id": "FPDPubId", - "ext": {"a": "FPDPubExt", "b": 2} - }, - "content": { - "id": "FPDContentId", - "ext": {"a": "FPDContentExt", "b": 2}, - "producer": { - "id": "FPDProducerId", - "ext": {"a": "FPDProducerExt", "b": 2} - }, - "network": { - "id": "FPDNetworkId", - "ext": {"a": "FPDNetworkExt", "b": 2} - }, - "channel": { - "id": "FPDChannelId", - "ext": {"a": "FPDChannelExt", "b": 2} - } - } -} -`) -) - func loadTestFile[T any](filename string) (T, error) { var testFile T diff --git a/go.mod b/go.mod index 94e2a2a4e3c..19043a0a557 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/julienschmidt/httprouter v1.3.0 github.com/lib/pq v1.10.4 github.com/mitchellh/copystructure v1.2.0 + github.com/modern-go/reflect2 v1.0.2 github.com/pkg/errors v0.9.1 github.com/prebid/go-gdpr v1.12.0 github.com/prebid/go-gpp v0.2.0 @@ -53,7 +54,6 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/ortb/clone.go b/ortb/clone.go index 3023169bc8c..fa55cbe124f 100644 --- a/ortb/clone.go +++ b/ortb/clone.go @@ -6,83 +6,6 @@ import ( "github.com/prebid/prebid-server/v2/util/sliceutil" ) -func CloneApp(s *openrtb2.App) *openrtb2.App { - if s == nil { - return nil - } - - // Shallow Copy (Value Fields) - c := *s - - // Deep Copy (Pointers) - c.Cat = sliceutil.Clone(s.Cat) - c.SectionCat = sliceutil.Clone(s.SectionCat) - c.PageCat = sliceutil.Clone(s.PageCat) - c.PrivacyPolicy = ptrutil.Clone(s.PrivacyPolicy) - c.Paid = ptrutil.Clone(s.Paid) - c.Publisher = ClonePublisher(s.Publisher) - c.Content = CloneContent(s.Content) - c.KwArray = sliceutil.Clone(s.KwArray) - c.Ext = sliceutil.Clone(s.Ext) - - return &c -} - -func ClonePublisher(s *openrtb2.Publisher) *openrtb2.Publisher { - if s == nil { - return nil - } - - // Shallow Copy (Value Fields) - c := *s - - // Deep Copy (Pointers) - c.Cat = sliceutil.Clone(s.Cat) - c.Ext = sliceutil.Clone(s.Ext) - - return &c -} - -func CloneContent(s *openrtb2.Content) *openrtb2.Content { - if s == nil { - return nil - } - - // Shallow Copy (Value Fields) - c := *s - - // Deep Copy (Pointers) - c.Producer = CloneProducer(s.Producer) - c.Cat = sliceutil.Clone(s.Cat) - c.ProdQ = ptrutil.Clone(s.ProdQ) - c.VideoQuality = ptrutil.Clone(s.VideoQuality) - c.KwArray = sliceutil.Clone(s.KwArray) - c.LiveStream = ptrutil.Clone(s.LiveStream) - c.SourceRelationship = ptrutil.Clone(s.SourceRelationship) - c.Embeddable = ptrutil.Clone(s.Embeddable) - c.Data = CloneDataSlice(s.Data) - c.Network = CloneNetwork(s.Network) - c.Channel = CloneChannel(s.Channel) - c.Ext = sliceutil.Clone(s.Ext) - - return &c -} - -func CloneProducer(s *openrtb2.Producer) *openrtb2.Producer { - if s == nil { - return nil - } - - // Shallow Copy (Value Fields) - c := *s - - // Deep Copy (Pointers) - c.Cat = sliceutil.Clone(s.Cat) - c.Ext = sliceutil.Clone(s.Ext) - - return &c -} - func CloneDataSlice(s []openrtb2.Data) []openrtb2.Data { if s == nil { return nil @@ -130,56 +53,6 @@ func CloneSegment(s openrtb2.Segment) openrtb2.Segment { return s } -func CloneNetwork(s *openrtb2.Network) *openrtb2.Network { - if s == nil { - return nil - } - - // Shallow Copy (Value Fields) - c := *s - - // Deep Copy (Pointers) - c.Ext = sliceutil.Clone(s.Ext) - - return &c -} - -func CloneChannel(s *openrtb2.Channel) *openrtb2.Channel { - if s == nil { - return nil - } - - // Shallow Copy (Value Fields) - c := *s - - // Deep Copy (Pointers) - c.Ext = sliceutil.Clone(s.Ext) - - return &c -} - -func CloneSite(s *openrtb2.Site) *openrtb2.Site { - if s == nil { - return nil - } - - // Shallow Copy (Value Fields) - c := *s - - // Deep Copy (Pointers) - c.Cat = sliceutil.Clone(s.Cat) - c.SectionCat = sliceutil.Clone(s.SectionCat) - c.PageCat = sliceutil.Clone(s.PageCat) - c.Mobile = ptrutil.Clone(s.Mobile) - c.PrivacyPolicy = ptrutil.Clone(s.PrivacyPolicy) - c.Publisher = ClonePublisher(s.Publisher) - c.Content = CloneContent(s.Content) - c.KwArray = sliceutil.Clone(s.KwArray) - c.Ext = sliceutil.Clone(s.Ext) - - return &c -} - func CloneUser(s *openrtb2.User) *openrtb2.User { if s == nil { return nil @@ -387,24 +260,6 @@ func CloneUID(s openrtb2.UID) openrtb2.UID { return s } -func CloneDOOH(s *openrtb2.DOOH) *openrtb2.DOOH { - if s == nil { - return nil - } - - // Shallow Copy (Value Fields) - c := *s - - // Deep Copy (Pointers) - c.VenueType = sliceutil.Clone(s.VenueType) - c.VenueTypeTax = ptrutil.Clone(s.VenueTypeTax) - c.Publisher = ClonePublisher(s.Publisher) - c.Content = CloneContent(s.Content) - c.Ext = sliceutil.Clone(s.Ext) - - return &c -} - // CloneBidRequestPartial performs a deep clone of just the bid request device, user, and source fields. func CloneBidRequestPartial(s *openrtb2.BidRequest) *openrtb2.BidRequest { if s == nil { diff --git a/ortb/clone_test.go b/ortb/clone_test.go index 73d03614db4..21c19f170c6 100644 --- a/ortb/clone_test.go +++ b/ortb/clone_test.go @@ -11,236 +11,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestCloneApp(t *testing.T) { - t.Run("nil", func(t *testing.T) { - result := CloneApp(nil) - assert.Nil(t, result) - }) - - t.Run("empty", func(t *testing.T) { - given := &openrtb2.App{} - result := CloneApp(given) - assert.Equal(t, given, result) - assert.NotSame(t, given, result) - }) - - t.Run("populated", func(t *testing.T) { - given := &openrtb2.App{ - ID: "anyID", - Name: "anyName", - Bundle: "anyBundle", - Domain: "anyDomain", - StoreURL: "anyStoreURL", - CatTax: adcom1.CatTaxIABContent10, - Cat: []string{"cat1"}, - SectionCat: []string{"sectionCat1"}, - PageCat: []string{"pageCat1"}, - Ver: "anyVer", - PrivacyPolicy: ptrutil.ToPtr[int8](1), - Paid: ptrutil.ToPtr[int8](2), - Publisher: &openrtb2.Publisher{ID: "anyPublisher", Ext: json.RawMessage(`{"publisher":1}`)}, - Content: &openrtb2.Content{ID: "anyContent", Ext: json.RawMessage(`{"content":1}`)}, - Keywords: "anyKeywords", - KwArray: []string{"key1"}, - InventoryPartnerDomain: "anyInventoryPartnerDomain", - Ext: json.RawMessage(`{"anyField":1}`), - } - result := CloneApp(given) - assert.Equal(t, given, result, "equality") - assert.NotSame(t, given, result, "pointer") - assert.NotSame(t, given.Cat, result.Cat, "cat") - assert.NotSame(t, given.SectionCat, result.SectionCat, "sectioncat") - assert.NotSame(t, given.PageCat, result.PageCat, "pagecat") - assert.NotSame(t, given.PrivacyPolicy, result.PrivacyPolicy, "privacypolicy") - assert.NotSame(t, given.Paid, result.Paid, "paid") - assert.NotSame(t, given.Publisher, result.Publisher, "publisher") - assert.NotSame(t, given.Publisher.Ext, result.Publisher.Ext, "publisher-ext") - assert.NotSame(t, given.Content, result.Content, "content") - assert.NotSame(t, given.Content.Ext, result.Content.Ext, "content-ext") - assert.NotSame(t, given.KwArray, result.KwArray, "kwarray") - assert.NotSame(t, given.Ext, result.Ext, "ext") - }) - - t.Run("assumptions", func(t *testing.T) { - assert.ElementsMatch(t, discoverPointerFields(reflect.TypeOf(openrtb2.App{})), - []string{ - "Cat", - "SectionCat", - "PageCat", - "PrivacyPolicy", - "Paid", - "Publisher", - "Content", - "KwArray", - "Ext", - }) - }) -} - -func TestClonePublisher(t *testing.T) { - t.Run("nil", func(t *testing.T) { - result := ClonePublisher(nil) - assert.Nil(t, result) - }) - - t.Run("empty", func(t *testing.T) { - given := &openrtb2.Publisher{} - result := ClonePublisher(given) - assert.Equal(t, given, result) - assert.NotSame(t, given, result) - }) - - t.Run("populated", func(t *testing.T) { - given := &openrtb2.Publisher{ - ID: "anyID", - Name: "anyName", - CatTax: adcom1.CatTaxIABContent20, - Cat: []string{"cat1"}, - Domain: "anyDomain", - Ext: json.RawMessage(`{"anyField":1}`), - } - result := ClonePublisher(given) - assert.Equal(t, given, result, "equality") - assert.NotSame(t, given, result, "pointer") - assert.NotSame(t, given.Cat, result.Cat, "cat") - assert.NotSame(t, given.Ext, result.Ext, "ext") - }) - - t.Run("assumptions", func(t *testing.T) { - assert.ElementsMatch(t, discoverPointerFields(reflect.TypeOf(openrtb2.Publisher{})), - []string{ - "Cat", - "Ext", - }) - }) -} - -func TestCloneContent(t *testing.T) { - t.Run("nil", func(t *testing.T) { - result := CloneContent(nil) - assert.Nil(t, result) - }) - - t.Run("empty", func(t *testing.T) { - given := &openrtb2.Content{} - result := CloneContent(given) - assert.Equal(t, given, result) - assert.NotSame(t, given, result) - }) - - t.Run("populated", func(t *testing.T) { - given := &openrtb2.Content{ - ID: "anyID", - Episode: 1, - Title: "anyTitle", - Series: "anySeries", - Season: "anySeason", - Artist: "anyArtist", - Genre: "anyGenre", - Album: "anyAlbum", - ISRC: "anyIsrc", - Producer: &openrtb2.Producer{ID: "anyID", Cat: []string{"anyCat"}}, - URL: "anyUrl", - CatTax: adcom1.CatTaxIABContent10, - Cat: []string{"cat1"}, - ProdQ: ptrutil.ToPtr(adcom1.ProductionProsumer), - VideoQuality: ptrutil.ToPtr(adcom1.ProductionProfessional), - Context: adcom1.ContentApp, - ContentRating: "anyContentRating", - UserRating: "anyUserRating", - QAGMediaRating: adcom1.MediaRatingAll, - Keywords: "anyKeywords", - KwArray: []string{"key1"}, - LiveStream: ptrutil.ToPtr[int8](2), - SourceRelationship: ptrutil.ToPtr[int8](3), - Len: 4, - Language: "anyLanguage", - LangB: "anyLangB", - Embeddable: ptrutil.ToPtr[int8](5), - Data: []openrtb2.Data{{ID: "1", Ext: json.RawMessage(`{"data":1}`)}}, - Network: &openrtb2.Network{ID: "anyNetwork", Ext: json.RawMessage(`{"network":1}`)}, - Channel: &openrtb2.Channel{ID: "anyChannel", Ext: json.RawMessage(`{"channel":1}`)}, - Ext: json.RawMessage(`{"anyField":1}`), - } - result := CloneContent(given) - assert.Equal(t, given, result, "equality") - assert.NotSame(t, given, result, "pointer") - assert.NotSame(t, given.Producer, result.Producer, "producer") - assert.NotSame(t, given.Producer.Cat, result.Producer.Cat, "producer-cat") - assert.NotSame(t, given.Cat, result.Cat, "cat") - assert.NotSame(t, given.ProdQ, result.ProdQ, "prodq") - assert.NotSame(t, given.VideoQuality, result.VideoQuality, "videoquality") - assert.NotSame(t, given.KwArray, result.KwArray, "kwarray") - assert.NotSame(t, given.LiveStream, result.LiveStream, "livestream") - assert.NotSame(t, given.SourceRelationship, result.SourceRelationship, "sourcerelationship") - assert.NotSame(t, given.Embeddable, result.Embeddable, "embeddable") - assert.NotSame(t, given.Data, result.Data, "data") - assert.NotSame(t, given.Data[0], result.Data[0], "data-item") - assert.NotSame(t, given.Data[0].Ext, result.Data[0].Ext, "data-item-ext") - assert.NotSame(t, given.Network, result.Network, "network") - assert.NotSame(t, given.Network.Ext, result.Network.Ext, "network-ext") - assert.NotSame(t, given.Channel, result.Channel, "channel") - assert.NotSame(t, given.Channel.Ext, result.Channel.Ext, "channel-ext") - assert.NotSame(t, given.Ext, result.Ext, "ext") - }) - - t.Run("assumptions", func(t *testing.T) { - assert.ElementsMatch(t, discoverPointerFields(reflect.TypeOf(openrtb2.Content{})), - []string{ - "Producer", - "Cat", - "ProdQ", - "VideoQuality", - "KwArray", - "LiveStream", - "SourceRelationship", - "Embeddable", - "Data", - "Network", - "Channel", - "Ext", - }) - }) -} - -func TestCloneProducer(t *testing.T) { - t.Run("nil", func(t *testing.T) { - result := CloneProducer(nil) - assert.Nil(t, result) - }) - - t.Run("empty", func(t *testing.T) { - given := &openrtb2.Producer{} - result := CloneProducer(given) - assert.Equal(t, given, result) - assert.NotSame(t, given, result) - }) - - t.Run("populated", func(t *testing.T) { - given := &openrtb2.Producer{ - ID: "anyID", - Name: "anyName", - CatTax: adcom1.CatTaxIABContent20, - Cat: []string{"cat1"}, - Domain: "anyDomain", - Ext: json.RawMessage(`{"anyField":1}`), - } - result := CloneProducer(given) - assert.Equal(t, given, result, "equality") - assert.NotSame(t, given, result, "pointer") - assert.NotSame(t, given.Cat, result.Cat, "cat") - assert.NotSame(t, given.Ext, result.Ext, "ext") - }) - - t.Run("assumptions", func(t *testing.T) { - assert.ElementsMatch(t, discoverPointerFields(reflect.TypeOf(openrtb2.Producer{})), - []string{ - "Cat", - "Ext", - }) - }) -} - func TestCloneDataSlice(t *testing.T) { t.Run("nil", func(t *testing.T) { result := CloneDataSlice(nil) @@ -363,140 +133,6 @@ func TestCloneSegment(t *testing.T) { }) } -func TestCloneNetwork(t *testing.T) { - t.Run("nil", func(t *testing.T) { - result := CloneNetwork(nil) - assert.Nil(t, result) - }) - - t.Run("empty", func(t *testing.T) { - given := &openrtb2.Network{} - result := CloneNetwork(given) - assert.Empty(t, result) - assert.NotSame(t, given, result) - }) - - t.Run("populated", func(t *testing.T) { - given := &openrtb2.Network{ - ID: "anyID", - Name: "anyName", - Domain: "anyDomain", - Ext: json.RawMessage(`{"anyField":1}`), - } - result := CloneNetwork(given) - assert.Equal(t, given, result, "equality") - assert.NotSame(t, given, result, "pointer") - assert.NotSame(t, given.Ext, result.Ext, "ext") - }) - - t.Run("assumptions", func(t *testing.T) { - assert.ElementsMatch(t, discoverPointerFields(reflect.TypeOf(openrtb2.Network{})), - []string{ - "Ext", - }) - }) -} - -func TestCloneChannel(t *testing.T) { - t.Run("nil", func(t *testing.T) { - result := CloneChannel(nil) - assert.Nil(t, result) - }) - - t.Run("empty", func(t *testing.T) { - given := &openrtb2.Channel{} - result := CloneChannel(given) - assert.Empty(t, result) - assert.NotSame(t, given, result) - }) - - t.Run("populated", func(t *testing.T) { - given := &openrtb2.Channel{ - ID: "anyID", - Name: "anyName", - Domain: "anyDomain", - Ext: json.RawMessage(`{"anyField":1}`), - } - result := CloneChannel(given) - assert.Equal(t, given, result, "equality") - assert.NotSame(t, given, result, "pointer") - assert.NotSame(t, given.Ext, result.Ext, "ext") - }) - - t.Run("assumptions", func(t *testing.T) { - assert.ElementsMatch(t, discoverPointerFields(reflect.TypeOf(openrtb2.Channel{})), - []string{ - "Ext", - }) - }) -} - -func TestCloneSite(t *testing.T) { - t.Run("nil", func(t *testing.T) { - result := CloneSite(nil) - assert.Nil(t, result) - }) - - t.Run("empty", func(t *testing.T) { - given := &openrtb2.Site{} - result := CloneSite(given) - assert.Empty(t, result) - assert.NotSame(t, given, result) - }) - - t.Run("populated", func(t *testing.T) { - given := &openrtb2.Site{ - ID: "anyID", - Name: "anyName", - Domain: "anyDomain", - CatTax: adcom1.CatTaxIABContent10, - Cat: []string{"cat1"}, - SectionCat: []string{"sectionCat1"}, - PageCat: []string{"pageCat1"}, - Page: "anyPage", - Ref: "anyRef", - Search: "anySearch", - Mobile: ptrutil.ToPtr[int8](1), - PrivacyPolicy: ptrutil.ToPtr[int8](2), - Publisher: &openrtb2.Publisher{ID: "anyPublisher", Ext: json.RawMessage(`{"publisher":1}`)}, - Content: &openrtb2.Content{ID: "anyContent", Ext: json.RawMessage(`{"content":1}`)}, - Keywords: "anyKeywords", - KwArray: []string{"key1"}, - InventoryPartnerDomain: "anyInventoryPartnerDomain", - Ext: json.RawMessage(`{"anyField":1}`), - } - result := CloneSite(given) - assert.Equal(t, given, result, "equality") - assert.NotSame(t, given, result, "pointer") - assert.NotSame(t, given.Cat, result.Cat, "cat") - assert.NotSame(t, given.SectionCat, result.SectionCat, "sectioncat") - assert.NotSame(t, given.PageCat, result.PageCat, "pagecat") - assert.NotSame(t, given.Mobile, result.Mobile, "mobile") - assert.NotSame(t, given.PrivacyPolicy, result.PrivacyPolicy, "privacypolicy") - assert.NotSame(t, given.Publisher, result.Publisher, "publisher") - assert.NotSame(t, given.Publisher.Ext, result.Publisher.Ext, "publisher-ext") - assert.NotSame(t, given.Content, result.Content, "content") - assert.NotSame(t, given.Content.Ext, result.Content.Ext, "content-ext") - assert.NotSame(t, given.KwArray, result.KwArray, "kwarray") - assert.NotSame(t, given.Ext, result.Ext, "ext") - }) - - t.Run("assumptions", func(t *testing.T) { - assert.ElementsMatch(t, discoverPointerFields(reflect.TypeOf(openrtb2.Site{})), - []string{ - "Cat", - "SectionCat", - "PageCat", - "Mobile", - "PrivacyPolicy", - "Publisher", - "Content", - "KwArray", - "Ext", - }) - }) -} - func TestCloneUser(t *testing.T) { t.Run("nil", func(t *testing.T) { result := CloneUser(nil) @@ -1080,55 +716,6 @@ func TestCloneUID(t *testing.T) { }) } -func TestCloneDOOH(t *testing.T) { - t.Run("nil", func(t *testing.T) { - result := CloneDOOH(nil) - assert.Nil(t, result) - }) - - t.Run("empty", func(t *testing.T) { - given := &openrtb2.DOOH{} - result := CloneDOOH(given) - assert.Empty(t, result) - assert.NotSame(t, given, result) - }) - - t.Run("populated", func(t *testing.T) { - given := &openrtb2.DOOH{ - ID: "anyID", - Name: "anyName", - VenueType: []string{"venue1"}, - VenueTypeTax: ptrutil.ToPtr(adcom1.VenueTaxonomyAdCom), - Publisher: &openrtb2.Publisher{ID: "anyPublisher", Ext: json.RawMessage(`{"publisher":1}`)}, - Domain: "anyDomain", - Keywords: "anyKeywords", - Content: &openrtb2.Content{ID: "anyContent", Ext: json.RawMessage(`{"content":1}`)}, - Ext: json.RawMessage(`{"anyField":1}`), - } - result := CloneDOOH(given) - assert.Equal(t, given, result, "equality") - assert.NotSame(t, given, result, "pointer") - assert.NotSame(t, given.VenueType, result.VenueType, "venuetype") - assert.NotSame(t, given.VenueTypeTax, result.VenueTypeTax, "venuetypetax") - assert.NotSame(t, given.Publisher, result.Publisher, "publisher") - assert.NotSame(t, given.Publisher.Ext, result.Publisher.Ext, "publisher-ext") - assert.NotSame(t, given.Content, result.Content, "content") - assert.NotSame(t, given.Content.Ext, result.Content.Ext, "content-ext") - assert.NotSame(t, given.Ext, result.Ext, "ext") - }) - - t.Run("assumptions", func(t *testing.T) { - assert.ElementsMatch(t, discoverPointerFields(reflect.TypeOf(openrtb2.DOOH{})), - []string{ - "VenueType", - "VenueTypeTax", - "Publisher", - "Content", - "Ext", - }) - }) -} - func TestCloneBidderReq(t *testing.T) { t.Run("nil", func(t *testing.T) { result := CloneBidRequestPartial(nil) diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 695ccd8a5c1..1aed24bc8a5 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -228,7 +228,7 @@ func (e *RawMessageExtension) CreateEncoder(typ reflect2.Type) jsoniter.ValEncod return nil } -var jsonRawMessageType = reflect2.TypeOfPtr(&json.RawMessage{}).Elem() +var jsonRawMessageType = reflect2.TypeOfPtr((*json.RawMessage)(nil)).Elem() // rawMessageCodec implements jsoniter.ValEncoder interface so we can override the default json.RawMessage Encode() // function with our implementation diff --git a/util/jsonutil/merge.go b/util/jsonutil/merge.go new file mode 100644 index 00000000000..1a5543bce1d --- /dev/null +++ b/util/jsonutil/merge.go @@ -0,0 +1,185 @@ +package jsonutil + +import ( + "encoding/json" + "errors" + "reflect" + "unsafe" + + jsoniter "github.com/json-iterator/go" + "github.com/modern-go/reflect2" + jsonpatch "gopkg.in/evanphx/json-patch.v4" + + "github.com/prebid/prebid-server/v2/errortypes" + "github.com/prebid/prebid-server/v2/util/reflectutil" +) + +// jsonConfigMergeClone uses the same configuration as the `ConfigCompatibleWithStandardLibrary` profile +// with extensions added to support the merge clone behavior. +var jsonConfigMergeClone = jsoniter.Config{ + EscapeHTML: true, + SortMapKeys: true, + ValidateJsonRawMessage: true, +}.Froze() + +func init() { + jsonConfigMergeClone.RegisterExtension(&mergeCloneExtension{}) +} + +// MergeClone unmarshals json data on top of an existing object and clones pointers of +// the existing object before setting new values. Slices and maps are also cloned. +// Fields of type json.RawMessage are merged rather than replaced. +func MergeClone(v any, data json.RawMessage) error { + err := jsonConfigMergeClone.Unmarshal(data, v) + if err != nil { + return &errortypes.FailedToUnmarshal{ + Message: tryExtractErrorMessage(err), + } + } + return err +} + +type mergeCloneExtension struct { + jsoniter.DummyExtension +} + +func (e *mergeCloneExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { + if typ == jsonRawMessageType { + return &extMergeDecoder{sliceType: typ.(*reflect2.UnsafeSliceType)} + } + return nil +} + +var extMergeDecoderType = reflect2.TypeOf(extMergeDecoder{}) + +func (e *mergeCloneExtension) DecorateDecoder(typ reflect2.Type, decoder jsoniter.ValDecoder) jsoniter.ValDecoder { + if typ.Kind() == reflect.Ptr { + ptrType := typ.(*reflect2.UnsafePtrType) + return &ptrCloneDecoder{valueDecoder: decoder, elemType: ptrType.Elem()} + } + + // don't use json.RawMessage on fields handled by extMergeDecoder + if typ.Kind() == reflect.Slice && reflect2.TypeOfPtr(decoder).Elem() != extMergeDecoderType { + return &sliceCloneDecoder{valueDecoder: decoder, sliceType: typ.(*reflect2.UnsafeSliceType)} + } + + if typ.Kind() == reflect.Map { + return &mapCloneDecoder{valueDecoder: decoder, mapType: typ.(*reflect2.UnsafeMapType)} + } + + return decoder +} + +type ptrCloneDecoder struct { + elemType reflect2.Type + valueDecoder jsoniter.ValDecoder +} + +func (d *ptrCloneDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + // don't clone if field is being set to nil. checking for nil "consumes" the null + // token, so must be handled in this decoder. + if iter.ReadNil() { + *((*unsafe.Pointer)(ptr)) = nil + return + } + + // clone if there is an existing object. creation of new objects is handled by the + // original decoder. + if *((*unsafe.Pointer)(ptr)) != nil { + obj := d.elemType.UnsafeNew() + d.elemType.UnsafeSet(obj, *((*unsafe.Pointer)(ptr))) + *((*unsafe.Pointer)(ptr)) = obj + } + + d.valueDecoder.Decode(ptr, iter) +} + +type sliceCloneDecoder struct { + sliceType *reflect2.UnsafeSliceType + valueDecoder jsoniter.ValDecoder +} + +func (d *sliceCloneDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + // don't clone if field is being set to nil. checking for nil "consumes" the null + // token, so must be handled in this decoder. + if iter.ReadNil() { + d.sliceType.UnsafeSetNil(ptr) + return + } + + // clone if there is an existing object. creation of new objects is handled by the + // original decoder. + if !d.sliceType.UnsafeIsNil(ptr) { + clone := reflectutil.UnsafeSliceClone(ptr, d.sliceType) + d.sliceType.UnsafeSet(ptr, clone) + } + + d.valueDecoder.Decode(ptr, iter) +} + +type mapCloneDecoder struct { + mapType *reflect2.UnsafeMapType + valueDecoder jsoniter.ValDecoder +} + +func (d *mapCloneDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + // don't clone if field is being set to nil. checking for nil "consumes" the null + // token, so must be handled in this decoder. + if iter.ReadNil() { + *(*unsafe.Pointer)(ptr) = nil + d.mapType.UnsafeSet(ptr, d.mapType.UnsafeNew()) + return + } + + // clone if there is an existing object. creation of new objects is handled by the + // original decoder. + if !d.mapType.UnsafeIsNil(ptr) { + clone := d.mapType.UnsafeMakeMap(0) + mapIter := d.mapType.UnsafeIterate(ptr) + for mapIter.HasNext() { + key, elem := mapIter.UnsafeNext() + d.mapType.UnsafeSetIndex(clone, key, elem) + } + d.mapType.UnsafeSet(ptr, clone) + } + + d.valueDecoder.Decode(ptr, iter) +} + +type extMergeDecoder struct { + sliceType *reflect2.UnsafeSliceType +} + +func (d *extMergeDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + // incoming nil value, keep existing + if iter.ReadNil() { + return + } + + existing := *((*json.RawMessage)(ptr)) + incoming := iter.SkipAndReturnBytes() + + // check for read errors to avoid calling jsonpatch.MergePatch on bad data. + if iter.Error != nil { + return + } + + // existing empty value, use incoming + if len(existing) == 0 { + *((*json.RawMessage)(ptr)) = incoming + return + } + + // non-empty incoming and existing values, merge + merged, err := jsonpatch.MergePatch(existing, incoming) + if err != nil { + if errors.Is(err, jsonpatch.ErrBadJSONDoc) { + iter.ReportError("merge", "invalid json on existing object") + } else { + iter.ReportError("merge", err.Error()) + } + return + } + + *((*json.RawMessage)(ptr)) = merged +} diff --git a/util/jsonutil/merge_test.go b/util/jsonutil/merge_test.go new file mode 100644 index 00000000000..181247518f7 --- /dev/null +++ b/util/jsonutil/merge_test.go @@ -0,0 +1,443 @@ +package jsonutil + +import ( + "encoding/json" + "testing" + + "github.com/prebid/prebid-server/v2/util/sliceutil" + + "github.com/prebid/openrtb/v20/openrtb2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMergeClonePtr(t *testing.T) { + t.Run("root", func(t *testing.T) { + var ( + banner = &openrtb2.Banner{ID: "1"} + imp = &openrtb2.Imp{Banner: banner} + impOriginal = imp + ) + + // root objects are not cloned + err := MergeClone(imp, []byte(`{"banner":{"id":"4"}}`)) + require.NoError(t, err) + + assert.Same(t, impOriginal, imp, "imp-ref") + assert.NotSame(t, imp.Banner, banner, "banner-ref") + }) + + t.Run("embedded-nil", func(t *testing.T) { + var ( + banner = &openrtb2.Banner{ID: "1"} + video = &openrtb2.Video{PodID: "a"} + imp = &openrtb2.Imp{Banner: banner, Video: video} + ) + + err := MergeClone(imp, []byte(`{"banner":null}`)) + require.NoError(t, err) + + assert.NotSame(t, banner, imp.Banner, "banner-ref") + assert.Same(t, video, imp.Video, "video") + assert.Nil(t, imp.Banner, "banner-nil") + }) + + t.Run("embedded-struct", func(t *testing.T) { + var ( + banner = &openrtb2.Banner{ID: "1"} + video = &openrtb2.Video{PodID: "a"} + imp = &openrtb2.Imp{Banner: banner, Video: video} + ) + + err := MergeClone(imp, []byte(`{"banner":{"id":"2"}}`)) + require.NoError(t, err) + + assert.NotSame(t, banner, imp.Banner, "banner-ref") + assert.Same(t, video, imp.Video, "video-ref") + assert.Equal(t, "1", banner.ID, "id-original") + assert.Equal(t, "2", imp.Banner.ID, "id-clone") + }) + + t.Run("embedded-int", func(t *testing.T) { + var ( + clickbrowser = int8(1) + imp = &openrtb2.Imp{ClickBrowser: &clickbrowser} + ) + + err := MergeClone(imp, []byte(`{"clickbrowser":2}`)) + require.NoError(t, err) + + require.NotNil(t, imp.ClickBrowser, "clickbrowser-nil") + assert.NotSame(t, clickbrowser, imp.ClickBrowser, "clickbrowser-ref") + assert.Equal(t, int8(2), *imp.ClickBrowser, "clickbrowser-val") + }) + + t.Run("invalid-null", func(t *testing.T) { + var ( + banner = &openrtb2.Banner{ID: "1"} + imp = &openrtb2.Imp{Banner: banner} + ) + + err := MergeClone(imp, []byte(`{"banner":nul}`)) + require.EqualError(t, err, "cannot unmarshal openrtb2.Imp.Banner: expect ull") + }) + + t.Run("invalid-malformed", func(t *testing.T) { + var ( + banner = &openrtb2.Banner{ID: "1"} + imp = &openrtb2.Imp{Banner: banner} + ) + + err := MergeClone(imp, []byte(`{"banner":malformed}`)) + require.EqualError(t, err, "cannot unmarshal openrtb2.Imp.Banner: expect { or n, but found m") + }) +} + +func TestMergeCloneSlice(t *testing.T) { + t.Run("null", func(t *testing.T) { + var ( + iframeBuster = []string{"a", "b"} + imp = &openrtb2.Imp{IframeBuster: iframeBuster} + ) + + err := MergeClone(imp, []byte(`{"iframeBuster":null}`)) + require.NoError(t, err) + + assert.Equal(t, []string{"a", "b"}, iframeBuster, "iframeBuster-val") + assert.Nil(t, imp.IframeBuster, "iframeBuster-nil") + }) + + t.Run("one", func(t *testing.T) { + var ( + iframeBuster = []string{"a"} + imp = &openrtb2.Imp{IframeBuster: iframeBuster} + ) + + err := MergeClone(imp, []byte(`{"iframeBuster":["b"]}`)) + require.NoError(t, err) + + assert.NotSame(t, iframeBuster, imp.IframeBuster, "ref") + assert.Equal(t, []string{"a"}, iframeBuster, "original-val") + assert.Equal(t, []string{"b"}, imp.IframeBuster, "new-val") + }) + + t.Run("many", func(t *testing.T) { + var ( + iframeBuster = []string{"a"} + imp = &openrtb2.Imp{IframeBuster: iframeBuster} + ) + + err := MergeClone(imp, []byte(`{"iframeBuster":["b", "c"]}`)) + require.NoError(t, err) + + assert.NotSame(t, iframeBuster, imp.IframeBuster, "ref") + assert.Equal(t, []string{"a"}, iframeBuster, "original-val") + assert.Equal(t, []string{"b", "c"}, imp.IframeBuster, "new-val") + }) + + t.Run("invalid-null", func(t *testing.T) { + var ( + iframeBuster = []string{"a"} + imp = &openrtb2.Imp{IframeBuster: iframeBuster} + ) + + err := MergeClone(imp, []byte(`{"iframeBuster":nul}`)) + require.EqualError(t, err, "cannot unmarshal openrtb2.Imp.IframeBuster: expect ull") + }) + + t.Run("invalid-malformed", func(t *testing.T) { + var ( + iframeBuster = []string{"a"} + imp = &openrtb2.Imp{IframeBuster: iframeBuster} + ) + + err := MergeClone(imp, []byte(`{"iframeBuster":malformed}`)) + require.EqualError(t, err, "cannot unmarshal openrtb2.Imp.IframeBuster: decode slice: expect [ or n, but found m") + }) +} + +func TestMergeCloneMap(t *testing.T) { + t.Run("null", func(t *testing.T) { + var ( + testMap = map[string]int{"a": 1, "b": 2} + test = &struct { + Foo map[string]int `json:"foo"` + }{Foo: testMap} + ) + + err := MergeClone(test, []byte(`{"foo":null}`)) + require.NoError(t, err) + + assert.NotSame(t, testMap, test.Foo, "ref") + assert.Equal(t, map[string]int{"a": 1, "b": 2}, testMap, "val") + assert.Nil(t, test.Foo, "nil") + }) + + t.Run("key-string", func(t *testing.T) { + var ( + testMap = map[string]int{"a": 1, "b": 2} + test = &struct { + Foo map[string]int `json:"foo"` + }{Foo: testMap} + ) + + err := MergeClone(test, []byte(`{"foo":{"c":3}}`)) + require.NoError(t, err) + + assert.NotSame(t, testMap, test.Foo) + assert.Equal(t, map[string]int{"a": 1, "b": 2}, testMap, "original-val") + assert.Equal(t, map[string]int{"a": 1, "b": 2, "c": 3}, test.Foo, "new-val") + + // verify modifications don't corrupt original + testMap["a"] = 10 + assert.Equal(t, map[string]int{"a": 10, "b": 2}, testMap, "mod-original-val") + assert.Equal(t, map[string]int{"a": 1, "b": 2, "c": 3}, test.Foo, "mod-ew-val") + }) + + t.Run("key-numeric", func(t *testing.T) { + var ( + testMap = map[int]string{1: "a", 2: "b"} + test = &struct { + Foo map[int]string `json:"foo"` + }{Foo: testMap} + ) + + err := MergeClone(test, []byte(`{"foo":{"3":"c"}}`)) + require.NoError(t, err) + + assert.NotSame(t, testMap, test.Foo) + assert.Equal(t, map[int]string{1: "a", 2: "b"}, testMap, "original-val") + assert.Equal(t, map[int]string{1: "a", 2: "b", 3: "c"}, test.Foo, "new-val") + + // verify modifications don't corrupt original + testMap[1] = "z" + assert.Equal(t, map[int]string{1: "z", 2: "b"}, testMap, "mod-original-val") + assert.Equal(t, map[int]string{1: "a", 2: "b", 3: "c"}, test.Foo, "mod-ew-val") + }) + + t.Run("invalid-null", func(t *testing.T) { + var ( + testMap = map[int]string{1: "a", 2: "b"} + test = &struct { + Foo map[int]string `json:"foo"` + }{Foo: testMap} + ) + + err := MergeClone(test, []byte(`{"foo":nul}`)) + require.EqualError(t, err, "cannot unmarshal Foo: expect ull") + }) + + t.Run("invalid-malformed", func(t *testing.T) { + var ( + testMap = map[int]string{1: "a", 2: "b"} + test = &struct { + Foo map[int]string `json:"foo"` + }{Foo: testMap} + ) + + err := MergeClone(test, []byte(`{"foo":malformed}`)) + require.EqualError(t, err, "cannot unmarshal Foo: expect { or n, but found m") + }) +} + +func TestMergeCloneExt(t *testing.T) { + testCases := []struct { + name string + givenExisting json.RawMessage + givenIncoming json.RawMessage + expectedExt json.RawMessage + expectedErr string + }{ + { + name: "both-populated", + givenExisting: json.RawMessage(`{"a":1,"b":2}`), + givenIncoming: json.RawMessage(`{"b":200,"c":3}`), + expectedExt: json.RawMessage(`{"a":1,"b":200,"c":3}`), + }, + { + name: "both-omitted", + givenExisting: nil, + givenIncoming: nil, + expectedExt: nil, + }, + { + name: "both-nil", + givenExisting: nil, + givenIncoming: json.RawMessage(`null`), + expectedExt: nil, + }, + { + name: "both-empty", + givenExisting: nil, + givenIncoming: json.RawMessage(`{}`), + expectedExt: json.RawMessage(`{}`), + }, + { + name: "ext-omitted", + givenExisting: json.RawMessage(`{"b":2}`), + givenIncoming: nil, + expectedExt: json.RawMessage(`{"b":2}`), + }, + { + name: "ext-nil", + givenExisting: json.RawMessage(`{"b":2}`), + givenIncoming: json.RawMessage(`null`), + expectedExt: json.RawMessage(`{"b":2}`), + }, + { + name: "ext-empty", + givenExisting: json.RawMessage(`{"b":2}`), + givenIncoming: json.RawMessage(`{}`), + expectedExt: json.RawMessage(`{"b":2}`), + }, + { + name: "ext-malformed", + givenExisting: json.RawMessage(`{"b":2}`), + givenIncoming: json.RawMessage(`malformed`), + expectedErr: "openrtb2.BidRequest.Ext", + }, + { + name: "existing-nil", + givenExisting: nil, + givenIncoming: json.RawMessage(`{"a":1}`), + expectedExt: json.RawMessage(`{"a":1}`), + }, + { + name: "existing-empty", + givenExisting: json.RawMessage(`{}`), + givenIncoming: json.RawMessage(`{"a":1}`), + expectedExt: json.RawMessage(`{"a":1}`), + }, + { + name: "existing-omitted", + givenExisting: nil, + givenIncoming: json.RawMessage(`{"b":2}`), + expectedExt: json.RawMessage(`{"b":2}`), + }, + { + name: "existing-malformed", + givenExisting: json.RawMessage(`malformed`), + givenIncoming: json.RawMessage(`{"a":1}`), + expectedErr: "cannot unmarshal openrtb2.BidRequest.Ext: invalid json on existing object", + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + // copy original values to check at the end for no modification + originalExisting := sliceutil.Clone(test.givenExisting) + originalIncoming := sliceutil.Clone(test.givenIncoming) + + // build request + request := &openrtb2.BidRequest{Ext: test.givenExisting} + + // build data + data := test.givenIncoming + if len(data) > 0 { + data = []byte(`{"ext":` + string(data) + `}`) // wrap in ext + } else { + data = []byte(`{}`) // omit ext + } + + err := MergeClone(request, data) + + // assert error + if test.expectedErr == "" { + assert.NoError(t, err, "err") + } else { + assert.ErrorContains(t, err, test.expectedErr, "err") + } + + // assert ext + if test.expectedErr != "" { + // expect no change in case of error + assert.Equal(t, string(test.givenExisting), string(request.Ext), "json") + } else { + // compare as strings instead of json in case of nil or malformed ext + assert.Equal(t, string(test.expectedExt), string(request.Ext), "json") + } + + // assert no modifications + // - can't use `assert.Same`` comparison checks since that's expected if + // either existing or incoming are nil / omitted / empty. + assert.Equal(t, originalExisting, []byte(test.givenExisting), "existing") + assert.Equal(t, originalIncoming, []byte(test.givenIncoming), "incoming") + }) + } +} + +func TestMergeCloneCombinations(t *testing.T) { + t.Run("slice-of-ptr", func(t *testing.T) { + var ( + imp = &openrtb2.Imp{ID: "1"} + impSlice = []*openrtb2.Imp{imp} + test = &struct { + Imps []*openrtb2.Imp `json:"imps"` + }{Imps: impSlice} + ) + + err := MergeClone(test, []byte(`{"imps":[{"id":"2"}]}`)) + require.NoError(t, err) + + assert.NotSame(t, impSlice, test.Imps, "slice-ref") + require.Len(t, test.Imps, 1, "slice-len") + + assert.NotSame(t, imp, test.Imps[0], "item-ref") + assert.Equal(t, "1", imp.ID, "original-val") + assert.Equal(t, "2", test.Imps[0].ID, "new-val") + }) + + // special case of "slice-of-ptr" + t.Run("jsonrawmessage-ptr", func(t *testing.T) { + var ( + testJson = json.RawMessage(`{"a":1}`) + test = &struct { + Foo *json.RawMessage `json:"foo"` + }{Foo: &testJson} + ) + + err := MergeClone(test, []byte(`{"foo":{"b":2}}`)) + require.NoError(t, err) + + assert.NotSame(t, &testJson, test.Foo, "ref") + assert.Equal(t, json.RawMessage(`{"a":1}`), testJson) + assert.Equal(t, json.RawMessage(`{"a":1,"b":2}`), *test.Foo) + }) + + t.Run("struct-ptr", func(t *testing.T) { + var ( + imp = &openrtb2.Imp{ID: "1"} + test = &struct { + Imp *openrtb2.Imp `json:"imp"` + }{Imp: imp} + ) + + err := MergeClone(test, []byte(`{"imp":{"id":"2"}}`)) + require.NoError(t, err) + + assert.NotSame(t, imp, test.Imp, "ref") + assert.Equal(t, "1", imp.ID, "original-val") + assert.Equal(t, "2", test.Imp.ID, "new-val") + }) + + t.Run("map-of-ptrs", func(t *testing.T) { + var ( + imp = &openrtb2.Imp{ID: "1"} + impMap = map[string]*openrtb2.Imp{"a": imp} + test = &struct { + Imps map[string]*openrtb2.Imp `json:"imps"` + }{Imps: impMap} + ) + + err := MergeClone(test, []byte(`{"imps":{"a":{"id":"2"}}}`)) + require.NoError(t, err) + + assert.NotSame(t, impMap, test.Imps, "map-ref") + assert.NotSame(t, imp, test.Imps["a"], "imp-ref") + + assert.Same(t, impMap["a"], imp, "imp-map-ref") + + assert.Equal(t, "1", imp.ID, "original-val") + assert.Equal(t, "2", test.Imps["a"].ID, "new-val") + }) +} diff --git a/util/maputil/maputil.go b/util/maputil/maputil.go index c33740d456c..19224801a14 100644 --- a/util/maputil/maputil.go +++ b/util/maputil/maputil.go @@ -49,11 +49,12 @@ func HasElement(m map[string]interface{}, k ...string) bool { return exists } -// Clone creates an indepent copy of a map, +// Clone creates an independent copy of a map, func Clone[K comparable, V any](m map[K]V) map[K]V { if m == nil { return nil } + clone := make(map[K]V, len(m)) for key, value := range m { clone[key] = value diff --git a/util/reflectutil/slice.go b/util/reflectutil/slice.go new file mode 100644 index 00000000000..6da1aed2808 --- /dev/null +++ b/util/reflectutil/slice.go @@ -0,0 +1,49 @@ +package reflectutil + +import ( + "unsafe" + + "github.com/modern-go/reflect2" +) + +// UnsafeSliceClone clones an existing slice using unsafe.Pointer conventions. Intended +// for use by json iterator extensions and should likely be used no where else. Nil +// behavior is undefined as checks are expected upstream. +func UnsafeSliceClone(ptr unsafe.Pointer, sliceType reflect2.SliceType) unsafe.Pointer { + // it's also possible to use `sliceType.Elem().RType`, but that returns a `uintptr` + // which causes `go vet` to emit a warning even though the usage is safe. this approach + // of copying some internals from the reflect2 package avoids the cast of `uintptr` to + // `unsafe.Pointer` which keeps `go vet` output clean. + elemRType := unpackEFace(sliceType.Elem().Type1()).data + + header := (*sliceHeader)(ptr) + newHeader := (*sliceHeader)(sliceType.UnsafeMakeSlice(header.Len, header.Cap)) + typedslicecopy(elemRType, *newHeader, *header) + return unsafe.Pointer(newHeader) +} + +// sliceHeader is copied from the reflect2 package v1.0.2. +type sliceHeader struct { + Data unsafe.Pointer + Len int + Cap int +} + +// typedslicecopyis copied from the reflect2 package v1.0.2. +// it copies a slice of elemType values from src to dst, +// returning the number of elements copied. +// +//go:linkname typedslicecopy reflect.typedslicecopy +//go:noescape +func typedslicecopy(elemType unsafe.Pointer, dst, src sliceHeader) int + +// eface is copied from the reflect2 package v1.0.2. +type eface struct { + rtype unsafe.Pointer + data unsafe.Pointer +} + +// unpackEFace is copied from the reflect2 package v1.0.2. +func unpackEFace(obj interface{}) *eface { + return (*eface)(unsafe.Pointer(&obj)) +} diff --git a/util/reflectutil/slice_test.go b/util/reflectutil/slice_test.go new file mode 100644 index 00000000000..4f1785b3065 --- /dev/null +++ b/util/reflectutil/slice_test.go @@ -0,0 +1,41 @@ +package reflectutil + +import ( + "testing" + + "github.com/modern-go/reflect2" + "github.com/stretchr/testify/assert" +) + +func TestUnsafeSliceClone(t *testing.T) { + testCases := []struct { + name string + given []int + }{ + { + name: "empty", + given: []int{}, + }, + { + name: "one", + given: []int{1}, + }, + { + name: "many", + given: []int{1, 2}, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + original := test.given + clonePtr := UnsafeSliceClone(reflect2.PtrOf(test.given), reflect2.TypeOf([]int{}).(*reflect2.UnsafeSliceType)) + clone := *(*[]int)(clonePtr) + + assert.NotSame(t, original, clone, "reference") + assert.Equal(t, original, clone, "equality") + assert.Equal(t, len(original), len(clone), "len") + assert.Equal(t, cap(original), cap(clone), "cap") + }) + } +} diff --git a/util/sliceutil/clone.go b/util/sliceutil/clone.go index 2077a9336b2..64faea32a4e 100644 --- a/util/sliceutil/clone.go +++ b/util/sliceutil/clone.go @@ -5,7 +5,7 @@ func Clone[T any](s []T) []T { return nil } - c := make([]T, len(s)) + c := make([]T, len(s), cap(s)) copy(c, s) return c From c016c4557c87297dd1b5b808ee12a8d881efb7d5 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 28 Mar 2024 01:45:28 -0400 Subject: [PATCH 2/5] Improved Slice Clone Avoidance Check --- util/jsonutil/merge.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/util/jsonutil/merge.go b/util/jsonutil/merge.go index 1a5543bce1d..19090dd8d25 100644 --- a/util/jsonutil/merge.go +++ b/util/jsonutil/merge.go @@ -50,8 +50,6 @@ func (e *mergeCloneExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecod return nil } -var extMergeDecoderType = reflect2.TypeOf(extMergeDecoder{}) - func (e *mergeCloneExtension) DecorateDecoder(typ reflect2.Type, decoder jsoniter.ValDecoder) jsoniter.ValDecoder { if typ.Kind() == reflect.Ptr { ptrType := typ.(*reflect2.UnsafePtrType) @@ -59,8 +57,8 @@ func (e *mergeCloneExtension) DecorateDecoder(typ reflect2.Type, decoder jsonite } // don't use json.RawMessage on fields handled by extMergeDecoder - if typ.Kind() == reflect.Slice && reflect2.TypeOfPtr(decoder).Elem() != extMergeDecoderType { - return &sliceCloneDecoder{valueDecoder: decoder, sliceType: typ.(*reflect2.UnsafeSliceType)} + if typ.Kind() == reflect.Slice && typ != jsonRawMessageType { + return &sliceCloneDecoder{valueDecoder: decoder, sliceType: s} } if typ.Kind() == reflect.Map { From 127a78ce00fd9f53e6a75d0422c29f335eb0aa08 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 28 Mar 2024 02:34:54 -0400 Subject: [PATCH 3/5] Remove Old Comments --- firstpartydata/first_party_data_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/firstpartydata/first_party_data_test.go b/firstpartydata/first_party_data_test.go index c7a05951819..44c9405f1bc 100644 --- a/firstpartydata/first_party_data_test.go +++ b/firstpartydata/first_party_data_test.go @@ -14,8 +14,6 @@ import ( "github.com/stretchr/testify/require" ) -// todo: fpd error formatting in resolve methods - func TestExtractGlobalFPD(t *testing.T) { testCases := []struct { description string From 56887ff152757df77793dcd2b280711404839790 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Mon, 22 Apr 2024 11:22:32 -0400 Subject: [PATCH 4/5] "complex" test for slice reflectutil --- util/reflectutil/slice_test.go | 39 +++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/util/reflectutil/slice_test.go b/util/reflectutil/slice_test.go index 4f1785b3065..d5183892369 100644 --- a/util/reflectutil/slice_test.go +++ b/util/reflectutil/slice_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestUnsafeSliceClone(t *testing.T) { +func TestUnsafeSliceCloneSimple(t *testing.T) { testCases := []struct { name string given []int @@ -39,3 +39,40 @@ func TestUnsafeSliceClone(t *testing.T) { }) } } + +func TestUnsafeSliceCloneComplex(t *testing.T) { + type foo struct { + value string + } + + testCases := []struct { + name string + given []foo + }{ + { + name: "empty", + given: []foo{}, + }, + { + name: "one", + given: []foo{{value: "a"}}, + }, + { + name: "many", + given: []foo{{value: "a"}, {value: "b"}}, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + original := test.given + clonePtr := UnsafeSliceClone(reflect2.PtrOf(test.given), reflect2.TypeOf([]foo{}).(*reflect2.UnsafeSliceType)) + clone := *(*[]foo)(clonePtr) + + assert.NotSame(t, original, clone, "reference") + assert.Equal(t, original, clone, "equality") + assert.Equal(t, len(original), len(clone), "len") + assert.Equal(t, cap(original), cap(clone), "cap") + }) + } +} From 48f5b8cf782887f6e113b01978c2b3642c0979fc Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 25 Apr 2024 14:43:01 -0400 Subject: [PATCH 5/5] strange error explanation comments --- util/jsonutil/merge_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/util/jsonutil/merge_test.go b/util/jsonutil/merge_test.go index 181247518f7..8c85d6235c1 100644 --- a/util/jsonutil/merge_test.go +++ b/util/jsonutil/merge_test.go @@ -79,6 +79,10 @@ func TestMergeClonePtr(t *testing.T) { ) err := MergeClone(imp, []byte(`{"banner":nul}`)) + + // json-iter will produce an error since "nul" is not a valid json value. the + // parsing code will see the "n" and then expect "ull" to follow. the strange + // "expect ull" error being asserted is generated by json-iter. require.EqualError(t, err, "cannot unmarshal openrtb2.Imp.Banner: expect ull") }) @@ -142,6 +146,10 @@ func TestMergeCloneSlice(t *testing.T) { ) err := MergeClone(imp, []byte(`{"iframeBuster":nul}`)) + + // json-iter will produce an error since "nul" is not a valid json value. the + // parsing code will see the "n" and then expect "ull" to follow. the strange + // "expect ull" error being asserted is generated by json-iter. require.EqualError(t, err, "cannot unmarshal openrtb2.Imp.IframeBuster: expect ull") }) @@ -224,6 +232,10 @@ func TestMergeCloneMap(t *testing.T) { ) err := MergeClone(test, []byte(`{"foo":nul}`)) + + // json-iter will produce an error since "nul" is not a valid json value. the + // parsing code will see the "n" and then expect "ull" to follow. the strange + // "expect ull" error being asserted is generated by json-iter. require.EqualError(t, err, "cannot unmarshal Foo: expect ull") })