Default TCF1 GVL in anticipation of IAB no longer hosting the v1 GVL (p…

…rebid#1433)

config/config.go

@@ -156,6 +156,7 @@ type GDPR struct {
 	Timeouts                GDPRTimeouts `mapstructure:"timeouts_ms"`
 	NonStandardPublishers   []string     `mapstructure:"non_standard_publishers,flow"`
 	NonStandardPublisherMap map[string]int
+	TCF1                    TCF1 `mapstructure:"tcf1"`
 	TCF2                    TCF2 `mapstructure:"tcf2"`
 	AMPException            bool `mapstructure:"amp_exception"`
 }
@@ -180,6 +181,12 @@ func (t *GDPRTimeouts) ActiveTimeout() time.Duration {
 	return time.Duration(t.ActiveVendorlistFetch) * time.Millisecond
 }
 
+// TCF1 defines the TCF1 specific configurations for GDPR
+type TCF1 struct {
+	FetchGVL        bool   `mapstructure:"fetch_gvl"`
+	FallbackGVLPath string `mapstructure:"fallback_gvl_path"`
+}
+
 // TCF2 defines the TCF2 specific configurations for GDPR
 type TCF2 struct {
 	Enabled             bool                 `mapstructure:"enabled"`
@@ -885,6 +892,8 @@ func SetupViper(v *viper.Viper, filename string) {
 	v.SetDefault("gdpr.timeouts_ms.init_vendorlist_fetches", 0)
 	v.SetDefault("gdpr.timeouts_ms.active_vendorlist_fetch", 0)
 	v.SetDefault("gdpr.non_standard_publishers", []string{""})
+	v.SetDefault("gdpr.tcf1.fetch_gvl", true)
+	v.SetDefault("gdpr.tcf1.fallback_gvl_path", "./static/tcf1/fallback_gvl.json")
 	v.SetDefault("gdpr.tcf2.enabled", true)
 	v.SetDefault("gdpr.tcf2.purpose1.enabled", true)
 	v.SetDefault("gdpr.tcf2.purpose2.enabled", true)

gdpr/vendorlist-fetching.go

@@ -27,8 +27,20 @@ type saveVendors func(uint16, api.VendorList)
 // Nothing in this file is exported. Public APIs can be found in gdpr.go
 
 func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16, uint8) string, TCFVer uint8) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
+	var fallbackVL api.VendorList = nil
+
+	if TCFVer == tCF1 && len(cfg.TCF1.FallbackGVLPath) > 0 {
+		fallbackVL = loadFallbackGVL(cfg.TCF1.FallbackGVLPath)
+	}
+
+	// If we are not going to try fetching the GVL dynamically, we have a simple fetcher
+	if !cfg.TCF1.FetchGVL && TCFVer == tCF1 && fallbackVL != nil {
+		return func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
+			return fallbackVL, nil
+		}
+	}
 	// These save and load functions can be used to store & retrieve lists from our cache.
-	save, load := newVendorListCache()
+	save, load := newVendorListCache(fallbackVL)
 
 	withTimeout, cancel := context.WithTimeout(initCtx, cfg.Timeouts.InitTimeout())
 	defer cancel()
@@ -46,6 +58,9 @@ func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http
 		if list != nil {
 			return list, nil
 		}
+		if fallbackVL != nil {
+			return fallbackVL, nil
+		}
 		return nil, fmt.Errorf("gdpr vendor list version %d does not exist, or has not been loaded yet. Try again in a few minutes", id)
 	}
 }
@@ -132,7 +147,7 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
 	return newList.Version()
 }
 
-func newVendorListCache() (save func(id uint16, list api.VendorList), load func(id uint16) api.VendorList) {
+func newVendorListCache(fallbackVL api.VendorList) (save func(id uint16, list api.VendorList), load func(id uint16) api.VendorList) {
 	cache := &sync.Map{}
 
 	save = func(id uint16, list api.VendorList) {
@@ -143,7 +158,19 @@ func newVendorListCache() (save func(id uint16, list api.VendorList), load func(
 		if ok {
 			return list.(vendorlist.VendorList)
 		}
-		return nil
+		return fallbackVL
 	}
 	return
 }
+
+func loadFallbackGVL(fallbackGVLPath string) vendorlist.VendorList {
+	fallbackVLbody, err := ioutil.ReadFile(fallbackGVLPath)
+	if err != nil {
+		glog.Fatalf("Error reading from file %s: %v", fallbackGVLPath, err)
+	}
+	fallbackVL, err := vendorlist.ParseEagerly(fallbackVLbody)
+	if err != nil {
+		glog.Fatalf("Error processing default GVL from %s: %v", fallbackGVLPath, err)
+	}
+	return fallbackVL
+}

gdpr/vendorlist-fetching_test.go

@@ -9,6 +9,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/prebid/prebid-server/config"
 )
 
@@ -139,6 +141,101 @@ func TestVendorListMaker(t *testing.T) {
 	assertStringsEqual(t, "https://vendorlist.consensu.org/v2/archives/vendor-list-v7.json", vendorListURLMaker(7, 2))
 }
 
+func TestDefaultVendorList(t *testing.T) {
+	firstVendorList := mockVendorListData(t, 1, map[uint16]*purposes{
+		32: {
+			purposes: []int{1, 2},
+		},
+	})
+	secondVendorList := mockVendorListData(t, 2, map[uint16]*purposes{
+		12: {
+			purposes: []int{2},
+		},
+	})
+	server := httptest.NewServer(http.HandlerFunc(mockServer(2, map[int]string{
+		1: firstVendorList,
+		2: secondVendorList,
+	})))
+	defer server.Close()
+
+	testcfg := testConfig()
+	testcfg.TCF1.FetchGVL = true
+	testcfg.TCF1.FallbackGVLPath = "../static/tcf1/fallback_gvl.json"
+	fetcher := newVendorListFetcher(context.Background(), testcfg, server.Client(), testURLMaker(server), 1)
+
+	list, err := fetcher(context.Background(), 12)
+	assert.NoError(t, err, "Error with fetching default vendorlist: %v", err)
+	assert.Equal(t, uint16(214), list.Version(), "Expected to fetch default version 214, got %d", list.Version())
+
+	// Testing that we got the default vendorlist data, and not the version off the server.
+	vendor := list.Vendor(12)
+	assert.Equal(t, true, vendor.Purpose(1))
+	assert.Equal(t, false, vendor.Purpose(2))
+}
+
+func TestDefaultVendorListPassthrough(t *testing.T) {
+	firstVendorList := mockVendorListData(t, 1, map[uint16]*purposes{
+		32: {
+			purposes: []int{1, 2},
+		},
+	})
+	secondVendorList := mockVendorListData(t, 2, map[uint16]*purposes{
+		12: {
+			purposes: []int{2},
+		},
+	})
+	server := httptest.NewServer(http.HandlerFunc(mockServer(2, map[int]string{
+		1: firstVendorList,
+		2: secondVendorList,
+	})))
+	defer server.Close()
+
+	testcfg := testConfig()
+	testcfg.TCF1.FetchGVL = true
+	testcfg.TCF1.FallbackGVLPath = "../static/tcf1/fallback_gvl.json"
+	fetcher := newVendorListFetcher(context.Background(), testcfg, server.Client(), testURLMaker(server), 1)
+	list, err := fetcher(context.Background(), 2)
+	assert.NoError(t, err, "Error with fetching existing vendorlist: %v", err)
+	assert.Equal(t, uint16(2), list.Version(), "Expected to fetch mock list version 2, got version %d", list.Version())
+
+	// Testing that we got the testing vendorlist data, and not the default.
+	vendor := list.Vendor(12)
+	assert.Equal(t, false, vendor.Purpose(1))
+	assert.Equal(t, true, vendor.Purpose(2))
+}
+
+func TestDefaultVendorListNoFetch(t *testing.T) {
+	firstVendorList := mockVendorListData(t, 1, map[uint16]*purposes{
+		32: {
+			purposes: []int{1, 2},
+		},
+	})
+	secondVendorList := mockVendorListData(t, 2, map[uint16]*purposes{
+		12: {
+			purposes: []int{2},
+		},
+	})
+	server := httptest.NewServer(http.HandlerFunc(mockServer(1, map[int]string{
+		1: firstVendorList,
+		2: secondVendorList,
+	})))
+	defer server.Close()
+
+	testcfg := testConfig()
+	testcfg.TCF1.FetchGVL = false
+	testcfg.TCF1.FallbackGVLPath = "../static/tcf1/fallback_gvl.json"
+	fetcher := newVendorListFetcher(context.Background(), testcfg, server.Client(), testURLMaker(server), 1)
+	list, err := fetcher(context.Background(), 2)
+	assert.NoError(t, err, "Error with fetching default vendorlist: %v", err)
+	assert.Equal(t, uint16(214), list.Version(), "Expected to fetch default version 214, got %d", list.Version())
+
+	// Testing that we got the default vendorlist data, and not the version off the server.
+	vendor := list.Vendor(12)
+	assert.Equal(t, true, vendor.Purpose(1))
+	assert.Equal(t, false, vendor.Purpose(2))
+
+}
+
 // mockServer returns a handler which returns the given response for each global vendor list version.
 // The latestVersion param can be used to mock "updates" which occur after PBS has been turned on.
 // For example, if latestVersion is 3, but the responses map has data at "4", the server will return