IP scrubber (#2990)

* Added anonymizeIpv6 function to transmitPreciseGeo activitiy to match PBS-Java implementation

* Rename and unit test refactoring

* Replaced existing ip scrubbing function with new scrubIp function. Works both for ipv4 and ipv6

* Minor refactoring

* Added ipv4 and ipv6 scrubbing config

* Pointers refactoring

* Naming refactoring

* Unit tests

* Naming refactoring

* refactored IPv6 and IPv4 masking config to have just one value `left_mask_bits`

* Unit tests

* Test rename

* Config refactoring, rename left-mask-bits to anon-keep-bits

* Merge fixes

* Refactoring

* Refactoring

* Constant rename and comments remove

* Code refactoring and default IPv6 and IPv4 values set

account/account.go

@@ -4,14 +4,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-
 	"github.com/buger/jsonparser"
 	"github.com/prebid/go-gdpr/consentconstants"
 	"github.com/prebid/prebid-server/config"
 	"github.com/prebid/prebid-server/errortypes"
 	"github.com/prebid/prebid-server/metrics"
 	"github.com/prebid/prebid-server/openrtb_ext"
 	"github.com/prebid/prebid-server/stored_requests"
+	"github.com/prebid/prebid-server/util/iputil"
 	jsonpatch "gopkg.in/evanphx/json-patch.v4"
 )
 
@@ -103,6 +103,14 @@ func GetAccount(ctx context.Context, cfg *config.Configuration, fetcher stored_r
 		return nil, errs
 	}
 
+	if ipV6Err := account.Privacy.IPv6Config.Validate(nil); len(ipV6Err) > 0 {
+		account.Privacy.IPv6Config.AnonKeepBits = iputil.IPv6DefaultMaskingBitSize
+	}
+
+	if ipV4Err := account.Privacy.IPv4Config.Validate(nil); len(ipV4Err) > 0 {
+		account.Privacy.IPv4Config.AnonKeepBits = iputil.IPv4DefaultMaskingBitSize
+	}
+
 	// set the value of events.enabled field based on deprecated events_enabled field and ensure backward compatibility
 	deprecateEventsEnabledField(account)
 

account/account_test.go

@@ -12,17 +12,19 @@ import (
 	"github.com/prebid/prebid-server/metrics"
 	"github.com/prebid/prebid-server/openrtb_ext"
 	"github.com/prebid/prebid-server/stored_requests"
+	"github.com/prebid/prebid-server/util/iputil"
 	"github.com/prebid/prebid-server/util/ptrutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 )
 
 var mockAccountData = map[string]json.RawMessage{
-	"valid_acct":                json.RawMessage(`{"disabled":false}`),
-	"disabled_acct":             json.RawMessage(`{"disabled":true}`),
-	"malformed_acct":            json.RawMessage(`{"disabled":"invalid type"}`),
-	"gdpr_channel_enabled_acct": json.RawMessage(`{"disabled":false,"gdpr":{"channel_enabled":{"amp":true}}}`),
-	"ccpa_channel_enabled_acct": json.RawMessage(`{"disabled":false,"ccpa":{"channel_enabled":{"amp":true}}}`),
+	"valid_acct":                                   json.RawMessage(`{"disabled":false}`),
+	"invalid_acct_ipv6_ipv4":                       json.RawMessage(`{"disabled":false, "privacy": {"ipv6": {"anon_keep_bits": -32}, "ipv4": {"anon_keep_bits": -16}}}`),
+	"disabled_acct":                                json.RawMessage(`{"disabled":true}`),
+	"malformed_acct":                               json.RawMessage(`{"disabled":"invalid type"}`),
+	"gdpr_channel_enabled_acct":                    json.RawMessage(`{"disabled":false,"gdpr":{"channel_enabled":{"amp":true}}}`),
+	"ccpa_channel_enabled_acct":                    json.RawMessage(`{"disabled":false,"ccpa":{"channel_enabled":{"amp":true}}}`),
 	"gdpr_channel_enabled_deprecated_purpose_acct": json.RawMessage(`{"disabled":false,"gdpr":{"purpose1":{"enforce_purpose":"full"}, "channel_enabled":{"amp":true}}}`),
 	"gdpr_deprecated_purpose1":                     json.RawMessage(`{"disabled":false,"gdpr":{"purpose1":{"enforce_purpose":"full"}}}`),
 	"gdpr_deprecated_purpose2":                     json.RawMessage(`{"disabled":false,"gdpr":{"purpose2":{"enforce_purpose":"full"}}}`),
@@ -54,6 +56,8 @@ func TestGetAccount(t *testing.T) {
 		required bool
 		// account_defaults.disabled
 		disabled bool
+		// checkDefaultIP indicates IPv6 and IPv6 should be set to default values
+		checkDefaultIP bool
 		// expected error, or nil if account should be found
 		err error
 	}{
@@ -78,6 +82,8 @@ func TestGetAccount(t *testing.T) {
 		{accountID: "valid_acct", required: false, disabled: true, err: nil},
 		{accountID: "valid_acct", required: true, disabled: true, err: nil},
 
+		{accountID: "invalid_acct_ipv6_ipv4", required: true, disabled: false, err: nil, checkDefaultIP: true},
+
 		// pubID given and matches a host account explicitly disabled (Disabled: true on account json)
 		{accountID: "disabled_acct", required: false, disabled: false, err: &errortypes.BlacklistedAcct{}},
 		{accountID: "disabled_acct", required: true, disabled: false, err: &errortypes.BlacklistedAcct{}},
@@ -129,6 +135,10 @@ func TestGetAccount(t *testing.T) {
 				assert.Nil(t, account, "return account must be nil on error")
 				assert.IsType(t, test.err, errors[0], "error is of unexpected type")
 			}
+			if test.checkDefaultIP {
+				assert.Equal(t, account.Privacy.IPv6Config.AnonKeepBits, iputil.IPv6DefaultMaskingBitSize, "ipv6 should be set to default value")
+				assert.Equal(t, account.Privacy.IPv4Config.AnonKeepBits, iputil.IPv4DefaultMaskingBitSize, "ipv4 should be set to default value")
+			}
 		})
 	}
 }

config/account.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/prebid/go-gdpr/consentconstants"
 	"github.com/prebid/prebid-server/openrtb_ext"
+	"github.com/prebid/prebid-server/util/iputil"
 )
 
 // ChannelType enumerates the values of integrations Prebid Server can configure for an account
@@ -40,7 +41,7 @@ type Account struct {
 	Validations             Validations                                 `mapstructure:"validations" json:"validations"`
 	DefaultBidLimit         int                                         `mapstructure:"default_bid_limit" json:"default_bid_limit"`
 	BidAdjustments          *openrtb_ext.ExtRequestPrebidBidAdjustments `mapstructure:"bidadjustments" json:"bidadjustments"`
-	Privacy                 *AccountPrivacy                             `mapstructure:"privacy" json:"privacy"`
+	Privacy                 AccountPrivacy                              `mapstructure:"privacy" json:"privacy"`
 }
 
 // CookieSync represents the account-level defaults for the cookie sync endpoint.
@@ -295,5 +296,31 @@ func (a *AccountChannel) IsSet() bool {
 }
 
 type AccountPrivacy struct {
-	AllowActivities AllowActivities `mapstructure:"allowactivities" json:"allowactivities"`
+	AllowActivities *AllowActivities `mapstructure:"allowactivities" json:"allowactivities"`
+	IPv6Config      IPv6             `mapstructure:"ipv6" json:"ipv6"`
+	IPv4Config      IPv4             `mapstructure:"ipv4" json:"ipv4"`
+}
+
+type IPv6 struct {
+	AnonKeepBits int `mapstructure:"anon_keep_bits" json:"anon_keep_bits"`
+}
+
+type IPv4 struct {
+	AnonKeepBits int `mapstructure:"anon_keep_bits" json:"anon_keep_bits"`
+}
+
+func (ip *IPv6) Validate(errs []error) []error {
+	if ip.AnonKeepBits > iputil.IPv6BitSize || ip.AnonKeepBits < 0 {
+		err := fmt.Errorf("bits cannot exceed %d in ipv6 address, or be less than 0", iputil.IPv6BitSize)
+		errs = append(errs, err)
+	}
+	return errs
+}
+
+func (ip *IPv4) Validate(errs []error) []error {
+	if ip.AnonKeepBits > iputil.IPv4BitSize || ip.AnonKeepBits < 0 {
+		err := fmt.Errorf("bits cannot exceed %d in ipv4 address, or be less than 0", iputil.IPv4BitSize)
+		errs = append(errs, err)
+	}
+	return errs
 }

config/account_test.go

@@ -910,3 +910,49 @@ func TestAccountPriceFloorsValidate(t *testing.T) {
 		})
 	}
 }
+
+func TestIPMaskingValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		privacy AccountPrivacy
+		want    []error
+	}{
+		{
+			name: "valid",
+			privacy: AccountPrivacy{
+				IPv4Config: IPv4{AnonKeepBits: 1},
+				IPv6Config: IPv6{AnonKeepBits: 0},
+			},
+		},
+		{
+			name: "invalid",
+			privacy: AccountPrivacy{
+				IPv4Config: IPv4{AnonKeepBits: -100},
+				IPv6Config: IPv6{AnonKeepBits: -200},
+			},
+			want: []error{
+				errors.New("bits cannot exceed 32 in ipv4 address, or be less than 0"),
+				errors.New("bits cannot exceed 128 in ipv6 address, or be less than 0"),
+			},
+		},
+		{
+			name: "mixed",
+			privacy: AccountPrivacy{
+				IPv4Config: IPv4{AnonKeepBits: 10},
+				IPv6Config: IPv6{AnonKeepBits: -10},
+			},
+			want: []error{
+				errors.New("bits cannot exceed 128 in ipv6 address, or be less than 0"),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var errs []error
+			errs = tt.privacy.IPv4Config.Validate(errs)
+			errs = tt.privacy.IPv6Config.Validate(errs)
+			assert.ElementsMatch(t, errs, tt.want)
+		})
+	}
+}

config/config.go

@@ -149,12 +149,11 @@ func (cfg *Configuration) validate(v *viper.Viper) []error {
 		errs = append(errs, errors.New("account_defaults.Events.VASTEvents has no effect as the feature is under development."))
 	}
 
-	if cfg.AccountDefaults.Privacy != nil {
-		glog.Warning("account_defaults.Privacy has no effect as the feature is under development.")
-	}
-
 	errs = cfg.Experiment.validate(errs)
 	errs = cfg.BidderInfos.validate(errs)
+	errs = cfg.AccountDefaults.Privacy.IPv6Config.Validate(errs)
+	errs = cfg.AccountDefaults.Privacy.IPv4Config.Validate(errs)
+
 	return errs
 }
 
@@ -1019,6 +1018,8 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
 	v.SetDefault("account_defaults.price_floors.max_rules", 100)
 	v.SetDefault("account_defaults.price_floors.max_schema_dims", 3)
 	v.SetDefault("account_defaults.events_enabled", false)
+	v.SetDefault("account_defaults.privacy.ipv6.anon_keep_bits", 56)
+	v.SetDefault("account_defaults.privacy.ipv4.anon_keep_bits", 24)
 
 	v.SetDefault("compression.response.enable_gzip", false)
 	v.SetDefault("compression.request.enable_gzip", false)

config/config_test.go

@@ -206,6 +206,9 @@ func TestDefaults(t *testing.T) {
 	cmpUnsignedInts(t, "tmax_adjustments.bidder_network_latency_buffer_ms", 0, cfg.TmaxAdjustments.BidderNetworkLatencyBuffer)
 	cmpUnsignedInts(t, "tmax_adjustments.pbs_response_preparation_duration_ms", 0, cfg.TmaxAdjustments.PBSResponsePreparationDuration)
 
+	cmpInts(t, "account_defaults.privacy.ipv6.anon_keep_bits", 56, cfg.AccountDefaults.Privacy.IPv6Config.AnonKeepBits)
+	cmpInts(t, "account_defaults.privacy.ipv4.anon_keep_bits", 24, cfg.AccountDefaults.Privacy.IPv4Config.AnonKeepBits)
+
 	//Assert purpose VendorExceptionMap hash tables were built correctly
 	expectedTCF2 := TCF2{
 		Enabled: true,
@@ -477,6 +480,11 @@ account_defaults:
         use_dynamic_data: true
         max_rules: 120
         max_schema_dims: 5
+    privacy:
+        ipv6:
+            anon_keep_bits: 50
+        ipv4:
+            anon_keep_bits: 20
 tmax_adjustments:
   enabled: true
   bidder_response_duration_min_ms: 700
@@ -583,6 +591,9 @@ func TestFullConfig(t *testing.T) {
 	cmpBools(t, "account_defaults.events_enabled", *cfg.AccountDefaults.EventsEnabled, true)
 	cmpNils(t, "account_defaults.events.enabled", cfg.AccountDefaults.Events.Enabled)
 
+	cmpInts(t, "account_defaults.privacy.ipv6.anon_keep_bits", 50, cfg.AccountDefaults.Privacy.IPv6Config.AnonKeepBits)
+	cmpInts(t, "account_defaults.privacy.ipv4.anon_keep_bits", 20, cfg.AccountDefaults.Privacy.IPv4Config.AnonKeepBits)
+
 	// Assert compression related defaults
 	cmpBools(t, "enable_gzip", false, cfg.EnableGzip)
 	cmpBools(t, "compression.request.enable_gzip", true, cfg.Compression.Request.GZIP)

endpoints/cookie_sync.go

@@ -149,7 +149,7 @@ func (c *cookieSyncEndpoint) parseRequest(r *http.Request) (usersync.Request, pr
 		}
 	}
 
-	activityControl, activitiesErr := privacy.NewActivityControl(account.Privacy)
+	activityControl, activitiesErr := privacy.NewActivityControl(&account.Privacy)
 	if activitiesErr != nil {
 		if errortypes.ContainsFatalError([]error{activitiesErr}) {
 			activityControl = privacy.ActivityControl{}

endpoints/cookie_sync_test.go

@@ -2108,7 +2108,7 @@ func (p *fakePermissions) AuctionActivitiesAllowed(ctx context.Context, bidderCo
 
 func getDefaultActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
 	return &config.AccountPrivacy{
-		AllowActivities: config.AllowActivities{
+		AllowActivities: &config.AllowActivities{
 			SyncUser: config.Activity{
 				Default: ptrutil.ToPtr(true),
 				Rules: []config.ActivityRule{

endpoints/openrtb2/amp_auction.go

@@ -229,7 +229,7 @@ func (deps *endpointDeps) AmpAuction(w http.ResponseWriter, r *http.Request, _ h
 
 	tcf2Config := gdpr.NewTCF2Config(deps.cfg.GDPR.TCF2, account.GDPR)
 
-	activities, activitiesErr := privacy.NewActivityControl(account.Privacy)
+	activities, activitiesErr := privacy.NewActivityControl(&account.Privacy)
 	if activitiesErr != nil {
 		errL = append(errL, activitiesErr)
 		writeError(errL, w, &labels)

endpoints/openrtb2/auction.go

@@ -193,7 +193,7 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http
 
 	tcf2Config := gdpr.NewTCF2Config(deps.cfg.GDPR.TCF2, account.GDPR)
 
-	activities, activitiesErr := privacy.NewActivityControl(account.Privacy)
+	activities, activitiesErr := privacy.NewActivityControl(&account.Privacy)
 	if activitiesErr != nil {
 		errL = append(errL, activitiesErr)
 		if errortypes.ContainsFatalError(errL) {

endpoints/openrtb2/video_auction.go

@@ -303,7 +303,7 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	activities, activitiesErr := privacy.NewActivityControl(account.Privacy)
+	activities, activitiesErr := privacy.NewActivityControl(&account.Privacy)
 	if activitiesErr != nil {
 		errL = append(errL, activitiesErr)
 		if errortypes.ContainsFatalError(errL) {

endpoints/openrtb2/video_auction_test.go

@@ -1299,8 +1299,8 @@ func mockDepsInvalidPrivacy(t *testing.T, ex *mockExchangeVideo) *endpointDeps {
 		&mockAccountFetcher{data: mockVideoAccountData},
 		&config.Configuration{MaxRequestSize: maxSize,
 			AccountDefaults: config.Account{
-				Privacy: &config.AccountPrivacy{
-					AllowActivities: config.AllowActivities{
+				Privacy: config.AccountPrivacy{
+					AllowActivities: &config.AllowActivities{
 						TransmitPreciseGeo: config.Activity{Rules: []config.ActivityRule{
 							{Condition: config.ActivityCondition{ComponentName: []string{"bidderA.BidderB.bidderC"}}},
 						}},

endpoints/setuid.go

@@ -104,7 +104,7 @@ func NewSetUIDEndpoint(cfg *config.Configuration, syncersByBidder map[string]use
 			return
 		}
 
-		activities, activitiesErr := privacy.NewActivityControl(account.Privacy)
+		activities, activitiesErr := privacy.NewActivityControl(&account.Privacy)
 		if activitiesErr != nil {
 			if errortypes.ContainsFatalError([]error{activitiesErr}) {
 				activities = privacy.ActivityControl{}

exchange/utils.go

@@ -214,7 +214,7 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context,
 
 		privacyEnforcement.TID = !auctionReq.Activities.Allow(privacy.ActivityTransmitTids, scopedName)
 
-		privacyEnforcement.Apply(bidderRequest.BidRequest)
+		privacyEnforcement.Apply(bidderRequest.BidRequest, auctionReq.Account.Privacy)
 		allowedBidderRequests = append(allowedBidderRequests, bidderRequest)
 
 		// GPP downgrade: always downgrade unless we can confirm GPP is supported

exchange/utils_test.go

@@ -4272,7 +4272,7 @@ func TestCleanOpenRTBRequestsActivities(t *testing.T) {
 	testCases := []struct {
 		name                 string
 		req                  *openrtb2.BidRequest
-		privacyConfig        *config.AccountPrivacy
+		privacyConfig        config.AccountPrivacy
 		componentName        string
 		allow                bool
 		expectedReqNumber    int
@@ -4364,7 +4364,7 @@ func TestCleanOpenRTBRequestsActivities(t *testing.T) {
 	}
 
 	for _, test := range testCases {
-		activities, err := privacy.NewActivityControl(test.privacyConfig)
+		activities, err := privacy.NewActivityControl(&test.privacyConfig)
 		assert.NoError(t, err, "")
 		auctionReq := AuctionRequest{
 			BidRequestWrapper: &openrtb_ext.RequestWrapper{BidRequest: test.req},
@@ -4410,33 +4410,33 @@ func buildDefaultActivityConfig(componentName string, allow bool) config.Activit
 	}
 }
 
-func getFetchBidsActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
-	return &config.AccountPrivacy{
-		AllowActivities: config.AllowActivities{
+func getFetchBidsActivityConfig(componentName string, allow bool) config.AccountPrivacy {
+	return config.AccountPrivacy{
+		AllowActivities: &config.AllowActivities{
 			FetchBids: buildDefaultActivityConfig(componentName, allow),
 		},
 	}
 }
 
-func getTransmitUFPDActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
-	return &config.AccountPrivacy{
-		AllowActivities: config.AllowActivities{
+func getTransmitUFPDActivityConfig(componentName string, allow bool) config.AccountPrivacy {
+	return config.AccountPrivacy{
+		AllowActivities: &config.AllowActivities{
 			TransmitUserFPD: buildDefaultActivityConfig(componentName, allow),
 		},
 	}
 }
 
-func getTransmitPreciseGeoActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
-	return &config.AccountPrivacy{
-		AllowActivities: config.AllowActivities{
+func getTransmitPreciseGeoActivityConfig(componentName string, allow bool) config.AccountPrivacy {
+	return config.AccountPrivacy{
+		AllowActivities: &config.AllowActivities{
 			TransmitPreciseGeo: buildDefaultActivityConfig(componentName, allow),
 		},
 	}
 }
 
-func getTransmitTIDActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
-	return &config.AccountPrivacy{
-		AllowActivities: config.AllowActivities{
+func getTransmitTIDActivityConfig(componentName string, allow bool) config.AccountPrivacy {
+	return config.AccountPrivacy{
+		AllowActivities: &config.AllowActivities{
 			TransmitTids: buildDefaultActivityConfig(componentName, allow),
 		},
 	}

privacy/activitycontrol.go

@@ -22,7 +22,7 @@ func NewActivityControl(privacyConf *config.AccountPrivacy) (ActivityControl, er
 	ac := ActivityControl{}
 	var err error
 
-	if privacyConf == nil {
+	if privacyConf == nil || privacyConf.AllowActivities == nil {
 		return ac, nil
 	}