IP scrubber

account/account.go

@@ -103,6 +103,11 @@ func GetAccount(ctx context.Context, cfg *config.Configuration, fetcher stored_r
 		return nil, errs
 	}
 
+	errs = account.Privacy.IPMasking.Validate(errs)
+	if len(errs) > 0 {
+		return nil, errs
+	}
+
 	// set the value of events.enabled field based on deprecated events_enabled field and ensure backward compatibility
 	deprecateEventsEnabledField(account)
 

config/account.go

@@ -21,6 +21,11 @@ const (
 	ChannelWeb   ChannelType = "web"
 )
 
+const (
+	IPv4 = 32
+	IPv6 = 128
+)
+
 // Account represents a publisher account configuration
 type Account struct {
 	ID                      string                                      `mapstructure:"id" json:"id"`
@@ -40,7 +45,7 @@ type Account struct {
 	Validations             Validations                                 `mapstructure:"validations" json:"validations"`
 	DefaultBidLimit         int                                         `mapstructure:"default_bid_limit" json:"default_bid_limit"`
 	BidAdjustments          *openrtb_ext.ExtRequestPrebidBidAdjustments `mapstructure:"bidadjustments" json:"bidadjustments"`
-	Privacy                 *AccountPrivacy                             `mapstructure:"privacy" json:"privacy"`
+	Privacy                 AccountPrivacy                              `mapstructure:"privacy" json:"privacy"`
 }
 
 // CookieSync represents the account-level defaults for the cookie sync endpoint.
@@ -295,5 +300,47 @@ func (a *AccountChannel) IsSet() bool {
 }
 
 type AccountPrivacy struct {
-	AllowActivities AllowActivities `mapstructure:"allowactivities" json:"allowactivities"`
+	AllowActivities *AllowActivities `mapstructure:"allowactivities" json:"allowactivities"`
+	IPMasking       IPMasking        `mapstructure:"ip_masking" json:"ip_masking"`
+}
+
+type IPMasking struct {
+	IpV6 IPMasks `mapstructure:"ipv6" json:"ipv6"`
+	IpV4 IPMasks `mapstructure:"ipv4" json:"ipv4"`
+}
+
+type IPMasks struct {
+	ActivityLeftMaskBits    int `mapstructure:"activity_left_mask_bits" json:"activity_left_mask_bits"`
+	GdprLeftMaskBitsLowest  int `mapstructure:"gdpr_left_mask_bits_lowest" json:"gdpr_left_mask_bits_lowest"`
+	GdprLeftMaskBitsHighest int `mapstructure:"gdpr_left_mask_bits_highest" json:"gdpr_left_mask_bits_highest"`
+}
+
+func (ipMasking *IPMasking) Validate(errs []error) []error {
+	if e := ipMasking.IpV6.validate(IPv6, "ipv6"); e != nil {
+		errs = append(errs, e...)
+	}
+	if e := ipMasking.IpV4.validate(IPv4, "ipv4"); e != nil {
+		errs = append(errs, e...)
+	}
+	return errs
+}
+
+func (ipMasks *IPMasks) validate(maxBits int, ipVersion string) []error {
+	var errs []error
+
+	if ipMasks.ActivityLeftMaskBits > maxBits || ipMasks.ActivityLeftMaskBits < 0 {
+		err := fmt.Errorf("activity left mask bits cannot exceed %d in %s address, or be less than 0", maxBits, ipVersion)
+		errs = append(errs, err)
+	}
+
+	if ipMasks.GdprLeftMaskBitsLowest > maxBits || ipMasks.GdprLeftMaskBitsLowest < 0 {
+		err := fmt.Errorf("gdpr left mask bits lowest cannot exceed %d in %s address, or be less than 0", maxBits, ipVersion)
+		errs = append(errs, err)
+	}
+
+	if ipMasks.GdprLeftMaskBitsHighest > maxBits || ipMasks.GdprLeftMaskBitsHighest < 0 {
+		err := fmt.Errorf("gdpr left mask bits highest cannot exceed %d in %s address, or be less than 0", maxBits, ipVersion)
+		errs = append(errs, err)
+	}
+	return errs
 }

config/account_test.go

@@ -910,3 +910,78 @@ func TestAccountPriceFloorsValidate(t *testing.T) {
 		})
 	}
 }
+
+func TestIPMaskingValidate(t *testing.T) {
+
+	tests := []struct {
+		name    string
+		masking *IPMasking
+		want    []error
+	}{
+		{
+			name: "valid configuration",
+			masking: &IPMasking{
+				IpV4: IPMasks{
+					ActivityLeftMaskBits:    1,
+					GdprLeftMaskBitsLowest:  2,
+					GdprLeftMaskBitsHighest: 0,
+				},
+				IpV6: IPMasks{
+					ActivityLeftMaskBits:    0,
+					GdprLeftMaskBitsLowest:  2,
+					GdprLeftMaskBitsHighest: 3,
+				},
+			},
+		},
+		{
+			name: "invalid configuration",
+			masking: &IPMasking{
+				IpV4: IPMasks{
+					ActivityLeftMaskBits:    100,
+					GdprLeftMaskBitsLowest:  -200,
+					GdprLeftMaskBitsHighest: 300,
+				},
+				IpV6: IPMasks{
+					ActivityLeftMaskBits:    -100,
+					GdprLeftMaskBitsLowest:  200,
+					GdprLeftMaskBitsHighest: -300,
+				},
+			},
+			want: []error{
+				errors.New("activity left mask bits cannot exceed 32 in ipv4 address, or be less than 0"),
+				errors.New("gdpr left mask bits lowest cannot exceed 32 in ipv4 address, or be less than 0"),
+				errors.New("gdpr left mask bits highest cannot exceed 32 in ipv4 address, or be less than 0"),
+				errors.New("activity left mask bits cannot exceed 128 in ipv6 address, or be less than 0"),
+				errors.New("gdpr left mask bits lowest cannot exceed 128 in ipv6 address, or be less than 0"),
+				errors.New("gdpr left mask bits highest cannot exceed 128 in ipv6 address, or be less than 0"),
+			},
+		},
+		{
+			name: "mixed valid and invalid configuration",
+			masking: &IPMasking{
+				IpV4: IPMasks{
+					ActivityLeftMaskBits:    10,
+					GdprLeftMaskBitsLowest:  -20,
+					GdprLeftMaskBitsHighest: 30,
+				},
+				IpV6: IPMasks{
+					ActivityLeftMaskBits:    10,
+					GdprLeftMaskBitsLowest:  20,
+					GdprLeftMaskBitsHighest: -30,
+				},
+			},
+			want: []error{
+				errors.New("gdpr left mask bits lowest cannot exceed 32 in ipv4 address, or be less than 0"),
+				errors.New("gdpr left mask bits highest cannot exceed 128 in ipv6 address, or be less than 0"),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var errs []error
+			got := tt.masking.Validate(errs)
+			assert.ElementsMatch(t, got, tt.want)
+		})
+	}
+}

config/config.go

@@ -155,12 +155,14 @@ func (cfg *Configuration) validate(v *viper.Viper) []error {
 		cfg.TmaxAdjustments.Enabled = false
 	}
 
-	if cfg.AccountDefaults.Privacy != nil {
+	if cfg.AccountDefaults.Privacy.AllowActivities != nil {
 		glog.Warning("account_defaults.Privacy has no effect as the feature is under development.")
 	}
 
 	errs = cfg.Experiment.validate(errs)
 	errs = cfg.BidderInfos.validate(errs)
+	errs = cfg.AccountDefaults.Privacy.IPMasking.Validate(errs)
+
 	return errs
 }
 
@@ -1025,6 +1027,10 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
 	v.SetDefault("account_defaults.price_floors.max_rules", 100)
 	v.SetDefault("account_defaults.price_floors.max_schema_dims", 3)
 	v.SetDefault("account_defaults.events_enabled", false)
+	v.SetDefault("account_defaults.privacy.ip_masking.ipv6.activity_left_mask_bits", 56)
+	v.SetDefault("account_defaults.privacy.ip_masking.ipv6.gdpr_left_mask_bits_lowest", 112)
+	v.SetDefault("account_defaults.privacy.ip_masking.ipv6.gdpr_left_mask_bits_highest", 96)
+	v.SetDefault("account_defaults.privacy.ip_masking.ipv4.gdpr_left_mask_bits_lowest", 24)
 
 	v.SetDefault("compression.response.enable_gzip", false)
 	v.SetDefault("compression.request.enable_gzip", false)

config/config_test.go

@@ -206,6 +206,11 @@ func TestDefaults(t *testing.T) {
 	cmpUnsignedInts(t, "tmax_adjustments.bidder_network_latency_buffer_ms", 0, cfg.TmaxAdjustments.BidderNetworkLatencyBuffer)
 	cmpUnsignedInts(t, "tmax_adjustments.pbs_response_preparation_duration_ms", 0, cfg.TmaxAdjustments.PBSResponsePreparationDuration)
 
+	cmpInts(t, "account_defaults.privacy.ip_masking.ipv6.activity_left_mask_bits", 56, cfg.AccountDefaults.Privacy.IPMasking.IpV6.ActivityLeftMaskBits)
+	cmpInts(t, "account_defaults.privacy.ip_masking.ipv6.gdpr_left_mask_bits_lowest", 112, cfg.AccountDefaults.Privacy.IPMasking.IpV6.GdprLeftMaskBitsLowest)
+	cmpInts(t, "account_defaults.privacy.ip_masking.ipv6.gdpr_left_mask_bits_highest", 96, cfg.AccountDefaults.Privacy.IPMasking.IpV6.GdprLeftMaskBitsHighest)
+	cmpInts(t, "account_defaults.privacy.ip_masking.ipv4.gdpr_left_mask_bits_lowest", 24, cfg.AccountDefaults.Privacy.IPMasking.IpV4.GdprLeftMaskBitsLowest)
+
 	//Assert purpose VendorExceptionMap hash tables were built correctly
 	expectedTCF2 := TCF2{
 		Enabled: true,
@@ -477,6 +482,14 @@ account_defaults:
         use_dynamic_data: true
         max_rules: 120
         max_schema_dims: 5
+    privacy:
+        ip_masking:
+            ipv6:
+                activity_left_mask_bits: 50
+                gdpr_left_mask_bits_lowest: 110
+                gdpr_left_mask_bits_highest: 90
+            ipv4:
+                gdpr_left_mask_bits_lowest: 20
 tmax_adjustments:
   enabled: true
   bidder_response_duration_min_ms: 700
@@ -583,6 +596,11 @@ func TestFullConfig(t *testing.T) {
 	cmpBools(t, "account_defaults.events_enabled", *cfg.AccountDefaults.EventsEnabled, true)
 	cmpNils(t, "account_defaults.events.enabled", cfg.AccountDefaults.Events.Enabled)
 
+	cmpInts(t, "account_defaults.privacy.ip_masking.ipv6.activity_left_mask_bits", 50, cfg.AccountDefaults.Privacy.IPMasking.IpV6.ActivityLeftMaskBits)
+	cmpInts(t, "account_defaults.privacy.ip_masking.ipv6.gdpr_left_mask_bits_lowest", 110, cfg.AccountDefaults.Privacy.IPMasking.IpV6.GdprLeftMaskBitsLowest)
+	cmpInts(t, "account_defaults.privacy.ip_masking.ipv6.gdpr_left_mask_bits_highest", 90, cfg.AccountDefaults.Privacy.IPMasking.IpV6.GdprLeftMaskBitsHighest)
+	cmpInts(t, "account_defaults.privacy.ip_masking.ipv4.gdpr_left_mask_bits_lowest", 20, cfg.AccountDefaults.Privacy.IPMasking.IpV4.GdprLeftMaskBitsLowest)
+
 	// Assert compression related defaults
 	cmpBools(t, "enable_gzip", false, cfg.EnableGzip)
 	cmpBools(t, "compression.request.enable_gzip", true, cfg.Compression.Request.GZIP)

endpoints/cookie_sync_test.go

@@ -2101,9 +2101,9 @@ func (p *fakePermissions) AuctionActivitiesAllowed(ctx context.Context, bidderCo
 	}, nil
 }
 
-func getDefaultActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
-	return &config.AccountPrivacy{
-		AllowActivities: config.AllowActivities{
+func getDefaultActivityConfig(componentName string, allow bool) config.AccountPrivacy {
+	return config.AccountPrivacy{
+		AllowActivities: &config.AllowActivities{
 			SyncUser: config.Activity{
 				Default: ptrutil.ToPtr(true),
 				Rules: []config.ActivityRule{

exchange/utils.go

@@ -212,7 +212,7 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context,
 			applyFPD(auctionReq.FirstPartyData[bidderRequest.BidderName], bidderRequest.BidRequest)
 		}
 
-		privacyEnforcement.Apply(bidderRequest.BidRequest)
+		privacyEnforcement.Apply(bidderRequest.BidRequest, auctionReq.Account.Privacy.IPMasking)
 		allowedBidderRequests = append(allowedBidderRequests, bidderRequest)
 
 		// GPP downgrade: always downgrade unless we can confirm GPP is supported

exchange/utils_test.go

@@ -4272,7 +4272,7 @@ func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) {
 	testCases := []struct {
 		name                 string
 		req                  *openrtb2.BidRequest
-		privacyConfig        *config.AccountPrivacy
+		privacyConfig        config.AccountPrivacy
 		componentName        string
 		allow                bool
 		expectedReqNumber    int
@@ -4382,25 +4382,25 @@ func buildDefaultActivityConfig(componentName string, allow bool) config.Activit
 	}
 }
 
-func getFetchBidsActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
-	return &config.AccountPrivacy{
-		AllowActivities: config.AllowActivities{
+func getFetchBidsActivityConfig(componentName string, allow bool) config.AccountPrivacy {
+	return config.AccountPrivacy{
+		AllowActivities: &config.AllowActivities{
 			FetchBids: buildDefaultActivityConfig(componentName, allow),
 		},
 	}
 }
 
-func getTransmitUFPDActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
-	return &config.AccountPrivacy{
-		AllowActivities: config.AllowActivities{
+func getTransmitUFPDActivityConfig(componentName string, allow bool) config.AccountPrivacy {
+	return config.AccountPrivacy{
+		AllowActivities: &config.AllowActivities{
 			TransmitUserFPD: buildDefaultActivityConfig(componentName, allow),
 		},
 	}
 }
 
-func getTransmitPreciseGeoActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
-	return &config.AccountPrivacy{
-		AllowActivities: config.AllowActivities{
+func getTransmitPreciseGeoActivityConfig(componentName string, allow bool) config.AccountPrivacy {
+	return config.AccountPrivacy{
+		AllowActivities: &config.AllowActivities{
 			TransmitPreciseGeo: buildDefaultActivityConfig(componentName, allow),
 		},
 	}

privacy/enforcement.go

@@ -1,6 +1,9 @@
 package privacy
 
-import "github.com/prebid/openrtb/v19/openrtb2"
+import (
+	"github.com/prebid/openrtb/v19/openrtb2"
+	"github.com/prebid/prebid-server/config"
+)
 
 // Enforcement represents the privacy policies to enforce for an OpenRTB bid request.
 type Enforcement struct {
@@ -27,8 +30,8 @@ func (e Enforcement) AnyActivities() bool {
 }
 
 // Apply cleans personally identifiable information from an OpenRTB bid request.
-func (e Enforcement) Apply(bidRequest *openrtb2.BidRequest) {
-	e.apply(bidRequest, NewScrubber())
+func (e Enforcement) Apply(bidRequest *openrtb2.BidRequest, ipMasking config.IPMasking) {
+	e.apply(bidRequest, NewScrubber(ipMasking))
 }
 
 func (e Enforcement) apply(bidRequest *openrtb2.BidRequest, scrubber Scrubber) {

privacy/enforcer.go

@@ -26,11 +26,11 @@ type ActivityControl struct {
 	plans map[Activity]ActivityPlan
 }
 
-func NewActivityControl(privacyConf *config.AccountPrivacy) (ActivityControl, error) {
+func NewActivityControl(privacyConf config.AccountPrivacy) (ActivityControl, error) {
 	ac := ActivityControl{}
 	var err error
 
-	if privacyConf == nil {
+	if privacyConf.AllowActivities == nil {
 		return ac, nil
 	}