From 1bb5c661ace54e68680ae05767ba8dafcd6e4844 Mon Sep 17 00:00:00 2001 From: Jacob Baungard Hansen Date: Wed, 27 Sep 2023 16:01:22 +0200 Subject: [PATCH 01/10] Query: add optional tenancy enforcement With this commit it's now possible to enable enforcement of tenancy. If tenancy is enabled, a tenant label will be added to queries based on the tenant information provided by the tenant header, and the tenant-label-name. The implementation for query APIs are done by using prom-label-proxy as library, while the implementation for non-query APIs are written from scratch. Signed-off-by: Jacob Baungard Hansen --- cmd/thanos/query.go | 8 ++ docs/components/query.md | 6 + go.mod | 3 + go.sum | 10 ++ pkg/api/query/v1.go | 123 ++++++++++++----- pkg/tenancy/tenancy.go | 33 ++++- test/e2e/e2ethanos/services.go | 10 ++ test/e2e/query_test.go | 234 +++++++++++++++++++++++++++++++++ 8 files changed, 391 insertions(+), 36 deletions(-) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index f7a76cc0bd..d3db82581b 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -220,6 +220,8 @@ func registerQuery(app *extkingpin.App) { tenantHeader := cmd.Flag("query.tenant-header", "HTTP header to determine tenant.").Default(tenancy.DefaultTenantHeader).String() defaultTenant := cmd.Flag("query.default-tenant-id", "Default tenant ID to use if tenant header is not present").Default(tenancy.DefaultTenant).String() tenantCertField := cmd.Flag("query.tenant-certificate-field", "Use TLS client's certificate field to determine tenant for write requests. Must be one of "+tenancy.CertificateFieldOrganization+", "+tenancy.CertificateFieldOrganizationalUnit+" or "+tenancy.CertificateFieldCommonName+". This setting will cause the query.tenant-header flag value to be ignored.").Default("").Enum("", tenancy.CertificateFieldOrganization, tenancy.CertificateFieldOrganizationalUnit, tenancy.CertificateFieldCommonName) + enforceTenancy := cmd.Flag("query.enable-tenancy", "Enable tenancy. Only responses where the value of the configured tenant-label-name and value of the tenant header matches are returned.").Default("false").Bool() + tenantLabel := cmd.Flag("query.tenant-label-name", "Label name to use when enforce tenancy when -querier.tenancy is enabled").Default(tenancy.DefaultTenantLabel).String() var storeRateLimits store.SeriesSelectLimits storeRateLimits.RegisterFlags(cmd) @@ -343,6 +345,8 @@ func registerQuery(app *extkingpin.App) { *tenantHeader, *defaultTenant, *tenantCertField, + *enforceTenancy, + *tenantLabel, ) }) } @@ -422,6 +426,8 @@ func runQuery( tenantHeader string, defaultTenant string, tenantCertField string, + enforceTenancy bool, + tenantLabel string, ) error { if alertQueryURL == "" { lastColon := strings.LastIndex(httpBindAddr, ":") @@ -759,6 +765,8 @@ func runQuery( tenantHeader, defaultTenant, tenantCertField, + enforceTenancy, + tenantLabel, ) api.Register(router.WithPrefix("/api/v1"), tracer, logger, ins, logMiddleware) diff --git a/docs/components/query.md b/docs/components/query.md index 2c4f473464..7a22bd12c0 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -363,6 +363,9 @@ Flags: --query.default-tenant-id="default-tenant" Default tenant ID to use if tenant header is not present + --query.enable-tenancy Enable tenancy. Only responses where the value + of the configured tenant-label-name and value + of the tenant header matches are returned. --query.lookback-delta=QUERY.LOOKBACK-DELTA The maximum lookback duration for retrieving metrics during expression evaluations. @@ -415,6 +418,9 @@ Flags: flag value to be ignored. --query.tenant-header="THANOS-TENANT" HTTP header to determine tenant. + --query.tenant-label-name="tenant_id" + Label name to use when enforce tenancy when + -querier.tenancy is enabled --query.timeout=2m Maximum time to process query by query node. --request.logging-config= Alternative to 'request.logging-config-file' diff --git a/go.mod b/go.mod index 594cd579a0..6312f6e221 100644 --- a/go.mod +++ b/go.mod @@ -118,15 +118,18 @@ require ( require ( github.com/onsi/gomega v1.27.10 + github.com/prometheus-community/prom-label-proxy v0.7.0 go.opentelemetry.io/contrib/propagators/autoprop v0.38.0 go4.org/intern v0.0.0-20230525184215-6c62f75575cb golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b ) require ( + github.com/go-openapi/runtime v0.26.0 // indirect github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible // indirect + github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39 // indirect diff --git a/go.sum b/go.sum index b90a294b96..bdd3331978 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,7 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.8.3 h1:i84ZOPT35YCJROyuf97VP/VEdYhQce/8NTLOWq5tqJw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.8.3/go.mod h1:3+qm+VCJbVmQ9uscVz+8h1rRkJEy9ZNFGgpT1XB9mPg= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.32.3 h1:FhsH8qgWFkkPlPXBZ68uuT/FH/R+DLTtVPxjLEBs1v4= @@ -347,6 +348,8 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= github.com/go-openapi/loads v0.21.2 h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8enro= github.com/go-openapi/loads v0.21.2/go.mod h1:Jq58Os6SSGz0rzh62ptiu8Z31I+OTHqmULx5e/gJbNw= +github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc= +github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= @@ -646,6 +649,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -719,6 +723,8 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a h1:0usWxe5SGXKQovz3p+BiQ81Jy845xSMu2CWKuXsXuUM= +github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a/go.mod h1:3OETvrxfELvGsU2RoGGWercfeZ4bCL3+SOwzIWtJH/Q= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= @@ -844,10 +850,13 @@ github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4 github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/alertmanager v0.26.0 h1:uOMJWfIwJguc3NaM3appWNbbrh6G/OjvaHMk22aBBYc= github.com/prometheus/alertmanager v0.26.0/go.mod h1:rVcnARltVjavgVaNnmevxK7kOn7IZavyf0KNgHkbEpU= +github.com/prometheus-community/prom-label-proxy v0.7.0 h1:1iNHXF7V8z2iOCinEyxKDUHu2jppPAAd6PmBCi3naok= +github.com/prometheus-community/prom-label-proxy v0.7.0/go.mod h1:wR9C/Mwp5aBbiqM6gQ+FZdFRwL8pCzzhsje8lTAx/aA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= @@ -865,6 +874,7 @@ github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index 9f654b549b..baebdaeead 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -175,6 +175,8 @@ type QueryAPI struct { tenantHeader string defaultTenant string tenantCertField string + enforceTenancy bool + tenantLabel string } // NewQueryAPI returns an initialized QueryAPI type. @@ -208,6 +210,8 @@ func NewQueryAPI( tenantHeader string, defaultTenant string, tenantCertField string, + enforceTenancy bool, + tenantLabel string, ) *QueryAPI { if statsAggregatorFactory == nil { statsAggregatorFactory = &store.NoopSeriesStatsAggregatorFactory{} @@ -241,6 +245,8 @@ func NewQueryAPI( tenantHeader: tenantHeader, defaultTenant: defaultTenant, tenantCertField: tenantCertField, + enforceTenancy: enforceTenancy, + tenantLabel: tenantLabel, queryRangeHist: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ Name: "thanos_query_range_requested_timespan_duration_seconds", @@ -652,6 +658,15 @@ func (qapi *QueryAPI) query(r *http.Request) (interface{}, []error, *api.ApiErro } ctx = context.WithValue(ctx, tenancy.TenantKey, tenant) + queryStr := r.FormValue("query") + + if qapi.enforceTenancy { + queryStr, err = tenancy.EnforceQueryTenancy(qapi.tenantLabel, tenant, queryStr) + if err != nil { + return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} + } + } + // We are starting promQL tracing span here, because we have no control over promQL code. span, ctx := tracing.StartSpan(ctx, "promql_instant_query") defer span.Finish() @@ -671,7 +686,7 @@ func (qapi *QueryAPI) query(r *http.Request) (interface{}, []error, *api.ApiErro query.NewAggregateStatsReporter(&seriesStats), ), promql.NewPrometheusQueryOpts(false, lookbackDelta), - r.FormValue("query"), + queryStr, ts, ) @@ -954,6 +969,15 @@ func (qapi *QueryAPI) queryRange(r *http.Request) (interface{}, []error, *api.Ap // Record the query range requested. qapi.queryRangeHist.Observe(end.Sub(start).Seconds()) + queryStr := r.FormValue("query") + + if qapi.enforceTenancy { + queryStr, err = tenancy.EnforceQueryTenancy(qapi.tenantLabel, tenant, queryStr) + if err != nil { + return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} + } + } + // We are starting promQL tracing span here, because we have no control over promQL code. span, ctx := tracing.StartSpan(ctx, "promql_range_query") defer span.Finish() @@ -973,7 +997,7 @@ func (qapi *QueryAPI) queryRange(r *http.Request) (interface{}, []error, *api.Ap query.NewAggregateStatsReporter(&seriesStats), ), promql.NewPrometheusQueryOpts(false, lookbackDelta), - r.FormValue("query"), + queryStr, start, end, step, @@ -1049,15 +1073,6 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A return nil, nil, apiErr, func() {} } - var matcherSets [][]*labels.Matcher - for _, s := range r.Form[MatcherParam] { - matchers, err := parser.ParseMetricSelector(s) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} - } - matcherSets = append(matcherSets, matchers) - } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, qapi.tenantCertField) if err != nil { apiErr = &api.ApiError{Typ: api.ErrorBadData, Err: err} @@ -1065,6 +1080,11 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A } ctx = context.WithValue(ctx, tenancy.TenantKey, tenant) + matcherSets, apiErr := qapi.getLabelMatchers(r.Form[MatcherParam], tenant) + if apiErr != nil { + return nil, nil, apiErr, func() {} + } + q, err := qapi.queryableCreate( true, nil, @@ -1132,13 +1152,16 @@ func (qapi *QueryAPI) series(r *http.Request) (interface{}, []error, *api.ApiErr return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} } - var matcherSets [][]*labels.Matcher - for _, s := range r.Form[MatcherParam] { - matchers, err := parser.ParseMetricSelector(s) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} - } - matcherSets = append(matcherSets, matchers) + tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, "") + if err != nil { + apiErr := &api.ApiError{Typ: api.ErrorBadData, Err: err} + return nil, nil, apiErr, func() {} + } + ctx := context.WithValue(r.Context(), tenancy.TenantKey, tenant) + + matcherSets, apiErr := qapi.getLabelMatchers(r.Form[MatcherParam], tenant) + if apiErr != nil { + return nil, nil, apiErr, func() {} } enableDedup, apiErr := qapi.parseEnableDedupParam(r) @@ -1161,13 +1184,6 @@ func (qapi *QueryAPI) series(r *http.Request) (interface{}, []error, *api.ApiErr return nil, nil, apiErr, func() {} } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, "") - if err != nil { - apiErr = &api.ApiError{Typ: api.ErrorBadData, Err: err} - return nil, nil, apiErr, func() {} - } - ctx := context.WithValue(r.Context(), tenancy.TenantKey, tenant) - q, err := qapi.queryableCreate( enableDedup, replicaLabels, @@ -1219,15 +1235,6 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap return nil, nil, apiErr, func() {} } - var matcherSets [][]*labels.Matcher - for _, s := range r.Form[MatcherParam] { - matchers, err := parser.ParseMetricSelector(s) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} - } - matcherSets = append(matcherSets, matchers) - } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, "") if err != nil { apiErr = &api.ApiError{Typ: api.ErrorBadData, Err: err} @@ -1235,6 +1242,11 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap } ctx := context.WithValue(r.Context(), tenancy.TenantKey, tenant) + matcherSets, apiErr := qapi.getLabelMatchers(r.Form[MatcherParam], tenant) + if apiErr != nil { + return nil, nil, apiErr, func() {} + } + q, err := qapi.queryableCreate( true, nil, @@ -1301,6 +1313,49 @@ func (qapi *QueryAPI) stores(_ *http.Request) (interface{}, []error, *api.ApiErr return statuses, nil, nil, func() {} } +func (qapi *QueryAPI) getLabelMatchers(matchers []string, tenant string) ([][]*labels.Matcher, *api.ApiError) { + tenantLabelMatcher := &labels.Matcher{ + Name: qapi.tenantLabel, + Type: labels.MatchEqual, + Value: tenant, + } + + matcherSets := make([][]*labels.Matcher, 0, len(matchers)) + + // If tenancy is enforced, but there are no matchers at all, add the tenant matcher + if len(matchers) == 0 && qapi.enforceTenancy { + var matcher []*labels.Matcher + matcher = append(matcher, tenantLabelMatcher) + matcherSets = append(matcherSets, matcher) + return matcherSets, nil + } + + for _, s := range matchers { + matchers, err := parser.ParseMetricSelector(s) + if err != nil { + return nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} + } + if qapi.enforceTenancy { + // first check if there's a tenant matcher already, in which case we overwrite it + // if there are multiple tenant matchers, we overwrite all of them + found := false + for idx, matchValue := range matchers { + if matchValue.Name == qapi.tenantLabel { + matchers[idx] = tenantLabelMatcher + found = true + } + } + // if there are no pre-existing tenant matchers, add it. + if !found { + matchers = append(matchers, tenantLabelMatcher) + } + } + matcherSets = append(matcherSets, matchers) + } + + return matcherSets, nil +} + // NewTargetsHandler created handler compatible with HTTP /api/v1/targets https://prometheus.io/docs/prometheus/latest/querying/api/#targets // which uses gRPC Unary Targets API. func NewTargetsHandler(client targets.UnaryClient, enablePartialResponse bool) func(*http.Request) (interface{}, []error, *api.ApiError, func()) { diff --git a/pkg/tenancy/tenancy.go b/pkg/tenancy/tenancy.go index f8b54bcc48..3462fca271 100644 --- a/pkg/tenancy/tenancy.go +++ b/pkg/tenancy/tenancy.go @@ -8,9 +8,11 @@ import ( "net/http" "path" - "google.golang.org/grpc/metadata" - "github.com/pkg/errors" + "github.com/prometheus-community/prom-label-proxy/injectproxy" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/promql/parser" + "google.golang.org/grpc/metadata" ) type contextKey int @@ -136,3 +138,30 @@ func GetTenantFromGRPCMetadata(ctx context.Context) (string, bool) { } return md.Get(DefaultTenantHeader)[0], true } + +func EnforceQueryTenancy(tenantLabel string, tenant string, query string) (string, error) { + labelMatcher := &labels.Matcher{ + Name: tenantLabel, + Type: labels.MatchEqual, + Value: tenant, + } + + e := injectproxy.NewEnforcer(false, labelMatcher) + + expr, err := parser.ParseExpr(query) + if err != nil { + return "", errors.Wrap(err, "error parsing query string, when enforcing tenenacy") + } + + if err := e.EnforceNode(expr); err != nil { + var illegalLabelMatcherError *injectproxy.IllegalLabelMatcherError + if errors.As(err, *illegalLabelMatcherError) { + return "", illegalLabelMatcherError + } + return "", errors.Wrap(err, "error enforcing label") + } + + queryStr := expr.String() + + return queryStr, nil +} diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 0401868d89..3362375a81 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -264,6 +264,8 @@ type QuerierBuilder struct { telemetrySamplesQuantiles []float64 telemetrySeriesQuantiles []float64 + enforceTenancy bool + e2e.Linkable f e2e.FutureRunnable } @@ -385,6 +387,11 @@ func (q *QuerierBuilder) WithTelemetryQuantiles(duration []float64, samples []fl return q } +func (q *QuerierBuilder) WithTenancy(enforceTenancy bool) *QuerierBuilder { + q.enforceTenancy = enforceTenancy + return q +} + func (q *QuerierBuilder) Init() *e2eobs.Observable { args, err := q.collectArgs() if err != nil { @@ -485,6 +492,9 @@ func (q *QuerierBuilder) collectArgs() ([]string, error) { for _, bucket := range q.telemetrySeriesQuantiles { args = append(args, "--query.telemetry.request-series-seconds-quantiles="+strconv.FormatFloat(bucket, 'f', -1, 64)) } + if q.enforceTenancy { + args = append(args, "--query.enable-tenancy") + } return args, nil } diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 6584c7b842..3545ee281a 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -59,6 +59,7 @@ import ( "github.com/thanos-io/thanos/pkg/store/labelpb" prompb_copy "github.com/thanos-io/thanos/pkg/store/storepb/prompb" "github.com/thanos-io/thanos/pkg/targets/targetspb" + "github.com/thanos-io/thanos/pkg/tenancy" "github.com/thanos-io/thanos/pkg/testutil/e2eutil" "github.com/thanos-io/thanos/test/e2e/e2ethanos" ) @@ -2420,3 +2421,236 @@ func TestTenantHTTPMetrics(t *testing.T) { e2emon.WaitMissingMetrics(), )) } + +func TestQueryTenancyEnforcement(t *testing.T) { + t.Parallel() + + // Build up. + e, err := e2e.New(e2e.WithName("tenancyEnforce")) + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + + bucket := "store-gw-test" + minio := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS()) + testutil.Ok(t, e2e.StartAndWaitReady(minio)) + + l := log.NewLogfmtLogger(os.Stdout) + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + testutil.Ok(t, err) + + // Add series from different tenants + now := time.Now() + tenantLabel01 := labels.FromStrings(tenancy.DefaultTenantLabel, "tenant-01") + tenantLabel02 := labels.FromStrings(tenancy.DefaultTenantLabel, "tenant-02") + tenantLabel03 := labels.FromStrings(tenancy.DefaultTenantLabel, "default-tenant") + dir := filepath.Join(e.SharedDir(), "tmp") + testutil.Ok(t, os.MkdirAll(filepath.Join(e.SharedDir(), dir), os.ModePerm)) + + series1 := []labels.Labels{labels.FromStrings("a", "1")} + series2 := []labels.Labels{labels.FromStrings("b", "2")} + series3 := []labels.Labels{labels.FromStrings("c", "3")} + + blockID1, err := e2eutil.CreateBlockWithBlockDelay(ctx, + dir, + series1, + 10, + timestamp.FromTime(now), + timestamp.FromTime(now.Add(2*time.Hour)), + 30*time.Minute, + tenantLabel01, + 0, + metadata.NoneFunc, + ) + testutil.Ok(t, err) + + blockID2, err := e2eutil.CreateBlockWithBlockDelay(ctx, + dir, + series2, + 10, + timestamp.FromTime(now), + timestamp.FromTime(now.Add(2*time.Hour)), + 30*time.Minute, + tenantLabel02, + 0, + metadata.NoneFunc, + ) + testutil.Ok(t, err) + + blockID3, err := e2eutil.CreateBlockWithBlockDelay(ctx, + dir, + series3, + 10, + timestamp.FromTime(now), + timestamp.FromTime(now.Add(2*time.Hour)), + 30*time.Minute, + tenantLabel03, + 0, + metadata.NoneFunc, + ) + testutil.Ok(t, err) + + testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, blockID1.String()), blockID1.String())) + testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, blockID2.String()), blockID2.String())) + testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, blockID3.String()), blockID3.String())) + + storeGW := e2ethanos.NewStoreGW( + e, + "s1", + client.BucketConfig{ + Type: client.S3, + Config: e2ethanos.NewS3Config(bucket, minio.InternalEndpoint("http"), minio.InternalDir()), + }, + "", + "", + nil, + ) + + querierEnforce := e2ethanos.NewQuerierBuilder(e, "1", storeGW.InternalEndpoint("grpc")).WithTenancy(true).Init() + querierNoEnforce := e2ethanos.NewQuerierBuilder(e, "2", storeGW.InternalEndpoint("grpc")).Init() + testutil.Ok(t, e2e.StartAndWaitReady(storeGW, querierEnforce, querierNoEnforce)) + testutil.Ok(t, storeGW.WaitSumMetrics(e2emon.Equals(3), "thanos_blocks_meta_synced")) + + // default-tenant should only see part of the results + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{c=\"3\"}" }, + time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, + []model.Metric{ + { + "c": "3", + "tenant_id": "default-tenant", + }, + }, + ) + + // With no enforcement enabled, default tenant can see everything + queryAndAssertSeries(t, ctx, querierNoEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, + time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, + []model.Metric{ + { + "a": "1", + "tenant_id": "tenant-01", + }, + }, + ) + + // Default tenant don't see "a" when tenancy is enforced + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, + time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, + nil, + ) + + // default-tenant cannot attempt to view other tenants data, by setting the tenant id + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{tenant_id=\"tenant-01\"}" }, + time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, + nil, + ) + + rangeQuery(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3600, + promclient.QueryOptions{ + Deduplicate: true, + }, func(res model.Matrix) error { + if res.Len() == 0 { + return nil + } else { + return errors.New("default-tenant shouldn't be able to see results with label a") + } + }) + + rangeQuery(t, ctx, querierNoEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3600, + promclient.QueryOptions{ + Deduplicate: true, + }, func(res model.Matrix) error { + if res[0].Metric["a"] == "1" { + return nil + } else { + return errors.New("default-tenant should be able to see results with label a when enforcement is off") + } + }) + + rangeQuery(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{c=\"3\"}" }, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3600, + promclient.QueryOptions{ + Deduplicate: true, + }, func(res model.Matrix) error { + if res[0].Metric["c"] == "3" { + return nil + } else { + return errors.New("default-tenant should be able to see its own data when enforcement is enabled") + } + }) + + // default-tenant should only see two labels when enforcing is on (c,tenant_id) + labelNames(t, ctx, querierEnforce.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 2 + }) + + // default-tenant should only see all labels when enforcing is not on (a,b,c,tenant_id) + labelNames(t, ctx, querierNoEnforce.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 4 + }) + + // default tenant can just the value of the C label + labelValues(t, ctx, querierEnforce.Endpoint("http"), "c", nil, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 1 + }, + ) + labelValues(t, ctx, querierEnforce.Endpoint("http"), "a", nil, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 0 + }, + ) + + // Series endpoint tests + var matcherSetC []*labels.Matcher + labelMatcherC := &labels.Matcher{ + Name: "c", + Type: labels.MatchEqual, + Value: "3", + } + matcherSetC = append(matcherSetC, labelMatcherC) + + var matcherSetB []*labels.Matcher + labelMatcher := &labels.Matcher{ + Name: "b", + Type: labels.MatchEqual, + Value: "2", + } + matcherSetB = append(matcherSetB, labelMatcher) + + // default-tenant can see series with matcher C + series(t, ctx, querierEnforce.Endpoint("http"), matcherSetC, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []map[string]string) bool { + var expected = []map[string]string{ + { + "c": "3", + "tenant_id": "default-tenant", + }, + } + return reflect.DeepEqual(res, expected) + }) + + // default-tenant cannot see series with matcher B when tenancy is enabled + series(t, ctx, querierEnforce.Endpoint("http"), matcherSetB, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []map[string]string) bool { + return len(res) == 0 + }) + + // default-tenant can see series with matcher B when tenancy is not enabled + series(t, ctx, querierNoEnforce.Endpoint("http"), matcherSetB, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []map[string]string) bool { + var expected = []map[string]string{ + { + "b": "2", + "tenant_id": "tenant-02", + }, + } + return reflect.DeepEqual(res, expected) + }) +} From 4cee3ca93cfa07ee7a038c10ba4358302aac3407 Mon Sep 17 00:00:00 2001 From: Jacob Baungard Hansen Date: Thu, 5 Oct 2023 15:02:41 +0200 Subject: [PATCH 02/10] Add changelog entry Signed-off-by: Jacob Baungard Hansen --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f5dd2d4f8..f9f11e51eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Added +- [#6756](https://github.com/thanos-io/thanos/pull/6756) Query: Add the following options to allow enforcement of tenancy on the query path: `query.enable-tenancy`, `query.tenant-label-name`. + ### Changed ### Removed From 017c7ce3de47ba13506ac96279aa6ad5ed034e71 Mon Sep 17 00:00:00 2001 From: Jacob Baungard Hansen Date: Thu, 5 Oct 2023 15:02:56 +0200 Subject: [PATCH 03/10] Query: Add non-default tenant testcase Signed-off-by: Jacob Baungard Hansen --- test/e2e/query_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 3545ee281a..991b93652d 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -2513,6 +2513,12 @@ func TestQueryTenancyEnforcement(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(storeGW, querierEnforce, querierNoEnforce)) testutil.Ok(t, storeGW.WaitSumMetrics(e2emon.Equals(3), "thanos_blocks_meta_synced")) + tenant1Header := make(http.Header) + tenant1Header.Add("thanos-tenant", "tenant-01") + + tenant2Header := make(http.Header) + tenant2Header.Add("thanos-tenant", "tenant-02") + // default-tenant should only see part of the results queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{c=\"3\"}" }, time.Now, promclient.QueryOptions{ @@ -2526,6 +2532,20 @@ func TestQueryTenancyEnforcement(t *testing.T) { }, ) + // tenant-01 should only see part of the results + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, + time.Now, promclient.QueryOptions{ + Deduplicate: false, + HTTPHeaders: tenant1Header, + }, + []model.Metric{ + { + "a": "1", + "tenant_id": "tenant-01", + }, + }, + ) + // With no enforcement enabled, default tenant can see everything queryAndAssertSeries(t, ctx, querierNoEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, time.Now, promclient.QueryOptions{ @@ -2547,6 +2567,15 @@ func TestQueryTenancyEnforcement(t *testing.T) { nil, ) + // tenant-2 don't see "a" when tenancy is enforced + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, + time.Now, promclient.QueryOptions{ + Deduplicate: false, + HTTPHeaders: tenant2Header, + }, + nil, + ) + // default-tenant cannot attempt to view other tenants data, by setting the tenant id queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{tenant_id=\"tenant-01\"}" }, time.Now, promclient.QueryOptions{ From a2820854ae0dccb34a9feb69b8b05ab1968a0be7 Mon Sep 17 00:00:00 2001 From: Jacob Baungard Hansen Date: Thu, 5 Oct 2023 15:20:07 +0200 Subject: [PATCH 04/10] Test: make query a constant to make linter happy Signed-off-by: Jacob Baungard Hansen --- test/e2e/query_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 991b93652d..74d2dde8f0 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -64,6 +64,8 @@ import ( "github.com/thanos-io/thanos/test/e2e/e2ethanos" ) +const testQueryA = "{a=\"1\"}" + func defaultWebConfig() string { // username: test, secret: test(bcrypt hash) return ` @@ -2533,7 +2535,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { ) // tenant-01 should only see part of the results - queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return testQueryA }, time.Now, promclient.QueryOptions{ Deduplicate: false, HTTPHeaders: tenant1Header, @@ -2547,7 +2549,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { ) // With no enforcement enabled, default tenant can see everything - queryAndAssertSeries(t, ctx, querierNoEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, + queryAndAssertSeries(t, ctx, querierNoEnforce.Endpoint("http"), func() string { return testQueryA }, time.Now, promclient.QueryOptions{ Deduplicate: false, }, @@ -2560,7 +2562,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { ) // Default tenant don't see "a" when tenancy is enforced - queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return testQueryA }, time.Now, promclient.QueryOptions{ Deduplicate: false, }, @@ -2568,7 +2570,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { ) // tenant-2 don't see "a" when tenancy is enforced - queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return testQueryA }, time.Now, promclient.QueryOptions{ Deduplicate: false, HTTPHeaders: tenant2Header, @@ -2584,7 +2586,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { nil, ) - rangeQuery(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3600, + rangeQuery(t, ctx, querierEnforce.Endpoint("http"), func() string { return testQueryA }, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3600, promclient.QueryOptions{ Deduplicate: true, }, func(res model.Matrix) error { @@ -2595,7 +2597,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { } }) - rangeQuery(t, ctx, querierNoEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3600, + rangeQuery(t, ctx, querierNoEnforce.Endpoint("http"), func() string { return testQueryA }, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3600, promclient.QueryOptions{ Deduplicate: true, }, func(res model.Matrix) error { From 689948ecaefc09c0d38bf33e036198fc521bb6fb Mon Sep 17 00:00:00 2001 From: Jacob Baungard Hansen Date: Tue, 17 Oct 2023 10:08:24 +0200 Subject: [PATCH 05/10] Address review comments - Remove empty lines - If multiple tenant matchers are found in the original query, we only replace the first one with the header provided tenant, and remove any subsequent ones. Signed-off-by: Jacob Baungard Hansen --- pkg/api/query/v1.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index baebdaeead..5618d6c175 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -659,7 +659,6 @@ func (qapi *QueryAPI) query(r *http.Request) (interface{}, []error, *api.ApiErro ctx = context.WithValue(ctx, tenancy.TenantKey, tenant) queryStr := r.FormValue("query") - if qapi.enforceTenancy { queryStr, err = tenancy.EnforceQueryTenancy(qapi.tenantLabel, tenant, queryStr) if err != nil { @@ -970,7 +969,6 @@ func (qapi *QueryAPI) queryRange(r *http.Request) (interface{}, []error, *api.Ap qapi.queryRangeHist.Observe(end.Sub(start).Seconds()) queryStr := r.FormValue("query") - if qapi.enforceTenancy { queryStr, err = tenancy.EnforceQueryTenancy(qapi.tenantLabel, tenant, queryStr) if err != nil { @@ -1337,12 +1335,17 @@ func (qapi *QueryAPI) getLabelMatchers(matchers []string, tenant string) ([][]*l } if qapi.enforceTenancy { // first check if there's a tenant matcher already, in which case we overwrite it - // if there are multiple tenant matchers, we overwrite all of them + // if there are multiple tenant matchers, we remove the subsequent ones found := false for idx, matchValue := range matchers { if matchValue.Name == qapi.tenantLabel { - matchers[idx] = tenantLabelMatcher - found = true + if found { + // remove any additional tenant matchers. + matchers = append(matchers[:idx], matchers[idx+1:]...) + } else { + matchers[idx] = tenantLabelMatcher + found = true + } } } // if there are no pre-existing tenant matchers, add it. From b2043ad003fd2db53fb82ce678d2c57498389f9f Mon Sep 17 00:00:00 2001 From: Jacob Baungard Hansen Date: Thu, 26 Oct 2023 16:30:58 +0200 Subject: [PATCH 06/10] Address review comments - `--enable-tenancy` -> `--enforce-tenancy` - Create `RewritePromQL` and `RewriteLabelMatchers` to clean up code in query api. Also move getLabelMatchers to tenancy pkg. - Use prom-label-proxys `EnforceMatchers` to rewrite labels on non-query APIs instead of own solution - Don't specifically handle `illegalLabelMatcherError` Signed-off-by: Jacob Baungard Hansen --- cmd/thanos/query.go | 2 +- docs/components/query.md | 7 ++- pkg/api/query/v1.go | 102 +++------------------------------ pkg/tenancy/tenancy.go | 79 +++++++++++++++++++++++-- test/e2e/e2ethanos/services.go | 2 +- 5 files changed, 87 insertions(+), 105 deletions(-) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index d3db82581b..9c44f94db5 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -220,7 +220,7 @@ func registerQuery(app *extkingpin.App) { tenantHeader := cmd.Flag("query.tenant-header", "HTTP header to determine tenant.").Default(tenancy.DefaultTenantHeader).String() defaultTenant := cmd.Flag("query.default-tenant-id", "Default tenant ID to use if tenant header is not present").Default(tenancy.DefaultTenant).String() tenantCertField := cmd.Flag("query.tenant-certificate-field", "Use TLS client's certificate field to determine tenant for write requests. Must be one of "+tenancy.CertificateFieldOrganization+", "+tenancy.CertificateFieldOrganizationalUnit+" or "+tenancy.CertificateFieldCommonName+". This setting will cause the query.tenant-header flag value to be ignored.").Default("").Enum("", tenancy.CertificateFieldOrganization, tenancy.CertificateFieldOrganizationalUnit, tenancy.CertificateFieldCommonName) - enforceTenancy := cmd.Flag("query.enable-tenancy", "Enable tenancy. Only responses where the value of the configured tenant-label-name and value of the tenant header matches are returned.").Default("false").Bool() + enforceTenancy := cmd.Flag("query.enforce-tenancy", "Enforce tenancy on Query APIs. Only responses where the value of the configured tenant-label-name and value of the tenant header matches are returned.").Default("false").Bool() tenantLabel := cmd.Flag("query.tenant-label-name", "Label name to use when enforce tenancy when -querier.tenancy is enabled").Default(tenancy.DefaultTenantLabel).String() var storeRateLimits store.SeriesSelectLimits diff --git a/docs/components/query.md b/docs/components/query.md index 7a22bd12c0..57f6e51e74 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -363,9 +363,10 @@ Flags: --query.default-tenant-id="default-tenant" Default tenant ID to use if tenant header is not present - --query.enable-tenancy Enable tenancy. Only responses where the value - of the configured tenant-label-name and value - of the tenant header matches are returned. + --query.enforce-tenancy Enforce tenancy on Query APIs. Only + responses where the value of the configured + tenant-label-name and value of the tenant + header matches are returned. --query.lookback-delta=QUERY.LOOKBACK-DELTA The maximum lookback duration for retrieving metrics during expression evaluations. diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index 5618d6c175..1401f48314 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -651,19 +651,9 @@ func (qapi *QueryAPI) query(r *http.Request) (interface{}, []error, *api.ApiErro lookbackDelta = lookbackDeltaFromReq } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, qapi.tenantCertField) + queryStr, tenant, ctx, err := tenancy.RewritePromQL(ctx, r, qapi.tenantHeader, qapi.defaultTenant, qapi.tenantCertField, qapi.enforceTenancy, qapi.tenantLabel, r.FormValue("query")) if err != nil { - apiErr = &api.ApiError{Typ: api.ErrorBadData, Err: err} - return nil, nil, apiErr, func() {} - } - ctx = context.WithValue(ctx, tenancy.TenantKey, tenant) - - queryStr := r.FormValue("query") - if qapi.enforceTenancy { - queryStr, err = tenancy.EnforceQueryTenancy(qapi.tenantLabel, tenant, queryStr) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} - } + return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} } // We are starting promQL tracing span here, because we have no control over promQL code. @@ -958,24 +948,14 @@ func (qapi *QueryAPI) queryRange(r *http.Request) (interface{}, []error, *api.Ap lookbackDelta = lookbackDeltaFromReq } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, qapi.tenantCertField) + queryStr, tenant, ctx, err := tenancy.RewritePromQL(ctx, r, qapi.tenantHeader, qapi.defaultTenant, qapi.tenantCertField, qapi.enforceTenancy, qapi.tenantLabel, r.FormValue("query")) if err != nil { - apiErr = &api.ApiError{Typ: api.ErrorBadData, Err: err} - return nil, nil, apiErr, func() {} + return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} } - ctx = context.WithValue(ctx, tenancy.TenantKey, tenant) // Record the query range requested. qapi.queryRangeHist.Observe(end.Sub(start).Seconds()) - queryStr := r.FormValue("query") - if qapi.enforceTenancy { - queryStr, err = tenancy.EnforceQueryTenancy(qapi.tenantLabel, tenant, queryStr) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} - } - } - // We are starting promQL tracing span here, because we have no control over promQL code. span, ctx := tracing.StartSpan(ctx, "promql_range_query") defer span.Finish() @@ -1071,17 +1051,11 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A return nil, nil, apiErr, func() {} } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, qapi.tenantCertField) + matcherSets, ctx, err := tenancy.RewriteLabelMatchers(ctx, r, qapi.tenantHeader, qapi.defaultTenant, qapi.tenantCertField, qapi.enforceTenancy, qapi.tenantLabel, r.Form[MatcherParam]) if err != nil { apiErr = &api.ApiError{Typ: api.ErrorBadData, Err: err} return nil, nil, apiErr, func() {} } - ctx = context.WithValue(ctx, tenancy.TenantKey, tenant) - - matcherSets, apiErr := qapi.getLabelMatchers(r.Form[MatcherParam], tenant) - if apiErr != nil { - return nil, nil, apiErr, func() {} - } q, err := qapi.queryableCreate( true, @@ -1150,17 +1124,11 @@ func (qapi *QueryAPI) series(r *http.Request) (interface{}, []error, *api.ApiErr return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, "") + matcherSets, ctx, err := tenancy.RewriteLabelMatchers(r.Context(), r, qapi.tenantHeader, qapi.defaultTenant, qapi.tenantCertField, qapi.enforceTenancy, qapi.tenantLabel, r.Form[MatcherParam]) if err != nil { apiErr := &api.ApiError{Typ: api.ErrorBadData, Err: err} return nil, nil, apiErr, func() {} } - ctx := context.WithValue(r.Context(), tenancy.TenantKey, tenant) - - matcherSets, apiErr := qapi.getLabelMatchers(r.Form[MatcherParam], tenant) - if apiErr != nil { - return nil, nil, apiErr, func() {} - } enableDedup, apiErr := qapi.parseEnableDedupParam(r) if apiErr != nil { @@ -1233,15 +1201,9 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap return nil, nil, apiErr, func() {} } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, "") + matcherSets, ctx, err := tenancy.RewriteLabelMatchers(r.Context(), r, qapi.tenantHeader, qapi.defaultTenant, qapi.tenantCertField, qapi.enforceTenancy, qapi.tenantLabel, r.Form[MatcherParam]) if err != nil { - apiErr = &api.ApiError{Typ: api.ErrorBadData, Err: err} - return nil, nil, apiErr, func() {} - } - ctx := context.WithValue(r.Context(), tenancy.TenantKey, tenant) - - matcherSets, apiErr := qapi.getLabelMatchers(r.Form[MatcherParam], tenant) - if apiErr != nil { + apiErr := &api.ApiError{Typ: api.ErrorBadData, Err: err} return nil, nil, apiErr, func() {} } @@ -1311,54 +1273,6 @@ func (qapi *QueryAPI) stores(_ *http.Request) (interface{}, []error, *api.ApiErr return statuses, nil, nil, func() {} } -func (qapi *QueryAPI) getLabelMatchers(matchers []string, tenant string) ([][]*labels.Matcher, *api.ApiError) { - tenantLabelMatcher := &labels.Matcher{ - Name: qapi.tenantLabel, - Type: labels.MatchEqual, - Value: tenant, - } - - matcherSets := make([][]*labels.Matcher, 0, len(matchers)) - - // If tenancy is enforced, but there are no matchers at all, add the tenant matcher - if len(matchers) == 0 && qapi.enforceTenancy { - var matcher []*labels.Matcher - matcher = append(matcher, tenantLabelMatcher) - matcherSets = append(matcherSets, matcher) - return matcherSets, nil - } - - for _, s := range matchers { - matchers, err := parser.ParseMetricSelector(s) - if err != nil { - return nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} - } - if qapi.enforceTenancy { - // first check if there's a tenant matcher already, in which case we overwrite it - // if there are multiple tenant matchers, we remove the subsequent ones - found := false - for idx, matchValue := range matchers { - if matchValue.Name == qapi.tenantLabel { - if found { - // remove any additional tenant matchers. - matchers = append(matchers[:idx], matchers[idx+1:]...) - } else { - matchers[idx] = tenantLabelMatcher - found = true - } - } - } - // if there are no pre-existing tenant matchers, add it. - if !found { - matchers = append(matchers, tenantLabelMatcher) - } - } - matcherSets = append(matcherSets, matchers) - } - - return matcherSets, nil -} - // NewTargetsHandler created handler compatible with HTTP /api/v1/targets https://prometheus.io/docs/prometheus/latest/querying/api/#targets // which uses gRPC Unary Targets API. func NewTargetsHandler(client targets.UnaryClient, enablePartialResponse bool) func(*http.Request) (interface{}, []error, *api.ApiError, func()) { diff --git a/pkg/tenancy/tenancy.go b/pkg/tenancy/tenancy.go index 3462fca271..13592833a8 100644 --- a/pkg/tenancy/tenancy.go +++ b/pkg/tenancy/tenancy.go @@ -154,14 +154,81 @@ func EnforceQueryTenancy(tenantLabel string, tenant string, query string) (strin } if err := e.EnforceNode(expr); err != nil { - var illegalLabelMatcherError *injectproxy.IllegalLabelMatcherError - if errors.As(err, *illegalLabelMatcherError) { - return "", illegalLabelMatcherError - } return "", errors.Wrap(err, "error enforcing label") } - queryStr := expr.String() + return expr.String(), nil +} + +func getLabelMatchers(formMatchers []string, tenant string, enforceTenancy bool, tenantLabel string) ([][]*labels.Matcher, error) { + tenantLabelMatcher := &labels.Matcher{ + Name: tenantLabel, + Type: labels.MatchEqual, + Value: tenant, + } + + matcherSets := make([][]*labels.Matcher, 0, len(formMatchers)) + + // If tenancy is enforced, but there are no matchers at all, add the tenant matcher + if len(formMatchers) == 0 && enforceTenancy { + var matcher []*labels.Matcher + matcher = append(matcher, tenantLabelMatcher) + matcherSets = append(matcherSets, matcher) + return matcherSets, nil + } + + for _, s := range formMatchers { + matchers, err := parser.ParseMetricSelector(s) + if err != nil { + return nil, err + } + + if enforceTenancy { + e := injectproxy.NewEnforcer(false, tenantLabelMatcher) + matchers, err = e.EnforceMatchers(matchers) + if err != nil { + return nil, err + } + } + + matcherSets = append(matcherSets, matchers) + } + + return matcherSets, nil +} + +// This function will: +// - Get tenant from HTTP header and add it to context. +// - if tenancy is enforce, add a tenant matcher. +func RewritePromQL(ctx context.Context, r *http.Request, tenantHeader string, defaultTenantID string, certTenantField string, enforceTenancy bool, tenantLabel string, queryStr string) (string, string, context.Context, error) { + tenant, err := GetTenantFromHTTP(r, tenantHeader, defaultTenantID, certTenantField) + if err != nil { + return "", "", ctx, err + } + ctx = context.WithValue(ctx, TenantKey, tenant) + + if enforceTenancy { + queryStr, err = EnforceQueryTenancy(tenantLabel, tenant, queryStr) + return queryStr, tenant, ctx, err + } + return queryStr, tenant, ctx, nil +} + +// This function will: +// - Get tenant from HTTP header and add it to context. +// - Parse all labels matchers provided. +// - If tenancy is enforced, make sure a tenant matcher is present. +func RewriteLabelMatchers(ctx context.Context, r *http.Request, tenantHeader string, defaultTenantID string, certTenantField string, enforceTenancy bool, tenantLabel string, formMatchers []string) ([][]*labels.Matcher, context.Context, error) { + tenant, err := GetTenantFromHTTP(r, tenantHeader, defaultTenantID, certTenantField) + if err != nil { + return nil, ctx, err + } + ctx = context.WithValue(ctx, TenantKey, tenant) + + matcherSets, err := getLabelMatchers(formMatchers, tenant, enforceTenancy, tenantLabel) + if err != nil { + return nil, ctx, err + } - return queryStr, nil + return matcherSets, ctx, nil } diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 3362375a81..ead6a639be 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -493,7 +493,7 @@ func (q *QuerierBuilder) collectArgs() ([]string, error) { args = append(args, "--query.telemetry.request-series-seconds-quantiles="+strconv.FormatFloat(bucket, 'f', -1, 64)) } if q.enforceTenancy { - args = append(args, "--query.enable-tenancy") + args = append(args, "--query.enforce-tenancy") } return args, nil } From d2da06cbf4de8c6fd181ed450cf77109c22edd9b Mon Sep 17 00:00:00 2001 From: Jacob Baungard Hansen Date: Mon, 18 Dec 2023 11:23:04 +0100 Subject: [PATCH 07/10] Re-arrage go.mod to make linter happy. Signed-off-by: Jacob Baungard Hansen --- go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 36021bc256..73ae55a2b7 100644 --- a/go.mod +++ b/go.mod @@ -126,14 +126,14 @@ require ( ) require ( - github.com/go-openapi/runtime v0.26.0 // indirect github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect + github.com/go-openapi/runtime v0.26.0 // indirect github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible // indirect - github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39 // indirect From e10aa4c164bdb4b926c3befd61f50ac97a67cf5d Mon Sep 17 00:00:00 2001 From: Jacob Baungard Hansen Date: Mon, 8 Jan 2024 11:05:21 +0100 Subject: [PATCH 08/10] Address review comments Minor changes to CLI docs, code-comments and changelog. Signed-off-by: Jacob Baungard Hansen --- CHANGELOG.md | 3 +-- cmd/thanos/query.go | 4 ++-- docs/components/query.md | 12 ++++++------ pkg/tenancy/tenancy.go | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63ec8c6e1b..a0bccbbaf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Added -- [#6756](https://github.com/thanos-io/thanos/pull/6756) Query: Add the following options to allow enforcement of tenancy on the query path: `query.enable-tenancy`, `query.tenant-label-name`. +- [#6756](https://github.com/thanos-io/thanos/pull/6756) Query: Add `query.enable-tenancy` & `query.tenant-label-name` options to allow enforcement of tenancy on the query path, by injecting labels into queries (uses prom-label-proxy internally). - [#6944](https://github.com/thanos-io/thanos/pull/6944) Receive: Added a new flag for maximum retention bytes. - [#6891](https://github.com/thanos-io/thanos/pull/6891) Objstore: Bump `objstore` which adds support for Azure Workload Identity. - [#6453](https://github.com/thanos-io/thanos/pull/6453) Sidecar: Added `--reloader.method` to support configuration reloads via SIHUP signal. @@ -27,7 +27,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6943](https://github.com/thanos-io/thanos/pull/6943) Ruler: Added `keep_firing_for` field in alerting rule. - [#6972](https://github.com/thanos-io/thanos/pull/6972) Store Gateway: Apply series limit when streaming series for series actually matched if lazy postings is enabled. - [#6984](https://github.com/thanos-io/thanos/pull/6984) Store Gateway: Added `--store.index-header-lazy-download-strategy` to specify how to lazily download index headers when lazy mmap is enabled. - - [#6887](https://github.com/thanos-io/thanos/pull/6887) Query Frontend: *breaking :warning:* Add tenant label to relevant exported metrics. Note that this change may cause some pre-existing custom dashboard queries to be incorrect due to the added label. - [#7028](https://github.com/thanos-io/thanos/pull/7028) Query|Query Frontend: Add new `--query-frontend.enable-x-functions` flag to enable experimental extended functions. diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 5d9bd1bf5e..4d831ab6d1 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -220,8 +220,8 @@ func registerQuery(app *extkingpin.App) { tenantHeader := cmd.Flag("query.tenant-header", "HTTP header to determine tenant.").Default(tenancy.DefaultTenantHeader).String() defaultTenant := cmd.Flag("query.default-tenant-id", "Default tenant ID to use if tenant header is not present").Default(tenancy.DefaultTenant).String() tenantCertField := cmd.Flag("query.tenant-certificate-field", "Use TLS client's certificate field to determine tenant for write requests. Must be one of "+tenancy.CertificateFieldOrganization+", "+tenancy.CertificateFieldOrganizationalUnit+" or "+tenancy.CertificateFieldCommonName+". This setting will cause the query.tenant-header flag value to be ignored.").Default("").Enum("", tenancy.CertificateFieldOrganization, tenancy.CertificateFieldOrganizationalUnit, tenancy.CertificateFieldCommonName) - enforceTenancy := cmd.Flag("query.enforce-tenancy", "Enforce tenancy on Query APIs. Only responses where the value of the configured tenant-label-name and value of the tenant header matches are returned.").Default("false").Bool() - tenantLabel := cmd.Flag("query.tenant-label-name", "Label name to use when enforce tenancy when -querier.tenancy is enabled").Default(tenancy.DefaultTenantLabel).String() + enforceTenancy := cmd.Flag("query.enforce-tenancy", "Enforce tenancy on Query APIs. Responses are returned only if the label value of the configured tenant-label-name and the value of the tenant header matches.").Default("false").Bool() + tenantLabel := cmd.Flag("query.tenant-label-name", "Label name to use when enforcing tenancy (if --query.enforce-tenancy is enabled).").Default(tenancy.DefaultTenantLabel).String() var storeRateLimits store.SeriesSelectLimits storeRateLimits.RegisterFlags(cmd) diff --git a/docs/components/query.md b/docs/components/query.md index 2d08b7d39a..c70b749ef1 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -367,10 +367,10 @@ Flags: Whether to enable extended rate functions (xrate, xincrease and xdelta). Only has effect when used with Thanos engine. - --query.enforce-tenancy Enforce tenancy on Query APIs. Only - responses where the value of the configured - tenant-label-name and value of the tenant - header matches are returned. + --query.enforce-tenancy Enforce tenancy on Query APIs. Responses + are returned only if the label value of the + configured tenant-label-name and the value of + the tenant header matches. --query.lookback-delta=QUERY.LOOKBACK-DELTA The maximum lookback duration for retrieving metrics during expression evaluations. @@ -424,8 +424,8 @@ Flags: --query.tenant-header="THANOS-TENANT" HTTP header to determine tenant. --query.tenant-label-name="tenant_id" - Label name to use when enforce tenancy when - -querier.tenancy is enabled + Label name to use when enforcing tenancy (if + --query.enforce-tenancy is enabled). --query.timeout=2m Maximum time to process query by query node. --request.logging-config= Alternative to 'request.logging-config-file' diff --git a/pkg/tenancy/tenancy.go b/pkg/tenancy/tenancy.go index 13592833a8..aec0bad86a 100644 --- a/pkg/tenancy/tenancy.go +++ b/pkg/tenancy/tenancy.go @@ -199,7 +199,7 @@ func getLabelMatchers(formMatchers []string, tenant string, enforceTenancy bool, // This function will: // - Get tenant from HTTP header and add it to context. -// - if tenancy is enforce, add a tenant matcher. +// - if tenancy is enforced, add a tenant matcher to the promQL expression. func RewritePromQL(ctx context.Context, r *http.Request, tenantHeader string, defaultTenantID string, certTenantField string, enforceTenancy bool, tenantLabel string, queryStr string) (string, string, context.Context, error) { tenant, err := GetTenantFromHTTP(r, tenantHeader, defaultTenantID, certTenantField) if err != nil { From 62bbbe219e3fdac9973f0f2a435a6364b1424ea1 Mon Sep 17 00:00:00 2001 From: Jacob Baungard Hansen Date: Mon, 8 Jan 2024 11:09:49 +0100 Subject: [PATCH 09/10] Query: Add tenancy docs This commit adds documentation for the tenancy features. Signed-off-by: Jacob Baungard Hansen --- docs/components/query.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/components/query.md b/docs/components/query.md index c70b749ef1..1b021f8995 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -260,6 +260,20 @@ Example file SD file in YAML: `--query.active-query-path` is an option which allows the user to specify a directory which will contain a `queries.active` file to track active queries. To enable this feature, the user has to specify a directory other than "", since that is skipped being the default. +## Tenancy + +### Tenant Metrics + +Tenant information is captured in relevant Thanos exported metrics in the Querier, Query Frontend and Store. In order make use of this functionality requests to the Query/Query Frontend component should include the tenant-id in the appropriate HTTP request header as configured with `--query.tenant-header`. The tenant information is passed through components (including Query Frontend), down to the Thanos Store, enabling per-tenant metrics in these components also. If no tenant header is set to requests to the query component, the default tenant as defined by `--query.tenant-default-id` will be used. + +### Tenant Enforcement + +Enforcement of tenancy can be enabled using `--query.enforce-tenancy`. If enabled, queries will only return results which contains a matching label, where the label name is `--query.tenant-label-name` and the label value matches the tenant as sent to the querier in the HTTP header configured with `--query-tenant-header`. This functionality requires that metrics are injected with a tenant label when ingested into Thanos. This can be done for example by enabling tenancy in the Thanos Receive component. + +In case of nested Thanos Query components, it's important to note that tenancy enforcement will only occur in the querier which the initial request is sent to, the layered queriers will not perform any enforcement. + +Further, note that there are no authentication mechanisms in Thanos, so anyone can set an arbitrary tenant in the HTTP header. It is recommended to use a proxy in front of the querier in case an authentication mechanism is needed. The Query UI also includes an option to set an arbitrary tenant, and should therefore not be exposed to end-users if users should not be able to see each others data. + ## Flags ```$ mdox-exec="thanos query --help" From 28ade315893ab6cfd300739c5e6f17eb1b0c5919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacob=20Baung=C3=A5rd=20Hansen?= Date: Mon, 8 Jan 2024 11:24:44 +0100 Subject: [PATCH 10/10] Update docs/components/query.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review comment Co-authored-by: Saswata Mukherjee Signed-off-by: Jacob Baungård Hansen --- docs/components/query.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/components/query.md b/docs/components/query.md index 1b021f8995..4584363ba3 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -268,7 +268,7 @@ Tenant information is captured in relevant Thanos exported metrics in the Querie ### Tenant Enforcement -Enforcement of tenancy can be enabled using `--query.enforce-tenancy`. If enabled, queries will only return results which contains a matching label, where the label name is `--query.tenant-label-name` and the label value matches the tenant as sent to the querier in the HTTP header configured with `--query-tenant-header`. This functionality requires that metrics are injected with a tenant label when ingested into Thanos. This can be done for example by enabling tenancy in the Thanos Receive component. +Enforcement of tenancy can be enabled using `--query.enforce-tenancy`. If enabled, queries will only fetch series containing a specific matcher, while evaluating PromQL expressions. The matcher label name is `--query.tenant-label-name` and the matcher value matches the tenant, as sent to the querier in the HTTP header configured with `--query-tenant-header`. This functionality requires that metrics are injected with a tenant label when ingested into Thanos. This can be done for example by enabling tenancy in the Thanos Receive component. In case of nested Thanos Query components, it's important to note that tenancy enforcement will only occur in the querier which the initial request is sent to, the layered queriers will not perform any enforcement.