diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/postgres/PostgresBasedRequestHandler.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/postgres/PostgresBasedRequestHandler.java index cd194341..1e0dede4 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/postgres/PostgresBasedRequestHandler.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/postgres/PostgresBasedRequestHandler.java @@ -19,6 +19,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -63,9 +64,11 @@ public class PostgresBasedRequestHandler implements RequestHandler { private static final String COUNT_COLUMN_NAME_CONFIG_KEY = "countColumnName"; private static final String START_TIME_ATTRIBUTE_NAME_CONFIG_KEY = "startTimeAttributeName"; private static final String SLOW_QUERY_THRESHOLD_MS_CONFIG = "slowQueryThresholdMs"; + private static final String MIN_REQUEST_DURATION_KEY = "minRequestDuration"; private static final int DEFAULT_SLOW_QUERY_THRESHOLD_MS = 3000; private static final Set GTE_OPERATORS = Set.of(Operator.GE, Operator.GT, Operator.EQ); + private static final Set LTE_OPERATORS = Set.of(Operator.LE, Operator.LT); // string values equivalent for null value of different data types // this is required to keep null values equivalent to default values for @@ -92,6 +95,7 @@ public class PostgresBasedRequestHandler implements RequestHandler { private Timer postgresQueryExecutionTimer; private int slowQueryThreshold = DEFAULT_SLOW_QUERY_THRESHOLD_MS; + private Duration minRequestDuration = Duration.ZERO; PostgresBasedRequestHandler(String name, Config config) { this(name, config, PostgresClientFactory.get()); @@ -152,6 +156,11 @@ private void processConfig(Config config) { if (config.hasPath(SLOW_QUERY_THRESHOLD_MS_CONFIG)) { this.slowQueryThreshold = config.getInt(SLOW_QUERY_THRESHOLD_MS_CONFIG); } + + if (config.hasPath(MIN_REQUEST_DURATION_KEY)) { + this.minRequestDuration = config.getDuration(MIN_REQUEST_DURATION_KEY); + } + LOG.info( "Using {}ms as the threshold for logging slow queries of handler: {}", slowQueryThreshold, @@ -196,6 +205,13 @@ public QueryCost canHandle(QueryRequest request, ExecutionContext executionConte cost = tableDefinition.getTimeGranularityMillis() / (Long.MAX_VALUE * 2D); } + long requestEndTime = getRequestEndTime(request.getFilter()); + Duration requestDuration = Duration.ofMillis(requestEndTime - requestStartTime); + + // choose this handler if requestDuration >= minRequestDuration + if (requestDuration.compareTo(minRequestDuration) >= 0) { + cost /= 2; + } return new QueryCost(cost); } @@ -224,6 +240,23 @@ && rhsHasLongValue(filter.getRhs())) { return requestStartTime; } + private long getRequestEndTime(Filter filter) { + long requestEndTime = Long.MIN_VALUE; + + if (lhsIsStartTimeAttribute(filter.getLhs()) + && LTE_OPERATORS.contains(filter.getOperator()) + && rhsHasLongValue(filter.getRhs())) { + long filterEndTime = filter.getRhs().getLiteral().getValue().getLong(); + requestEndTime = Math.max(requestEndTime, filterEndTime); + } + + for (Filter childFilter : filter.getChildFilterList()) { + requestEndTime = Math.max(requestEndTime, getRequestEndTime(childFilter)); + } + + return requestEndTime; + } + private boolean lhsIsStartTimeAttribute(Expression lhs) { return startTimeAttributeName.isPresent() && startTimeAttributeName.equals(getLogicalColumnName(lhs)); diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryServiceConfigTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryServiceConfigTest.java index a067c2ca..f9a923f5 100644 --- a/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryServiceConfigTest.java +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryServiceConfigTest.java @@ -33,7 +33,7 @@ public void testQueryServiceImplConfigParser() { assertEquals("query-service", appConfig.getString("service.name")); assertEquals(8091, appConfig.getInt("service.admin.port")); assertEquals(8090, appConfig.getInt("service.port")); - assertEquals(6, queryServiceConfig.getQueryRequestHandlersConfigs().size()); + assertEquals(9, queryServiceConfig.getQueryRequestHandlersConfigs().size()); assertEquals(2, queryServiceConfig.getRequestHandlerClientConfigs().size()); RequestHandlerConfig handler0 = queryServiceConfig.getQueryRequestHandlersConfigs().get(0); diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/postgres/PostgresBasedRequestHandlerTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/postgres/PostgresBasedRequestHandlerTest.java new file mode 100644 index 00000000..2f901abd --- /dev/null +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/postgres/PostgresBasedRequestHandlerTest.java @@ -0,0 +1,199 @@ +package org.hypertrace.core.query.service.postgres; + +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createTimeFilter; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import java.time.Duration; +import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import org.hypertrace.core.query.service.ExecutionContext; +import org.hypertrace.core.query.service.QueryRequestBuilderUtils; +import org.hypertrace.core.query.service.RequestHandler; +import org.hypertrace.core.query.service.RequestHandlerRegistry; +import org.hypertrace.core.query.service.RequestHandlerSelector; +import org.hypertrace.core.query.service.api.Filter; +import org.hypertrace.core.query.service.api.Operator; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class PostgresBasedRequestHandlerTest { + private final Config serviceConfig = + ConfigFactory.parseURL( + Objects.requireNonNull( + QueryRequestToPostgresSQLConverterTest.class + .getClassLoader() + .getResource("application.conf"))) + .getConfig("service.config"); + private final Set requestHandlers = preparePostgresBasedRequestHandler(); + + @Test + /** + * Test the scenarios among handlers where range fits between two minRequestTime:[ actual < + * minReqTime1 < minReqTime2 < minReqTime3 ] + */ + public void testOneHourQueryRequestTimeRangeRequestHandler() { + RequestHandlerRegistry mockRegistry = mock(RequestHandlerRegistry.class); + RequestHandlerSelector requestHandlerSelector = new RequestHandlerSelector(mockRegistry); + when(mockRegistry.getAll()).thenReturn(requestHandlers); + + // prepare 1 hrs query range + QueryRequest request = prepareOneHourTimeRangeQueryRequest(); + ExecutionContext context = new ExecutionContext("__default", request); + + // select an handler + Optional selectedRequestHandler = + requestHandlerSelector.select(request, context); + + // verify that default handler is selected + Assertions.assertFalse(selectedRequestHandler.isEmpty()); + Assertions.assertEquals( + "backend-traces-from-bare-span-event-view-aggr-handler", + selectedRequestHandler.get().getName()); + } + + @Test + /** + * Test the scenarios among handlers where range fits between two minRequestTime:[ minReqTime1 < + * minReqTime2 < minReqTime3 < actual ] + */ + public void testTwelveHoursQueryRequestTimeRangeRequestHandler() { + RequestHandlerRegistry mockRegistry = mock(RequestHandlerRegistry.class); + RequestHandlerSelector requestHandlerSelector = new RequestHandlerSelector(mockRegistry); + when(mockRegistry.getAll()).thenReturn(requestHandlers); + + // prepare 12 hrs query range + QueryRequest request = prepareTwelveHourTimeRangeQueryRequest(); + ExecutionContext context = new ExecutionContext("__default", request); + + // select an handler + Optional selectedRequestHandler = + requestHandlerSelector.select(request, context); + + // verify that default handler is selected + Assertions.assertFalse(selectedRequestHandler.isEmpty()); + Assertions.assertEquals( + "backend-traces-from-bare-span-event-view-5min-aggr-handler", + selectedRequestHandler.get().getName()); + } + + @Test + /** + * Test the scenarios among handlers where range fits between two minRequestTime:[ minReqTime1 < + * minReqTime2 < actual < minReqTime2 ] + */ + public void testFourHoursQueryRequestTimeRangeRequestHandler() { + RequestHandlerRegistry mockRegistry = mock(RequestHandlerRegistry.class); + RequestHandlerSelector requestHandlerSelector = new RequestHandlerSelector(mockRegistry); + when(mockRegistry.getAll()).thenReturn(requestHandlers); + + // prepare 4 hrs query range + QueryRequest request = prepareFourHourTimeRangeQueryRequest(); + ExecutionContext context = new ExecutionContext("__default", request); + + // select an handler + Optional selectedRequestHandler = + requestHandlerSelector.select(request, context); + + // verify that default handler is selected + Assertions.assertFalse(selectedRequestHandler.isEmpty()); + Assertions.assertEquals( + "backend-traces-from-bare-span-event-view-3hrs-aggr-handler", + selectedRequestHandler.get().getName()); + } + + private boolean isPostgresConfig(Config config) { + return config.getString("type").equals("postgres"); + } + + private Set preparePostgresBasedRequestHandler() { + Set requestHandlers = new LinkedHashSet<>(); + for (Config config : serviceConfig.getConfigList("queryRequestHandlersConfig")) { + if (!isPostgresConfig(config)) { + continue; + } + PostgresBasedRequestHandler handler = + new PostgresBasedRequestHandler( + config.getString("name"), config.getConfig("requestHandlerInfo")); + requestHandlers.add(handler); + } + return requestHandlers; + } + + private QueryRequest prepareOneHourTimeRangeQueryRequest() { + QueryRequest.Builder builder = QueryRequest.newBuilder(); + long startTimeInMillis = System.currentTimeMillis(); + Filter startTimeFilter = + createTimeFilter("BACKEND_TRACE.startTime", Operator.GT, startTimeInMillis); + Filter endTimeFilter = + createTimeFilter( + "BACKEND_TRACE.startTime", + Operator.LT, + startTimeInMillis + Duration.ofHours(1).toMillis()); + + Filter andFilter = + Filter.newBuilder() + .setOperator(Operator.AND) + .addChildFilter(startTimeFilter) + .addChildFilter(endTimeFilter) + .build(); + builder.setFilter(andFilter); + builder.addSelection( + QueryRequestBuilderUtils.createColumnExpression("BACKEND_TRACE.backendId")); + + return builder.build(); + } + + private QueryRequest prepareFourHourTimeRangeQueryRequest() { + QueryRequest.Builder builder = QueryRequest.newBuilder(); + long startTimeInMillis = System.currentTimeMillis(); + Filter startTimeFilter = + createTimeFilter("BACKEND_TRACE.startTime", Operator.GT, startTimeInMillis); + Filter endTimeFilter = + createTimeFilter( + "BACKEND_TRACE.startTime", + Operator.LT, + startTimeInMillis + Duration.ofHours(4).toMillis()); + + Filter andFilter = + Filter.newBuilder() + .setOperator(Operator.AND) + .addChildFilter(startTimeFilter) + .addChildFilter(endTimeFilter) + .build(); + builder.setFilter(andFilter); + builder.addSelection( + QueryRequestBuilderUtils.createColumnExpression("BACKEND_TRACE.backendId")); + + return builder.build(); + } + + private QueryRequest prepareTwelveHourTimeRangeQueryRequest() { + QueryRequest.Builder builder = QueryRequest.newBuilder(); + long startTimeInMillis = System.currentTimeMillis(); + Filter startTimeFilter = + createTimeFilter("BACKEND_TRACE.startTime", Operator.GT, startTimeInMillis); + Filter endTimeFilter = + createTimeFilter( + "BACKEND_TRACE.startTime", + Operator.LT, + startTimeInMillis + Duration.ofHours(12).toMillis()); + + Filter andFilter = + Filter.newBuilder() + .setOperator(Operator.AND) + .addChildFilter(startTimeFilter) + .addChildFilter(endTimeFilter) + .build(); + builder.setFilter(andFilter); + builder.addSelection( + QueryRequestBuilderUtils.createColumnExpression("BACKEND_TRACE.backendId")); + + return builder.build(); + } +} diff --git a/query-service-impl/src/test/resources/application.conf b/query-service-impl/src/test/resources/application.conf index 62f7fc97..4ea3ab97 100644 --- a/query-service-impl/src/test/resources/application.conf +++ b/query-service-impl/src/test/resources/application.conf @@ -229,6 +229,68 @@ service.config = { } } } + }, + { + name = backend-traces-from-bare-span-event-view-5min-aggr-handler + type = postgres + clientConfig = postgres + requestHandlerInfo = { + tenantColumnName: customer_id + countColumnName: call_count_aggr + startTimeAttributeName = "BACKEND_TRACE.startTime" + minRequestDuration: "12h" + tableDefinition = { + tableName = bare-span-event-view-aggr-5min + fieldMap = { + "BACKEND_TRACE.backendId" : "backend_id", + "BACKEND_TRACE.startTime": "bucketed_start_time_millis", + "EVENT.environment": "environment", + "EVENT.isBare": "is_bare", + "EVENT.isSampled": "is_sampled" + } + } + } + }, + { + name = backend-traces-from-bare-span-event-view-3hrs-aggr-handler + type = postgres + clientConfig = postgres + requestHandlerInfo = { + tenantColumnName: customer_id + countColumnName: call_count_aggr + startTimeAttributeName = "BACKEND_TRACE.startTime" + minRequestDuration: "3h" + tableDefinition = { + tableName = bare-span-event-view-aggr-3hrs + fieldMap = { + "BACKEND_TRACE.backendId" : "backend_id", + "BACKEND_TRACE.startTime": "bucketed_start_time_millis", + "EVENT.environment": "environment", + "EVENT.isBare": "is_bare", + "EVENT.isSampled": "is_sampled" + } + } + } + }, + { + name = backend-traces-from-bare-span-event-view-aggr-handler + type = postgres + clientConfig = postgres + requestHandlerInfo = { + tenantColumnName: customer_id + countColumnName: call_count_aggr + startTimeAttributeName = "BACKEND_TRACE.startTime" + tableDefinition = { + tableName = bare-span-event-view-aggr + fieldMap = { + "BACKEND_TRACE.backendId" : "backend_id", + "BACKEND_TRACE.startTime": "bucketed_start_time_millis", + "EVENT.environment": "environment", + "EVENT.isBare": "is_bare", + "EVENT.isSampled": "is_sampled" + } + } + } } ] } \ No newline at end of file