Skip to content

Commit

Permalink
Web console: add support for Dart engine (#17147)
Browse files Browse the repository at this point in the history
* add console support for Dart engine

This reverts commit 6e46edf15dd55e5c51a1a4068e83deba4f22529b.

* feedback fixes

* surface new fields

* prioratize error over results

* better metadata refresh

* feedback fixes
  • Loading branch information
vogievetsky authored Oct 2, 2024
1 parent 1fc82a9 commit c852929
Show file tree
Hide file tree
Showing 28 changed files with 823 additions and 235 deletions.
1 change: 1 addition & 0 deletions web-console/script/druid
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ function _build_distribution() {
&& echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\", \"druid-parquet-extensions\", \"druid-deltalake-extensions\"]" >> conf/druid/auto/_common/common.runtime.properties \
&& echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/auto/_common/common.runtime.properties \
&& echo -e "\n\ndruid.export.storage.baseDir=/" >> conf/druid/auto/_common/common.runtime.properties \
&& echo -e "\n\ndruid.msq.dart.enabled=true" >> conf/druid/auto/_common/common.runtime.properties \
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ exports[`HeaderBar matches snapshot 1`] = `
Capabilities {
"coordinator": true,
"maxTaskSlots": undefined,
"multiStageQueryDart": true,
"multiStageQueryTask": true,
"overlord": true,
"queryType": "nativeAndSql",
Expand Down
49 changes: 49 additions & 0 deletions web-console/src/druid-models/dart/dart-query-entry.mock.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import type { DartQueryEntry } from './dart-query-entry';

export const DART_QUERIES: DartQueryEntry[] = [
{
sqlQueryId: '77b2344c-0a1f-4aa0-b127-de6fbc0c2b57',
dartQueryId: '99cdba0d-ed77-433d-9adc-0562d816e105',
sql: 'SELECT\n "URL",\n COUNT(*)\nFROM "c"\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 50\n',
authenticator: 'allowAll',
identity: 'allowAll',
startTime: '2024-09-28T07:41:21.194Z',
state: 'RUNNING',
},
{
sqlQueryId: '45441cf5-d8b7-46cb-b6d8-682334f056ef',
dartQueryId: '25af9bff-004d-494e-b562-2752dc3779c8',
sql: 'SELECT\n "URL",\n COUNT(*)\nFROM "c"\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 50\n',
authenticator: 'allowAll',
identity: 'allowAll',
startTime: '2024-09-28T07:41:22.854Z',
state: 'CANCELED',
},
{
sqlQueryId: 'f7257c78-6bbe-439d-99ba-f4998b300770',
dartQueryId: 'f7c2d644-9c40-4d61-9fdb-7b0e15219886',
sql: 'SELECT\n "URL",\n COUNT(*)\nFROM "c"\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 50\n',
authenticator: 'allowAll',
identity: 'allowAll',
startTime: '2024-09-28T07:41:24.425Z',
state: 'ACCEPTED',
},
];
27 changes: 27 additions & 0 deletions web-console/src/druid-models/dart/dart-query-entry.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

export interface DartQueryEntry {
sqlQueryId: string;
dartQueryId: string;
sql: string;
authenticator: string;
identity: string;
startTime: string;
state: 'ACCEPTED' | 'RUNNING' | 'CANCELED';
}
9 changes: 7 additions & 2 deletions web-console/src/druid-models/druid-engine/druid-engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,14 @@
* limitations under the License.
*/

export type DruidEngine = 'native' | 'sql-native' | 'sql-msq-task';
export type DruidEngine = 'native' | 'sql-native' | 'sql-msq-task' | 'sql-msq-dart';

export const DRUID_ENGINES: DruidEngine[] = ['native', 'sql-native', 'sql-msq-task'];
export const DRUID_ENGINES: DruidEngine[] = [
'native',
'sql-native',
'sql-msq-task',
'sql-msq-dart',
];

export function validDruidEngine(
possibleDruidEngine: string | undefined,
Expand Down
1 change: 1 addition & 0 deletions web-console/src/druid-models/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ export * from './async-query/async-query';
export * from './compaction-config/compaction-config';
export * from './compaction-status/compaction-status';
export * from './coordinator-dynamic-config/coordinator-dynamic-config';
export * from './dart/dart-query-entry';
export * from './dimension-spec/dimension-spec';
export * from './druid-engine/druid-engine';
export * from './execution/execution';
Expand Down
23 changes: 7 additions & 16 deletions web-console/src/druid-models/stages/stages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import { max, sum } from 'd3-array';

import { AutoForm } from '../../components';
import { countBy, deleteKeys, filterMap, groupByAsMap, oneOf, zeroDivide } from '../../utils';
import type { InputFormat } from '../input-format/input-format';
import type { InputSource } from '../input-source/input-source';
Expand Down Expand Up @@ -252,26 +253,16 @@ export const CPUS_COUNTER_FIELDS: CpusCounterFields[] = [

export function cpusCounterFieldTitle(k: CpusCounterFields) {
switch (k) {
case 'main':
return 'Main';

case 'collectKeyStatistics':
return 'Collect key stats';

case 'mergeInput':
return 'Merge input';

case 'hashPartitionOutput':
return 'Hash partition out';

case 'mixOutput':
return 'Mix output';

case 'sortOutput':
return 'Sort output';

default:
return k;
// main
// mergeInput
// hashPartitionOutput
// mixOutput
// sortOutput
return AutoForm.makeLabelName(k);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -528,7 +528,7 @@ export class WorkbenchQuery {
};

let cancelQueryId: string | undefined;
if (engine === 'sql-native') {
if (engine === 'sql-native' || engine === 'sql-msq-dart') {
cancelQueryId = apiQuery.context.sqlQueryId;
if (!cancelQueryId) {
// If the sqlQueryId is not explicitly set on the context generate one, so it is possible to cancel the query.
Expand All @@ -550,6 +550,10 @@ export class WorkbenchQuery {
apiQuery.context.sqlStringifyArrays ??= false;
}

if (engine === 'sql-msq-dart') {
apiQuery.context.fullReport ??= true;
}

if (Array.isArray(queryParameters) && queryParameters.length) {
apiQuery.parameters = queryParameters;
}
Expand Down
32 changes: 31 additions & 1 deletion web-console/src/helpers/capabilities.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ export type QueryType = 'none' | 'nativeOnly' | 'nativeAndSql';
export interface CapabilitiesValue {
queryType: QueryType;
multiStageQueryTask: boolean;
multiStageQueryDart: boolean;
coordinator: boolean;
overlord: boolean;
maxTaskSlots?: number;
Expand All @@ -53,6 +54,7 @@ export class Capabilities {

private readonly queryType: QueryType;
private readonly multiStageQueryTask: boolean;
private readonly multiStageQueryDart: boolean;
private readonly coordinator: boolean;
private readonly overlord: boolean;
private readonly maxTaskSlots?: number;
Expand Down Expand Up @@ -139,6 +141,15 @@ export class Capabilities {
}
}

static async detectMultiStageQueryDart(): Promise<boolean> {
try {
const resp = await Api.instance.get(`/druid/v2/sql/dart/enabled?capabilities`);
return Boolean(resp.data.enabled);
} catch {
return false;
}
}

static async detectCapabilities(): Promise<Capabilities | undefined> {
const queryType = await Capabilities.detectQueryType();
if (typeof queryType === 'undefined') return;
Expand All @@ -154,11 +165,15 @@ export class Capabilities {
coordinator = overlord = await Capabilities.detectManagementProxy();
}

const multiStageQueryTask = await Capabilities.detectMultiStageQueryTask();
const [multiStageQueryTask, multiStageQueryDart] = await Promise.all([
Capabilities.detectMultiStageQueryTask(),
Capabilities.detectMultiStageQueryDart(),
]);

return new Capabilities({
queryType,
multiStageQueryTask,
multiStageQueryDart,
coordinator,
overlord,
});
Expand All @@ -179,6 +194,7 @@ export class Capabilities {
constructor(value: CapabilitiesValue) {
this.queryType = value.queryType;
this.multiStageQueryTask = value.multiStageQueryTask;
this.multiStageQueryDart = value.multiStageQueryDart;
this.coordinator = value.coordinator;
this.overlord = value.overlord;
this.maxTaskSlots = value.maxTaskSlots;
Expand All @@ -188,6 +204,7 @@ export class Capabilities {
return {
queryType: this.queryType,
multiStageQueryTask: this.multiStageQueryTask,
multiStageQueryDart: this.multiStageQueryDart,
coordinator: this.coordinator,
overlord: this.overlord,
maxTaskSlots: this.maxTaskSlots,
Expand Down Expand Up @@ -248,6 +265,10 @@ export class Capabilities {
return this.multiStageQueryTask;
}

public hasMultiStageQueryDart(): boolean {
return this.multiStageQueryDart;
}

public getSupportedQueryEngines(): DruidEngine[] {
const queryEngines: DruidEngine[] = ['native'];
if (this.hasSql()) {
Expand All @@ -256,6 +277,9 @@ export class Capabilities {
if (this.hasMultiStageQueryTask()) {
queryEngines.push('sql-msq-task');
}
if (this.hasMultiStageQueryDart()) {
queryEngines.push('sql-msq-dart');
}
return queryEngines;
}

Expand All @@ -282,36 +306,42 @@ export class Capabilities {
Capabilities.FULL = new Capabilities({
queryType: 'nativeAndSql',
multiStageQueryTask: true,
multiStageQueryDart: true,
coordinator: true,
overlord: true,
});
Capabilities.NO_SQL = new Capabilities({
queryType: 'nativeOnly',
multiStageQueryTask: false,
multiStageQueryDart: false,
coordinator: true,
overlord: true,
});
Capabilities.COORDINATOR_OVERLORD = new Capabilities({
queryType: 'none',
multiStageQueryTask: false,
multiStageQueryDart: false,
coordinator: true,
overlord: true,
});
Capabilities.COORDINATOR = new Capabilities({
queryType: 'none',
multiStageQueryTask: false,
multiStageQueryDart: false,
coordinator: true,
overlord: false,
});
Capabilities.OVERLORD = new Capabilities({
queryType: 'none',
multiStageQueryTask: false,
multiStageQueryDart: false,
coordinator: false,
overlord: true,
});
Capabilities.NO_PROXY = new Capabilities({
queryType: 'nativeAndSql',
multiStageQueryTask: true,
multiStageQueryDart: false,
coordinator: false,
overlord: false,
});
13 changes: 13 additions & 0 deletions web-console/src/utils/druid-query.ts
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,19 @@ export async function queryDruidSql<T = any>(
return sqlResultResp.data;
}

export async function queryDruidSqlDart<T = any>(
sqlQueryPayload: Record<string, any>,
cancelToken?: CancelToken,
): Promise<T[]> {
let sqlResultResp: AxiosResponse;
try {
sqlResultResp = await Api.instance.post('/druid/v2/sql/dart', sqlQueryPayload, { cancelToken });
} catch (e) {
throw new Error(getDruidErrorMessage(e));
}
return sqlResultResp.data;
}

export interface QueryExplanation {
query: any;
signature: { name: string; type: string }[];
Expand Down
1 change: 1 addition & 0 deletions web-console/src/utils/local-storage-keys.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ export const LocalStorageKeys = {
WORKBENCH_PANE_SIZE: 'workbench-pane-size' as const,
WORKBENCH_HISTORY: 'workbench-history' as const,
WORKBENCH_TASK_PANEL: 'workbench-task-panel' as const,
WORKBENCH_DART_PANEL: 'workbench-dart-panel' as const,

SQL_DATA_LOADER_CONTENT: 'sql-data-loader-content' as const,

Expand Down
Loading

0 comments on commit c852929

Please sign in to comment.