Backport #17147 and #17213 to 31 (#17222)

* Web console: add support for Dart engine (#17147)

* add console support for Dart engine

This reverts commit 6e46edf15dd55e5c51a1a4068e83deba4f22529b.

* feedback fixes

* surface new fields

* prioratize error over results

* better metadata refresh

* feedback fixes

* Web console: misc fixes to the Explore view (#17213)

* make record table able to hide column

* stickyness

* refactor query log

* fix measure drag

* start nested column dialog

* nested expand

* fix filtering on Measures

* use output name

* fix scrolling

* select all / none

* use ARRAY_CONCAT_AGG

* no need to limit if aggregating

* remove magic number

* better search

* update arg list

* add, don't replace

web-console/script/druid

@@ -67,6 +67,7 @@ function _build_distribution() {
     && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\", \"druid-parquet-extensions\", \"druid-deltalake-extensions\"]" >> conf/druid/auto/_common/common.runtime.properties \
     && echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/auto/_common/common.runtime.properties \
     && echo -e "\n\ndruid.export.storage.baseDir=/" >> conf/druid/auto/_common/common.runtime.properties \
+    && echo -e "\n\ndruid.msq.dart.enabled=true" >> conf/druid/auto/_common/common.runtime.properties \
   )
 }
 

web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap

@@ -213,6 +213,7 @@ exports[`HeaderBar matches snapshot 1`] = `
         Capabilities {
           "coordinator": true,
           "maxTaskSlots": undefined,
+          "multiStageQueryDart": true,
           "multiStageQueryTask": true,
           "overlord": true,
           "queryType": "nativeAndSql",

web-console/src/druid-models/dart/dart-query-entry.mock.ts

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { DartQueryEntry } from './dart-query-entry';
+
+export const DART_QUERIES: DartQueryEntry[] = [
+  {
+    sqlQueryId: '77b2344c-0a1f-4aa0-b127-de6fbc0c2b57',
+    dartQueryId: '99cdba0d-ed77-433d-9adc-0562d816e105',
+    sql: 'SELECT\n  "URL",\n  COUNT(*)\nFROM "c"\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 50\n',
+    authenticator: 'allowAll',
+    identity: 'allowAll',
+    startTime: '2024-09-28T07:41:21.194Z',
+    state: 'RUNNING',
+  },
+  {
+    sqlQueryId: '45441cf5-d8b7-46cb-b6d8-682334f056ef',
+    dartQueryId: '25af9bff-004d-494e-b562-2752dc3779c8',
+    sql: 'SELECT\n  "URL",\n  COUNT(*)\nFROM "c"\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 50\n',
+    authenticator: 'allowAll',
+    identity: 'allowAll',
+    startTime: '2024-09-28T07:41:22.854Z',
+    state: 'CANCELED',
+  },
+  {
+    sqlQueryId: 'f7257c78-6bbe-439d-99ba-f4998b300770',
+    dartQueryId: 'f7c2d644-9c40-4d61-9fdb-7b0e15219886',
+    sql: 'SELECT\n  "URL",\n  COUNT(*)\nFROM "c"\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 50\n',
+    authenticator: 'allowAll',
+    identity: 'allowAll',
+    startTime: '2024-09-28T07:41:24.425Z',
+    state: 'ACCEPTED',
+  },
+];

web-console/src/druid-models/dart/dart-query-entry.ts

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface DartQueryEntry {
+  sqlQueryId: string;
+  dartQueryId: string;
+  sql: string;
+  authenticator: string;
+  identity: string;
+  startTime: string;
+  state: 'ACCEPTED' | 'RUNNING' | 'CANCELED';
+}

web-console/src/druid-models/druid-engine/druid-engine.ts

@@ -16,9 +16,14 @@
  * limitations under the License.
  */
 
-export type DruidEngine = 'native' | 'sql-native' | 'sql-msq-task';
+export type DruidEngine = 'native' | 'sql-native' | 'sql-msq-task' | 'sql-msq-dart';
 
-export const DRUID_ENGINES: DruidEngine[] = ['native', 'sql-native', 'sql-msq-task'];
+export const DRUID_ENGINES: DruidEngine[] = [
+  'native',
+  'sql-native',
+  'sql-msq-task',
+  'sql-msq-dart',
+];
 
 export function validDruidEngine(
   possibleDruidEngine: string | undefined,

web-console/src/druid-models/index.ts

@@ -20,6 +20,7 @@ export * from './async-query/async-query';
 export * from './compaction-config/compaction-config';
 export * from './compaction-status/compaction-status';
 export * from './coordinator-dynamic-config/coordinator-dynamic-config';
+export * from './dart/dart-query-entry';
 export * from './dimension-spec/dimension-spec';
 export * from './druid-engine/druid-engine';
 export * from './execution/execution';

web-console/src/druid-models/stages/stages.ts

@@ -18,6 +18,7 @@
 
 import { max, sum } from 'd3-array';
 
+import { AutoForm } from '../../components';
 import { countBy, deleteKeys, filterMap, groupByAsMap, oneOf, zeroDivide } from '../../utils';
 import type { InputFormat } from '../input-format/input-format';
 import type { InputSource } from '../input-source/input-source';
@@ -252,26 +253,16 @@ export const CPUS_COUNTER_FIELDS: CpusCounterFields[] = [
 
 export function cpusCounterFieldTitle(k: CpusCounterFields) {
   switch (k) {
-    case 'main':
-      return 'Main';
-
     case 'collectKeyStatistics':
       return 'Collect key stats';
 
-    case 'mergeInput':
-      return 'Merge input';
-
-    case 'hashPartitionOutput':
-      return 'Hash partition out';
-
-    case 'mixOutput':
-      return 'Mix output';
-
-    case 'sortOutput':
-      return 'Sort output';
-
     default:
-      return k;
+      // main
+      // mergeInput
+      // hashPartitionOutput
+      // mixOutput
+      // sortOutput
+      return AutoForm.makeLabelName(k);
   }
 }
 

web-console/src/druid-models/workbench-query/workbench-query.ts

@@ -528,7 +528,7 @@ export class WorkbenchQuery {
     };
 
     let cancelQueryId: string | undefined;
-    if (engine === 'sql-native') {
+    if (engine === 'sql-native' || engine === 'sql-msq-dart') {
       cancelQueryId = apiQuery.context.sqlQueryId;
       if (!cancelQueryId) {
         // If the sqlQueryId is not explicitly set on the context generate one, so it is possible to cancel the query.
@@ -550,6 +550,10 @@ export class WorkbenchQuery {
       apiQuery.context.sqlStringifyArrays ??= false;
     }
 
+    if (engine === 'sql-msq-dart') {
+      apiQuery.context.fullReport ??= true;
+    }
+
     if (Array.isArray(queryParameters) && queryParameters.length) {
       apiQuery.parameters = queryParameters;
     }

web-console/src/helpers/capabilities.ts

@@ -37,6 +37,7 @@ export type QueryType = 'none' | 'nativeOnly' | 'nativeAndSql';
 export interface CapabilitiesValue {
   queryType: QueryType;
   multiStageQueryTask: boolean;
+  multiStageQueryDart: boolean;
   coordinator: boolean;
   overlord: boolean;
   maxTaskSlots?: number;
@@ -53,6 +54,7 @@ export class Capabilities {
 
   private readonly queryType: QueryType;
   private readonly multiStageQueryTask: boolean;
+  private readonly multiStageQueryDart: boolean;
   private readonly coordinator: boolean;
   private readonly overlord: boolean;
   private readonly maxTaskSlots?: number;
@@ -139,6 +141,15 @@ export class Capabilities {
     }
   }
 
+  static async detectMultiStageQueryDart(): Promise<boolean> {
+    try {
+      const resp = await Api.instance.get(`/druid/v2/sql/dart/enabled?capabilities`);
+      return Boolean(resp.data.enabled);
+    } catch {
+      return false;
+    }
+  }
+
   static async detectCapabilities(): Promise<Capabilities | undefined> {
     const queryType = await Capabilities.detectQueryType();
     if (typeof queryType === 'undefined') return;
@@ -154,11 +165,15 @@ export class Capabilities {
       coordinator = overlord = await Capabilities.detectManagementProxy();
     }
 
-    const multiStageQueryTask = await Capabilities.detectMultiStageQueryTask();
+    const [multiStageQueryTask, multiStageQueryDart] = await Promise.all([
+      Capabilities.detectMultiStageQueryTask(),
+      Capabilities.detectMultiStageQueryDart(),
+    ]);
 
     return new Capabilities({
       queryType,
       multiStageQueryTask,
+      multiStageQueryDart,
       coordinator,
       overlord,
     });
@@ -179,6 +194,7 @@ export class Capabilities {
   constructor(value: CapabilitiesValue) {
     this.queryType = value.queryType;
     this.multiStageQueryTask = value.multiStageQueryTask;
+    this.multiStageQueryDart = value.multiStageQueryDart;
     this.coordinator = value.coordinator;
     this.overlord = value.overlord;
     this.maxTaskSlots = value.maxTaskSlots;
@@ -188,6 +204,7 @@ export class Capabilities {
     return {
       queryType: this.queryType,
       multiStageQueryTask: this.multiStageQueryTask,
+      multiStageQueryDart: this.multiStageQueryDart,
       coordinator: this.coordinator,
       overlord: this.overlord,
       maxTaskSlots: this.maxTaskSlots,
@@ -248,6 +265,10 @@ export class Capabilities {
     return this.multiStageQueryTask;
   }
 
+  public hasMultiStageQueryDart(): boolean {
+    return this.multiStageQueryDart;
+  }
+
   public getSupportedQueryEngines(): DruidEngine[] {
     const queryEngines: DruidEngine[] = ['native'];
     if (this.hasSql()) {
@@ -256,6 +277,9 @@ export class Capabilities {
     if (this.hasMultiStageQueryTask()) {
       queryEngines.push('sql-msq-task');
     }
+    if (this.hasMultiStageQueryDart()) {
+      queryEngines.push('sql-msq-dart');
+    }
     return queryEngines;
   }
 
@@ -282,36 +306,42 @@ export class Capabilities {
 Capabilities.FULL = new Capabilities({
   queryType: 'nativeAndSql',
   multiStageQueryTask: true,
+  multiStageQueryDart: true,
   coordinator: true,
   overlord: true,
 });
 Capabilities.NO_SQL = new Capabilities({
   queryType: 'nativeOnly',
   multiStageQueryTask: false,
+  multiStageQueryDart: false,
   coordinator: true,
   overlord: true,
 });
 Capabilities.COORDINATOR_OVERLORD = new Capabilities({
   queryType: 'none',
   multiStageQueryTask: false,
+  multiStageQueryDart: false,
   coordinator: true,
   overlord: true,
 });
 Capabilities.COORDINATOR = new Capabilities({
   queryType: 'none',
   multiStageQueryTask: false,
+  multiStageQueryDart: false,
   coordinator: true,
   overlord: false,
 });
 Capabilities.OVERLORD = new Capabilities({
   queryType: 'none',
   multiStageQueryTask: false,
+  multiStageQueryDart: false,
   coordinator: false,
   overlord: true,
 });
 Capabilities.NO_PROXY = new Capabilities({
   queryType: 'nativeAndSql',
   multiStageQueryTask: true,
+  multiStageQueryDart: false,
   coordinator: false,
   overlord: false,
 });

web-console/src/utils/druid-query.ts

@@ -342,6 +342,19 @@ export async function queryDruidSql<T = any>(
   return sqlResultResp.data;
 }
 
+export async function queryDruidSqlDart<T = any>(
+  sqlQueryPayload: Record<string, any>,
+  cancelToken?: CancelToken,
+): Promise<T[]> {
+  let sqlResultResp: AxiosResponse;
+  try {
+    sqlResultResp = await Api.instance.post('/druid/v2/sql/dart', sqlQueryPayload, { cancelToken });
+  } catch (e) {
+    throw new Error(getDruidErrorMessage(e));
+  }
+  return sqlResultResp.data;
+}
+
 export interface QueryExplanation {
   query: any;
   signature: { name: string; type: string }[];

web-console/src/utils/local-storage-keys.tsx

@@ -53,10 +53,12 @@ export const LocalStorageKeys = {
   WORKBENCH_PANE_SIZE: 'workbench-pane-size' as const,
   WORKBENCH_HISTORY: 'workbench-history' as const,
   WORKBENCH_TASK_PANEL: 'workbench-task-panel' as const,
+  WORKBENCH_DART_PANEL: 'workbench-dart-panel' as const,
 
   SQL_DATA_LOADER_CONTENT: 'sql-data-loader-content' as const,
 
   EXPLORE_STATE: 'explore-state' as const,
+  EXPLORE_STICKY: 'explore-sticky' as const,
 };
 export type LocalStorageKeys = (typeof LocalStorageKeys)[keyof typeof LocalStorageKeys];
 

web-console/src/utils/table-helpers.ts

@@ -35,6 +35,7 @@ export function changePage(pagination: Pagination, page: number): Pagination {
 export interface ColumnHint {
   displayName?: string;
   group?: string;
+  hidden?: boolean;
   expressionForWhere?: SqlExpression;
   formatter?: (x: any) => string;
 }