Skip to content

Commit

Permalink
Web console: add a nice UI for overlord dynamic configs and improve t…
Browse files Browse the repository at this point in the history
…he docs (#13993)

* in progress

* better form

* doc updates

* doc changes

* add inline docs

* fix tests

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: 317brian <[email protected]>

* final fixes

* fix case

* Update docs/configuration/index.md

Co-authored-by: Kashif Faraz <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: Kashif Faraz <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: Kashif Faraz <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: Kashif Faraz <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: Kashif Faraz <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: Kashif Faraz <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: Kashif Faraz <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: Kashif Faraz <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: Kashif Faraz <[email protected]>

* Update docs/configuration/index.md

Co-authored-by: Kashif Faraz <[email protected]>

* fix overflow

* fix spelling

---------

Co-authored-by: 317brian <[email protected]>
Co-authored-by: Kashif Faraz <[email protected]>
  • Loading branch information
3 people authored Mar 31, 2023
1 parent e3211e3 commit 981662e
Show file tree
Hide file tree
Showing 6 changed files with 320 additions and 80 deletions.
152 changes: 94 additions & 58 deletions docs/configuration/index.md

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,11 @@
}
}
}

.#{$bp-ns}-popover2-content {
.code-block {
white-space: pre;
overflow: auto;
font-family: monospace;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ exports[`OverlordDynamicConfigDialog matches snapshot 1`] = `
className="overlord-dynamic-config-dialog"
onClose={[Function]}
onSave={[Function]}
saveDisabled={false}
title="Overlord dynamic config"
>
<Memo(Loader) />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@

.loader {
position: relative;
height: 60vh;
width: 100%;
}

.#{$bp-ns}-dialog-body {
Expand All @@ -37,14 +35,6 @@
.auto-form {
max-height: 60vh;
overflow: auto;

.ace_editor {
height: 25vh !important;
}
}

.html-select {
width: 195px;
}

.config-comment {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ import { Intent } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import React, { useState } from 'react';

import { AutoForm, ExternalLink, Loader } from '../../components';
import type { FormJsonTabs } from '../../components';
import { AutoForm, ExternalLink, FormJsonSelector, JsonInput, Loader } from '../../components';
import type { OverlordDynamicConfig } from '../../druid-models';
import { OVERLORD_DYNAMIC_CONFIG_FIELDS } from '../../druid-models';
import { useQueryManager } from '../../hooks';
Expand All @@ -39,7 +40,9 @@ export const OverlordDynamicConfigDialog = React.memo(function OverlordDynamicCo
props: OverlordDynamicConfigDialogProps,
) {
const { onClose } = props;
const [currentTab, setCurrentTab] = useState<FormJsonTabs>('form');
const [dynamicConfig, setDynamicConfig] = useState<OverlordDynamicConfig | undefined>();
const [jsonError, setJsonError] = useState<Error | undefined>();

const [historyRecordsState] = useQueryManager<null, any[]>({
initQuery: null,
Expand Down Expand Up @@ -93,15 +96,16 @@ export const OverlordDynamicConfigDialog = React.memo(function OverlordDynamicCo
return (
<SnitchDialog
className="overlord-dynamic-config-dialog"
onSave={saveConfig}
saveDisabled={Boolean(jsonError)}
onSave={comment => void saveConfig(comment)}
onClose={onClose}
title="Overlord dynamic config"
historyRecords={historyRecordsState.data}
>
{dynamicConfig ? (
<>
<p>
Edit the overlord dynamic configuration on the fly. For more information please refer to
Edit the overlord dynamic configuration at runtime. For more information please refer to
the{' '}
<ExternalLink
href={`${getLink('DOCS')}/configuration/index.html#overlord-dynamic-configuration`}
Expand All @@ -110,11 +114,24 @@ export const OverlordDynamicConfigDialog = React.memo(function OverlordDynamicCo
</ExternalLink>
.
</p>
<AutoForm
fields={OVERLORD_DYNAMIC_CONFIG_FIELDS}
model={dynamicConfig}
onChange={setDynamicConfig}
/>
<FormJsonSelector tab={currentTab} onChange={setCurrentTab} />
{currentTab === 'form' ? (
<AutoForm
fields={OVERLORD_DYNAMIC_CONFIG_FIELDS}
model={dynamicConfig}
onChange={setDynamicConfig}
/>
) : (
<JsonInput
value={dynamicConfig}
height="50vh"
onChange={v => {
setDynamicConfig(v);
setJsonError(undefined);
}}
onError={setJsonError}
/>
)}
</>
) : (
<Loader />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,208 @@
* limitations under the License.
*/

import { Callout } from '@blueprintjs/core';
import React from 'react';

import type { Field } from '../../components';
import { deepGet, oneOf } from '../../utils';

export interface OverlordDynamicConfig {
selectStrategy?: Record<string, any>;
autoScaler?: Record<string, any>;
selectStrategy?: {
type: string;
affinityConfig?: AffinityConfig;
workerCategorySpec?: WorkerCategorySpec;
};
autoScaler?: AutoScalerConfig;
}

export interface AffinityConfig {
affinity?: Record<string, string[]>;
strong?: boolean;
}

export interface WorkerCategorySpec {
categoryMap?: Record<string, any>;
strong?: boolean;
}

export interface AutoScalerConfig {
type: string;
minNumWorkers: number;
maxNumWorkers: number;
envConfig: {
// ec2
availabilityZone?: string;
nodeData?: Record<string, any>;
userData?: Record<string, any>;

// gce
numInstances?: number;
projectId?: string;
zoneName?: string;
managedInstanceGroupName?: string;
};
}

export const OVERLORD_DYNAMIC_CONFIG_FIELDS: Field<OverlordDynamicConfig>[] = [
{
name: 'selectStrategy',
name: 'selectStrategy.type',
type: 'string',
defaultValue: 'equalDistribution',
suggestions: [
'equalDistribution',
'equalDistributionWithCategorySpec',
'fillCapacity',
'fillCapacityWithCategorySpec',
],
},

// AffinityConfig
{
name: 'selectStrategy.affinityConfig.affinity',
type: 'json',
placeholder: `{"datasource1":["host1:port","host2:port"], "datasource2":["host3:port"]}`,
defined: c =>
oneOf(
deepGet(c, 'selectStrategy.type') ?? 'equalDistribution',
'equalDistribution',
'fillCapacity',
),
info: (
<>
<p>An example affinity config might look like:</p>
<Callout className="code-block">
{`{
"datasource1": ["host1:port", "host2:port"],
"datasource2": ["host3:port"]
}`}
</Callout>
</>
),
},
{
name: 'selectStrategy.affinityConfig.strong',
type: 'boolean',
defaultValue: false,
defined: c =>
oneOf(
deepGet(c, 'selectStrategy.type') ?? 'equalDistribution',
'equalDistribution',
'fillCapacity',
),
},

// WorkerCategorySpec
{
name: 'selectStrategy.workerCategorySpec.categoryMap',
type: 'json',
defaultValue: '{}',
defined: c =>
oneOf(
deepGet(c, 'selectStrategy.type'),
'equalDistributionWithCategorySpec',
'fillCapacityWithCategorySpec',
),
info: (
<>
<p>An example category map might look like:</p>
<Callout className="code-block">
{`{
"index_kafka": {
"defaultCategory": "category1",
"categoryAffinity": {
"datasource1": "category2"
}
}
}`}
</Callout>
</>
),
},
{
name: 'selectStrategy.workerCategorySpec.strong',
type: 'boolean',
defaultValue: false,
defined: c =>
oneOf(
deepGet(c, 'selectStrategy.type'),
'equalDistributionWithCategorySpec',
'fillCapacityWithCategorySpec',
),
},

// javascript
{
name: 'selectStrategy.workerCategorySpec.function',
type: 'string',
multiline: true,
placeholder: `function(config, zkWorkers, task) { ... }`,
defined: c => deepGet(c, 'selectStrategy.type') === 'javascript',
},

{
name: 'autoScaler.type',
label: 'Auto scaler type',
type: 'string',
suggestions: [undefined, 'ec2', 'gce'],
defined: c => oneOf(deepGet(c, 'selectStrategy.type'), 'fillCapacity', 'javascript'),
},
{
name: 'autoScaler.minNumWorkers',
label: 'Auto scaler min workers',
type: 'number',
defaultValue: 0,
defined: c => oneOf(deepGet(c, 'autoScaler.type'), 'ec2', 'gce'),
},
{
name: 'autoScaler.maxNumWorkers',
label: 'Auto scaler max workers',
type: 'number',
defaultValue: 0,
defined: c => oneOf(deepGet(c, 'autoScaler.type'), 'ec2', 'gce'),
},

// EC2
{
name: 'autoScaler.envConfig.availabilityZone',
label: 'Auto scaler availability zone',
type: 'string',
defined: c => deepGet(c, 'autoScaler.type') === 'ec2',
},
{
name: 'autoScaler.envConfig.nodeData',
label: 'Auto scaler node data',
type: 'json',
defined: c => deepGet(c, 'autoScaler.type') === 'ec2',
required: true,
info: 'A JSON object that describes how to launch new nodes.',
},
{
name: 'autoScaler',
name: 'autoScaler.envConfig.userData',
label: 'Auto scaler user data',
type: 'json',
defined: c => deepGet(c, 'autoScaler.type') === 'ec2',
},

// GCE
{
name: 'autoScaler.envConfig.numInstances',
type: 'number',
defined: c => deepGet(c, 'autoScaler.type') === 'gce',
},
{
name: 'autoScaler.envConfig.projectId',
type: 'string',
defined: c => deepGet(c, 'autoScaler.type') === 'gce',
},
{
name: 'autoScaler.envConfig.zoneName',
type: 'string',
defined: c => deepGet(c, 'autoScaler.type') === 'gce',
},
{
name: 'autoScaler.envConfig.managedInstanceGroupName',
type: 'string',
defined: c => deepGet(c, 'autoScaler.type') === 'gce',
},
];

0 comments on commit 981662e

Please sign in to comment.