From 1196e855c5bbd4d2251b3ab778a1a231d2e94e51 Mon Sep 17 00:00:00 2001 From: Yizhe Liu <59710443+yizheliu-amazon@users.noreply.github.com> Date: Thu, 31 Dec 2020 10:01:49 -0800 Subject: [PATCH] Refactor AnomalyHistory Chart to improve performance for HC detector (#350) --- .../containers/AnomaliesChart.tsx | 31 +- .../containers/AnomalyHeatmapChart.tsx | 174 ++-- .../__tests__/AnomaliesChart.test.tsx | 52 +- .../__tests__/AnomalyDetailsChart.test.tsx | 77 ++ .../__tests__/AnomalyHeatmapChart.test.tsx | 71 ++ .../__tests__/AnomalyOccurrenceChart.test.tsx | 77 ++ .../AnomalyHeatmapChart.test.tsx.snap | 826 ++++++++++++++++++ .../AnomalyCharts/utils/anomalyChartUtils.ts | 183 +++- public/pages/Dashboard/utils/utils.tsx | 6 +- .../containers/AnomalyHistory.tsx | 282 ++++-- public/pages/utils/__tests__/constants.ts | 79 ++ public/pages/utils/anomalyResultUtils.ts | 336 ++++++- public/pages/utils/constants.ts | 9 + server/models/interfaces.ts | 37 +- server/routes/ad.ts | 40 +- server/utils/constants.ts | 24 + 16 files changed, 2087 insertions(+), 217 deletions(-) create mode 100644 public/pages/AnomalyCharts/containers/__tests__/AnomalyDetailsChart.test.tsx create mode 100644 public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx create mode 100644 public/pages/AnomalyCharts/containers/__tests__/AnomalyOccurrenceChart.test.tsx create mode 100644 public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap create mode 100644 public/pages/utils/__tests__/constants.ts diff --git a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx index 48567065..a6d16959 100644 --- a/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomaliesChart.tsx @@ -24,6 +24,7 @@ import { import { get } from 'lodash'; import moment, { DurationInputArg2 } from 'moment'; import React, { useState } from 'react'; +import { EntityAnomalySummaries } from '../../../../server/models/interfaces'; import ContentPanel from '../../../components/ContentPanel/ContentPanel'; import { useDelayedLoader } from '../../../hooks/useDelayedLoader'; import { @@ -38,6 +39,7 @@ import { AnomalyDetailsChart } from '../containers/AnomalyDetailsChart'; import { AnomalyHeatmapChart, HeatmapCell, + HeatmapDisplayOption, } from '../containers/AnomalyHeatmapChart'; import { getAnomalyGradeWording, @@ -71,10 +73,13 @@ interface AnomaliesChartProps { isHCDetector?: boolean; detectorCategoryField?: string[]; onHeatmapCellSelected?(heatmapCell: HeatmapCell): void; + onDisplayOptionChanged?(heatmapDisplayOption: HeatmapDisplayOption): void; selectedHeatmapCell?: HeatmapCell; newDetector?: Detector; zoomRange?: DateRange; anomaliesResult: Anomalies | undefined; + heatmapDisplayOption?: HeatmapDisplayOption; + entityAnomalySummaries?: EntityAnomalySummaries[]; } export const AnomaliesChart = React.memo((props: AnomaliesChartProps) => { @@ -172,6 +177,21 @@ export const AnomaliesChart = React.memo((props: AnomaliesChartProps) => { ); }; + const hasValidHCProps = () => { + return ( + props.isHCDetector && + props.onHeatmapCellSelected && + props.detectorCategoryField && + // For Non-Sample HC detector case, aka realtime HC detector(showAlert == true), + // we use anomaly summaries data to render heatmap + // we must have function onDisplayOptionChanged and entityAnomalySummaries defined + // so that heatmap can work as expected. + (props.showAlerts !== true || + (props.showAlerts && + props.onDisplayOptionChanged && + props.entityAnomalySummaries)) + ); + }; return ( { } > - {props.isHCDetector && - props.onHeatmapCellSelected && - props.detectorCategoryField ? ( + {hasValidHCProps() ? (
{ props.detector, 'detectionInterval.period.unit' )} + //@ts-ignore onHeatmapCellSelected={props.onHeatmapCellSelected} + entityAnomalySummaries={props.entityAnomalySummaries} + onDisplayOptionChanged={props.onDisplayOptionChanged} + heatmapDisplayOption={props.heatmapDisplayOption} + // TODO use props.isNotSample after Tyler's change is merged + // https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/350#discussion_r547009140 + isNotSample={props.showAlerts === true} />, props.showAlerts !== true ? [ diff --git a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx index 105346ca..d179d907 100644 --- a/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx +++ b/public/pages/AnomalyCharts/containers/AnomalyHeatmapChart.tsx @@ -16,9 +16,9 @@ import React, { useState } from 'react'; import moment from 'moment'; -import { PlotData } from 'plotly.js'; -import Plot from 'react-plotly.js'; -import { get, isEmpty } from 'lodash'; +import Plotly, { PlotData } from 'plotly.js-dist'; +import plotComponentFactory from 'react-plotly.js/factory'; +import { get, isEmpty, uniq } from 'lodash'; import { EuiFlexItem, EuiFlexGroup, @@ -41,21 +41,26 @@ import { AnomalyHeatmapSortType, sortHeatmapPlotData, filterHeatmapPlotDataByY, + getEntitytAnomaliesHeatmapData, } from '../utils/anomalyChartUtils'; import { MIN_IN_MILLI_SECS } from '../../../../server/utils/constants'; +import { EntityAnomalySummaries } from '../../../../server/models/interfaces'; interface AnomalyHeatmapChartProps { title: string; detectorId: string; detectorName: string; - anomalies: any[]; + anomalies?: any[]; dateRange: DateRange; isLoading: boolean; - showAlerts?: boolean; monitor?: Monitor; detectorInterval?: number; unit?: string; onHeatmapCellSelected(cell: HeatmapCell | undefined): void; + onDisplayOptionChanged?(option: HeatmapDisplayOption | undefined): void; + heatmapDisplayOption?: HeatmapDisplayOption; + entityAnomalySummaries?: EntityAnomalySummaries[]; + isNotSample?: boolean; } export interface HeatmapCell { @@ -63,6 +68,25 @@ export interface HeatmapCell { entityValue: string; } +export interface HeatmapDisplayOption { + sortType: AnomalyHeatmapSortType; + entityOption: { label: string; value: number }; +} + +const COMBINED_OPTIONS = { + label: 'Combined options', + options: [ + { label: 'Top 10', value: 10 }, + { label: 'Top 20', value: 20 }, + { label: 'Top 30', value: 30 }, + ], +}; + +export const INITIAL_HEATMAP_DISPLAY_OPTION = { + sortType: AnomalyHeatmapSortType.SEVERITY, + entityOption: COMBINED_OPTIONS.options[0], +} as HeatmapDisplayOption; + export const AnomalyHeatmapChart = React.memo( (props: AnomalyHeatmapChartProps) => { const showLoader = useDelayedLoader(props.isLoading); @@ -80,14 +104,7 @@ export const AnomalyHeatmapChart = React.memo( }, ]; - const COMBINED_OPTIONS = { - label: 'Combined options', - options: [ - { label: 'Top 10', value: 10 }, - { label: 'Top 20', value: 20 }, - { label: 'Top 30', value: 30 }, - ], - }; + const PlotComponent = plotComponentFactory(Plotly); const getViewEntityOptions = (inputHeatmapData: PlotData[]) => { let individualEntities = []; @@ -107,28 +124,57 @@ export const AnomalyHeatmapChart = React.memo( }); return [ - COMBINED_OPTIONS, + getViewableCombinedOptions( + COMBINED_OPTIONS, + props.heatmapDisplayOption?.entityOption + ), { label: 'Individual entities', - options: individualEntityOptions, + options: individualEntityOptions.reverse(), }, ]; }; + const getViewableCombinedOptions = ( + existingOptions: any, + selectedCombinedOption: any | undefined + ) => { + if (!selectedCombinedOption) { + return existingOptions; + } + return { + label: existingOptions.label, + options: uniq([selectedCombinedOption, ...existingOptions.options]), + }; + }; + const [originalHeatmapData, setOriginalHeatmapData] = useState( - getAnomaliesHeatmapData( - props.anomalies, - props.dateRange, - AnomalyHeatmapSortType.SEVERITY, - COMBINED_OPTIONS.options[0].value - ) + props.isNotSample + ? // use anomaly summary data in case of realtime result + getEntitytAnomaliesHeatmapData( + props.dateRange, + props.entityAnomalySummaries, + props.heatmapDisplayOption.entityOption.value + ) + : // use anomalies data in case of sample result + getAnomaliesHeatmapData( + props.anomalies, + props.dateRange, + AnomalyHeatmapSortType.SEVERITY, + COMBINED_OPTIONS.options[0].value + ) ); + const [heatmapData, setHeatmapData] = useState( originalHeatmapData ); - const [sortByFieldValue, setSortByFieldValue] = useState( - SORT_BY_FIELD_OPTIONS[0].value + const [sortByFieldValue, setSortByFieldValue] = useState< + AnomalyHeatmapSortType + >( + props.isNotSample + ? props.heatmapDisplayOption.sortType + : SORT_BY_FIELD_OPTIONS[0].value ); const [currentViewOptions, setCurrentViewOptions] = useState([ @@ -191,14 +237,14 @@ export const AnomalyHeatmapChart = React.memo( ); setHeatmapData([transparentHeatmapData, ...selectedHeatmapData]); - const selectedEndDate = moment( + const selectedStartDate = moment( //@ts-ignore heatmapData[0].x[selectedCellIndices[1]], HEATMAP_X_AXIS_DATE_FORMAT ).valueOf(); - const selectedStartDate = - selectedEndDate - + const selectedEndDate = + selectedStartDate + get(selectedHeatmapData, '[0].cellTimeInterval', MIN_IN_MILLI_SECS); props.onHeatmapCellSelected({ dateRange: { @@ -226,18 +272,25 @@ export const AnomalyHeatmapChart = React.memo( if (isEmpty(selectedViewOptions)) { // when `clear` is hit for combo box setCurrentViewOptions([COMBINED_OPTIONS.options[0]]); - const displayTopEntityNum = get(COMBINED_OPTIONS.options[0], 'value'); - const updateHeatmapPlotData = getAnomaliesHeatmapData( - props.anomalies, - props.dateRange, - sortByFieldValue, - displayTopEntityNum - ); - setOriginalHeatmapData(updateHeatmapPlotData); - setHeatmapData(updateHeatmapPlotData); - setNumEntities(updateHeatmapPlotData[0].y.length); - setEntityViewOptions(getViewEntityOptions(updateHeatmapPlotData)); + if (props.isNotSample && props.onDisplayOptionChanged) { + props.onDisplayOptionChanged({ + sortType: sortByFieldValue, + entityOption: COMBINED_OPTIONS.options[0], + }); + } else { + const displayTopEntityNum = get(COMBINED_OPTIONS.options[0], 'value'); + const updateHeatmapPlotData = getAnomaliesHeatmapData( + props.anomalies, + props.dateRange, + sortByFieldValue, + displayTopEntityNum + ); + setOriginalHeatmapData(updateHeatmapPlotData); + setHeatmapData(updateHeatmapPlotData); + setNumEntities(updateHeatmapPlotData[0].y.length); + setEntityViewOptions(getViewEntityOptions(updateHeatmapPlotData)); + } return; } const nonCombinedOptions = [] as any[]; @@ -251,17 +304,26 @@ export const AnomalyHeatmapChart = React.memo( if (isCombinedViewEntityOption(option)) { // only allow 1 combined option setCurrentViewOptions([option]); - const displayTopEntityNum = get(option, 'value'); - const updateHeatmapPlotData = getAnomaliesHeatmapData( - props.anomalies, - props.dateRange, - sortByFieldValue, - displayTopEntityNum - ); - setOriginalHeatmapData(updateHeatmapPlotData); - setHeatmapData(updateHeatmapPlotData); - setNumEntities(updateHeatmapPlotData[0].y.length); - setEntityViewOptions(getViewEntityOptions(updateHeatmapPlotData)); + if (props.isNotSample && props.onDisplayOptionChanged) { + props.onDisplayOptionChanged({ + sortType: sortByFieldValue, + entityOption: option, + }); + } else { + const displayTopEntityNum = get(option, 'value'); + const updateHeatmapPlotData = getAnomaliesHeatmapData( + props.anomalies, + props.dateRange, + sortByFieldValue, + displayTopEntityNum + ); + + setOriginalHeatmapData(updateHeatmapPlotData); + setHeatmapData(updateHeatmapPlotData); + setNumEntities(updateHeatmapPlotData[0].y.length); + setEntityViewOptions(getViewEntityOptions(updateHeatmapPlotData)); + } + return; } else { nonCombinedOptions.push(option); @@ -294,6 +356,19 @@ export const AnomalyHeatmapChart = React.memo( const handleSortByFieldChange = (value: any) => { setSortByFieldValue(value); + props.onHeatmapCellSelected(undefined); + if ( + props.isNotSample && + props.onDisplayOptionChanged && + currentViewOptions.length === 1 && + isCombinedViewEntityOption(currentViewOptions[0]) + ) { + props.onDisplayOptionChanged({ + sortType: value, + entityOption: currentViewOptions[0], + }); + return; + } const sortedHeatmapData = sortHeatmapPlotData( heatmapData[0], value, @@ -303,7 +378,6 @@ export const AnomalyHeatmapChart = React.memo( opacity: 1, }); setHeatmapData([updatedHeatmapData]); - props.onHeatmapCellSelected(undefined); }; return ( @@ -439,7 +513,7 @@ export const AnomalyHeatmapChart = React.memo( ) : ( - ({ ...render( - + diff --git a/public/pages/AnomalyCharts/containers/__tests__/AnomalyDetailsChart.test.tsx b/public/pages/AnomalyCharts/containers/__tests__/AnomalyDetailsChart.test.tsx new file mode 100644 index 00000000..f9521ff9 --- /dev/null +++ b/public/pages/AnomalyCharts/containers/__tests__/AnomalyDetailsChart.test.tsx @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import React from 'react'; +import { render } from '@testing-library/react'; +import { Provider } from 'react-redux'; +import { mockedStore } from '../../../../redux/utils/testUtils'; +import { CoreServicesContext } from '../../../../components/CoreServices/CoreServices'; +import { coreServicesMock } from '../../../../../test/mocks'; +import { AnomalyDetailsChart } from '../AnomalyDetailsChart'; +import { + FAKE_ANOMALY_DATA, + FAKE_DATE_RANGE, +} from '../../../../pages/utils/__tests__/constants'; +import { INITIAL_ANOMALY_SUMMARY } from '../../utils/constants'; +import { getRandomDetector } from '../../../../redux/reducers/__tests__/utils'; + +const renderAnomalyOccurenceChart = ( + isNotSample: boolean, + isHCDetector: boolean +) => ({ + ...render( + + + + + + ), +}); + +describe(' spec', () => { + test('renders the component in case of Sample Anomaly', () => { + console.error = jest.fn(); + const { getByText } = renderAnomalyOccurenceChart(false, false); + expect(getByText('Sample anomaly grade')).not.toBeNull(); + }); + + test('renders the component in case of Realtime Anomaly', () => { + console.error = jest.fn(); + const { getByText, queryByText } = renderAnomalyOccurenceChart(true, false); + expect(getByText('Anomaly grade')).not.toBeNull(); + expect(queryByText('Sample anomaly grade')).toBeNull(); + expect(getByText('Alert')).not.toBeNull(); + }); + + test('renders the component in case of HC Detector', () => { + console.error = jest.fn(); + const { getByText, queryByText } = renderAnomalyOccurenceChart(true, true); + expect(getByText('Anomaly grade')).not.toBeNull(); + expect(queryByText('Sample anomaly grade')).toBeNull(); + expect(queryByText('Alert')).toBeNull(); + }); +}); diff --git a/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx b/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx new file mode 100644 index 00000000..f4813189 --- /dev/null +++ b/public/pages/AnomalyCharts/containers/__tests__/AnomalyHeatmapChart.test.tsx @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { render } from '@testing-library/react'; +import React from 'react'; +import { + AnomalyHeatmapChart, + INITIAL_HEATMAP_DISPLAY_OPTION, +} from '../AnomalyHeatmapChart'; +import { + FAKE_ANOMALY_DATA, + FAKE_DATE_RANGE, + FAKE_ENTITY_ANOMALY_SUMMARIES, +} from '../../../../pages/utils/__tests__/constants'; + +describe(' spec', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('AnomalyHeatmapChart with Sample anomaly data', () => { + const { container } = render( + + ); + expect(container).toMatchSnapshot(); + }); + + test('AnomalyHeatmapChart with anomaly summaries data', () => { + const { container } = render( + + ); + expect(container).toMatchSnapshot(); + }); +}); diff --git a/public/pages/AnomalyCharts/containers/__tests__/AnomalyOccurrenceChart.test.tsx b/public/pages/AnomalyCharts/containers/__tests__/AnomalyOccurrenceChart.test.tsx new file mode 100644 index 00000000..db9c5ede --- /dev/null +++ b/public/pages/AnomalyCharts/containers/__tests__/AnomalyOccurrenceChart.test.tsx @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import React from 'react'; +import { render } from '@testing-library/react'; +import { Provider } from 'react-redux'; +import { mockedStore } from '../../../../redux/utils/testUtils'; +import { CoreServicesContext } from '../../../../components/CoreServices/CoreServices'; +import { coreServicesMock } from '../../../../../test/mocks'; +import { AnomalyOccurrenceChart } from '../AnomalyOccurrenceChart'; +import { + FAKE_ANOMALY_DATA, + FAKE_DATE_RANGE, +} from '../../../../pages/utils/__tests__/constants'; +import { INITIAL_ANOMALY_SUMMARY } from '../../utils/constants'; +import { getRandomDetector } from '../../../../redux/reducers/__tests__/utils'; + +const renderAnomalyOccurenceChart = ( + isNotSample: boolean, + isHCDetector: boolean +) => ({ + ...render( + + + + + + ), +}); + +describe(' spec', () => { + test('renders the component in case of Sample Anomaly', () => { + console.error = jest.fn(); + const { getByText } = renderAnomalyOccurenceChart(false, false); + expect(getByText('Sample anomaly grade')).not.toBeNull(); + }); + + test('renders the component in case of Realtime Anomaly', () => { + console.error = jest.fn(); + const { getByText, queryByText } = renderAnomalyOccurenceChart(true, false); + expect(getByText('Anomaly grade')).not.toBeNull(); + expect(queryByText('Sample anomaly grade')).toBeNull(); + }); + + test('renders the component in case of HC Detector', () => { + console.error = jest.fn(); + const { getByText, queryByText } = renderAnomalyOccurenceChart(true, true); + expect(getByText('Anomaly grade')).not.toBeNull(); + expect(queryByText('Sample anomaly grade')).toBeNull(); + expect(getByText('Click on an anomaly entity to view data')).not.toBeNull(); + }); +}); diff --git a/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap b/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap new file mode 100644 index 00000000..3d1758dd --- /dev/null +++ b/public/pages/AnomalyCharts/containers/__tests__/__snapshots__/AnomalyHeatmapChart.test.tsx.snap @@ -0,0 +1,826 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[` spec AnomalyHeatmapChart with Sample anomaly data 1`] = ` +
+
+
+
+
+ + + Choose a filled rectangle in the heat map for a more detailed view of anomalies within that entity. + +
+
+
+
+
+
+
+
+
+

+ test-tile +

+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+ + Select an option: By severity, is selected + + +
+ + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Anomaly grade + + + + +
+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+
+ + 0.0 + + (None) +
+
+
+
+ (Critical) + + 1.0 + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+`; + +exports[` spec AnomalyHeatmapChart with anomaly summaries data 1`] = ` +
+
+
+
+
+ + + No anomalies found in the specified date range. + +
+
+
+
+
+
+
+
+
+

+ test-tile +

+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+ + Select an option: By severity, is selected + + +
+ + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Anomaly grade + + + + + + +
+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+
+ + 0.0 + + (None) +
+
+
+
+ (Critical) + + 1.0 + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+`; diff --git a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts b/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts index 8fad176d..cef2dc50 100644 --- a/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts +++ b/public/pages/AnomalyCharts/utils/anomalyChartUtils.ts @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -import { cloneDeep, get, isEmpty, orderBy } from 'lodash'; +import { cloneDeep, defaultTo, get, isEmpty, orderBy } from 'lodash'; import { DateRange, Detector, @@ -28,6 +28,12 @@ import { Datum, PlotData } from 'plotly.js'; import moment from 'moment'; import { calculateTimeWindowsWithMaxDataPoints } from '../../utils/anomalyResultUtils'; import { HeatmapCell } from '../containers/AnomalyHeatmapChart'; +import { + EntityAnomalySummaries, + EntityAnomalySummary, +} from '../../../../server/models/interfaces'; +import { toFixedNumberForAnomaly } from '../../../../server/utils/helpers'; +import { ENTITY_VALUE_PATH_FIELD } from '../../../../server/utils/constants'; export const convertAlerts = (response: any): MonitorAlert[] => { const alerts = get(response, 'data.response.alerts', []); @@ -195,7 +201,7 @@ const getHeatmapColorByValue = (value: number) => { } }; -const NUM_CELLS = 20; +export const NUM_CELLS = 20; export const HEATMAP_X_AXIS_DATE_FORMAT = 'MM-DD HH:mm YYYY'; @@ -208,7 +214,7 @@ const buildBlankStringWithLength = (length: number) => { }; export const getAnomaliesHeatmapData = ( - anomalies: any[], + anomalies: any[] | undefined, dateRange: DateRange, sortType: AnomalyHeatmapSortType = AnomalyHeatmapSortType.SEVERITY, displayTopNum: number @@ -265,38 +271,155 @@ export const getAnomaliesHeatmapData = ( numAnomalyGrades.push(numAnomalyGradesForEntity); }); - const plotTimes = timeWindows.map((timeWindow) => timeWindow.endDate); - const plotData = - //@ts-ignore - { - x: plotTimes.map((timestamp) => - moment(timestamp).format(HEATMAP_X_AXIS_DATE_FORMAT) - ), - y: entityValues, - z: maxAnomalyGrades, - colorscale: ANOMALY_HEATMAP_COLORSCALE, - //@ts-ignore - zmin: 0, - zmax: 1, - type: 'heatmap', - showscale: false, - xgap: 2, - ygap: 2, - opacity: 1, - text: numAnomalyGrades, - hovertemplate: - 'Time: %{x}
' + - 'Max anomaly grade: %{z}
' + - 'Anomaly occurrences: %{text}' + - '', - cellTimeInterval: timeWindows[0].endDate - timeWindows[0].startDate, - } as PlotData; + const plotTimes = timeWindows.map((timeWindow) => timeWindow.startDate); + const plotTimesInString = plotTimes.map((timestamp) => + moment(timestamp).format(HEATMAP_X_AXIS_DATE_FORMAT) + ); + const cellTimeInterval = timeWindows[0].endDate - timeWindows[0].startDate; + const plotData = buildHeatmapPlotData( + plotTimesInString, + entityValues, + maxAnomalyGrades, + numAnomalyGrades, + cellTimeInterval + ); const resultPlotData = sortHeatmapPlotData(plotData, sortType, displayTopNum); return [resultPlotData]; }; -const getEntityAnomaliesMap = (anomalies: any[]): Map => { +const buildHeatmapPlotData = ( + x: any[], + y: any[], + z: any[], + text: any[], + cellTimeInterval: number +): PlotData => { + //@ts-ignore + return { + x: x, + y: y, + z: z, + colorscale: ANOMALY_HEATMAP_COLORSCALE, + zmin: 0, + zmax: 1, + type: 'heatmap', + showscale: false, + xgap: 2, + ygap: 2, + opacity: 1, + text: text, + hovertemplate: + 'Time: %{x}
' + + 'Max anomaly grade: %{z}
' + + 'Anomaly occurrences: %{text}' + + '', + cellTimeInterval: cellTimeInterval, + } as PlotData; +}; + +export const getEntitytAnomaliesHeatmapData = ( + dateRange: DateRange, + entitiesAnomalySummaryResult: EntityAnomalySummaries[], + displayTopNum: number +) => { + const entityValues = [] as string[]; + const maxAnomalyGrades = [] as any[]; + const numAnomalyGrades = [] as any[]; + + const timeWindows = calculateTimeWindowsWithMaxDataPoints( + NUM_CELLS, + dateRange + ); + + let entitiesAnomalySummaries = [] as EntityAnomalySummaries[]; + + if (isEmpty(entitiesAnomalySummaryResult)) { + // put placeholder data so that heatmap won't look empty + for (let i = 0; i < displayTopNum; i++) { + // using blank string with different length as entity values instead of + // only 1 whitesapce for all entities, to avoid heatmap with single row + const blankStrValue = buildBlankStringWithLength(i); + entitiesAnomalySummaries.push({ + entity: { + value: blankStrValue, + }, + } as EntityAnomalySummaries); + } + } else { + entitiesAnomalySummaries = entitiesAnomalySummaryResult; + } + + entitiesAnomalySummaries.forEach((entityAnomalySummaries) => { + const maxAnomalyGradesForEntity = [] as number[]; + const numAnomalyGradesForEntity = [] as number[]; + + const entityValue = get( + entityAnomalySummaries, + ENTITY_VALUE_PATH_FIELD, + '' + ) as string; + const anomaliesSummary = get( + entityAnomalySummaries, + 'anomalySummaries', + [] + ) as EntityAnomalySummary[]; + entityValues.push(entityValue); + + timeWindows.forEach((timeWindow) => { + const anomalySummaryInTimeRange = anomaliesSummary.filter( + (singleAnomalySummary) => + singleAnomalySummary.startTime >= timeWindow.startDate && + singleAnomalySummary.startTime < timeWindow.endDate + ); + + if (isEmpty(anomalySummaryInTimeRange)) { + maxAnomalyGradesForEntity.push(0); + numAnomalyGradesForEntity.push(0); + return; + } + + const maxAnomalies = anomalySummaryInTimeRange.map((anomalySummary) => { + return toFixedNumberForAnomaly( + defaultTo(get(anomalySummary, 'maxAnomaly'), 0) + ); + }); + const countAnomalies = anomalySummaryInTimeRange.map((anomalySummary) => { + return defaultTo(get(anomalySummary, 'anomalyCount'), 0); + }); + + maxAnomalyGradesForEntity.push(Math.max(...maxAnomalies)); + numAnomalyGradesForEntity.push( + countAnomalies.reduce((a, b) => { + return a + b; + }) + ); + }); + + maxAnomalyGrades.push(maxAnomalyGradesForEntity); + numAnomalyGrades.push(numAnomalyGradesForEntity); + }); + + const plotTimes = timeWindows.map((timeWindow) => timeWindow.startDate); + const timeStamps = plotTimes.map((timestamp) => + moment(timestamp).format(HEATMAP_X_AXIS_DATE_FORMAT) + ); + const plotData = buildHeatmapPlotData( + timeStamps, + entityValues.reverse(), + maxAnomalyGrades.reverse(), + numAnomalyGrades.reverse(), + timeWindows[0].endDate - timeWindows[0].startDate + ); + return [plotData]; +}; + +const getEntityAnomaliesMap = ( + anomalies: any[] | undefined +): Map => { const entityAnomaliesMap = new Map(); + if (anomalies == undefined) { + return entityAnomaliesMap; + } anomalies.forEach((anomaly) => { const entity = get(anomaly, 'entity', [] as EntityData[]); if (isEmpty(entity)) { diff --git a/public/pages/Dashboard/utils/utils.tsx b/public/pages/Dashboard/utils/utils.tsx index 7dae7acf..4a3af538 100644 --- a/public/pages/Dashboard/utils/utils.tsx +++ b/public/pages/Dashboard/utils/utils.tsx @@ -19,6 +19,8 @@ import { AD_DOC_FIELDS, SORT_DIRECTION, MIN_IN_MILLI_SECS, + KEY_FIELD, + DOC_COUNT_FIELD, } from '../../../../server/utils/constants'; import { Detector, @@ -624,7 +626,7 @@ export const getAnomalyDistributionForDetectorsByTimeRange = async ( const finalDetectorDistributionResult = [] as object[]; for (let detectorResult of detectorsAggResults) { - const detectorId = get(detectorResult, 'key', ''); + const detectorId = get(detectorResult, KEY_FIELD, ''); if (detectorAndIdMap.has(detectorId)) { const detector = detectorAndIdMap.get(detectorId); finalDetectorDistributionResult.push({ @@ -639,7 +641,7 @@ export const getAnomalyDistributionForDetectorsByTimeRange = async ( AD_DOC_FIELDS.INDICES, '' ).toString(), - count: get(detectorResult, 'doc_count', 0), + count: get(detectorResult, DOC_COUNT_FIELD, 0), }); } } diff --git a/public/pages/DetectorResults/containers/AnomalyHistory.tsx b/public/pages/DetectorResults/containers/AnomalyHistory.tsx index 01b8f7d8..806cb8c5 100644 --- a/public/pages/DetectorResults/containers/AnomalyHistory.tsx +++ b/public/pages/DetectorResults/containers/AnomalyHistory.tsx @@ -45,6 +45,10 @@ import { buildParamsForGetAnomalyResultsWithDateRange, FEATURE_DATA_CHECK_WINDOW_OFFSET, filterWithHeatmapFilter, + getTopAnomalousEntitiesQuery, + parseTopEntityAnomalySummaryResults, + getEntityAnomalySummariesQuery, + parseEntityAnomalySummaryResults, } from '../../utils/anomalyResultUtils'; import { AnomalyResultsTable } from './AnomalyResultsTable'; import { AnomaliesChart } from '../../AnomalyCharts/containers/AnomaliesChart'; @@ -57,8 +61,23 @@ import { MAX_ANOMALIES } from '../../../utils/constants'; import { getDetectorResults } from '../../../redux/reducers/anomalyResults'; import { searchResults } from '../../../redux/reducers/anomalyResults'; import { AnomalyOccurrenceChart } from '../../AnomalyCharts/containers/AnomalyOccurrenceChart'; -import { HeatmapCell } from '../../AnomalyCharts/containers/AnomalyHeatmapChart'; -import { getAnomalyHistoryWording } from '../../AnomalyCharts/utils/anomalyChartUtils'; +import { + HeatmapCell, + HeatmapDisplayOption, + INITIAL_HEATMAP_DISPLAY_OPTION, +} from '../../AnomalyCharts/containers/AnomalyHeatmapChart'; +import { + getAnomalyHistoryWording, + NUM_CELLS, +} from '../../AnomalyCharts/utils/anomalyChartUtils'; +import { darkModeEnabled } from '../../../utils/kibanaUtils'; +import { + EntityAnomalySummaries, + Entity, +} from '../../../../server/models/interfaces'; +// @ts-ignore +import { toastNotifications } from 'ui/notify'; +import { prettifyErrorMessage } from '../../../../server/utils/helpers'; interface AnomalyHistoryProps { detector: Detector; @@ -98,57 +117,68 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { const [selectedHeatmapCell, setSelectedHeatmapCell] = useState(); + const [entityAnomalySummaries, setEntityAnomalySummaries] = useState< + EntityAnomalySummaries[] + >(); + + const [heatmapDisplayOption, setHeatmapDisplayOption] = useState< + HeatmapDisplayOption + >(INITIAL_HEATMAP_DISPLAY_OPTION); + const detectorCategoryField = get(props.detector, 'categoryField', []); const isHCDetector = !isEmpty(detectorCategoryField); - useEffect(() => { - // We load at most 10k AD result data points for one call. If user choose - // a big time range which may have more than 10k AD results, will use bucket - // aggregation to load data points in whole time range with larger interval. - async function getBucketizedAnomalyResults() { - try { - setIsLoadingAnomalyResults(true); - const anomalySummaryResult = await dispatch( - searchResults( - getAnomalySummaryQuery( - dateRange.startDate, - dateRange.endDate, - props.detector.id - ) + // We load at most 10k AD result data points for one call. If user choose + // a big time range which may have more than 10k AD results, will use bucket + // aggregation to load data points in whole time range with larger interval. + // If entity is specified, we only query AD result data points for this entity. + async function getBucketizedAnomalyResults( + entity: Entity | undefined = undefined + ) { + try { + setIsLoadingAnomalyResults(true); + const anomalySummaryResult = await dispatch( + searchResults( + getAnomalySummaryQuery( + dateRange.startDate, + dateRange.endDate, + props.detector.id, + entity ) - ); + ) + ); + + setPureAnomalies(parsePureAnomalies(anomalySummaryResult)); + setBucketizedAnomalySummary(parseAnomalySummary(anomalySummaryResult)); - setPureAnomalies(parsePureAnomalies(anomalySummaryResult)); - setBucketizedAnomalySummary(parseAnomalySummary(anomalySummaryResult)); - - const result = await dispatch( - searchResults( - getBucketizedAnomalyResultsQuery( - dateRange.startDate, - dateRange.endDate, - 1, - props.detector.id - ) + const result = await dispatch( + searchResults( + getBucketizedAnomalyResultsQuery( + dateRange.startDate, + dateRange.endDate, + props.detector.id, + entity ) - ); + ) + ); - setBucketizedAnomalyResults(parseBucketizedAnomalyResults(result)); - } catch (err) { - console.error( - `Failed to get anomaly results for ${props.detector.id}`, - err - ); - } finally { - setIsLoadingAnomalyResults(false); - } + setBucketizedAnomalyResults(parseBucketizedAnomalyResults(result)); + } catch (err) { + console.error( + `Failed to get anomaly results for ${props.detector.id}`, + err + ); + } finally { + setIsLoadingAnomalyResults(false); } + } + useEffect(() => { fetchRawAnomalyResults(isHCDetector); if ( !isHCDetector && - dateRange.endDate - dateRange.startDate > - detectorInterval * MIN_IN_MILLI_SECS * MAX_ANOMALIES + isDateRangeOversize(dateRange, detectorInterval, MAX_ANOMALIES) ) { getBucketizedAnomalyResults(); } else { @@ -161,6 +191,16 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { 'detectionInterval.period.interval', 1 ); + const isDateRangeOversize = ( + dateRange: DateRange, + intervalInMinute: number, + maxSize: number + ) => { + return ( + dateRange.endDate - dateRange.startDate > + intervalInMinute * MIN_IN_MILLI_SECS * maxSize + ); + }; const fetchRawAnomalyResults = async (showLoading: boolean) => { if (showLoading) { @@ -191,8 +231,11 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { featureData: get(rawAnomaliesData, 'featureResults', []), } as Anomalies; setAtomicAnomalyResults(rawAnomaliesResult); - setRawAnomalyResults(rawAnomaliesResult); - setHCDetectorAnomalyResults(getAnomalyResultForHC(rawAnomaliesResult)); + if (isHCDetector) { + setHCDetectorAnomalyResults(rawAnomaliesResult); + } else { + setRawAnomalyResults(rawAnomaliesResult); + } } catch (err) { console.error( `Failed to get atomic anomaly results for ${props.detector.id}`, @@ -205,47 +248,138 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { } }; - //TODO current implementation can bring in performance issue, will work issue below - // https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/issues/313 - const getAnomalyResultForHC = (rawAnomalyResults: Anomalies) => { - const resultAnomaly = rawAnomalyResults.anomalies.filter( - (anomaly) => get(anomaly, 'anomalyGrade', 0) > 0 + useEffect(() => { + if (isHCDetector) { + fetchHCAnomalySummaries(); + } + }, [dateRange, heatmapDisplayOption]); + + useEffect(() => { + if (selectedHeatmapCell) { + fetchEntityAnomalyData(selectedHeatmapCell); + } else { + setAtomicAnomalyResults(hcDetectorAnomalyResults); + } + }, [selectedHeatmapCell]); + + const fetchHCAnomalySummaries = async () => { + setIsLoadingAnomalyResults(true); + const query = getTopAnomalousEntitiesQuery( + dateRange.startDate, + dateRange.endDate, + props.detector.id, + heatmapDisplayOption.entityOption.value, + heatmapDisplayOption.sortType + ); + const result = await dispatch(searchResults(query)); + const topEnityAnomalySummaries = parseTopEntityAnomalySummaryResults( + result ); + const entities = topEnityAnomalySummaries.map((summary) => summary.entity); - const anomaliesFeatureData = resultAnomaly.map( - (anomaly) => anomaly.features + const promises = entities.map(async (entity: Entity) => { + const entityResultQuery = getEntityAnomalySummariesQuery( + dateRange.startDate, + dateRange.endDate, + props.detector.id, + NUM_CELLS, + get(props.detector, 'categoryField[0]', ''), + entity.value + ); + return dispatch(searchResults(entityResultQuery)); + }); + + const allEntityAnomalySummaries = await Promise.all(promises).catch( + (error) => { + const errorMessage = `Error getting anomaly summaries for all entities: ${error}`; + console.error(errorMessage); + toastNotifications.addDanger(prettifyErrorMessage(errorMessage)); + } ); + const entitiesAnomalySummaries = [] as EntityAnomalySummaries[]; - const resultAnomalyFeatureData: { - [key: string]: FeatureAggregationData[]; - } = {}; - anomaliesFeatureData.forEach((anomalyFeatureData) => { - if (anomalyFeatureData) { - for (const [featureId, featureAggData] of Object.entries( - anomalyFeatureData - )) { - if (!resultAnomalyFeatureData[featureId]) { - resultAnomalyFeatureData[featureId] = []; - } - resultAnomalyFeatureData[featureId].push(featureAggData); - } + if (!isEmpty(allEntityAnomalySummaries)) { + //@ts-ignore + allEntityAnomalySummaries.forEach((entityResponse, i) => { + const entityAnomalySummariesResult = parseEntityAnomalySummaryResults( + entityResponse, + entities[i] + ); + entitiesAnomalySummaries.push(entityAnomalySummariesResult); + }); + } + setEntityAnomalySummaries(entitiesAnomalySummaries); + setIsLoadingAnomalyResults(false); + }; + + const fetchEntityAnomalyData = async (heatmapCell: HeatmapCell) => { + setIsLoadingAnomalyResults(true); + try { + if ( + isDateRangeOversize( + heatmapCell.dateRange, + detectorInterval, + MAX_ANOMALIES + ) + ) { + fetchBucketizedEntityAnomalyData(heatmapCell); + } else { + fetchAllEntityAnomalyData(heatmapCell); + setBucketizedAnomalyResults(undefined); } - }); - return { - anomalies: resultAnomaly, - featureData: resultAnomalyFeatureData, + } catch (err) { + console.error( + `Failed to get anomaly results for entity ${heatmapCell.entityValue}`, + err + ); + } finally { + setIsLoadingAnomalyResults(false); + } + }; + + const fetchAllEntityAnomalyData = async (heatmapCell: HeatmapCell) => { + const params = buildParamsForGetAnomalyResultsWithDateRange( + heatmapCell.dateRange.startDate, + heatmapCell.dateRange.endDate, + false, + { + //@ts-ignore + name: props.detector.categoryField[0], + value: heatmapCell.entityValue, + } + ); + + const entityAnomalyResultResponse = await dispatch( + getDetectorResults(props.detector.id, params) + ); + + const entityAnomaliesData = get( + entityAnomalyResultResponse, + 'response', + [] + ); + const entityAnomaliesResult = { + anomalies: get(entityAnomaliesData, 'results', []), + featureData: get(entityAnomaliesData, 'featureResults', []), } as Anomalies; + + setAtomicAnomalyResults(entityAnomaliesResult); }; + const fetchBucketizedEntityAnomalyData = async (heatmapCell: HeatmapCell) => { + getBucketizedAnomalyResults({ + //@ts-ignore + name: props.detector.categoryField[0], + value: heatmapCell.entityValue, + }); + }; const [atomicAnomalyResults, setAtomicAnomalyResults] = useState(); const [rawAnomalyResults, setRawAnomalyResults] = useState(); const [hcDetectorAnomalyResults, setHCDetectorAnomalyResults] = useState< Anomalies >(); - const anomalyResults = isHCDetector - ? hcDetectorAnomalyResults - : bucketizedAnomalyResults + const anomalyResults = bucketizedAnomalyResults ? bucketizedAnomalyResults : atomicAnomalyResults; const handleDateRangeChange = useCallback( @@ -269,6 +403,13 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { setSelectedHeatmapCell(heatmapCell); }, []); + const handleHeatmapDisplayOptionChanged = useCallback( + (option: HeatmapDisplayOption) => { + setHeatmapDisplayOption(option); + }, + [] + ); + const annotations = anomalyResults ? get(anomalyResults, 'anomalies', []) //@ts-ignore @@ -335,6 +476,9 @@ export const AnomalyHistory = (props: AnomalyHistoryProps) => { onHeatmapCellSelected={handleHeatmapCellSelected} selectedHeatmapCell={selectedHeatmapCell} anomaliesResult={anomalyResults} + onDisplayOptionChanged={handleHeatmapDisplayOptionChanged} + heatmapDisplayOption={heatmapDisplayOption} + entityAnomalySummaries={entityAnomalySummaries} > {renderTabs()} diff --git a/public/pages/utils/__tests__/constants.ts b/public/pages/utils/__tests__/constants.ts new file mode 100644 index 00000000..a02ed22a --- /dev/null +++ b/public/pages/utils/__tests__/constants.ts @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import moment from 'moment'; +import { + EntityAnomalySummaries, + EntityAnomalySummary, +} from '../../../../server/models/interfaces'; +import { AnomalyData } from '../../../models/interfaces'; + +export const FAKE_START_TIME = moment('2019-10-10T09:00:00'); +export const FAKE_END_TIME = FAKE_START_TIME.clone().add(2, 'd'); +export const FAKE_ANOMALY_START_TIME = FAKE_START_TIME.add( + 1, + 'minutes' +).valueOf(); +export const FAKE_ANOMALY_END_TIME = FAKE_START_TIME.add( + 2, + 'minutes' +).valueOf(); +export const FAKE_ANOMALY_PLOT_TIME = FAKE_START_TIME.add( + 90, + 'seconds' +).valueOf(); +export const FAKE_DATE_RANGE = { + startDate: FAKE_START_TIME.valueOf(), + endDate: FAKE_END_TIME.valueOf(), +}; +export const FAKE_SINGLE_FEATURE_VALUE = { + data: 10, + endTime: FAKE_ANOMALY_END_TIME, + startTime: FAKE_ANOMALY_START_TIME, + plotTime: FAKE_ANOMALY_PLOT_TIME, +}; +export const FAKE_FEATURE_DATA = { + testFeatureId: FAKE_SINGLE_FEATURE_VALUE, +}; +export const FAKE_ENTITY = { name: 'entityName', value: 'entityValue' }; +export const FAKE_ANOMALY_DATA = [ + { + anomalyGrade: 0.3, + confidence: 0.8, + startTime: FAKE_ANOMALY_START_TIME, + endTime: FAKE_ANOMALY_END_TIME, + plotTime: FAKE_ANOMALY_PLOT_TIME, + entity: [FAKE_ENTITY], + features: FAKE_FEATURE_DATA, + } as AnomalyData, +]; + +export const FAKE_ANOMALIES_RESULT = { + anomalies: FAKE_ANOMALY_DATA, + featureData: { + testFeatureId: [FAKE_SINGLE_FEATURE_VALUE], + }, +}; + +export const FAKE_ENTITY_ANOMALY_SUMMARY = { + startTime: FAKE_ANOMALY_START_TIME, + maxAnomaly: 0.9, + anomalyCount: 1, +} as EntityAnomalySummary; + +export const FAKE_ENTITY_ANOMALY_SUMMARIES = { + entity: FAKE_ENTITY, + anomalySummaries: [FAKE_ENTITY_ANOMALY_SUMMARY], +} as EntityAnomalySummaries; diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index 08529618..bca42865 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -16,8 +16,18 @@ import { get, isEmpty, orderBy } from 'lodash'; import moment from 'moment'; import { Dispatch } from 'redux'; +import { + EntityAnomalySummaries, + EntityAnomalySummary, + Entity, +} from '../../../server/models/interfaces'; import { AD_DOC_FIELDS, + DOC_COUNT_FIELD, + ENTITY_FIELD, + ENTITY_NAME_PATH_FIELD, + ENTITY_VALUE_PATH_FIELD, + KEY_FIELD, MIN_IN_MILLI_SECS, SORT_DIRECTION, } from '../../../server/utils/constants'; @@ -38,7 +48,17 @@ import { MISSING_FEATURE_DATA_SEVERITY, } from '../../utils/constants'; import { HeatmapCell } from '../AnomalyCharts/containers/AnomalyHeatmapChart'; +import { AnomalyHeatmapSortType } from '../AnomalyCharts/utils/anomalyChartUtils'; import { DETECTOR_INIT_FAILURES } from '../DetectorDetail/utils/constants'; +import { + COUNT_ANOMALY_AGGS, + ENTITY_DATE_BUCKET_ANOMALY_AGGS, + MAX_ANOMALY_AGGS, + MAX_ANOMALY_SORT_AGGS, + TOP_ANOMALY_GRADE_SORT_AGGS, + TOP_ENTITIES_FIELD, + TOP_ENTITY_AGGS, +} from './constants'; import { dateFormatter, minuteDateFormatter } from './helpers'; export const getQueryParamsForLiveAnomalyResults = ( @@ -77,19 +97,22 @@ export const getLiveAnomalyResults = ( export const buildParamsForGetAnomalyResultsWithDateRange = ( startTime: number, endTime: number, - anomalyOnly: boolean = false + anomalyOnly: boolean = false, + entity: Entity | undefined = undefined ) => { return { from: 0, size: MAX_ANOMALIES, sortDirection: SORT_DIRECTION.DESC, sortField: AD_DOC_FIELDS.DATA_START_TIME, - dateRangeFilter: { + dateRangeFilter: JSON.stringify({ startTime: startTime, endTime: endTime, fieldName: AD_DOC_FIELDS.DATA_START_TIME, - }, + }), anomalyThreshold: anomalyOnly ? 0 : -1, + entityName: entity?.name, + entityValue: entity?.value, }; }; @@ -284,7 +307,8 @@ export const RETURNED_AD_RESULT_FIELDS = [ export const getAnomalySummaryQuery = ( startTime: number, endTime: number, - detectorId: string + detectorId: string, + entity: Entity | undefined = undefined ) => { return { size: MAX_ANOMALIES, @@ -311,6 +335,34 @@ export const getAnomalySummaryQuery = ( detector_id: detectorId, }, }, + ...(entity + ? [ + { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_VALUE_PATH_FIELD]: { + value: entity.value, + }, + }, + }, + }, + }, + { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_NAME_PATH_FIELD]: { + value: entity.name, + }, + }, + }, + }, + }, + ] + : []), ], }, }, @@ -355,11 +407,11 @@ export const getAnomalySummaryQuery = ( export const getBucketizedAnomalyResultsQuery = ( startTime: number, endTime: number, - interval: number, - detectorId: string + detectorId: string, + entity: Entity | undefined = undefined ) => { const fixedInterval = Math.ceil( - (endTime - startTime) / (interval * MIN_IN_MILLI_SECS * MAX_DATA_POINTS) + (endTime - startTime) / (MIN_IN_MILLI_SECS * MAX_DATA_POINTS) ); return { size: 0, @@ -379,6 +431,34 @@ export const getBucketizedAnomalyResultsQuery = ( detector_id: detectorId, }, }, + ...(entity + ? [ + { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_VALUE_PATH_FIELD]: { + value: entity.value, + }, + }, + }, + }, + }, + { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_NAME_PATH_FIELD]: { + value: entity.name, + }, + }, + }, + }, + }, + ] + : []), ], }, }, @@ -862,3 +942,245 @@ export const filterWithHeatmapFilter = ( } return filterWithDateRange(data, heatmapCell.dateRange, timeField); }; + +export const getTopAnomalousEntitiesQuery = ( + startTime: number, + endTime: number, + detectorId: string, + size: number, + sortType: AnomalyHeatmapSortType +) => { + return { + size: 0, + query: { + bool: { + filter: [ + { + range: { + [AD_DOC_FIELDS.ANOMALY_GRADE]: { + gt: 0, + }, + }, + }, + { + range: { + data_end_time: { + gte: startTime, + lte: endTime, + }, + }, + }, + { + term: { + detector_id: detectorId, + }, + }, + ], + }, + }, + aggs: { + [TOP_ENTITIES_FIELD]: { + nested: { + path: ENTITY_FIELD, + }, + aggs: { + [TOP_ENTITY_AGGS]: { + terms: { + field: ENTITY_VALUE_PATH_FIELD, + size: size, + ...(sortType === AnomalyHeatmapSortType.SEVERITY + ? { + order: { + [TOP_ANOMALY_GRADE_SORT_AGGS]: SORT_DIRECTION.DESC, + }, + } + : {}), + }, + aggs: { + [TOP_ANOMALY_GRADE_SORT_AGGS]: { + reverse_nested: {}, + aggs: { + [MAX_ANOMALY_AGGS]: { + max: { + field: AD_DOC_FIELDS.ANOMALY_GRADE, + }, + }, + }, + }, + ...(sortType === AnomalyHeatmapSortType.SEVERITY + ? { + [MAX_ANOMALY_SORT_AGGS]: { + bucket_sort: { + sort: [ + { + [`${TOP_ANOMALY_GRADE_SORT_AGGS}.${MAX_ANOMALY_AGGS}`]: { + order: SORT_DIRECTION.DESC, + }, + }, + ], + }, + }, + } + : {}), + }, + }, + }, + }, + }, + }; +}; + +export const parseTopEntityAnomalySummaryResults = ( + result: any +): EntityAnomalySummaries[] => { + const rawEntityAnomalySummaries = get( + result, + `response.aggregations.${TOP_ENTITIES_FIELD}.${TOP_ENTITY_AGGS}.buckets`, + [] + ) as any[]; + let topEntityAnomalySummaries = [] as EntityAnomalySummaries[]; + rawEntityAnomalySummaries.forEach((item) => { + const anomalyCount = get(item, DOC_COUNT_FIELD, 0); + const entityValue = get(item, KEY_FIELD, 0); + const entity = { + value: entityValue, + } as Entity; + const maxAnomalyGrade = get( + item, + [TOP_ANOMALY_GRADE_SORT_AGGS, MAX_ANOMALY_AGGS].join('.'), + 0 + ); + const enityAnomalySummary = { + maxAnomaly: maxAnomalyGrade, + anomalyCount: anomalyCount, + } as EntityAnomalySummary; + const enityAnomaliSummaries = { + entity: entity, + anomalySummaries: [enityAnomalySummary], + } as EntityAnomalySummaries; + topEntityAnomalySummaries.push(enityAnomaliSummaries); + }); + return topEntityAnomalySummaries; +}; + +export const getEntityAnomalySummariesQuery = ( + startTime: number, + endTime: number, + detectorId: string, + size: number, + categoryField: string, + entityValue: string +) => { + const fixedInterval = Math.max( + Math.ceil((endTime - startTime) / (size * MIN_IN_MILLI_SECS)), + 1 + ); + // bucket key is calculated below + // https://www.elastic.co/guide/en/elasticsearch/reference/7.10/search-aggregations-bucket-datehistogram-aggregation.html + // bucket_key = Math.floor(value / interval) * interval + // if startTime is not divisible by fixedInterval, there will be remainder, + // this can be offset for bucket_key + const offsetInMillisec = startTime % (fixedInterval * MIN_IN_MILLI_SECS); + return { + size: 0, + query: { + bool: { + filter: [ + { + range: { + [AD_DOC_FIELDS.ANOMALY_GRADE]: { + gt: 0, + }, + }, + }, + { + range: { + data_end_time: { + gte: startTime, + lte: endTime, + }, + }, + }, + { + term: { + detector_id: detectorId, + }, + }, + { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_VALUE_PATH_FIELD]: { + value: entityValue, + }, + }, + }, + }, + }, + { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_NAME_PATH_FIELD]: { + value: categoryField, + }, + }, + }, + }, + }, + ], + }, + }, + aggs: { + [ENTITY_DATE_BUCKET_ANOMALY_AGGS]: { + date_histogram: { + field: AD_DOC_FIELDS.DATA_END_TIME, + fixed_interval: `${fixedInterval}m`, + offset: `${offsetInMillisec}ms`, + }, + aggs: { + [MAX_ANOMALY_AGGS]: { + max: { + field: AD_DOC_FIELDS.ANOMALY_GRADE, + }, + }, + [COUNT_ANOMALY_AGGS]: { + value_count: { + field: AD_DOC_FIELDS.ANOMALY_GRADE, + }, + }, + }, + }, + }, + }; +}; + +export const parseEntityAnomalySummaryResults = ( + result: any, + entity: Entity +): EntityAnomalySummaries => { + const rawEntityAnomalySummaries = get( + result, + `response.aggregations.${ENTITY_DATE_BUCKET_ANOMALY_AGGS}.buckets`, + [] + ) as any[]; + let anomalySummaries = [] as EntityAnomalySummary[]; + rawEntityAnomalySummaries.forEach((item) => { + const anomalyCount = get(item, `${COUNT_ANOMALY_AGGS}.value`, 0); + const startTime = get(item, 'key', 0); + const maxAnomalyGrade = get(item, `${MAX_ANOMALY_AGGS}.value`, 0); + const enityAnomalySummary = { + startTime: startTime, + maxAnomaly: maxAnomalyGrade, + anomalyCount: anomalyCount, + } as EntityAnomalySummary; + anomalySummaries.push(enityAnomalySummary); + }); + const enityAnomalySummaries = { + entity: entity, + anomalySummaries: anomalySummaries, + } as EntityAnomalySummaries; + return enityAnomalySummaries; +}; diff --git a/public/pages/utils/constants.ts b/public/pages/utils/constants.ts index 90ca6fbe..cbbeeeda 100644 --- a/public/pages/utils/constants.ts +++ b/public/pages/utils/constants.ts @@ -74,3 +74,12 @@ export const GET_SAMPLE_DETECTORS_QUERY_PARAMS = { }; export const GET_SAMPLE_INDICES_QUERY = 'opendistro-sample-*'; + +export const TOP_ENTITIES_FIELD = 'top_entities'; + +export const TOP_ENTITY_AGGS = 'top_entity_aggs'; +export const TOP_ANOMALY_GRADE_SORT_AGGS = 'top_anomaly_grade_sort_aggs'; +export const MAX_ANOMALY_AGGS = 'max_anomaly_aggs'; +export const COUNT_ANOMALY_AGGS = 'count_anomaly_aggs'; +export const MAX_ANOMALY_SORT_AGGS = 'max_anomaly_sort_aggs'; +export const ENTITY_DATE_BUCKET_ANOMALY_AGGS = 'entity_date_bucket_anomaly'; diff --git a/server/models/interfaces.ts b/server/models/interfaces.ts index 075df7d5..0305223c 100644 --- a/server/models/interfaces.ts +++ b/server/models/interfaces.ts @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -22,12 +22,12 @@ export interface DefaultHeaders { export interface SearchResponse { hits: { total: { value: number }; - hits: { + hits: Array<{ _source: T; _id: string; _seq_no?: number; _primary_term?: number; - }[]; + }>; }; } @@ -40,11 +40,11 @@ export interface AlertingApis { [API_ROUTE: string]: string; readonly ALERTING_BASE: string; } -export type Entity = { +export interface Entity { name: string; value: string; -}; -export type Anomaly = { +} +export interface Anomaly { anomalyGrade: number; confidence: number; anomalyScore: number; @@ -52,17 +52,34 @@ export type Anomaly = { endTime: number; plotTime: number; entity?: Entity[]; -}; -//Plot time is middle of start and end time to provide better visualization to customers +} +// Plot time is middle of start and end time to provide better visualization to customers // Example, if window is 10 mins, in a given startTime and endTime of 12:10 to 12:20 respectively. // plotTime will be 12:15. -export type FeatureData = { +export interface FeatureData { startTime: number; endTime: number; plotTime: number; data: number; -}; +} export interface AnomalyResults { anomalies: Anomaly[]; featureData: { [key: string]: FeatureData[] }; } + +export interface InitProgress { + percentageStr: string; + estimatedMinutesLeft: number; + neededShingles: number; +} + +export interface EntityAnomalySummary { + startTime: number; + maxAnomaly: number; + anomalyCount: number; +} + +export interface EntityAnomalySummaries { + entity: Entity; + anomalySummaries: EntityAnomalySummary[]; +} diff --git a/server/routes/ad.ts b/server/routes/ad.ts index 2b8c5585..1912b082 100644 --- a/server/routes/ad.ts +++ b/server/routes/ad.ts @@ -31,7 +31,13 @@ import { DateRangeFilter, } from '../models/types'; import { Router } from '../router'; -import { SORT_DIRECTION, AD_DOC_FIELDS } from '../utils/constants'; +import { + SORT_DIRECTION, + AD_DOC_FIELDS, + ENTITY_FIELD, + ENTITY_NAME_PATH_FIELD, + ENTITY_VALUE_PATH_FIELD, +} from '../utils/constants'; import { mapKeysDeep, toCamel, @@ -593,6 +599,8 @@ const getAnomalyResults = async ( sortField = AD_DOC_FIELDS.DATA_START_TIME, dateRangeFilter = undefined, anomalyThreshold = -1, + entityName = undefined, + entityValue = undefined, //@ts-ignore } = req.query as { from: number; @@ -601,6 +609,8 @@ const getAnomalyResults = async ( sortField?: string; dateRangeFilter?: string; anomalyThreshold: number; + entityName: string; + entityValue: string; }; const { detectorId } = req.params; @@ -642,6 +652,34 @@ const getAnomalyResults = async ( }, }, }, + ...(entityName && entityValue + ? [ + { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_NAME_PATH_FIELD]: { + value: entityName, + }, + }, + }, + }, + }, + { + nested: { + path: ENTITY_FIELD, + query: { + term: { + [ENTITY_VALUE_PATH_FIELD]: { + value: entityValue, + }, + }, + }, + }, + }, + ] + : []), ], }, }, diff --git a/server/utils/constants.ts b/server/utils/constants.ts index 486d255e..34c7d8bb 100644 --- a/server/utils/constants.ts +++ b/server/utils/constants.ts @@ -58,3 +58,27 @@ export enum AD_DOC_FIELDS { export const MAX_MONITORS = 1000; export const MAX_ALERTS = 1000; + +// TODO: maybe move types/interfaces/constants/helpers shared between client and server +// side as many as possible into single place +export enum DETECTOR_STATE { + DISABLED = 'Stopped', + INIT = 'Initializing', + RUNNING = 'Running', + FEATURE_REQUIRED = 'Feature required', + INIT_FAILURE = 'Initialization failure', + UNEXPECTED_FAILURE = 'Unexpected failure', +} + +export enum SAMPLE_TYPE { + HTTP_RESPONSES = 'http-responses', + HOST_HEALTH = 'host-health', + ECOMMERCE = 'ecommerce', +} + +export const ENTITY_FIELD = 'entity'; +export const ENTITY_VALUE_PATH_FIELD = 'entity.value'; +export const ENTITY_NAME_PATH_FIELD = 'entity.name'; + +export const DOC_COUNT_FIELD = 'doc_count'; +export const KEY_FIELD = 'key';