diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/aggregateOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/aggregateOperator.ts new file mode 100644 index 0000000000000000000000000000000000000000..aa3c518ad926bbda777b1d79f5d70866a42354f7 --- /dev/null +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/aggregateOperator.ts @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitationsxw + * under the License. + */ +import { + getMetricLabel, + ensureIsArray, + PostProcessingAggregation, + QueryFormData, + Aggregates, +} from '@superset-ui/core'; +import { PostProcessingFactory } from './types'; + +export const aggregationOperator: PostProcessingFactory< + PostProcessingAggregation +> = (formData: QueryFormData, queryObject) => { + const { aggregation = 'LAST_VALUE' } = formData; + + if (aggregation === 'LAST_VALUE') { + return undefined; + } + + const metrics = ensureIsArray(queryObject.metrics); + if (metrics.length === 0) { + return undefined; + } + + const aggregates: Aggregates = {}; + metrics.forEach(metric => { + const metricLabel = getMetricLabel(metric); + aggregates[metricLabel] = { + operator: aggregation, + column: metricLabel, + }; + }); + + return { + operation: 'aggregate', + options: { + groupby: [], + aggregates, + }, + }; +}; diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/index.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/index.ts index cac7088a775a74d9f8c3fbb5f4574b73750cede4..0f6a01ee1276a73e3a4073927c396bfeed95d018 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/index.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/index.ts @@ -21,6 +21,7 @@ export { rollingWindowOperator } from './rollingWindowOperator'; export { timeCompareOperator } from './timeCompareOperator'; export { timeComparePivotOperator } from './timeComparePivotOperator'; export { sortOperator } from './sortOperator'; +export { aggregationOperator } from './aggregateOperator'; export { histogramOperator } from './histogramOperator'; export { pivotOperator } from './pivotOperator'; export { resampleOperator } from './resampleOperator'; diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/customControls.tsx b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/customControls.tsx index d25273c08e9be554aed093e865158dbc41e23779..bdd6d1b82cc78822416c962d15e59e2810316d93 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/customControls.tsx +++ b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/customControls.tsx @@ -61,6 +61,32 @@ const xAxisSortVisibility = ({ controls }: { controls: ControlStateMapping }) => ensureIsArray(controls?.groupby?.value).length === 0 && ensureIsArray(controls?.metrics?.value).length === 1; +// TODO: Expand this aggregation options list to include all backend-supported aggregations. +// TODO: Migrate existing chart types (Pivot Table, etc.) to use this shared control. +export const aggregationControl = { + name: 'aggregation', + config: { + type: 'SelectControl', + label: t('Aggregation Method'), + default: 'LAST_VALUE', + clearable: false, + renderTrigger: false, + choices: [ + ['LAST_VALUE', t('Last Value')], + ['sum', t('Total (Sum)')], + ['mean', t('Average (Mean)')], + ['min', t('Minimum')], + ['max', t('Maximum')], + ['median', t('Median')], + ], + description: t('Select an aggregation method to apply to the metric.'), + provideFormDataToProps: true, + mapStateToProps: ({ form_data }: ControlPanelState) => ({ + value: form_data.aggregation || 'LAST_VALUE', + }), + }, +}; + const xAxisMultiSortVisibility = ({ controls, }: { diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/index.ts b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/index.ts index 8ff85439bfa3b4b00dea5492448afe191532ee85..0deb6b398621d2ba2bed3e4a5749201ec3e30637 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/index.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/index.ts @@ -19,6 +19,7 @@ export { default as sharedControls } from './sharedControls'; // React control components export { default as sharedControlComponents } from './components'; +export { aggregationControl } from './customControls'; export * from './components'; export * from './customControls'; export * from './mixins'; diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/operators/aggregateOperator.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/operators/aggregateOperator.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..52e3e45407817cc3ab2d44b1e594ddd2dd82ee10 --- /dev/null +++ b/superset-frontend/packages/superset-ui-chart-controls/test/operators/aggregateOperator.test.ts @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { QueryObject, SqlaFormData, VizType } from '@superset-ui/core'; +import { aggregationOperator } from '@superset-ui/chart-controls'; + +describe('aggregationOperator', () => { + const formData: SqlaFormData = { + metrics: [ + 'count(*)', + { label: 'sum(val)', expressionType: 'SQL', sqlExpression: 'sum(val)' }, + ], + time_range: '2015 : 2016', + granularity: 'month', + datasource: 'foo', + viz_type: VizType.Table, + }; + + const queryObject: QueryObject = { + metrics: [ + 'count(*)', + { label: 'sum(val)', expressionType: 'SQL', sqlExpression: 'sum(val)' }, + ], + time_range: '2015 : 2016', + granularity: 'month', + }; + + test('should return undefined for LAST_VALUE aggregation', () => { + const formDataWithLastValue = { + ...formData, + aggregation: 'LAST_VALUE', + }; + + expect( + aggregationOperator(formDataWithLastValue, queryObject), + ).toBeUndefined(); + }); + + test('should return undefined when metrics is empty', () => { + const queryObjectWithoutMetrics = { + ...queryObject, + metrics: [], + }; + + const formDataWithSum = { + ...formData, + aggregation: 'sum', + }; + + expect( + aggregationOperator(formDataWithSum, queryObjectWithoutMetrics), + ).toBeUndefined(); + }); + + test('should apply sum aggregation to all metrics', () => { + const formDataWithSum = { + ...formData, + aggregation: 'sum', + }; + + expect(aggregationOperator(formDataWithSum, queryObject)).toEqual({ + operation: 'aggregate', + options: { + groupby: [], + aggregates: { + 'count(*)': { + operator: 'sum', + column: 'count(*)', + }, + 'sum(val)': { + operator: 'sum', + column: 'sum(val)', + }, + }, + }, + }); + }); + + test('should apply mean aggregation to all metrics', () => { + const formDataWithMean = { + ...formData, + aggregation: 'mean', + }; + + expect(aggregationOperator(formDataWithMean, queryObject)).toEqual({ + operation: 'aggregate', + options: { + groupby: [], + aggregates: { + 'count(*)': { + operator: 'mean', + column: 'count(*)', + }, + 'sum(val)': { + operator: 'mean', + column: 'sum(val)', + }, + }, + }, + }); + }); + + test('should use default aggregation when not specified', () => { + expect(aggregationOperator(formData, queryObject)).toBeUndefined(); + }); +}); diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/operators/timeCompareOperator.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/operators/timeCompareOperator.test.ts index c7861af2ee53d9d9a6c042648d3d66815ea2f451..437d3064bf5865d2c6b66ff04c31e1c7628f863f 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/test/operators/timeCompareOperator.test.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/test/operators/timeCompareOperator.test.ts @@ -54,7 +54,7 @@ const queryObject: QueryObject = { }, }, { - operation: 'aggregation', + operation: 'aggregate', options: { groupby: ['col1'], aggregates: {}, diff --git a/superset-frontend/packages/superset-ui-core/src/query/types/PostProcessing.ts b/superset-frontend/packages/superset-ui-core/src/query/types/PostProcessing.ts index a70d0111f2ec61e6b7afb43583dc85e8199d31a1..79bcabdaff12c8cf2a9f839c2dc658fba7522433 100644 --- a/superset-frontend/packages/superset-ui-core/src/query/types/PostProcessing.ts +++ b/superset-frontend/packages/superset-ui-core/src/query/types/PostProcessing.ts @@ -67,7 +67,7 @@ export interface Aggregates { export type DefaultPostProcessing = undefined; interface _PostProcessingAggregation { - operation: 'aggregation'; + operation: 'aggregate'; options: { groupby: string[]; aggregates: Aggregates; @@ -271,7 +271,7 @@ export type PostProcessingRule = export function isPostProcessingAggregation( rule?: PostProcessingRule, ): rule is PostProcessingAggregation { - return rule?.operation === 'aggregation'; + return rule?.operation === 'aggregate'; } export function isPostProcessingBoxplot( diff --git a/superset-frontend/packages/superset-ui-core/test/query/types/PostProcessing.test.ts b/superset-frontend/packages/superset-ui-core/test/query/types/PostProcessing.test.ts index 05c385fb4e6af798013c6331665982591a094f2d..4e4ff949cbd9abc2a411201602b9e194e97d5fd2 100644 --- a/superset-frontend/packages/superset-ui-core/test/query/types/PostProcessing.test.ts +++ b/superset-frontend/packages/superset-ui-core/test/query/types/PostProcessing.test.ts @@ -61,7 +61,7 @@ const AGGREGATES_OPTION: Aggregates = { }; const AGGREGATE_RULE: PostProcessingAggregation = { - operation: 'aggregation', + operation: 'aggregate', options: { groupby: ['foo'], aggregates: AGGREGATES_OPTION, diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts index 7a0ba462b88b2d33a4cd13b0b10d250d73ce6cca..398125719b1e49df67d261082567c6facb1e891e 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts @@ -24,6 +24,7 @@ import { QueryFormData, } from '@superset-ui/core'; import { + aggregationOperator, flattenOperator, pivotOperator, resampleOperator, @@ -47,5 +48,19 @@ export default function buildQuery(formData: QueryFormData) { flattenOperator(formData, baseQueryObject), ], }, + + { + ...baseQueryObject, + columns: [ + ...(isXAxisSet(formData) + ? ensureIsArray(getXAxisColumn(formData)) + : []), + ], + ...(isXAxisSet(formData) ? {} : { is_timeseries: true }), + post_processing: [ + pivotOperator(formData, baseQueryObject), + aggregationOperator(formData, baseQueryObject), + ], + }, ]); } diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/controlPanel.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/controlPanel.tsx index 83cf915c7ced055306b37d18157627cf1428e7e7..ea8f9c66f485623acae5cb8ae3f4c484e0e3677e 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/controlPanel.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/controlPanel.tsx @@ -18,6 +18,7 @@ */ import { SMART_DATE_ID, t } from '@superset-ui/core'; import { + aggregationControl, ControlPanelConfig, ControlSubSectionHeader, D3_FORMAT_DOCS, @@ -35,6 +36,7 @@ const config: ControlPanelConfig = { controlSetRows: [ ['x_axis'], ['time_grain_sqla'], + [aggregationControl], ['metric'], ['adhoc_filters'], ], diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts index d285a551b136c948a942389c38a0994e7842dc54..53a44d9e3b0aa112003dd1d992fd081bff7474d0 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts @@ -66,6 +66,7 @@ export default function transformProps( metric = 'value', showTimestamp, showTrendLine, + aggregation, startYAxisAtZero, subheader = '', subheaderFontSize, @@ -82,6 +83,15 @@ export default function transformProps( from_dttm: fromDatetime, to_dttm: toDatetime, } = queriesData[0]; + + const aggregatedQueryData = queriesData.length > 1 ? queriesData[1] : null; + + const hasAggregatedData = + aggregatedQueryData?.data && + aggregatedQueryData.data.length > 0 && + aggregation !== 'LAST_VALUE'; + + const aggregatedData = hasAggregatedData ? aggregatedQueryData.data[0] : null; const refs: Refs = {}; const metricName = getMetricLabel(metric); const compareLag = Number(compareLag_) || 0; @@ -95,18 +105,39 @@ export default function transformProps( let percentChange = 0; let bigNumber = data.length === 0 ? null : data[0][metricName]; let timestamp = data.length === 0 ? null : data[0][xAxisLabel]; - let bigNumberFallback; - - const metricColtypeIndex = colnames.findIndex(name => name === metricName); - const metricColtype = - metricColtypeIndex > -1 ? coltypes[metricColtypeIndex] : null; + let bigNumberFallback = null; + let sortedData: [number | null, number | null][] = []; if (data.length > 0) { - const sortedData = (data as BigNumberDatum[]) - .map(d => [d[xAxisLabel], parseMetricValue(d[metricName])]) + sortedData = (data as BigNumberDatum[]) + .map( + d => + [d[xAxisLabel], parseMetricValue(d[metricName])] as [ + number | null, + number | null, + ], + ) // sort in time descending order .sort((a, b) => (a[0] !== null && b[0] !== null ? b[0] - a[0] : 0)); + } + if (hasAggregatedData && aggregatedData) { + if ( + aggregatedData[metricName] !== null && + aggregatedData[metricName] !== undefined + ) { + bigNumber = aggregatedData[metricName]; + } else { + const metricKeys = Object.keys(aggregatedData).filter( + key => + key !== xAxisLabel && + aggregatedData[key] !== null && + typeof aggregatedData[key] === 'number', + ); + bigNumber = metricKeys.length > 0 ? aggregatedData[metricKeys[0]] : null; + } + timestamp = sortedData.length > 0 ? sortedData[0][0] : null; + } else if (sortedData.length > 0) { bigNumber = sortedData[0][1]; timestamp = sortedData[0][0]; @@ -115,25 +146,28 @@ export default function transformProps( bigNumber = bigNumberFallback ? bigNumberFallback[1] : null; timestamp = bigNumberFallback ? bigNumberFallback[0] : null; } + } - if (compareLag > 0) { - const compareIndex = compareLag; - if (compareIndex < sortedData.length) { - const compareValue = sortedData[compareIndex][1]; - // compare values must both be non-nulls - if (bigNumber !== null && compareValue !== null) { - percentChange = compareValue - ? (bigNumber - compareValue) / Math.abs(compareValue) - : 0; - formattedSubheader = `${formatPercentChange( - percentChange, - )} ${compareSuffix}`; - } + if (compareLag > 0 && sortedData.length > 0) { + const compareIndex = compareLag; + if (compareIndex < sortedData.length) { + const compareValue = sortedData[compareIndex][1]; + // compare values must both be non-nulls + if (bigNumber !== null && compareValue !== null) { + percentChange = compareValue + ? (Number(bigNumber) - compareValue) / Math.abs(compareValue) + : 0; + formattedSubheader = `${formatPercentChange( + percentChange, + )} ${compareSuffix}`; } } - sortedData.reverse(); + } + + if (data.length > 0) { + const reversedData = [...sortedData].reverse(); // @ts-ignore - trendLineData = showTrendLine ? sortedData : undefined; + trendLineData = showTrendLine ? reversedData : undefined; } let className = ''; @@ -143,6 +177,10 @@ export default function transformProps( className = 'negative'; } + const metricColtypeIndex = colnames.findIndex(name => name === metricName); + const metricColtype = + metricColtypeIndex > -1 ? coltypes[metricColtypeIndex] : null; + let metricEntry: Metric | undefined; if (chartProps.datasource?.metrics) { metricEntry = chartProps.datasource.metrics.find( diff --git a/superset-frontend/plugins/plugin-chart-echarts/test/BigNumber/transformProps.test.ts b/superset-frontend/plugins/plugin-chart-echarts/test/BigNumber/transformProps.test.ts index 8b0bf3552585931dfcff64c5793d395e438d9a53..4ccedd1e7f215c9dc52fc84b07e3974d859ae1d8 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/test/BigNumber/transformProps.test.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/test/BigNumber/transformProps.test.ts @@ -186,3 +186,188 @@ describe('BigNumberWithTrendline', () => { }); }); }); + +describe('BigNumberWithTrendline - Aggregation Tests', () => { + const baseProps = { + width: 800, + height: 600, + formData: { + colorPicker: { r: 0, g: 0, b: 0, a: 1 }, + metric: 'metric', + aggregation: 'LAST_VALUE', + }, + queriesData: [ + { + data: [ + { __timestamp: 1607558400000, metric: 10 }, + { __timestamp: 1607558500000, metric: 30 }, + { __timestamp: 1607558600000, metric: 50 }, + { __timestamp: 1607558700000, metric: 60 }, + ], + colnames: ['__timestamp', 'metric'], + coltypes: ['TIMESTAMP', 'BIGINT'], + }, + ], + hooks: {}, + filterState: {}, + datasource: { + columnFormats: {}, + currencyFormats: {}, + }, + rawDatasource: {}, + rawFormData: {}, + theme: { + colors: { + grayscale: { + light5: '#fafafa', + }, + }, + }, + } as unknown as BigNumberWithTrendlineChartProps; + + const propsWithEvenData = { + ...baseProps, + queriesData: [ + { + data: [ + { __timestamp: 1607558400000, metric: 10 }, + { __timestamp: 1607558500000, metric: 20 }, + { __timestamp: 1607558600000, metric: 30 }, + { __timestamp: 1607558700000, metric: 40 }, + ], + colnames: ['__timestamp', 'metric'], + coltypes: ['TIMESTAMP', 'BIGINT'], + }, + ], + } as unknown as BigNumberWithTrendlineChartProps; + + it('should correctly calculate SUM', () => { + const props = { + ...baseProps, + formData: { ...baseProps.formData, aggregation: 'sum' }, + queriesData: [ + baseProps.queriesData[0], + { + data: [{ metric: 150 }], + colnames: ['metric'], + coltypes: ['BIGINT'], + }, + ], + } as unknown as BigNumberWithTrendlineChartProps; + + const transformed = transformProps(props); + expect(transformed.bigNumber).toStrictEqual(150); + }); + + it('should correctly calculate AVG', () => { + const props = { + ...baseProps, + formData: { ...baseProps.formData, aggregation: 'mean' }, + queriesData: [ + baseProps.queriesData[0], + { + data: [{ metric: 37.5 }], + colnames: ['metric'], + coltypes: ['BIGINT'], + }, + ], + } as unknown as BigNumberWithTrendlineChartProps; + + const transformed = transformProps(props); + expect(transformed.bigNumber).toStrictEqual(37.5); + }); + + it('should correctly calculate MIN', () => { + const props = { + ...baseProps, + formData: { ...baseProps.formData, aggregation: 'min' }, + queriesData: [ + baseProps.queriesData[0], + { + data: [{ metric: 10 }], + colnames: ['metric'], + coltypes: ['BIGINT'], + }, + ], + } as unknown as BigNumberWithTrendlineChartProps; + + const transformed = transformProps(props); + expect(transformed.bigNumber).toStrictEqual(10); + }); + + it('should correctly calculate MAX', () => { + const props = { + ...baseProps, + formData: { ...baseProps.formData, aggregation: 'max' }, + queriesData: [ + baseProps.queriesData[0], + { + data: [{ metric: 60 }], + colnames: ['metric'], + coltypes: ['BIGINT'], + }, + ], + } as unknown as BigNumberWithTrendlineChartProps; + + const transformed = transformProps(props); + expect(transformed.bigNumber).toStrictEqual(60); + }); + + it('should correctly calculate MEDIAN (odd count)', () => { + const oddCountProps = { + ...baseProps, + queriesData: [ + { + data: [ + { __timestamp: 1607558300000, metric: 10 }, + { __timestamp: 1607558400000, metric: 20 }, + { __timestamp: 1607558500000, metric: 30 }, + { __timestamp: 1607558600000, metric: 40 }, + { __timestamp: 1607558700000, metric: 50 }, + ], + colnames: ['__timestamp', 'metric'], + coltypes: ['TIMESTAMP', 'BIGINT'], + }, + ], + } as unknown as BigNumberWithTrendlineChartProps; + + const props = { + ...oddCountProps, + formData: { ...oddCountProps.formData, aggregation: 'median' }, + queriesData: [ + oddCountProps.queriesData[0], + { + data: [{ metric: 30 }], + colnames: ['metric'], + coltypes: ['BIGINT'], + }, + ], + } as unknown as BigNumberWithTrendlineChartProps; + + const transformed = transformProps(props); + expect(transformed.bigNumber).toStrictEqual(30); + }); + + it('should correctly calculate MEDIAN (even count)', () => { + const props = { + ...propsWithEvenData, + formData: { ...propsWithEvenData.formData, aggregation: 'median' }, + queriesData: [ + propsWithEvenData.queriesData[0], + { + data: [{ metric: 25 }], + colnames: ['metric'], + coltypes: ['BIGINT'], + }, + ], + } as unknown as BigNumberWithTrendlineChartProps; + + const transformed = transformProps(props); + expect(transformed.bigNumber).toStrictEqual(25); + }); + + it('should return the LAST_VALUE correctly', () => { + const transformed = transformProps(baseProps); + expect(transformed.bigNumber).toStrictEqual(10); + }); +});