Skip to content

G
gitlab-ce
Commit 0ccf9639 authored by Martin Wortschack's avatar Martin Wortschack Committed by Filipa Lacerda
Browse files
Options

Improve scatterplot performance 

- Store transformed data in store
and optimize median computation
parent cc602cfa
Hide whitespace changes
Inline Side-by-side
Showing with 180 additions and 184 deletions
ee/app/assets/javascripts/analytics/productivity_analytics/store/modules/charts/actions.js
View file @ 0ccf9639
import axios from '~/lib/utils/axios_utils';
import * as types from './mutation_types';
import { chartKeys } from '../../../constants';
import { getDateInPast } from '~/lib/utils/datetime_utility';
import { chartKeys, scatterPlotAddonQueryDays } from '../../../constants';
import { transformScatterData } from '../../../utils';
/**
* Fetches data for all charts except for the main chart
......@@ -27,14 +29,28 @@ export const fetchChartData = ({ dispatch, getters, state, rootState }, chartKey
.get(rootState.endpoint, { params })
.then(response => {
const { data } = response;
dispatch('receiveChartDataSuccess', { chartKey, data });
if (chartKey === chartKeys.scatterplot) {
const transformedData = transformScatterData(
data,
new Date(getDateInPast(rootState.filters.startDate, scatterPlotAddonQueryDays)),
new Date(rootState.filters.endDate),
);
dispatch('receiveChartDataSuccess', { chartKey, data, transformedData });
} else {
dispatch('receiveChartDataSuccess', { chartKey, data });
}
})
.catch(error => dispatch('receiveChartDataError', { chartKey, error }));
}
};
export const receiveChartDataSuccess = ({ commit }, { chartKey, data = {} }) => {
commit(types.RECEIVE_CHART_DATA_SUCCESS, { chartKey, data });
export const receiveChartDataSuccess = (
{ commit },
{ chartKey, data = {}, transformedData = null },
) => {
commit(types.RECEIVE_CHART_DATA_SUCCESS, { chartKey, data, transformedData });
};
export const receiveChartDataError = ({ commit }, { chartKey, error }) => {
......
ee/app/assets/javascripts/analytics/productivity_analytics/store/modules/charts/getters.js
View file @ 0ccf9639
......@@ -54,28 +54,12 @@ export const getColumnChartData = state => chartKey => {
export const chartHasData = state => chartKey => !_.isEmpty(state.charts[chartKey].data);
/**
* Creates a series array of main data for the scatterplot chart.
*
* Takes an object of the form
* {
* "1": { "metric": 138", merged_at": "2019-07-09T14:58:07.756Z" },
* "2": { "metric": 139, "merged_at": "2019-07-10T11:13:23.557Z" },
* "3": { "metric": 24, "merged_at": "2019-07-01T07:06:23.193Z" }
* }
*
* and creates the following structure:
*
* [
* ["2019-07-01T07:06:23.193Z", 24],
* ["2019-07-09T14:58:07.756Z", 138],
* ["2019-07-10T11:13:23.557Z", 139],
* ]
*
* It eliminates items which were merged before the startDate (minus an additional days offset).
*/
export const getScatterPlotMainData = (state, getters, rootState) =>
getScatterPlotData(state.charts.scatterplot.data, rootState.filters.startDate);
getScatterPlotData(
state.charts.scatterplot.transformedData,
new Date(rootState.filters.startDate),
new Date(rootState.filters.endDate),
);
/**
* Creates a series array of median data for the scatterplot chart.
......@@ -83,10 +67,11 @@ export const getScatterPlotMainData = (state, getters, rootState) =>
* It calls getMedianLineData internally with the raw scatterplot data and the computed by getters.getScatterPlotMainData.
* scatterPlotAddonQueryDays is necessary since we query the API with an additional day offset to compute the median.
*/
export const getScatterPlotMedianData = (state, getters) =>
export const getScatterPlotMedianData = (state, getters, rootState) =>
getMedianLineData(
state.charts.scatterplot.data,
getters.getScatterPlotMainData,
state.charts.scatterplot.transformedData,
new Date(rootState.filters.startDate),
new Date(rootState.filters.endDate),
scatterPlotAddonQueryDays,
);
......
ee/app/assets/javascripts/analytics/productivity_analytics/store/modules/charts/mutations.js
View file @ 0ccf9639
import * as types from './mutation_types';
import { chartKeys } from '../../../constants';
export default {
[types.RESET_CHART_DATA](state, chartKey) {
......@@ -8,15 +9,23 @@ export default {
[types.REQUEST_CHART_DATA](state, chartKey) {
state.charts[chartKey].isLoading = true;
},
[types.RECEIVE_CHART_DATA_SUCCESS](state, { chartKey, data }) {
[types.RECEIVE_CHART_DATA_SUCCESS](state, { chartKey, data, transformedData }) {
state.charts[chartKey].isLoading = false;
state.charts[chartKey].errorCode = null;
state.charts[chartKey].data = data;
if (chartKey === chartKeys.scatterplot) {
state.charts[chartKey].transformedData = transformedData;
}
},
[types.RECEIVE_CHART_DATA_ERROR](state, { chartKey, status }) {
state.charts[chartKey].isLoading = false;
state.charts[chartKey].errorCode = status;
state.charts[chartKey].data = {};
if (chartKey === chartKeys.scatterplot) {
state.charts[chartKey].transformedData = [];
}
},
[types.SET_METRIC_TYPE](state, { chartKey, metricType }) {
state.charts[chartKey].params.metricType = metricType;
......
ee/app/assets/javascripts/analytics/productivity_analytics/store/modules/charts/state.js
View file @ 0ccf9639
......@@ -39,6 +39,7 @@ export default () => ({
errorCode: null,
enabled: true,
data: {},
transformedData: [],
selected: [],
params: {
chartType: chartTypes.scatterplot,
......
ee/app/assets/javascripts/analytics/productivity_analytics/utils.js
View file @ 0ccf9639
......@@ -90,38 +90,6 @@ export const transformScatterData = (data, startDate, endDate) => {
return result;
};
/**
* Transforms a given data object into an array
* which will be used as series data for the scatterplot chart.
* It eliminates items which were merged before a "dateInPast" and sorts
* the result by date (ascending)
*
* Takes an object of the form
* {
* "1": { "metric": 138", merged_at": "2019-07-09T14:58:07.756Z" },
* "2": { "metric": 139, "merged_at": "2019-07-10T11:13:23.557Z" },
* "3": { "metric": 24, "merged_at": "2019-07-01T07:06:23.193Z" }
* }
*
* and creates the following two-dimensional array
* where the first value is the "merged_at" date and the second value is the metric:
*
* [
* ["2019-07-01T07:06:23.193Z", 24],
* ["2019-07-09T14:58:07.756Z", 138],
* ["2019-07-10T11:13:23.557Z", 139],
* ]
*
* @param {Object} data The raw data which will be transformed
* @param {Date} dateInPast Date in the past
* @returns {Array} The transformed data array sorted by date ascending
*/
export const getScatterPlotData = (data, dateInPast) =>
Object.keys(data)
.filter(key => new Date(data[key].merged_at) >= dateInPast)
.map(key => [data[key].merged_at, data[key].metric])
.sort((a, b) => new Date(a[0]) - new Date(b[0]));
/**
* Brings the data the we receive from transformScatterData into a format that can be passed to the chart.
* Since transformScatterData contains more data than we actually want to display on the scatterplot
......@@ -145,7 +113,7 @@ export const getScatterPlotData = (data, dateInPast) =>
* @param {*} endDate - The end date selected by the user
* @returns {Array} An array with each item being another arry of two items (date, computed median)
*/
export const getScatterPlotDataNew = (data, startDate, endDate) => {
export const getScatterPlotData = (data, startDate, endDate) => {
if (!data.length) return [];
const startIndex = data.length - 1 - getDayDifference(startDate, endDate);
......@@ -161,39 +129,6 @@ export const getScatterPlotDataNew = (data, startDate, endDate) => {
return result;
};
/**
* Computes the moving median line data.
* It takes the raw data object (which contains historical data) and the scatterData (from getScatterPlotData)
* and computes the median for every date in scatterData.
* The median for a given date in scatterData (called item) is computed by taking all metrics of the raw data into account
* which are before (or eqaul to) the the item's merged_at date
* and after (or equal to) the item's merged_at date minus a given "daysOffset" (e.g., 30 days for "30 day rolling median")
*
* i.e., moving median for a given DAY is the median the range of values (DAY-30 ... DAY)
*
* @param {Object} data The raw data which will be used for computing the median
* @param {Array} scatterData The transformed data from getScatterPlotData
* @param {Number} daysOffset The number of days that is substracted from each date in scatterData (e.g. 30 days in the past)
* @returns {Array} An array with each item being another arry of two items (date, computed median)
*/
export const getMedianLineData = (data, scatterData, daysOffset) =>
scatterData.map(item => {
const [dateString] = item;
const values = Object.keys(data)
.filter(key => {
const mergedAtDate = new Date(data[key].merged_at);
const itemDate = new Date(dateString);
return (
mergedAtDate <= itemDate && mergedAtDate >= new Date(getDateInPast(itemDate, daysOffset))
);
})
.map(key => data[key].metric);
const computedMedian = values.length ? median(values) : 0;
return [dateString, computedMedian];
});
/**
* Computes the moving median line data, i.e, it computes the 30 day rolling median for every item displayd on the scatterplot
* For example the 30 day rolling median for startDate=2019-09-01 and endDate=2019-09-03 is computed as follows:
......@@ -210,7 +145,7 @@ export const getMedianLineData = (data, scatterData, daysOffset) =>
* @param {Number} daysOffset The number of days that to look up data in the past (e.g. 30 days in the past for 30 day rolling median)
* @returns {Array} An array with each item being another arry of two items (date, computed median)
*/
export const getMedianLineDataNew = (data, startDate, endDate, daysOffset) => {
export const getMedianLineData = (data, startDate, endDate, daysOffset) => {
const result = [];
const dayDiff = getDayDifference(startDate, endDate);
const transformedData = data.map(arr => arr.map(x => x.metric));
......
ee/app/assets/javascripts/analytics/shared/components/scatterplot.vue
View file @ 0ccf9639
......@@ -32,6 +32,7 @@ export default {
tooltipContent: '',
chartOption: {
xAxis: {
type: 'time',
axisLabel: {
formatter: date => dateFormat(date, dateFormats.defaultDate),
},
......@@ -67,9 +68,9 @@ export default {
},
methods: {
renderTooltip({ data }) {
const [xValue, yValue] = data;
this.tooltipTitle = yValue;
this.tooltipContent = dateFormat(xValue, dateFormats.defaultDateTime);
const [, metric, dateTime] = data;
this.tooltipTitle = metric;
this.tooltipContent = dateFormat(dateTime, dateFormats.defaultDateTime);
},
},
};
......
ee/spec/frontend/analytics/productivity_analytics/components/app_spec.js
View file @ 0ccf9639
......@@ -82,6 +82,11 @@ describe('ProductivityApp component', () => {
describe('with a group being selected', () => {
beforeEach(() => {
wrapper.vm.$store.dispatch('filters/setDateRange', {
skipFetch: true,
startDate: new Date('2019-09-01'),
endDate: new Date('2019-09-02'),
});
wrapper.vm.$store.dispatch('filters/setGroupNamespace', 'gitlab-org');
mock.onGet(wrapper.vm.$store.state.endpoint).replyOnce(200);
});
......@@ -277,9 +282,13 @@ describe('ProductivityApp component', () => {
wrapper.vm.$store.dispatch('charts/receiveChartDataSuccess', {
chartKey: chartKeys.scatterplot,
data: {
1: { metric: 2, merged_at: '2019-07-01T07:06:23.193Z' },
2: { metric: 3, merged_at: '2019-07-05T08:27:42.411Z' },
1: { metric: 2, merged_at: '2019-09-01T07:06:23.193Z' },
2: { metric: 3, merged_at: '2019-09-05T08:27:42.411Z' },
},
transformedData: [
[{ metric: 2, merged_at: '2019-09-01T07:06:23.193Z' }],
[{ metric: 3, merged_at: '2019-09-05T08:27:42.411Z' }],
],
});
});
......
ee/spec/frontend/analytics/productivity_analytics/store/modules/charts/actions_spec.js
View file @ 0ccf9639
......@@ -6,7 +6,13 @@ import * as actions from 'ee/analytics/productivity_analytics/store/modules/char
import * as types from 'ee/analytics/productivity_analytics/store/modules/charts/mutation_types';
import getInitialState from 'ee/analytics/productivity_analytics/store/modules/charts/state';
import { chartKeys } from 'ee/analytics/productivity_analytics/constants';
import { mockHistogramData } from '../../../mock_data';
import { mockHistogramData, mockScatterplotData } from '../../../mock_data';
jest.mock('ee/analytics/productivity_analytics/utils', () => ({
transformScatterData: jest
.fn()
.mockImplementation(() => [[{ merged_at: '2019-09-01T00:00:000Z', metric: 10 }]]),
}));
describe('Productivity analytics chart actions', () => {
let mockedContext;
......@@ -24,6 +30,10 @@ describe('Productivity analytics chart actions', () => {
dispatch() {},
rootState: {
endpoint: `${TEST_HOST}/analytics/productivity_analytics.json`,
filters: {
startDate: '2019-09-01',
endDate: '2091-09-05',
},
},
getters: {
getFilterParams: () => globalParams,
......@@ -49,33 +59,62 @@ describe('Productivity analytics chart actions', () => {
describe('fetchChartData', () => {
describe('when chart is enabled', () => {
describe('success', () => {
beforeEach(() => {
mock.onGet(mockedState.endpoint).replyOnce(200, mockHistogramData);
});
describe('histogram charts', () => {
beforeEach(() => {
mock.onGet(mockedState.endpoint).replyOnce(200, mockHistogramData);
});
it('calls API with params', () => {
jest.spyOn(axios, 'get');
it('calls API with params', () => {
jest.spyOn(axios, 'get');
actions.fetchChartData(mockedContext, chartKey);
actions.fetchChartData(mockedContext, chartKey);
expect(axios.get).toHaveBeenCalledWith(mockedState.endpoint, { params: globalParams });
expect(axios.get).toHaveBeenCalledWith(mockedState.endpoint, { params: globalParams });
});
it('dispatches success with received data', done =>
testAction(
actions.fetchChartData,
chartKey,
mockedState,
[],
[
{ type: 'requestChartData', payload: chartKey },
{
type: 'receiveChartDataSuccess',
payload: expect.objectContaining({ chartKey, data: mockHistogramData }),
},
],
done,
));
});
it('dispatches success with received data', done =>
testAction(
actions.fetchChartData,
chartKey,
mockedState,
[],
[
{ type: 'requestChartData', payload: chartKey },
{
type: 'receiveChartDataSuccess',
payload: expect.objectContaining({ chartKey, data: mockHistogramData }),
},
],
done,
));
describe('scatterplot chart', () => {
beforeEach(() => {
mock.onGet(mockedState.endpoint).replyOnce(200, mockScatterplotData);
});
it('dispatches success with received data and transformedData', done => {
testAction(
actions.fetchChartData,
chartKeys.scatterplot,
mockedState,
[],
[
{ type: 'requestChartData', payload: chartKeys.scatterplot },
{
type: 'receiveChartDataSuccess',
payload: {
chartKey: chartKeys.scatterplot,
data: mockScatterplotData,
transformedData: [[{ merged_at: '2019-09-01T00:00:000Z', metric: 10 }]],
},
},
],
done,
);
});
});
});
describe('error', () => {
......@@ -149,7 +188,7 @@ describe('Productivity analytics chart actions', () => {
[
{
type: types.RECEIVE_CHART_DATA_SUCCESS,
payload: { chartKey, data: mockHistogramData },
payload: { chartKey, data: mockHistogramData, transformedData: null },
},
],
[],
......
ee/spec/frontend/analytics/productivity_analytics/store/modules/charts/getters_spec.js
View file @ 0ccf9639
......@@ -8,7 +8,7 @@ import {
scatterPlotAddonQueryDays,
} from 'ee/analytics/productivity_analytics/constants';
import { getScatterPlotData, getMedianLineData } from 'ee/analytics/productivity_analytics/utils';
import { mockHistogramData, mockScatterplotData } from '../../../mock_data';
import { mockHistogramData } from '../../../mock_data';
jest.mock('ee/analytics/productivity_analytics/utils');
......@@ -17,6 +17,10 @@ describe('Productivity analytics chart getters', () => {
const groupNamespace = 'gitlab-org';
const projectPath = 'gitlab-org/gitlab-test';
const transformedData = [
[{ merged_at: '2019-09-01T00:00:000Z', metric: 10 }],
[{ merged_at: '2019-09-02T00:00:000Z', metric: 20 }],
];
beforeEach(() => {
state = createState();
......@@ -54,33 +58,42 @@ describe('Productivity analytics chart getters', () => {
describe('getScatterPlotMainData', () => {
it('calls getScatterPlotData with the raw scatterplot data and the date in past', () => {
state.charts.scatterplot.data = mockScatterplotData;
state.charts.scatterplot.transformedData = transformedData;
const rootState = {
filters: {
startDate: '2019-07-16',
startDate: '2019-09-01',
endDate: '2019-09-05',
},
};
getters.getScatterPlotMainData(state, null, rootState);
expect(getScatterPlotData).toHaveBeenCalledWith(mockScatterplotData, '2019-07-16');
expect(getScatterPlotData).toHaveBeenCalledWith(
transformedData,
new Date(rootState.filters.startDate),
new Date(rootState.filters.endDate),
);
});
});
describe('getScatterPlotMedianData', () => {
it('calls getMedianLineData with the raw scatterplot data, the getScatterPlotMainData getter and the an additional days offset', () => {
state.charts.scatterplot.data = mockScatterplotData;
state.charts.scatterplot.transformedData = transformedData;
const mockGetters = {
getScatterPlotMainData: jest.fn(),
const rootState = {
filters: {
startDate: '2019-09-01',
endDate: '2019-09-05',
},
};
getters.getScatterPlotMedianData(state, mockGetters);
getters.getScatterPlotMedianData(state, null, rootState);
expect(getMedianLineData).toHaveBeenCalledWith(
mockScatterplotData,
mockGetters.getScatterPlotMainData,
transformedData,
new Date(rootState.filters.startDate),
new Date(rootState.filters.endDate),
scatterPlotAddonQueryDays,
);
});
......
ee/spec/frontend/analytics/productivity_analytics/store/modules/charts/mutations_spec.js
View file @ 0ccf9639
......@@ -2,7 +2,7 @@ import * as types from 'ee/analytics/productivity_analytics/store/modules/charts
import mutations from 'ee/analytics/productivity_analytics/store/modules/charts/mutations';
import getInitialState from 'ee/analytics/productivity_analytics/store/modules/charts/state';
import { chartKeys } from 'ee/analytics/productivity_analytics/constants';
import { mockHistogramData } from '../../../mock_data';
import { mockHistogramData, mockScatterplotData } from '../../../mock_data';
describe('Productivity analytics chart mutations', () => {
let state;
......@@ -37,23 +37,54 @@ describe('Productivity analytics chart mutations', () => {
expect(state.charts[chartKey].errorCode).toBe(null);
expect(state.charts[chartKey].data).toEqual(mockHistogramData);
});
it('updates the transformedData when chartKey=scatterplot', () => {
const transformedData = [
[
{
metric: 139,
merged_at: '2019-08-18T22:00:00.000Z',
},
],
[
{
metric: 138,
merged_at: '2019-08-17T22:00:00.000Z',
},
],
];
mutations[types.RECEIVE_CHART_DATA_SUCCESS](state, {
chartKey: chartKeys.scatterplot,
data: mockScatterplotData,
transformedData,
});
expect(state.charts[chartKey].isLoading).toBe(false);
expect(state.charts[chartKey].errorCode).toBe(null);
expect(state.charts[chartKey].data).toEqual(mockScatterplotData);
expect(state.charts[chartKey].transformedData).toEqual(transformedData);
});
});
describe(types.RECEIVE_CHART_DATA_ERROR, () => {
const status = 500;
beforeEach(() => {
mutations[types.RECEIVE_CHART_DATA_ERROR](state, { chartKey, status });
});
it('sets errorCode to 500', () => {
mutations[types.RECEIVE_CHART_DATA_ERROR](state, { chartKey, status });
expect(state.charts[chartKey].isLoading).toBe(false);
expect(state.charts[chartKey].errorCode).toBe(status);
});
it('clears data', () => {
mutations[types.RECEIVE_CHART_DATA_ERROR](state, { chartKey, status });
expect(state.charts[chartKey].isLoading).toBe(false);
expect(state.charts[chartKey].data).toEqual({});
});
it('clears transformedData when chartKey=scatterplot', () => {
mutations[types.RECEIVE_CHART_DATA_ERROR](state, { chartKey: chartKeys.scatterplot, status });
expect(state.charts[chartKey].transformedData).toEqual([]);
});
});
describe(types.SET_METRIC_TYPE, () => {
......
ee/spec/frontend/analytics/productivity_analytics/utils_spec.js
View file @ 0ccf9639
......@@ -4,13 +4,9 @@ import {
initDateArray,
transformScatterData,
getScatterPlotData,
getScatterPlotDataNew,
getMedianLineData,
getMedianLineDataNew,
} from 'ee/analytics/productivity_analytics/utils';
import { mockScatterplotData } from './mock_data';
describe('Productivity Analytics utils', () => {
const namespacePath = 'gitlab-org';
const projectWithNamespace = 'gitlab-org/gitlab-test';
......@@ -72,26 +68,6 @@ describe('Productivity Analytics utils', () => {
});
describe('getScatterPlotData', () => {
it('filters out data before given "dateInPast", transforms the data and sorts by date ascending', () => {
const dateInPast = new Date(2019, 7, 9); // '2019-08-09T22:00:00.000Z';
const result = getScatterPlotData(mockScatterplotData, dateInPast);
const expected = [
['2019-08-09T22:00:00.000Z', 44],
['2019-08-10T22:00:00.000Z', 46],
['2019-08-11T22:00:00.000Z', 62],
['2019-08-12T22:00:00.000Z', 60],
['2019-08-13T22:00:00.000Z', 43],
['2019-08-14T22:00:00.000Z', 46],
['2019-08-15T22:00:00.000Z', 56],
['2019-08-16T22:00:00.000Z', 24],
['2019-08-17T22:00:00.000Z', 138],
['2019-08-18T22:00:00.000Z', 139],
];
expect(result).toEqual(expected);
});
});
describe('getScatterPlotDataNew', () => {
it('returns a subset of data for the given start and end date and flattens the data', () => {
const startDate = new Date('2019-08-02');
const endDate = new Date('2019-08-04');
......@@ -104,7 +80,7 @@ describe('Productivity Analytics utils', () => {
{ merged_at: '2019-08-04T16:00:00.000Z', metric: 60 },
],
];
const result = getScatterPlotDataNew(data, startDate, endDate);
const result = getScatterPlotData(data, startDate, endDate);
const expected = [
['2019-08-02', 30, '2019-08-02T13:00:00.000Z'],
['2019-08-03', 40, '2019-08-03T14:00:00.000Z'],
......@@ -116,25 +92,6 @@ describe('Productivity Analytics utils', () => {
});
describe('getMedianLineData', () => {
const daysOffset = 10;
it(`computes the median for every item in the scatterData array for the past ${daysOffset} days`, () => {
const scatterData = [
['2019-08-16T22:00:00.000Z', 24],
['2019-08-17T22:00:00.000Z', 138],
['2019-08-18T22:00:00.000Z', 139],
];
const result = getMedianLineData(mockScatterplotData, scatterData, daysOffset);
const expected = [
['2019-08-16T22:00:00.000Z', 51],
['2019-08-17T22:00:00.000Z', 51],
['2019-08-18T22:00:00.000Z', 56],
];
expect(result).toEqual(expected);
});
});
describe('getMedianLineDataNew', () => {
const daysOffset = 2;
it(`computes the median for every date in the data array based on the past ${daysOffset} days`, () => {
......@@ -151,7 +108,7 @@ describe('Productivity Analytics utils', () => {
[{ merged_at: '2019-08-05T17:00:00.000Z', metric: 70 }],
[{ merged_at: '2019-08-06T18:00:00.000Z', metric: 80 }],
];
const result = getMedianLineDataNew(data, startDate, endDate, daysOffset);
const result = getMedianLineData(data, startDate, endDate, daysOffset);
const expected = [['2019-08-04', 45], ['2019-08-05', 55], ['2019-08-06', 65]];
expect(result).toEqual(expected);
});
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7