Commit 4eeb3b33 authored by Kushal Pandya's avatar Kushal Pandya

Merge branch 'usage-statistics-integrate-with-backend-apis' into 'master'

Integrate usage statistics with new API

See merge request gitlab-org/gitlab!44945
parents 3ba3f447 e4adbbb7
......@@ -6,8 +6,8 @@ import UsageGraph from './usage_graph.vue';
import UsageStatistics from './usage_statistics.vue';
import query from '../queries/storage.query.graphql';
import TemporaryStorageIncreaseModal from './temporary_storage_increase_modal.vue';
import { numberToHumanSize } from '~/lib/utils/number_utils';
import { parseBoolean } from '~/lib/utils/common_utils';
import { formatUsageSize, parseGetStorageResults } from '../utils';
export default {
name: 'StorageCounterApp',
......@@ -51,23 +51,10 @@ export default {
variables() {
return {
fullPath: this.namespacePath,
withExcessStorageData: this.isAdditionalStorageFlagEnabled,
};
},
/**
* `rootStorageStatistics` will be sent as null until an
* event happens to trigger the storage count.
* For that reason we have to verify if `storageSize` is sent or
* if we should render N/A
*/
update: data => ({
projects: data.namespace.projects.edges.map(({ node }) => node),
totalUsage:
data.namespace.rootStorageStatistics && data.namespace.rootStorageStatistics.storageSize
? numberToHumanSize(data.namespace.rootStorageStatistics.storageSize)
: 'N/A',
rootStorageStatistics: data.namespace.rootStorageStatistics,
limit: data.namespace.storageSizeLimit,
}),
update: parseGetStorageResults,
},
},
data() {
......@@ -85,10 +72,19 @@ export default {
isAdditionalStorageFlagEnabled() {
return this.glFeatures.additionalRepoStorageByNamespace;
},
},
methods: {
formatSize(size) {
return numberToHumanSize(size);
formattedNamespaceLimit() {
return formatUsageSize(this.namespace.limit);
},
storageStatistics() {
if (!this.namespace) {
return null;
}
return {
totalRepositorySize: this.namespace.totalRepositorySize,
totalRepositorySizeExcess: this.namespace.totalRepositorySizeExcess,
additionalPurchasedStorageSize: this.namespace.additionalPurchasedStorageSize,
};
},
},
modalId: 'temporary-increase-storage-modal',
......@@ -96,8 +92,8 @@ export default {
</script>
<template>
<div>
<div v-if="isAdditionalStorageFlagEnabled && namespace.rootStorageStatistics">
<usage-statistics :root-storage-statistics="namespace.rootStorageStatistics" />
<div v-if="isAdditionalStorageFlagEnabled && storageStatistics">
<usage-statistics :root-storage-statistics="storageStatistics" />
</div>
<div v-else class="gl-py-4 gl-px-2 gl-m-0">
<div class="gl-display-flex gl-align-items-center">
......@@ -114,7 +110,7 @@ export default {
:message="s__('UsageQuota|out of %{formattedLimit} of your namespace storage')"
>
<template #formattedLimit>
<span class="gl-font-weight-bold">{{ formatSize(namespace.limit) }}</span>
<span class="gl-font-weight-bold">{{ formattedNamespaceLimit }}</span>
</template>
</gl-sprintf>
</template>
......@@ -156,7 +152,7 @@ export default {
<projects-table :projects="namespaceProjects" />
<temporary-storage-increase-modal
v-if="isStorageIncreaseModalVisible"
:limit="formatSize(namespace.limit)"
:limit="formattedNamespaceLimit"
:modal-id="$options.modalId"
/>
</div>
......
......@@ -8,8 +8,9 @@
import { GlLink, GlIcon, GlTooltipDirective } from '@gitlab/ui';
import { s__ } from '~/locale';
import ProjectAvatar from '~/vue_shared/components/project_avatar/default.vue';
import { numberToHumanSize } from '~/lib/utils/number_utils';
import { getIdFromGraphQLId } from '~/graphql_shared/utils';
import { formatUsageSize, usageRatioToThresholdLevel } from '../utils';
import { ALERT_THRESHOLD, ERROR_THRESHOLD, WARNING_THRESHOLD } from '../constants';
export default {
components: {
......@@ -40,29 +41,29 @@ export default {
return this.project.nameWithNamespace;
},
storageSize() {
return numberToHumanSize(this.project.statistics.storageSize);
return formatUsageSize(this.project.totalCalculatedUsedStorage);
},
excessStorageSize() {
return numberToHumanSize(this.project.statistics?.excessStorageSize ?? 0);
return formatUsageSize(this.project.repositorySizeExcess);
},
excessStorageRatio() {
return this.project.totalCalculatedUsedStorage / this.project.totalCalculatedStorageLimit;
},
thresholdLevel() {
return usageRatioToThresholdLevel(this.excessStorageRatio);
},
status() {
// The project default limit will be sent by backend.
// This is being added here just for testing purposes.
// This entire component is rendered behind the
// additional_repo_storage_by_namespace feature flag. This
// piece will be removed along with the flag and the logic
// will be mostly on the backend.
const PROJECT_DEFAULT_LIMIT = 10000000000;
const PROJECT_DEFAULT_WARNING_LIMIT = 9000000000;
if (this.project.statistics.storageSize > PROJECT_DEFAULT_LIMIT) {
if (this.thresholdLevel === ERROR_THRESHOLD) {
return {
bgColor: { 'gl-bg-red-50': true },
iconClass: { 'gl-text-red-500': true },
linkClass: 'gl-text-red-500!',
tooltipText: s__('UsageQuota|This project is locked.'),
};
} else if (this.project.statistics.storageSize > PROJECT_DEFAULT_WARNING_LIMIT) {
} else if (
this.thresholdLevel === WARNING_THRESHOLD ||
this.thresholdLevel === ALERT_THRESHOLD
) {
return {
bgColor: { 'gl-bg-orange-50': true },
iconClass: 'gl-text-orange-500',
......
......@@ -2,7 +2,7 @@
import { GlAlert } from '@gitlab/ui';
import { n__, __ } from '~/locale';
import { getFormatter, SUPPORTED_FORMATS } from '~/lib/utils/unit_format';
import { usageRatioToThresholdLevel } from '../usage_thresholds';
import { usageRatioToThresholdLevel } from '../utils';
import { ALERT_THRESHOLD, ERROR_THRESHOLD, WARNING_THRESHOLD } from '../constants';
export default {
......
......@@ -2,8 +2,7 @@
import { GlButton } from '@gitlab/ui';
import UsageStatisticsCard from './usage_statistics_card.vue';
import { s__ } from '~/locale';
import { bytesToKB } from '~/lib/utils/number_utils';
import { getFormatter, SUPPORTED_FORMATS } from '~/lib/utils/unit_format';
import { formatUsageSize } from '../utils';
export default {
components: {
......@@ -18,9 +17,8 @@ export default {
},
computed: {
totalUsage() {
const { repositorySize = 0, lfsObjectsSize = 0 } = this.rootStorageStatistics;
return {
usage: this.formatSize(repositorySize + lfsObjectsSize),
usage: this.formatSize(this.rootStorageStatistics.totalRepositorySize),
description: s__('UsageQuota|Total namespace storage used'),
link: {
text: s__('UsageQuota|Learn more about usage quotas'),
......@@ -30,7 +28,7 @@ export default {
},
excessUsage() {
return {
usage: this.formatSize(0),
usage: this.formatSize(this.rootStorageStatistics.totalRepositorySizeExcess),
description: s__('UsageQuota|Total excess storage used'),
link: {
text: s__('UsageQuota|Learn more about excess storage usage'),
......@@ -39,8 +37,15 @@ export default {
};
},
purchasedUsage() {
const {
totalRepositorySizeExcess,
additionalPurchasedStorageSize,
} = this.rootStorageStatistics;
return {
usage: this.formatSize(0),
usage: this.formatSize(
Math.max(0, additionalPurchasedStorageSize - totalRepositorySizeExcess),
),
usageTotal: this.formatSize(additionalPurchasedStorageSize),
description: s__('UsageQuota|Purchased storage available'),
link: {
text: s__('UsageQuota|Purchase more storage'),
......@@ -51,25 +56,20 @@ export default {
},
methods: {
/**
* The formatDecimalBytes method returns
* The formatUsageSize method returns
* value along with the unit. However, the unit
* and the value needs to be separated so that
* they can have different styles. The method
* splits the value into value and unit.
*
* We want to display all units above bytes. Hence
* converting bytesToKB before passing it to
* `getFormatter`
*
* @params {Number} size size in bytes
* @returns {Object} value and unit of formatted size
*/
formatSize(size) {
const formatDecimalBytes = getFormatter(SUPPORTED_FORMATS.kilobytes);
const formattedSize = formatDecimalBytes(bytesToKB(size), 1);
const formattedSize = formatUsageSize(size);
return {
value: formattedSize.slice(0, -2),
unit: formattedSize.slice(-2),
value: formattedSize.slice(0, -3),
unit: formattedSize.slice(-3),
};
},
},
......@@ -94,6 +94,7 @@ export default {
<usage-statistics-card
data-testid="purchasedUsage"
:usage="purchasedUsage.usage"
:usage-total="purchasedUsage.usageTotal"
:link="purchasedUsage.link"
:description="purchasedUsage.description"
css-class="gl-ml-4"
......
......@@ -21,6 +21,11 @@ export default {
type: Object,
required: true,
},
usageTotal: {
type: Object,
required: false,
default: null,
},
cssClass: {
type: String,
required: false,
......@@ -40,6 +45,17 @@ export default {
<span class="gl-font-lg gl-font-weight-bold">{{ usage.unit }}</span>
</template>
</gl-sprintf>
<template v-if="usageTotal">
<span class="gl-font-size-h-display gl-font-weight-bold">/</span>
<gl-sprintf :message="__('%{size} %{unit}')">
<template #size>
<span class="gl-font-size-h-display gl-font-weight-bold">{{ usageTotal.value }}</span>
</template>
<template #unit>
<span class="gl-font-lg gl-font-weight-bold">{{ usageTotal.unit }}</span>
</template>
</gl-sprintf>
</template>
</p>
<p class="gl-border-b-2 gl-border-b-solid gl-border-b-gray-100 gl-font-weight-bold gl-pb-3">
{{ description }}
......
......@@ -3,3 +3,11 @@ export const INFO_THRESHOLD = 'info';
export const WARNING_THRESHOLD = 'warning';
export const ALERT_THRESHOLD = 'alert';
export const ERROR_THRESHOLD = 'error';
export const STORAGE_USAGE_THRESHOLDS = {
[NONE_THRESHOLD]: 0.0,
[INFO_THRESHOLD]: 0.5,
[WARNING_THRESHOLD]: 0.75,
[ALERT_THRESHOLD]: 0.95,
[ERROR_THRESHOLD]: 1.0,
};
import {
ALERT_THRESHOLD,
ERROR_THRESHOLD,
INFO_THRESHOLD,
NONE_THRESHOLD,
WARNING_THRESHOLD,
} from './constants';
const STORAGE_USAGE_THRESHOLDS = {
[NONE_THRESHOLD]: 0.0,
[INFO_THRESHOLD]: 0.5,
[WARNING_THRESHOLD]: 0.75,
[ALERT_THRESHOLD]: 0.95,
[ERROR_THRESHOLD]: 1.0,
};
export function usageRatioToThresholdLevel(currentUsageRatio) {
let currentLevel = Object.keys(STORAGE_USAGE_THRESHOLDS)[0];
Object.keys(STORAGE_USAGE_THRESHOLDS).forEach(thresholdLevel => {
if (currentUsageRatio >= STORAGE_USAGE_THRESHOLDS[thresholdLevel])
currentLevel = thresholdLevel;
});
return currentLevel;
}
import { getFormatter, SUPPORTED_FORMATS } from '~/lib/utils/unit_format';
import { numberToHumanSize, bytesToKiB } from '~/lib/utils/number_utils';
import { STORAGE_USAGE_THRESHOLDS } from './constants';
export function usageRatioToThresholdLevel(currentUsageRatio) {
let currentLevel = Object.keys(STORAGE_USAGE_THRESHOLDS)[0];
Object.keys(STORAGE_USAGE_THRESHOLDS).forEach(thresholdLevel => {
if (currentUsageRatio >= STORAGE_USAGE_THRESHOLDS[thresholdLevel])
currentLevel = thresholdLevel;
});
return currentLevel;
}
/**
* Formats given bytes to formatted human readable size
*
* We want to display all units above bytes. Hence
* converting bytesToKiB before passing it to
* `getFormatter`
* @param {Number} size size in bytes
* @returns {String}
*/
export const formatUsageSize = size => {
const formatDecimalBytes = getFormatter(SUPPORTED_FORMATS.kibibytes);
return formatDecimalBytes(bytesToKiB(size), 1);
};
/**
* Parses each project to add additional purchased data
* equally so that locked projects can be unlocked.
*
* For example, if a group contains the below projects and
* project 2, 3 have exceeded the default 10.0 GB limit.
* 2 and 3 will remain locked until user purchases additional
* data.
*
* Project 1: 7.0GB
* Project 2: 13.0GB Locked
* Project 3: 12.0GB Locked
*
* If user purchases X GB, it will be equally available
* to all the locked projects for further use.
*
* @param {Object} data project
* @param {Number} purchasedStorageRemaining Remaining purchased data in bytes
* @returns {Object}
*/
export const calculateUsedAndRemStorage = (project, purchasedStorageRemaining) => {
// We only consider repo size and lfs object size as of %13.5
const totalCalculatedUsedStorage =
project.statistics.repositorySize + project.statistics.lfsObjectsSize;
// If a project size is above the default limit, then the remaining
// storage value will be calculated on top of the project size as
// opposed to the default limit.
// This
const totalCalculatedStorageLimit =
totalCalculatedUsedStorage > project.actualRepositorySizeLimit
? totalCalculatedUsedStorage + purchasedStorageRemaining
: project.actualRepositorySizeLimit + purchasedStorageRemaining;
return {
...project,
totalCalculatedUsedStorage,
totalCalculatedStorageLimit,
};
};
/**
* Parses projects coming in from GraphQL response
* and patches each project with purchased related
* data
*
* @param {Array} params.projects list of projects
* @param {Number} params.additionalPurchasedStorageSize Amt purchased in bytes
* @param {Number} params.totalRepositorySizeExcess Sum of excess amounts on all projects
* @returns {Array}
*/
export const parseProjects = ({
projects,
additionalPurchasedStorageSize = 0,
totalRepositorySizeExcess = 0,
}) => {
const purchasedStorageRemaining = Math.max(
0,
additionalPurchasedStorageSize - totalRepositorySizeExcess,
);
return projects.edges.map(({ node: project }) =>
calculateUsedAndRemStorage(project, purchasedStorageRemaining),
);
};
/**
* This method parses the results from `getStorageCounter`
* call.
*
* `rootStorageStatistics` will be sent as null until an
* event happens to trigger the storage count.
* For that reason we have to verify if `storageSize` is sent or
* if we should render N/A
*
* @param {Object} data graphql result
* @returns {Object}
*/
export const parseGetStorageResults = data => {
const {
namespace: {
projects,
storageSizeLimit,
totalRepositorySize,
containsLockedProjects,
totalRepositorySizeExcess,
rootStorageStatistics = {},
actualRepositorySizeLimit,
additionalPurchasedStorageSize,
repositorySizeExcessProjectCount,
},
} = data || {};
return {
projects: parseProjects({
projects,
additionalPurchasedStorageSize,
totalRepositorySizeExcess,
}),
additionalPurchasedStorageSize,
actualRepositorySizeLimit,
containsLockedProjects,
repositorySizeExcessProjectCount,
totalRepositorySize,
totalRepositorySizeExcess,
totalUsage: rootStorageStatistics.storageSize
? numberToHumanSize(rootStorageStatistics.storageSize)
: 'N/A',
rootStorageStatistics,
limit: storageSizeLimit,
};
};
......@@ -4,9 +4,9 @@ import Project from 'ee/storage_counter/components/project.vue';
import UsageGraph from 'ee/storage_counter/components/usage_graph.vue';
import UsageStatistics from 'ee/storage_counter/components/usage_statistics.vue';
import TemporaryStorageIncreaseModal from 'ee/storage_counter/components/temporary_storage_increase_modal.vue';
import { formatUsageSize } from 'ee/storage_counter/utils';
import { createMockDirective, getBinding } from 'helpers/vue_mock_directive';
import { namespaceData, withRootStorageStatistics } from '../mock_data';
import { numberToHumanSize } from '~/lib/utils/number_utils';
const TEST_LIMIT = 1000;
......@@ -73,7 +73,7 @@ describe('Storage counter app', () => {
await wrapper.vm.$nextTick();
expect(wrapper.text()).toContain(numberToHumanSize(namespaceData.limit));
expect(wrapper.text()).toContain(formatUsageSize(namespaceData.limit));
});
it('when limit is 0 it does not render limit information', async () => {
......@@ -83,7 +83,7 @@ describe('Storage counter app', () => {
await wrapper.vm.$nextTick();
expect(wrapper.text()).not.toContain(numberToHumanSize(0));
expect(wrapper.text()).not.toContain(formatUsageSize(0));
});
});
......@@ -200,7 +200,7 @@ describe('Storage counter app', () => {
it('renders modal', () => {
expect(wrapper.find(TemporaryStorageIncreaseModal).props()).toEqual({
limit: numberToHumanSize(TEST_LIMIT),
limit: formatUsageSize(TEST_LIMIT),
modalId: StorageApp.modalId,
});
});
......
......@@ -2,8 +2,8 @@ import { shallowMount } from '@vue/test-utils';
import { GlLink } from '@gitlab/ui';
import { createMockDirective, getBinding } from 'helpers/vue_mock_directive';
import ProjectWithExcessStorage from 'ee/storage_counter/components/project_with_excess_storage.vue';
import { formatUsageSize } from 'ee/storage_counter/utils';
import ProjectAvatar from '~/vue_shared/components/project_avatar/default.vue';
import { numberToHumanSize } from '~/lib/utils/number_utils';
import { projects } from '../mock_data';
let wrapper;
......@@ -43,7 +43,7 @@ describe('Storage Counter project component', () => {
});
it('renders formatted storage size', () => {
expect(wrapper.text()).toContain(numberToHumanSize(projects[0].statistics.storageSize));
expect(wrapper.text()).toContain(formatUsageSize(projects[0].statistics.storageSize));
});
it('does not render the warning icon if project is not in error state', () => {
......@@ -81,7 +81,7 @@ describe('Storage Counter project component', () => {
createComponent({ project: projects[1] });
});
it('with error state background', () => {
it('with warning state background', () => {
expect(findTableRow().classes('gl-bg-orange-50')).toBe(true);
});
......
......@@ -14,6 +14,9 @@ export const projects = [
buildArtifactsSize: 0,
packagesSize: 0,
},
actualRepositorySizeLimit: 100000,
totalCalculatedUsedStorage: 41943,
totalCalculatedStorageLimit: 41943000,
},
{
id: '8',
......@@ -24,12 +27,15 @@ export const projects = [
name: 'Html5 Boilerplate',
statistics: {
commitCount: 0,
storageSize: 9933460120,
storageSize: 99000,
repositorySize: 0,
lfsObjectsSize: 0,
buildArtifactsSize: 1272375,
packagesSize: 0,
},
actualRepositorySizeLimit: 100000,
totalCalculatedUsedStorage: 89000,
totalCalculatedStorageLimit: 99430,
},
{
id: '80',
......@@ -40,12 +46,15 @@ export const projects = [
name: 'Twitter',
statistics: {
commitCount: 0,
storageSize: 129334601203,
repositorySize: 0,
lfsObjectsSize: 0,
storageSize: 12933460,
repositorySize: 209710,
lfsObjectsSize: 209720,
buildArtifactsSize: 1272375,
packagesSize: 0,
},
actualRepositorySizeLimit: 100000,
totalCalculatedUsedStorage: 13143170,
totalCalculatedStorageLimit: 12143170,
},
];
......@@ -69,3 +78,7 @@ export const withRootStorageStatistics = {
snippetsSize: 10000,
},
};
export const mockGetStorageCounterGraphQLResponse = {
edges: projects.map(node => ({ node })),
};
import { usageRatioToThresholdLevel } from 'ee/storage_counter/usage_thresholds';
describe('UsageThreshold', () => {
it.each`
usageRatio | expectedLevel
${0} | ${'none'}
${0.4} | ${'none'}
${0.5} | ${'info'}
${0.9} | ${'warning'}
${0.99} | ${'alert'}
${1} | ${'error'}
${1.5} | ${'error'}
`('returns $expectedLevel from $usageRatio', ({ usageRatio, expectedLevel }) => {
expect(usageRatioToThresholdLevel(usageRatio)).toBe(expectedLevel);
});
});
import {
usageRatioToThresholdLevel,
formatUsageSize,
parseProjects,
calculateUsedAndRemStorage,
} from 'ee/storage_counter/utils';
import { projects as mockProjectsData, mockGetStorageCounterGraphQLResponse } from './mock_data';
describe('UsageThreshold', () => {
it.each`
usageRatio | expectedLevel
${0} | ${'none'}
${0.4} | ${'none'}
${0.5} | ${'info'}
${0.9} | ${'warning'}
${0.99} | ${'alert'}
${1} | ${'error'}
${1.5} | ${'error'}
`('returns $expectedLevel from $usageRatio', ({ usageRatio, expectedLevel }) => {
expect(usageRatioToThresholdLevel(usageRatio)).toBe(expectedLevel);
});
});
describe('formatUsageSize', () => {
it.each`
input | expected
${0} | ${'0.0KiB'}
${999} | ${'1.0KiB'}
${1000} | ${'1.0KiB'}
${10240} | ${'10.0KiB'}
${1024 * 10 ** 5} | ${'97.7MiB'}
${10 ** 6} | ${'976.6KiB'}
${1024 * 10 ** 6} | ${'976.6MiB'}
${10 ** 8} | ${'95.4MiB'}
${1024 * 10 ** 8} | ${'95.4GiB'}
${10 ** 10} | ${'9.3GiB'}
${10 ** 12} | ${'931.3GiB'}
${10 ** 15} | ${'909.5TiB'}
`('returns $expected from $input', ({ input, expected }) => {
expect(formatUsageSize(input)).toBe(expected);
});
});
describe('calculateUsedAndRemStorage', () => {
it.each`
description | project | purchasedStorageRemaining | totalCalculatedUsedStorage | totalCalculatedStorageLimit
${'project within limit and purchased 0'} | ${mockProjectsData[0]} | ${0} | ${41943} | ${100000}
${'project within limit and purchased 10000'} | ${mockProjectsData[0]} | ${100000} | ${41943} | ${200000}
${'project in warning state and purchased 0'} | ${mockProjectsData[1]} | ${0} | ${0} | ${100000}
${'project in warning state and purchased 10000'} | ${mockProjectsData[1]} | ${100000} | ${0} | ${200000}
${'project in error state and purchased 0'} | ${mockProjectsData[2]} | ${0} | ${419430} | ${419430}
${'project in error state and purchased 10000'} | ${mockProjectsData[2]} | ${100000} | ${419430} | ${519430}
`(
'returns used: $totalCalculatedUsedStorage and remaining: $totalCalculatedStorageLimit storage for $description',
({
project,
purchasedStorageRemaining,
totalCalculatedUsedStorage,
totalCalculatedStorageLimit,
}) => {
const result = calculateUsedAndRemStorage(project, purchasedStorageRemaining);
expect(result.totalCalculatedUsedStorage).toBe(totalCalculatedUsedStorage);
expect(result.totalCalculatedStorageLimit).toBe(totalCalculatedStorageLimit);
},
);
});
describe('parseProjects', () => {
it('ensures all projects have totalCalculatedUsedStorage and totalCalculatedStorageLimit', () => {
const projects = parseProjects({
projects: mockGetStorageCounterGraphQLResponse,
additionalPurchasedStorageSize: 10000,
totalRepositorySizeExcess: 5000,
});
projects.forEach(project => {
expect(project).toMatchObject({
totalCalculatedUsedStorage: expect.any(Number),
totalCalculatedStorageLimit: expect.any(Number),
});
});
});
});
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment