Merge branch '215517-add-dag-utils' into 'master'

Add DAG data parsing utils

See merge request gitlab-org/gitlab!32768

app/assets/javascripts/pipelines/components/dag/utils.js

+import { sankey, sankeyLeft } from 'd3-sankey';
+import { uniqWith, isEqual } from 'lodash';
+
+/*
+    The following functions are the main engine in transforming the data as
+    received from the endpoint into the format the d3 graph expects.
+
+    Input is of the form:
+    [stages]
+      stages: {name, groups}
+        groups: [{ name, size, jobs }]
+          name is a group name; in the case that the group has one job, it is
+            also the job name
+          size is the number of parallel jobs
+          jobs: [{ name, needs}]
+            job name is either the same as the group name or group x/y
+
+    Output is of the form:
+    { nodes: [node], links: [link] }
+      node: { name, category }, + unused info passed through
+      link: { source, target, value }, with source & target being node names
+        and value being a constant
+
+    We create nodes, create links, and then dedupe the links, so that in the case where
+    job 4 depends on job 1 and job 2, and job 2 depends on job 1, we show only a single link
+    from job 1 to job 2 then another from job 2 to job 4.
+
+    CREATE NODES
+    stage.name -> node.category
+    stage.group.name -> node.name (this is the group name if there are parallel jobs)
+    stage.group.jobs -> node.jobs
+    stage.group.size -> node.size
+
+    CREATE LINKS
+    stages.groups.name -> target
+    stages.groups.needs.each -> source (source is the name of the group, not the parallel job)
+    10 -> value (constant)
+  */
+
+export const createNodes = data => {
+  return data.flatMap(({ groups, name }) => {
+    return groups.map(group => {
+      return { ...group, category: name };
+    });
+  });
+};
+
+export const createNodeDict = nodes => {
+  return nodes.reduce((acc, node) => {
+    const newNode = {
+      ...node,
+      needs: node.jobs.map(job => job.needs || []).flat(),
+    };
+
+    if (node.size > 1) {
+      node.jobs.forEach(job => {
+        acc[job.name] = newNode;
+      });
+    }
+
+    acc[node.name] = newNode;
+    return acc;
+  }, {});
+};
+
+export const createNodesStructure = data => {
+  const nodes = createNodes(data);
+  const nodeDict = createNodeDict(nodes);
+
+  return { nodes, nodeDict };
+};
+
+export const makeLinksFromNodes = (nodes, nodeDict) => {
+  const constantLinkValue = 10; // all links are the same weight
+  return nodes
+    .map(group => {
+      return group.jobs.map(job => {
+        if (!job.needs) {
+          return [];
+        }
+
+        return job.needs.map(needed => {
+          return {
+            source: nodeDict[needed]?.name,
+            target: group.name,
+            value: constantLinkValue,
+          };
+        });
+      });
+    })
+    .flat(2);
+};
+
+export const getAllAncestors = (nodes, nodeDict) => {
+  const needs = nodes
+    .map(node => {
+      return nodeDict[node].needs || '';
+    })
+    .flat()
+    .filter(Boolean);
+
+  if (needs.length) {
+    return [...needs, ...getAllAncestors(needs, nodeDict)];
+  }
+
+  return [];
+};
+
+export const filterByAncestors = (links, nodeDict) =>
+  links.filter(({ target, source }) => {
+    /*
+
+    for every link, check out it's target
+    for every target, get the target node's needs
+    then drop the current link source from that list
+
+    call a function to get all ancestors, recursively
+    is the current link's source in the list of all parents?
+    then we drop this link
+
+  */
+    const targetNode = target;
+    const targetNodeNeeds = nodeDict[targetNode].needs;
+    const targetNodeNeedsMinusSource = targetNodeNeeds.filter(need => need !== source);
+
+    const allAncestors = getAllAncestors(targetNodeNeedsMinusSource, nodeDict);
+    return !allAncestors.includes(source);
+  });
+
+export const parseData = data => {
+  const { nodes, nodeDict } = createNodesStructure(data);
+  const allLinks = makeLinksFromNodes(nodes, nodeDict);
+  const filteredLinks = filterByAncestors(allLinks, nodeDict);
+  const links = uniqWith(filteredLinks, isEqual);
+
+  return { nodes, links };
+};
+
+/*
+    createSankey calls the d3 layout to generate the relationships and positioning
+    values for the nodes and links in the graph.
+  */
+
+export const createSankey = ({ width, height, nodeWidth, nodePadding, paddingForLabels }) => {
+  const sankeyGenerator = sankey()
+    .nodeId(({ name }) => name)
+    .nodeAlign(sankeyLeft)
+    .nodeWidth(nodeWidth)
+    .nodePadding(nodePadding)
+    .extent([
+      [paddingForLabels, paddingForLabels],
+      [width - paddingForLabels, height - paddingForLabels],
+    ]);
+  return ({ nodes, links }) =>
+    sankeyGenerator({
+      nodes: nodes.map(d => ({ ...d })),
+      links: links.map(d => ({ ...d })),
+    });
+};
+
+/*
+  The number of nodes in the most populous generation drives the height of the graph.
+*/
+
+export const getMaxNodes = nodes => {
+  const counts = nodes.reduce((acc, { layer }) => {
+    if (!acc[layer]) {
+      acc[layer] = 0;
+    }
+
+    acc[layer] += 1;
+
+    return acc;
+  }, []);
+
+  return Math.max(...counts);
+};
+
+/*
+  Because we cannot know if a node is part of a relationship until after we
+  generate the links with createSankey, this function is used after the first call
+  to find nodes that have no relations.
+*/
+
+export const removeOrphanNodes = sankeyfiedNodes => {
+  return sankeyfiedNodes.filter(node => node.sourceLinks.length || node.targetLinks.length);
+};

package.json

@@ -71,6 +71,7 @@
     "core-js": "^3.6.4",
     "cropper": "^2.3.0",
     "css-loader": "^2.1.1",
+    "d3-sankey": "^0.12.3",
     "d3-scale": "^2.2.2",
     "d3-selection": "^1.2.0",
     "dateformat": "^3.0.3",

spec/frontend/pipelines/components/dag/mock_data.js

+/*
+  It is important that the simple base include parallel jobs
+  as well as non-parallel jobs with spaces in the name to prevent
+  us relying on spaces as an indicator.
+*/
+export default {
+  stages: [
+    {
+      name: 'test',
+      groups: [
+        {
+          name: 'jest',
+          size: 2,
+          jobs: [{ name: 'jest 1/2', needs: ['frontend fixtures'] }, { name: 'jest 2/2' }],
+        },
+        {
+          name: 'rspec',
+          size: 1,
+          jobs: [{ name: 'rspec', needs: ['frontend fixtures'] }],
+        },
+      ],
+    },
+    {
+      name: 'fixtures',
+      groups: [
+        {
+          name: 'frontend fixtures',
+          size: 1,
+          jobs: [{ name: 'frontend fixtures' }],
+        },
+      ],
+    },
+    {
+      name: 'un-needed',
+      groups: [
+        {
+          name: 'un-needed',
+          size: 1,
+          jobs: [{ name: 'un-needed' }],
+        },
+      ],
+    },
+  ],
+};

spec/frontend/pipelines/components/dag/utils_spec.js

+import {
+  createNodesStructure,
+  makeLinksFromNodes,
+  filterByAncestors,
+  parseData,
+  createSankey,
+  removeOrphanNodes,
+  getMaxNodes,
+} from '~/pipelines/components/dag/utils';
+
+import mockGraphData from './mock_data';
+
+describe('DAG visualization parsing utilities', () => {
+  const { nodes, nodeDict } = createNodesStructure(mockGraphData.stages);
+  const unfilteredLinks = makeLinksFromNodes(nodes, nodeDict);
+  const parsed = parseData(mockGraphData.stages);
+
+  const layoutSettings = {
+    width: 200,
+    height: 200,
+    nodeWidth: 10,
+    nodePadding: 20,
+    paddingForLabels: 100,
+  };
+
+  const sankeyLayout = createSankey(layoutSettings)(parsed);
+
+  describe('createNodesStructure', () => {
+    const parallelGroupName = 'jest';
+    const parallelJobName = 'jest 1/2';
+    const singleJobName = 'frontend fixtures';
+
+    const { name, jobs, size } = mockGraphData.stages[0].groups[0];
+
+    it('returns the expected node structure', () => {
+      expect(nodes[0]).toHaveProperty('category', mockGraphData.stages[0].name);
+      expect(nodes[0]).toHaveProperty('name', name);
+      expect(nodes[0]).toHaveProperty('jobs', jobs);
+      expect(nodes[0]).toHaveProperty('size', size);
+    });
+
+    it('adds needs to top level of nodeDict entries', () => {
+      expect(nodeDict[parallelGroupName]).toHaveProperty('needs');
+      expect(nodeDict[parallelJobName]).toHaveProperty('needs');
+      expect(nodeDict[singleJobName]).toHaveProperty('needs');
+    });
+
+    it('makes entries in nodeDict for jobs and parallel jobs', () => {
+      const nodeNames = Object.keys(nodeDict);
+
+      expect(nodeNames.includes(parallelGroupName)).toBe(true);
+      expect(nodeNames.includes(parallelJobName)).toBe(true);
+      expect(nodeNames.includes(singleJobName)).toBe(true);
+    });
+  });
+
+  describe('makeLinksFromNodes', () => {
+    it('returns the expected link structure', () => {
+      expect(unfilteredLinks[0]).toHaveProperty('source', 'frontend fixtures');
+      expect(unfilteredLinks[0]).toHaveProperty('target', 'jest');
+      expect(unfilteredLinks[0]).toHaveProperty('value', 10);
+    });
+  });
+
+  describe('filterByAncestors', () => {
+    const allLinks = [
+      { source: 'job1', target: 'job4' },
+      { source: 'job1', target: 'job2' },
+      { source: 'job2', target: 'job4' },
+    ];
+
+    const dedupedLinks = [{ source: 'job1', target: 'job2' }, { source: 'job2', target: 'job4' }];
+
+    const nodeLookup = {
+      job1: {
+        name: 'job1',
+      },
+      job2: {
+        name: 'job2',
+        needs: ['job1'],
+      },
+      job4: {
+        name: 'job4',
+        needs: ['job1', 'job2'],
+        category: 'build',
+      },
+    };
+
+    it('dedupes links', () => {
+      expect(filterByAncestors(allLinks, nodeLookup)).toMatchObject(dedupedLinks);
+    });
+  });
+
+  describe('parseData parent function', () => {
+    it('returns an object containing a list of nodes and links', () => {
+      // an array of nodes exist and the values are defined
+      expect(parsed).toHaveProperty('nodes');
+      expect(Array.isArray(parsed.nodes)).toBe(true);
+      expect(parsed.nodes.filter(Boolean)).not.toHaveLength(0);
+
+      // an array of links exist and the values are defined
+      expect(parsed).toHaveProperty('links');
+      expect(Array.isArray(parsed.links)).toBe(true);
+      expect(parsed.links.filter(Boolean)).not.toHaveLength(0);
+    });
+  });
+
+  describe('createSankey', () => {
+    it('returns a nodes data structure with expected d3-added properties', () => {
+      expect(sankeyLayout.nodes[0]).toHaveProperty('sourceLinks');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('targetLinks');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('depth');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('layer');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('x0');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('x1');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('y0');
+      expect(sankeyLayout.nodes[0]).toHaveProperty('y1');
+    });
+
+    it('returns a links data structure with expected d3-added properties', () => {
+      expect(sankeyLayout.links[0]).toHaveProperty('source');
+      expect(sankeyLayout.links[0]).toHaveProperty('target');
+      expect(sankeyLayout.links[0]).toHaveProperty('width');
+      expect(sankeyLayout.links[0]).toHaveProperty('y0');
+      expect(sankeyLayout.links[0]).toHaveProperty('y1');
+    });
+
+    describe('data structure integrity', () => {
+      const newObject = { name: 'bad-actor' };
+
+      beforeEach(() => {
+        sankeyLayout.nodes.unshift(newObject);
+      });
+
+      it('sankey does not propagate changes back to the original', () => {
+        expect(sankeyLayout.nodes[0]).toBe(newObject);
+        expect(parsed.nodes[0]).not.toBe(newObject);
+      });
+
+      afterEach(() => {
+        sankeyLayout.nodes.shift();
+      });
+    });
+  });
+
+  describe('removeOrphanNodes', () => {
+    it('removes sankey nodes that have no needs and are not needed', () => {
+      const cleanedNodes = removeOrphanNodes(sankeyLayout.nodes);
+      expect(cleanedNodes).toHaveLength(sankeyLayout.nodes.length - 1);
+    });
+  });
+
+  describe('getMaxNodes', () => {
+    it('returns the number of nodes in the most populous generation', () => {
+      const layerNodes = [
+        { layer: 0 },
+        { layer: 0 },
+        { layer: 1 },
+        { layer: 1 },
+        { layer: 0 },
+        { layer: 3 },
+        { layer: 2 },
+        { layer: 4 },
+        { layer: 1 },
+        { layer: 3 },
+        { layer: 4 },
+      ];
+      expect(getMaxNodes(layerNodes)).toBe(3);
+    });
+  });
+});

yarn.lock

@@ -3355,7 +3355,7 @@ cyclist@~0.2.2:
   resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-0.2.2.tgz#1b33792e11e914a2fd6d6ed6447464444e5fa640"
   integrity sha1-GzN5LhHpFKL9bW7WRHRkRE5fpkA=
 
-d3-array@1, d3-array@^1.1.1, d3-array@^1.2.0:
+d3-array@1, "d3-array@1 - 2", d3-array@^1.1.1, d3-array@^1.2.0:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.1.tgz#d1ca33de2f6ac31efadb8e050a021d7e2396d5dc"
   integrity sha512-CyINJQ0SOUHojDdFDH4JEM0552vCR1utGyLHegJHyYH0JyCpSeTPxi4OBqHMA2jJZq4NH782LtaJWBImqI/HBw==
@@ -3489,6 +3489,14 @@ d3-random@1:
   resolved "https://registry.yarnpkg.com/d3-random/-/d3-random-1.1.0.tgz#6642e506c6fa3a648595d2b2469788a8d12529d3"
   integrity sha1-ZkLlBsb6OmSFldKyRpeIqNElKdM=
 
+d3-sankey@^0.12.3:
+  version "0.12.3"
+  resolved "https://registry.yarnpkg.com/d3-sankey/-/d3-sankey-0.12.3.tgz#b3c268627bd72e5d80336e8de6acbfec9d15d01d"
+  integrity sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==
+  dependencies:
+    d3-array "1 - 2"
+    d3-shape "^1.2.0"
+
 d3-scale-chromatic@1:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/d3-scale-chromatic/-/d3-scale-chromatic-1.3.3.tgz#dad4366f0edcb288f490128979c3c793583ed3c0"
@@ -3514,10 +3522,10 @@ d3-selection@1, d3-selection@^1.1.0, d3-selection@^1.2.0:
   resolved "https://registry.yarnpkg.com/d3-selection/-/d3-selection-1.3.0.tgz#d53772382d3dc4f7507bfb28bcd2d6aed2a0ad6d"
   integrity sha512-qgpUOg9tl5CirdqESUAu0t9MU/t3O9klYfGfyKsXEmhyxyzLpzpeh08gaxBUTQw1uXIOkr/30Ut2YRjSSxlmHA==
 
-d3-shape@1:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-1.2.0.tgz#45d01538f064bafd05ea3d6d2cb748fd8c41f777"
-  integrity sha1-RdAVOPBkuv0F6j1tLLdI/YxB93c=
+d3-shape@1, d3-shape@^1.2.0:
+  version "1.3.7"
+  resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-1.3.7.tgz#df63801be07bc986bc54f63789b4fe502992b5d7"
+  integrity sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==
   dependencies:
     d3-path "1"