bundle diff endpoint (#140)

* provide a `diff/$old_bundle/$new_bundle` endpoint on qontract-server that provides detailed diffs between two bundles currently present in the qontract-server

this endpoint will be used during PR checks to find involved schemas for an app-interface MR. the diff endpoint needs to expose diffs for resource files as well along with their relation to the referencing data file. this way, we don't have to guess the involved schemas anymore during PR checks but have an explicit and correct way to find them all.

The endpoint returns a list of changes with the following fields

* datafilepath - the path of the data file that changed
* datafileschema - the schema of the datafile that changed
* resourcepath - the path of the resource that changed in the context of the datafile
* action:
** `E` if a field changed between the two bundles
** `A` if a field was added in the new bundle
** `D` if a field was deleted in the new bundle
* jsonpath: the jsonpath pointing to the field or subtree that changed in the datafile
* new - holds the data object from the new bundle - empty if a data file is deleted
* old - holds the data object from the old bundle - empty if a new data file is introduced

resource file changes are reported in the context of their referencing data file.

```yaml
[
    {
      "datafilepath": "/services/api-designer/cicd/ci-int/secrets.yaml",
      "datafileschema": "/dependencies/jenkins-config-1.yml",
      "action": "E",
      "jsonpath": "config_path",
      "resourcepath": "/jenkins/api-designer/secrets.yaml",
      "old": { ... },
      "new": { ... }
    }
    {
      "datafilepath": "/integrations/user-validator.yml",
      "datafileschema": "/app-sre/integration-1.yml",
      "action": "E",
      "jsonpath": "upstream",
      "old": { ... },
      "new": { ... }
   }
]
```

depends loosely on app-sre/qontract-schemas#160 and app-sre/qontract-validator#37. if backrefs are not present in the bundle, the diff endpoint only exposes differences in data

part of https://issues.redhat.com/browse/APPSRE-5629

Signed-off-by: Gerd Oberlechner <[email protected]>

package.json

@@ -20,6 +20,7 @@
     "aws-sdk": "^2.364.0",
     "bufferutil": "^4.0.1",
     "dotenv": "^8.2.0",
+    "deep-diff": "1.0.2",
     "express": "^4.16.4",
     "express-prom-bundle": "^6.0.0",
     "fs": "0.0.1-security",

src/db.ts

@@ -15,10 +15,18 @@ export type Datafile = {
   [key: string]: any;
 };
 
+export type ResourcefileBackRef = {
+  path: string;
+  datafileSchema: string;
+  type: string;
+  jsonpath: string;
+};
+
 export type Resourcefile = {
   path: string;
   content: string;
-  shasum256: string;
+  sha256sum: string;
+  backrefs: ResourcefileBackRef[];
 };
 
 export type Bundle = {

src/server.ts

@@ -1,6 +1,9 @@
 import { ApolloServer } from 'apollo-server-express';
 import * as express from 'express';
 import promClient = require('prom-client');
+import * as im from 'immutable';
+
+const deepDiff = require('deep-diff');
 
 import * as db from './db';
 import * as metrics from './metrics';
@@ -179,6 +182,73 @@ export const appFromBundle = async (bundlePromises: Promise<db.Bundle>[]) => {
     res.send(req.app.get('bundles')[bundleSha].fileHash);
   });
 
+  app.get('/diff/:base_sha/:head_sha', (req: express.Request, res: express.Response) => {
+    const baseBundle: db.Bundle = req.app.get('bundles')[req.params.base_sha];
+    const headBundle: db.Bundle = req.app.get('bundles')[req.params.head_sha];
+    const dataDiffs = deepDiff(baseBundle.datafiles.toJS(), headBundle.datafiles.toJS());
+
+    const resourceDiffs = deepDiff(
+      im.Map(
+        Array.from(
+          baseBundle.resourcefiles, ([path, resource]) => [path, resource.sha256sum],
+        ),
+      ).toJS(),
+      im.Map(
+        Array.from(
+          headBundle.resourcefiles, ([path, resource]) => [path, resource.sha256sum],
+        ),
+      ).toJS(),
+    );
+
+    const changes = [];
+    for (const d in resourceDiffs) {
+      const diff = resourceDiffs[d];
+      const path = diff['path'][0];
+      const backrefs = diff['kind'] === 'D' ?
+        baseBundle.resourcefiles.get(path).backrefs :
+        headBundle.resourcefiles.get(path).backrefs;
+      for (const backrefIndex in headBundle.resourcefiles.get(path).backrefs) {
+        const backref = headBundle.resourcefiles.get(path).backrefs[backrefIndex];
+        const oldRes = baseBundle.datafiles.get(backref.path);
+        const newRes = headBundle.datafiles.get(backref.path);
+        changes.push(
+          {
+            resourcepath: path,
+            datafilepath: backref.path,
+            datafileschema: backref.datafileSchema,
+            action: diff['kind'],
+            jsonpath: backref.jsonpath,
+            old: oldRes,
+            new: newRes,
+          },
+        );
+      }
+    }
+
+    for (const d in dataDiffs) {
+      const diff = dataDiffs[d];
+      const oldRes = baseBundle.datafiles.get(diff['path'][0]);
+      const newRes = headBundle.datafiles.get(diff['path'][0]);
+      const path = diff['path'].slice(1);
+      for (const i in path) {
+        if (Number.isInteger(path[i])) {
+          path[i] = `[${path[i]}]`;
+        }
+      }
+      changes.push(
+        {
+          datafilepath: diff['path'][0],
+          datafileschema: (newRes !== undefined ? newRes : oldRes).$schema,
+          action: diff['kind'],
+          jsonpath: path.join('.'),
+          old: oldRes,
+          new: newRes,
+        },
+      );
+    }
+    res.send(changes);
+  });
+
   app.get('/git-commit', (req: express.Request, res: express.Response) => {
     const bundleSha = req.app.get('latestBundleSha');
     res.send(req.app.get('bundles')[bundleSha].gitCommit);

test/diff/diff.test.ts

@@ -0,0 +1,57 @@
+import * as http from 'http';
+
+import * as chai from 'chai';
+
+// Chai is bad with types. See:
+// https://github.com/DefinitelyTyped/DefinitelyTyped/issues/19480
+import chaiHttp = require('chai-http');
+chai.use(chaiHttp);
+
+import * as server from '../../src/server';
+import * as db from '../../src/db';
+import { logger } from '../../src/logger';
+
+const should = chai.should();
+
+const diskBundles = 'test/diff/old.data.json,test/diff/new.data.json';
+const oldSha = 'bf56095bf2ada36a6b2deca9cb9b6616d536b5c9ce230f0905296165d221a66b';
+const newSha = '302071115aa5dda8559f6e582fa7b6db7e0b64b5a9a6a9e3e9c22e2f86567f4b';
+
+describe('diff', async() => {
+  let srv: http.Server;
+  before(async() => {
+    process.env.INIT_DISK_BUNDLES = diskBundles;
+    const app = await server.appFromBundle(db.getInitialBundles());
+    srv = app.listen({ port: 4000 });
+  });
+
+  it('GET /sha256 returns a valid sha256', async () => {
+    const response = await chai.request(srv).get('/sha256');
+    return response.text.should.equal(newSha);
+  });
+
+  it('serve diff', async() => {
+    const resp = await chai.request(srv)
+                           .get(`/diff/${oldSha}/${newSha}`);
+    resp.should.have.status(200);
+    logger.info(JSON.stringify(resp.body));
+
+    resp.body.length.should.equal(2);
+
+    const changed = resp.body[0];
+    changed.datafilepath.should.equal('/cluster.yml');
+    changed.datafileschema.should.equal('/openshift/cluster-1.yml');
+    changed.action.should.equal('E');
+    changed.jsonpath.should.equal('important.resource');
+
+    const resource = resp.body[1];
+    resource.datafilepath.should.equal('/cluster.yml');
+    resource.datafileschema.should.equal('/openshift/cluster-1.yml');
+    resource.action.should.equal('E');
+    resource.jsonpath.should.equal('automationToken.path');
+  });
+
+  after(() => {
+    delete process.env.INIT_DISK_BUNDLES;
+  });
+});

test/diff/new.data.json

@@ -0,0 +1,100 @@
+{
+  "data": {
+    "/cluster.yml": {
+      "$schema": "/openshift/cluster-1.yml",
+      "labels": {},
+      "serverUrl": "https://example.com",
+      "description": "example cluster",
+      "name": "example cluster",
+      "automationToken": {
+        "path": "secret-new"
+      }
+    }
+  },
+  "graphql": [
+    {
+      "fields": [
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "schema"
+        },
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "path"
+        },
+        {
+          "type": "json",
+          "name": "labels"
+        },
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "name"
+        },
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "description"
+        },
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "serverUrl"
+        },
+        {
+          "type": "VaultSecret_v1",
+          "name": "automationToken"
+        }
+      ],
+      "name": "Cluster_v1"
+    },
+    {
+      "fields": [
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "path"
+        },
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "field"
+        },
+        {
+          "type": "string",
+          "name": "format"
+        }
+      ],
+      "name": "VaultSecret_v1"
+    },
+    {
+      "fields": [
+        {
+          "type": "Cluster_v1",
+          "name": "clusters_v1",
+          "isList": true,
+          "datafileSchema": "/openshift/cluster-1.yml"
+        }
+      ],
+      "name": "Query"
+    }
+  ],
+  "resources": {
+    "/changed_resource.yml": {
+      "path": "/changed_resource.yml",
+      "content": "",
+      "$schema": null,
+      "sha256sum": "new_sha",
+      "backrefs": [
+        {
+            "path": "/cluster.yml",
+            "datafileSchema": "/openshift/cluster-1.yml",
+            "type": "Cluster_v1",
+            "jsonpath": "important.resource"
+        }
+      ]
+    }
+  }
+}

test/diff/old.data.json

@@ -0,0 +1,100 @@
+{
+  "data": {
+    "/cluster.yml": {
+      "$schema": "/openshift/cluster-1.yml",
+      "labels": {},
+      "serverUrl": "https://example.com",
+      "description": "example cluster",
+      "name": "example cluster",
+      "automationToken": {
+        "path": "secret-old"
+      }
+    }
+  },
+  "graphql": [
+    {
+      "fields": [
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "schema"
+        },
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "path"
+        },
+        {
+          "type": "json",
+          "name": "labels"
+        },
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "name"
+        },
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "description"
+        },
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "serverUrl"
+        },
+        {
+          "type": "VaultSecret_v1",
+          "name": "automationToken"
+        }
+      ],
+      "name": "Cluster_v1"
+    },
+    {
+      "fields": [
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "path"
+        },
+        {
+          "isRequired": true,
+          "type": "string",
+          "name": "field"
+        },
+        {
+          "type": "string",
+          "name": "format"
+        }
+      ],
+      "name": "VaultSecret_v1"
+    },
+    {
+      "fields": [
+        {
+          "type": "Cluster_v1",
+          "name": "clusters_v1",
+          "isList": true,
+          "datafileSchema": "/openshift/cluster-1.yml"
+        }
+      ],
+      "name": "Query"
+    }
+  ],
+  "resources": {
+    "/changed_resource.yml": {
+      "path": "/changed_resource.yml",
+      "content": "",
+      "$schema": null,
+      "sha256sum": "old_sha",
+      "backrefs": [
+        {
+            "path": "/cluster.yml",
+            "datafileSchema": "/openshift/cluster-1.yml",
+            "type": "Cluster_v1",
+            "jsonpath": "important.resource"
+        }
+      ]
+    }
+  }
+}

test/schemas/schemas.test.ts

@@ -48,7 +48,7 @@ describe('clusters', async() => {
     return resp.text.should.eql('242acb1998e9d37c26186ba9be0262fb34e3ef388b503390d143164f7658c24e');
   });
 
-  before(() => {
+  after(() => {
     delete process.env.INIT_DISK_BUNDLES;
   });
 });

yarn.lock

@@ -1576,6 +1576,11 @@ decode-uri-component@^0.2.0:
   resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545"
   integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=
 
+[email protected]:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/deep-diff/-/deep-diff-1.0.2.tgz#afd3d1f749115be965e89c63edc7abb1506b9c26"
+  integrity sha512-aWS3UIVH+NPGCD1kki+DCU9Dua032iSsO43LqQpcs4R3+dVv7tX0qBGjiVHJHjplsoUM2XRO/KB92glqc68awg==
+
 deep-eql@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df"