fix(deps): Updates Mongo to 4.2 (LLC-234) (#1541)

BREAKING CHANGE: Removes maxScan parameter from API
BREAKING CHANGE: Requires Mongo 4.2

.circleci/config.yml

@@ -4,7 +4,7 @@ jobs:
   build:
     docker:
       - image: circleci/node:10
-      - image: mongo:3.4-jessie
+      - image: mongo:4.2-bionic
       - image: redis:4-alpine
     environment:
       TERM: xterm

.env.example

@@ -87,10 +87,6 @@ REDIS_PREFIX=LEARNINGLOCKER
 # https://docs.mongodb.com/manual/reference/operator/meta/maxTimeMS/
 #MAX_TIME_MS=
 
-# Maximum number of documents an aggregation can scan
-# https://docs.mongodb.com/manual/reference/operator/meta/maxScan/
-#MAX_SCAN=
-
 # Turn off fuzzy scoring on persona matching
 # Set to true to make persona workers much faster at scale (default is true)
 #DISABLE_PERSONA_SCORING=true

api/src/controllers/PersonaController.js

@@ -4,7 +4,7 @@ import getJSONFromQuery from 'api/utils/getJSONFromQuery';
 import getFromQuery from 'api/utils/getFromQuery';
 import getOrgFromAuthInfo from 'lib/services/auth/authInfoSelectors/getOrgFromAuthInfo';
 import getScopeFilter from 'lib/services/auth/filters/getScopeFilter';
-import { MAX_TIME_MS, MAX_SCAN } from 'lib/models/plugins/addCRUDFunctions';
+import { MAX_TIME_MS } from 'lib/models/plugins/addCRUDFunctions';
 import parseQuery from 'lib/helpers/parseQuery';
 import { CursorDirection } from '@learninglocker/persona-service/dist/service/constants';
 import { entityResponse, entitiesResponse } from 'api/controllers/utils/entitiesResponse';
@@ -47,8 +47,7 @@ const personaConnection = catchErrors(async (req, res) => {
     filter,
     project,
     hint,
-    maxTimeMS: MAX_TIME_MS,
-    maxScan: MAX_SCAN
+    maxTimeMS: MAX_TIME_MS
   };
 
   const result = await personaService.getPersonasConnection(params);

api/src/controllers/PersonaIdentifierController.js

@@ -9,7 +9,7 @@ import getAuthFromRequest from 'lib/helpers/getAuthFromRequest';
 import getScopeFilter from 'lib/services/auth/filters/getScopeFilter';
 import { CursorDirection } from '@learninglocker/persona-service/dist/service/constants';
 import Locked from '@learninglocker/persona-service/dist/errors/Locked';
-import { MAX_TIME_MS, MAX_SCAN } from 'lib/models/plugins/addCRUDFunctions';
+import { MAX_TIME_MS } from 'lib/models/plugins/addCRUDFunctions';
 import parseQuery from 'lib/helpers/parseQuery';
 import asignIdentifierToStatements from 'lib/services/persona/asignIdentifierToStatements';
 import identifierHasStatements from 'lib/services/persona/identifierHasStatements';
@@ -67,8 +67,7 @@ const personaIdentifierConnection = catchErrors(async (req, res) => {
     filter: filterNoUndefined,
     project,
     hint,
-    maxTimeMS: MAX_TIME_MS,
-    maxScan: MAX_SCAN
+    maxTimeMS: MAX_TIME_MS
   });
 
   return res.status(200).send(result);

api/src/controllers/StatementController.js

@@ -6,15 +6,13 @@ import catchErrors from 'api/controllers/utils/catchErrors';
 import defaultTo from 'lodash/defaultTo';
 
 const MAX_TIME_MS = defaultTo(Number(process.env.MAX_TIME_MS), 0);
-const MAX_SCAN = defaultTo(Number(process.env.MAX_SCAN), 0);
 
 const aggregate = (req) => {
   const authInfo = req.user.authInfo || {};
   const limit = Number(req.query.limit) || -1;
   const skip = Number(req.query.skip) || 0;
   const cache = (!!req.query.cache && req.query.cache !== 'false') || false;
   const maxTimeMS = Number(req.query.maxTimeMS) || MAX_TIME_MS;
-  const maxScan = Number(req.query.maxScan) || MAX_SCAN;
   const pipeline = JSON.parse(req.query.pipeline);
   const sampleSize = Number(req.query.sampleSize) || undefined;
   const out = statementsService.aggregate({
@@ -23,7 +21,6 @@ const aggregate = (req) => {
     skip,
     cache,
     maxTimeMS,
-    maxScan,
     pipeline,
     sampleSize
   });
@@ -87,15 +84,13 @@ const countStatements = catchErrors(async (req, res) => {
     getJSONFromQuery(req, 'query', {})
   );
   const maxTimeMS = getFromQuery(req, 'maxTimeMS', MAX_TIME_MS, Number);
-  const maxScan = getFromQuery(req, 'maxScan', MAX_SCAN, Number);
   const hint = getJSONFromQuery(req, 'hint', null);
   const authInfo = getAuthFromRequest(req);
 
   const count = await statementsService.countStatements({
     authInfo,
     filter,
     maxTimeMS,
-    maxScan,
     hint
   });
 

api/src/utils/exports.js

@@ -49,7 +49,6 @@ export const exportCSV = ({ authInfo, pipelines }) => new Promise((resolve, reje
         limit: 500000,
         getStream: true,
         maxTimeMS: 0,
-        maxScan: 0,
       }, next),
     (err, streams) => {
       if (err) reject(err);

cli/src/commands/v2-migrations/20171122100800_common_indexes.js

@@ -32,7 +32,6 @@ const createStatementIndexes = (connection) => {
   createIndexWithVoid('timestamp__id', { timestamp: -1, _id: -1 });
   createIndexWithVoid('stored__id', { stored: -1, _id: -1 });
   createIndex(stmts, 'statementId_lrs_id', { 'statement.id': 1, lrs_id: 1 });
-  createIndexWithOrg('timestamp__id', { timestamp: -1, _id: 1 });
   createIndexWithOrg('stored__id', { stored: -1, _id: 1 });
   createIndexWithOrg('objId', objectIdIndex);
   createIndexWithOrg('verbId_objId', { ...verbIdIndex, ...objectIdIndex });

lib/helpers/convert$personaIdent.js

@@ -1,7 +1,7 @@
 import _, { map, without } from 'lodash';
 import getService from 'lib/connections/personaService';
 import logger from 'lib/logger';
-import { MAX_TIME_MS, MAX_SCAN } from 'lib/models/plugins/addCRUDFunctions';
+import { MAX_TIME_MS } from 'lib/models/plugins/addCRUDFunctions';
 import getOrgFromAuthInfo from 'lib/services/auth/authInfoSelectors/getOrgFromAuthInfo';
 
 /**
@@ -30,8 +30,7 @@ const convertOpValue = async (identifier, {
       filter,
       organisation,
       limit: Number.MAX_SAFE_INTEGER,
-      maxTimeMS: MAX_TIME_MS,
-      maxScan: MAX_SCAN
+      maxTimeMS: MAX_TIME_MS
     });
     attributes = map(result.edges, ({ node }) => node);
   } catch (err) {

lib/models/plugins/addCRUDFunctions.js

@@ -13,7 +13,6 @@ const FORWARDS = 'forwards';
 export const MAX_TIME_MS = process.env.MAX_TIME_MS
   ? Number(process.env.MAX_TIME_MS)
   : 0;
-export const MAX_SCAN = process.env.MAX_SCAN ? Number(process.env.MAX_SCAN) : 0;
 
 const sortDirectionToOperator = (direction, paginationDirection) => {
   if (paginationDirection === FORWARDS) {
@@ -71,9 +70,9 @@ const modelToCursor = (model, sort) => {
   return cursor;
 };
 
-const getLrsStatementCount = ({ orgFilter, maxTimeMS, maxScan, hint }) => {
+const getLrsStatementCount = ({ orgFilter, maxTimeMS, hint }) => {
   const LRS = getDBConnection().model('Lrs');
-  const query = LRS.collection.find(orgFilter, {}, { maxTimeMS, maxScan });
+  const query = LRS.collection.find(orgFilter, {}, { maxTimeMS });
   if (hint) query.hint(hint);
   return query
     .project({ statementCount: 1 })
@@ -149,8 +148,7 @@ async function getConnection({
   authInfo,
   hint,
   project,
-  maxTimeMS = MAX_TIME_MS,
-  maxScan = MAX_SCAN
+  maxTimeMS = MAX_TIME_MS
 }) {
   const modelName = this.modelName;
   const actionName = 'view';
@@ -162,7 +160,7 @@ async function getConnection({
     actionName,
   });
 
-  const query = this.find(parsedScopeFilter, {}, { maxTimeMS, maxScan });
+  const query = this.find(parsedScopeFilter, {}, { maxTimeMS });
 
   // apply sort
   query.sort(sort);
@@ -265,7 +263,6 @@ async function getCount({
   filter,
   authInfo,
   maxTimeMS = MAX_TIME_MS,
-  maxScan = MAX_SCAN,
   hint
 }) {
   const parsedScopeFilter = await getParsedScopedFilter({
@@ -283,11 +280,10 @@ async function getCount({
     return getLrsStatementCount({
       orgFilter: parsedScopeFilter,
       maxTimeMS,
-      maxScan,
       hint
     });
   }
-  const query = this.find(parsedScopeFilter, {}, { maxTimeMS, maxScan });
+  const query = this.find(parsedScopeFilter, {}, { maxTimeMS });
   if (hint) query.hint(hint);
   return query.select({ _id: 0, organisation: 1 }).countDocuments();
 }

lib/models/statement.js

@@ -33,7 +33,6 @@ const ASYNC_AGGREGATION_CACHE_SECONDS = defaultTo(Number(process.env.ASYNC_AGGRE
 const ASYNC_AGGREGATION_TIMEOUT_MS = defaultTo(Number(process.env.ASYNC_AGGREGATION_TIMEOUT_MS), 0);
 const ASYNC_AGGREGATION_REFRESH_AFTER_SECONDS = defaultTo(Number(process.env.ASYNC_AGGREGATION_REFRESH_AFTER_SECONDS), 60); // 1 minute
 const MAX_TIME_MS = defaultTo(Number(process.env.MAX_TIME_MS), 0);
-const MAX_SCAN = defaultTo(Number(process.env.MAX_SCAN), 0);
 
 let Statement;
 const schema = new mongoose.Schema({
@@ -107,11 +106,10 @@ schema.post('remove', async (statement, next) => {
  *                limit
  *                batchSize
  *                maxTimeMS
- *                maxScan
  *              }
  * @return {Promise}
  */
-const streamAggregation = ({ pipeline, skip, limit, batchSize, maxTimeMS, maxScan }) => {
+const streamAggregation = ({ pipeline, skip, limit, batchSize, maxTimeMS }) => {
   let query = Statement
     .aggregate(pipeline)
     .read('secondaryPreferred')
@@ -122,7 +120,6 @@ const streamAggregation = ({ pipeline, skip, limit, batchSize, maxTimeMS, maxSca
     query.options = {};
   }
   query.options.maxTimeMS = maxTimeMS;
-  query.options.maxScan = maxScan;
 
   return Promise.resolve(query
     .cursor({ batchSize })
@@ -187,7 +184,6 @@ schema.statics.aggregateByAuth = function aggregateByAuth(
     batchSize = 100,
     getStream = false,
     maxTimeMS = MAX_TIME_MS,
-    maxScan = MAX_SCAN,
     sampleSize = undefined
   },
   cb = () => { }
@@ -220,13 +216,12 @@ schema.statics.aggregateByAuth = function aggregateByAuth(
         limit,
         batchSize,
         maxTimeMS,
-        maxScan,
       }).then(stream => cb(null, stream));
     }
 
     if (cache === false) {
       return streamAggregation({
-        pipeline: finalPipeline, skip, limit, batchSize, maxTimeMS, maxScan
+        pipeline: finalPipeline, skip, limit, batchSize, maxTimeMS
       }).then(stream =>
         streamToStringResult(stream)
       ).then((result) => {
@@ -322,7 +317,6 @@ export const runAggregationAsync = async (
       limit,
       batchSize: 100,
       maxTimeMS: ASYNC_AGGREGATION_TIMEOUT_MS,
-      maxScan: MAX_SCAN,
     }).then(streamToStringResult);
 
     const startedAt = await redisClient.get(`${prefix}-STARTED-AT`);

lib/services/statements/aggregate.js

@@ -62,13 +62,12 @@ export default async ({
   skip,
   cache,
   maxTimeMS,
-  maxScan,
   pipeline,
   sampleSize
 }) => {
   const pipelineWithDashboard = await dashboardPipeline({ authInfo, pipeline });
 
-  const options = { skip, limit, cache, batchSize: 100, maxTimeMS, maxScan, sampleSize };
+  const options = { skip, limit, cache, batchSize: 100, maxTimeMS, sampleSize };
   const organisationId = await checkDashboardAuth(authInfo, pipelineWithDashboard);
   const authInfoWithOrgId = setOrgIdOnAuthInfo(authInfo, organisationId);
 

lib/services/statements/countStatements.js

@@ -1,13 +1,12 @@
 import Statement from 'lib/models/statement';
 import parseQuery from 'lib/helpers/parseQuery';
 
-export default async ({ filter, authInfo, maxTimeMS, maxScan, hint }) => {
+export default async ({ filter, authInfo, maxTimeMS, hint }) => {
   const parsedFilter = await parseQuery(filter, { authInfo });
   return Statement.getCount({
     filter: parsedFilter,
     authInfo,
     maxTimeMS,
-    maxScan,
     hint
   });
 };