Add VT invalid tracker & handle x out of range (mapbox#114)

* add back v1 tile tracking * add invalid tile logging and update x out of rnage error handling * add proper fixture for testing filler tiles * uncomment tilecopy error handler * rm unnecessary var * add fixture and correct error handler * incorporating vtvalidator * fix async validation * handle v1 vs v2 validation * rm console logs * restructure local build of vtvalidate * add decompressor * fix error handling * throw error on malicious tiles * change throw to callback * add einvalid to errs * uncomment and update tilelive tests * rm test file * update package json and incorporate pr feedback * move migration stream * migration stream only runs for v1 tiles * add err for vtvalidate * declare new validator errors within scope * bump to use npm 5 * Fix dupe test for updated npm ls output * add user friendly error string
My-thesis · Jan 4, 2018 · 706c7a4 · 706c7a4
1 parent 180495f
commit 706c7a4
Show file tree

Hide file tree

Showing 10 changed files with 146 additions and 22 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,5 +2,6 @@
 /node_modules
 test/fixtures/valid.geotiff.tif.aux.xml
 v1-stats.json
+vt-invalid.json
 npm-debug.log
 .DS_Store
diff --git a/.travis.yml b/.travis.yml
@@ -6,6 +6,9 @@ node_js:
 
 sudo: false
 
+before_install:
+  - npm install --global npm@5
+
 addons:
   apt:
     sources: [ 'ubuntu-toolchain-r-test' ]

diff --git a/lib/migration-stream.js b/lib/migration-stream.js
@@ -2,7 +2,10 @@ var stream = require('stream');
 var tiletype = require('@mapbox/tiletype');
 var mapnik = require('mapnik');
 var fs = require('fs');
+var vtvalidate = require('@mapbox/vtvalidate');
+var zlib = require('zlib');
 
+process.env.LOG_INVALID_VT = true;
 module.exports = MigrationStream;
 module.exports.migrate = migrate;
 
@@ -29,32 +32,77 @@ function MigrationStream() {
     });
 
     if (v2 === true) {
-      // Check for errors on the v2 tile
-      var err = checkForErrors(info);
-      if (err) return callback(err);
-
-      migrationStream.push(tile);
-      return callback();
-    } else {
+      decompressBuffer(tile.buffer, function(err, decompressedBuffer){
+          if (err) {
+            err.code = 'EINVALID';
+            return callback(err);
+          } 
+
+          vtvalidate.isValid(decompressedBuffer, function(err, result){
+            // returns empty string if it's a valid tile
+            // if invalid tile - string that specifies why the tile is invalid
+            var newValidatorErrors; 
+
+            if(err){
+              err.code = 'EINVALID';
+              return callback(err);
+            }
+
+            if (result === 'ClosePath command count is not 1' || result === 'Max count too large'){
+              var error = new Error("Invalid tile based on the Mapbox Vector Tile spec: " + result);
+              error.code = 'EINVALID';
+              return callback(error);
+
+            }else if(result){
+
+              newValidatorErrors = result; 
+
+            };
+
+            // Check for errors on the v2 tile
+            var data = {};
+            var error = checkForErrors(info);
+
+            if (newValidatorErrors || error){
+
+              data['vtValidate'] = newValidatorErrors;
+              data['mapnikVectorTile'] = error ? error.message : '';
+
+              if (process.env.LOG_INVALID_VT){
+
+                fs.writeFileSync('vt-invalid.json', JSON.stringify(data));
+
+              };
+
+              if (error){
+                error.code = 'EINVALID';
+                return callback(error);
+              };
+            };
+
+            migrationStream.push(tile);
+            return callback(); 
+
+          }); 
+      }); 
+    }else{
       if (!v1TileDataLogged && process.env.LOG_V1_TILES) {
         fs.writeFileSync('v1-stats.json', JSON.stringify({ 'tileVersion': '1' }));
         v1TileDataLogged = true;
       };
-    }
 
+      migrate(tile, function(err, new_tile) {
+        if (err) return callback(err);
 
-    migrate(tile, function(err, new_tile) {
-      if (err) return callback(err);
+        // Check for errors on newly-minted v2 tile
+        var err = checkForErrors(mapnik.VectorTile.info(new_tile.buffer));
+        if (err) return callback(err);
 
-      // Check for errors on newly-minted v2 tile
-      var err = checkForErrors(mapnik.VectorTile.info(new_tile.buffer));
-      if (err) return callback(err);
-
-      migrationStream.push(new_tile);
-      return callback();
-    });
+        migrationStream.push(new_tile);
+        return callback();
+      });
+    }; 
   };
-
   return migrationStream;
 }
 
@@ -75,7 +123,6 @@ function migrate(tile, callback) {
 
 function checkForErrors(info) {
   var err = null;
-
   if (info.errors) {
     if (info.tile_errors) {
       err = new Error(info.tile_errors[0].replace(' message', ''));
@@ -92,6 +139,19 @@ function checkForErrors(info) {
     }
     err.code = 'EINVALID';
   }
-
   return err;
 }
+
+function decompressBuffer(buffer, callback){
+  if (buffer[0] === 0x1F && buffer[1] === 0x8B) {
+    zlib.gunzip(buffer, function(err, decompressed_buffer){
+      if (err) {
+        err.code = 'EINVALID';
+        return callback(err);
+      }
+      return callback(null, decompressed_buffer); 
+    });
+  }else{
+    return callback(null, buffer); 
+  } 
+}
diff --git a/lib/tilelivecopy.js b/lib/tilelivecopy.js
@@ -26,7 +26,7 @@ function tilelivecopy(srcUri, s3url, options, callback) {
         }
 
         // web mercator out of range
-        if (err && err.message.indexOf('required parameter y is out of range') != -1) {
+        if (err && err.message.indexOf('required parameter y is out of range') != -1 || err && err.message.indexOf('required parameter x is out of range') != -1) {
           err.code = 'EINVALID';
           err.message = 'Coordinates beyond web mercator range. Please check projection and lat/lon values.'
         }

diff --git a/package.json b/package.json
@@ -26,6 +26,7 @@
     "@mapbox/tilelive-s3": "6.5.1",
     "@mapbox/tilelive-vector": "~3.10.0",
     "@mapbox/tiletype": "^0.3.0",
+    "@mapbox/vtvalidate": "0.1.0-alpha1",
     "mapbox-studio-default-fonts": "https://mapbox-npm.s3.amazonaws.com/package/mapbox-studio-default-fonts-0.0.4-4afb5235f457bd1c1a5a70fce6c2aa83bf7a851e.tgz",
     "mapbox-studio-pro-fonts": "https://mapbox-npm.s3.amazonaws.com/package/mapbox-studio-pro-fonts-1.0.0-9870a90b713f307b9391829602f4d5857e419615.tgz",
     "mapnik": "~3.6.0",
@@ -36,6 +37,7 @@
     "tile-stat-stream": "^1.0.1"
   },
   "devDependencies": {
+    "@mapbox/mvt-fixtures": "^3.1.0",
     "aws-sdk": "^2.0.25",
     "request": "^2.47.0",
     "sinon": "^1.12.2",

diff --git a/test/copy.tilelive.test.js b/test/copy.tilelive.test.js
@@ -13,6 +13,7 @@ var fs = require('fs');
 var mtc = require('..'); // just to get protocols registered
 var sinon = require('sinon');
 var TileStatStream = require('tile-stat-stream');
+var mvtf = require('@mapbox/mvt-fixtures');
 
 process.env.MapboxAPIMaps = 'https://api.tiles.mapbox.com';
 
@@ -70,6 +71,56 @@ function tileVersion(dst, z, x, y, callback) {
   });
 }
 
+test('copy mbtiles with v1 tile logging', function(t) {    
+  process.env.LOG_V1_TILES = true;    
+  var fixture = path.resolve(__dirname, 'fixtures', 'valid.mbtiles');   
+  var src = 'mbtiles://' + fixture;   
+  var dst = dsturi('valid.mbtiles');    
+  sinon.spy(tilelive, 'copy');    
+
+  tileliveCopy(src, dst, {}, function(err) {    
+    t.ifError(err, 'copied');   
+    tileCount(dst, function(err, count) {   
+      t.equal(tilelive.copy.getCall(0).args[2].type, 'list', 'uses list scheme for mbtiles');   
+      t.equal(tilelive.copy.getCall(0).args[2].retry, undefined, 'passes options.retry to tilelive.copy');    
+      tilelive.copy.restore();    
+
+      tileVersion(dst, 0, 0, 0, function(err, version) {    
+        var path = './v1-stats.json';   
+        t.equal(fs.existsSync(path), true);   
+        process.env.LOG_V1_TILES = false;   
+        fs.unlinkSync(path);    
+        t.end();    
+      });   
+    });   
+  });   
+});    
+
+test('copy invalid mbtiles with v2 invalid tile logging', function(t) {    
+  process.env.LOG_INVALID_VT = true;    
+  var fixture = path.resolve(__dirname, 'fixtures', 'v2-throw.mbtiles');   
+  var src = 'mbtiles://' + fixture;   
+  var dst = dsturi('v2-throw.mbtiles');    
+  sinon.spy(tilelive, 'copy');    
+
+  tileliveCopy(src, dst, {}, function(err) {   
+    tileCount(dst, function(err, count) {   
+      t.equal(tilelive.copy.getCall(0).args[2].type, 'list', 'uses list scheme for mbtiles');   
+      t.equal(tilelive.copy.getCall(0).args[2].retry, undefined, 'passes options.retry to tilelive.copy');    
+      tilelive.copy.restore();    
+
+      tileVersion(dst, 0, 0, 0, function(err, version) {    
+        var path = './vt-invalid.json';   
+        t.equal(fs.existsSync(path), true);  
+        process.env.LOG_V1_TILES = false;
+        fs.unlinkSync(path);    
+        t.end();    
+      });   
+    });   
+  });   
+});    
+
+
 test('copy mbtiles without v1 tile logging', function(t) {
   var fixture = path.resolve(__dirname, 'fixtures', 'valid.mbtiles');
   var src = 'mbtiles://' + fixture;

diff --git a/test/duplicate.modules.test.js b/test/duplicate.modules.test.js
@@ -7,7 +7,14 @@ var test = require('tape').test;
     exec(cmd, function (error, stdout, stderr) {
       var pattern = new RegExp(mod + '@','g');
       var match = stdout.match(pattern);
-      t.ok(match && match.length === 1, 'one copy of ' + mod + ' (`npm ls ' + mod + '`)');
+      var pattern_dedupe = new RegExp('deduped','g');
+      var deduped = stdout.match(pattern_dedupe);
+
+      if (deduped) {
+      	t.ok(match && deduped && (match.length - 1 === deduped.length), 'one copy of ' + mod + ' (`npm ls ' + mod + '`)');
+      } else {
+      	t.ok(match && match.length === 1, 'one copy of ' + mod + ' (`npm ls ' + mod + '`)');
+      }
       t.end();
     });
   });

diff --git a/test/fixtures/fortythree.mbtiles b/test/fixtures/fortythree.mbtiles
diff --git a/test/fixtures/threenine.mbtiles b/test/fixtures/threenine.mbtiles
diff --git a/test/fixtures/v2-throw.mbtiles b/test/fixtures/v2-throw.mbtiles