fix: 2min timeouts for all http reqs, 20min for kmz

Author: advayDev1

commands/run.js

@@ -28,7 +28,6 @@ const os = require('os');
 const path = require('path');
 const promisify = require('util').promisify;
 const pug = require('pug');
-const rp = require('request-promise');
 const yaml = require('js-yaml');
 const {globalStats, MeasureUnit} = require('@opencensus/core');
 
@@ -152,10 +151,6 @@ exports.builder = {
     boolean: true,
     desc: 'Use real fire names not hashtags',
   },
-  archiveInciweb: {
-    boolean: true,
-    desc: 'Save InciWeb updates to web.archive.org',
-  },
   emergingNew: {
     boolean: true,
     default: false,
@@ -649,18 +644,6 @@ exports.handler = (argv) => {
       const perimWebpage = argv.outputdir + '/img/WEB-PERIM-' + updateId + '.html';
       const mainWebpageUrl = 'http://localhost:8080/updates/img/WEB-INFO-' + updateId + '.html';
       const perimWebpageUrl = 'http://localhost:8080/updates/img/WEB-PERIM-' + updateId + '.html';
-      if (inciWeb && argv.archiveInciweb) {
-        const u = 'https://web.archive.org/save/https://inciweb.nwcg.gov/incident/' + inciWeb + '/';
-        rp({uri: u, resolveWithFullResponse: true}).then((r) => {
-          logger.info('   ~~ Archived to web.archive.org: %s', r.headers ? ('https://web.archive.org/' + r.headers['content-location']) : 'unknown');
-        }).catch((err) => {
-          logger.info('   ~~ ERROR Archiving to web.archive.org: ' + u);
-          logger.info(err);
-          if (argv.failOnError) {
-            process.exit(13);
-          }
-        });
-      }
       let rr = null;
       if (perim.length > 1 && argv.locations) {
         rr = await maprender.getMapBounds(perim, 1450 / 2, 1200 / 2, 15, cur.DailyAcres || 0);

lib/calfire.js

@@ -113,13 +113,14 @@ Archived: https://web.archive.org/web/20190627202100/https://www.fire.ca.gov/con
 */
 
 // ua: user-agent
-exports.getFires = async function(ua) {
+exports.getFires = async function(ua, timeoutMs) {
   const firesDataOptions = {
     uri: 'https://www.fire.ca.gov/imapdata/mapdataactive.csv',
     qs: qs,
     headers: {
       'User-Agent': 'Request-Promise; ' + ua,
     },
+    timeout: timeoutMs || 120000,
     json: false,
   };
   const provenance = util.createProvenance(firesDataOptions);
@@ -137,13 +138,14 @@ exports.getFires = async function(ua) {
 };
 
 
-exports.getFireDetail = async function(fire, ua) {
+exports.getFireDetail = async function(fire, ua, timeoutMs) {
   const firesDataOptions = {
     uri: fire.Link,
     qs: qs,
     headers: {
       'User-Agent': 'Request-Promise; ' + ua,
     },
+    timeout: timeoutMs || 120000,
     json: false,
   };
   const provenance = util.createProvenance(firesDataOptions);

lib/geomac.js

@@ -99,13 +99,14 @@ const qs = {
 };
 
 
-exports.getPerimeters = async function(ua, latest) {
+exports.getPerimeters = async function(ua, latest, timeoutMs) {
   const perimDataOptions = {
     uri: 'https://wildfire.cr.usgs.gov/ArcGIS/rest/services/geomac_dyn/MapServer/' + (latest ? '3' : '2') + '/query',
     qs: qs,
     headers: {
       'User-Agent': 'Request-Promise; ' + ua,
     },
+    timeout: timeoutMs || 120000,
     json: true,
   };
   const provenance = util.createProvenance(perimDataOptions);
@@ -117,13 +118,14 @@ exports.getPerimeters = async function(ua, latest) {
 };
 
 
-exports.getFires = async function(ua) {
+exports.getFires = async function(ua, timeoutMs) {
   const firesDataOptions = {
     uri: 'https://wildfire.cr.usgs.gov/arcgis/rest/services/geomac_dyn/MapServer/0/query',
     qs: qs,
     headers: {
       'User-Agent': 'Request-Promise; ' + ua,
     },
+    timeout: timeoutMs || 120000,
     json: true,
   };
   const provenance = util.createProvenance(firesDataOptions);

lib/kmz.js

@@ -30,13 +30,14 @@ const DOMParser = require('xmldom').DOMParser;
 const togeojson = require('@publicdocs/togeojson');
 
 
-exports.loadKmz = async function(url, path, toGeoJson, includeOtherFiles) {
+exports.loadKmz = async function(url, path, toGeoJson, includeOtherFiles, timeoutMs) {
   try {
     await promisify(fs.unlink)(path);
   } catch (err) {}
   const body = await rp({
     url: url,
     encoding: null,
+    timeout: timeoutMs || 1200000,
   });
   const zip = await JSZip.loadAsync(body);
   await Promise.all(_.map(zip.files, (k) => {

lib/nfsa.js

@@ -51,14 +51,15 @@ const processNfsaFire = function(e, proj) {
   return ret;
 };
 
-exports.getFires = async function(userAgent, includeEmergingNew, includeEmergingOld) {
+exports.getFires = async function(userAgent, includeEmergingNew, includeEmergingOld, timeoutMs) {
   const dataOptions = {
     uri: 'https://maps.nwcg.gov/sa/publicData.json',
     qs: {
     },
     headers: {
       'User-Agent': 'Request-Promise; ' + userAgent,
     },
+    timeout: timeoutMs || 120000,
     json: true,
   };