clients(devtools): only use locales that have locale files to download (#13214)

Author: brendankenny

build/build-bundle.js

@@ -13,14 +13,12 @@
 const fs = require('fs');
 const path = require('path');
 const assert = require('assert').strict;
-const stream = require('stream');
 const mkdir = fs.promises.mkdir;
 const LighthouseRunner = require('../lighthouse-core/runner.js');
 const exorcist = require('exorcist');
 const browserify = require('browserify');
 const terser = require('terser');
 const {minifyFileTransform} = require('./build-utils.js');
-const {LH_ROOT} = require('../root.js');
 
 const COMMIT_HASH = require('child_process')
   .execSync('git rev-parse HEAD')
@@ -32,9 +30,6 @@ const audits = LighthouseRunner.getAuditList()
 const gatherers = LighthouseRunner.getGathererList()
     .map(f => './lighthouse-core/gather/gatherers/' + f.replace(/\.js$/, ''));
 
-const locales = fs.readdirSync(LH_ROOT + '/shared/localization/locales/')
-    .map(f => require.resolve(`../shared/localization/locales/${f}`));
-
 // HACK: manually include the lighthouse-plugin-publisher-ads audits.
 /** @type {Array<string>} */
 // @ts-expect-error
@@ -67,23 +62,7 @@ async function browserifyFile(entryPath, distPath) {
     })
     // Transform the fs.readFile etc into inline strings.
     .transform('@wardpeet/brfs', {
-      /** @param {string} file */
-      readFileTransform: (file) => {
-        // Don't include locales in DevTools.
-        if (isDevtools(entryPath) && locales.includes(file)) {
-          return new stream.Transform({
-            transform(chunk, enc, next) {
-              next();
-            },
-            final(next) {
-              this.push('{}');
-              next();
-            },
-          });
-        }
-
-        return minifyFileTransform(file);
-      },
+      readFileTransform: minifyFileTransform,
       global: true,
       parserOpts: {ecmaVersion: 12},
     })
@@ -107,6 +86,11 @@ async function browserifyFile(entryPath, distPath) {
     bundle.ignore(require.resolve('../report/generator/report-assets.js'));
   }
 
+  // Don't include locales in DevTools.
+  if (isDevtools(entryPath)) {
+    bundle.ignore(require.resolve('../shared/localization/locales.js'));
+  }
+
   // Expose the audits, gatherers, and computed artifacts so they can be dynamically loaded.
   // Exposed path must be a relative path from lighthouse-core/config/config-helpers.js (where loading occurs).
   const corePath = './lighthouse-core/';

clients/devtools-entry.js

@@ -9,7 +9,7 @@ const lighthouse = require('../lighthouse-core/index.js');
 const RawProtocol = require('../lighthouse-core/gather/connections/raw.js');
 const log = require('lighthouse-logger');
 const {lookupLocale} = require('../lighthouse-core/lib/i18n/i18n.js');
-const {registerLocaleData} = require('../shared/localization/format.js');
+const {registerLocaleData, getCanonicalLocales} = require('../shared/localization/format.js');
 const constants = require('../lighthouse-core/config/constants.js');
 
 /** @typedef {import('../lighthouse-core/gather/connections/connection.js')} Connection */
@@ -60,6 +60,17 @@ function listenForStatus(listenCallback) {
   log.events.addListener('status', listenCallback);
 }
 
+/**
+ * Does a locale lookup but limits the result to the *canonical* Lighthouse
+ * locales, which are only the locales with a messages locale file that can
+ * be downloaded and then used via `registerLocaleData`.
+ * @param {string|string[]=} locales
+ * @return {LH.Locale}
+ */
+function lookupCanonicalLocale(locales) {
+  return lookupLocale(locales, getCanonicalLocales());
+}
+
 // For the bundle smoke test.
 if (typeof module !== 'undefined' && module.exports) {
   // Ideally this could be exposed via browserify's `standalone`, but it doesn't
@@ -85,6 +96,7 @@ if (typeof self !== 'undefined') {
   self.listenForStatus = listenForStatus;
   // @ts-expect-error
   self.registerLocaleData = registerLocaleData;
+  // TODO: expose as lookupCanonicalLocale in LighthouseService.ts?
   // @ts-expect-error
-  self.lookupLocale = lookupLocale;
+  self.lookupLocale = lookupCanonicalLocale;
 }

lighthouse-core/lib/i18n/i18n.js

@@ -120,12 +120,22 @@ const UIStrings = {
  * - supported locales in Intl formatters
  *
  * If `locale` isn't provided or one could not be found, DEFAULT_LOCALE is returned.
+ *
+ * By default any of the locales Lighthouse has strings for can be returned, but this
+ * can be overriden with `possibleLocales`, useful e.g. when Lighthouse is bundled and
+ * only DEFAULT_LOCALE is available, but `possibleLocales` can be used to select a
+ * locale available to be downloaded on demand.
  * @param {string|string[]=} locales
+ * @param {Array<string>=} possibleLocales
  * @return {LH.Locale}
  */
-function lookupLocale(locales) {
-  // If Node was built with `--with-intl=none`, `Intl` won't exist.
+function lookupLocale(locales, possibleLocales) {
+  // TODO: lookupLocale may need to be split into two functions, one that canonicalizes
+  // locales and one that looks up the best locale filename for a given locale.
+  // e.g. `en-IE` is canonical, but uses `en-GB.json`. See TODO in locales.js
+
   if (typeof Intl !== 'object') {
+    // If Node was built with `--with-intl=none`, `Intl` won't exist.
     throw new Error('Lighthouse must be run in Node with `Intl` support. See https://nodejs.org/api/intl.html for help');
   }
 
@@ -135,7 +145,7 @@ function lookupLocale(locales) {
   const availableLocales = Intl.NumberFormat.supportedLocalesOf(canonicalLocales);
 
   // Get available locales and transform into object to match `lookupClosestLocale`'s API.
-  const localesWithMessages = getAvailableLocales();
+  const localesWithMessages = possibleLocales || getAvailableLocales();
   const localesWithmessagesObj = /** @type {Record<LH.Locale, LhlMessages>} */ (
     Object.fromEntries(localesWithMessages.map(l => [l, {}])));
 

lighthouse-core/test/lib/i18n/i18n-test.js

@@ -80,5 +80,31 @@ describe('i18n', () => {
     it('falls back to en-US if no match is available', () => {
       expect(i18n.lookupLocale(invalidLocale)).toEqual('en-US');
     });
+
+    describe('possibleLocales option', () => {
+      it('canonicalizes from the possible locales', () => {
+        expect(i18n.lookupLocale('en-xa', ['ar', 'en-XA'])).toEqual('en-XA');
+      });
+
+      it('takes multiple locale strings and returns a possible, canonicalized one', () => {
+        // COMPAT: Node 12 only has 'en-US' by default.
+        if (isNode12SmallIcu()) {
+          expect(i18n.lookupLocale([invalidLocale, 'eN-uS', 'en-xa'], ['ar', 'es', 'en-US']))
+            .toEqual('en-US');
+          return;
+        }
+
+        expect(i18n.lookupLocale([invalidLocale, 'eS', 'en-xa'], ['ar', 'es']))
+            .toEqual('es');
+      });
+
+      it('falls back to en-US if no possible match is available', () => {
+        expect(i18n.lookupLocale('es', ['en-US', 'ru', 'zh'])).toEqual('en-US');
+      });
+
+      it('falls back to en-US if no possible matchs are available at all', () => {
+        expect(i18n.lookupLocale('ru', [])).toEqual('en-US');
+      });
+    });
   });
 });