#1880 coverage configuration 'all' option adds coverage for untested files

docs/docs/test-runner/cli-and-configuration.md

@@ -104,6 +104,8 @@ interface CoverageConfig {
   report: boolean;
   reportDir: string;
   reporters?: ReportType[];
+  // whether to measure coverage of untested files
+  all?: boolean;
 }
 
 type MimeTypeMappings = Record<string, string>;

packages/test-runner-core/src/config/TestRunnerCoreConfig.ts

@@ -22,6 +22,7 @@ export interface CoverageConfig {
   report?: boolean;
   reportDir?: string;
   reporters?: ReportType[];
+  all?: boolean;
 }
 
 export interface TestRunnerCoreConfig {

packages/test-runner-core/src/coverage/getTestCoverage.ts

@@ -178,11 +178,15 @@ function addingMissingCoverageItems(coverages: CoverageMapData[]) {
 export function getTestCoverage(
   sessions: Iterable<TestSession>,
   config?: CoverageConfig,
+  allFilesCoverage?: CoverageMapData
 ): TestCoverage {
   const coverageMap = createCoverageMap();
   let coverages = Array.from(sessions)
     .map(s => s.testCoverage)
     .filter(c => c) as CoverageMapData[];
+  if (allFilesCoverage) {
+    coverages.unshift(allFilesCoverage);
+  }
   // istanbul mutates the coverage objects, which pollutes coverage in watch mode
   // cloning prevents this. JSON stringify -> parse is faster than a fancy library
   // because we're only working with objects and arrays

packages/test-runner-core/src/runner/TestRunner.ts

@@ -11,6 +11,7 @@ import { createDebugSessions } from './createDebugSessions.js';
 import { TestRunnerServer } from '../server/TestRunnerServer.js';
 import { BrowserLauncher } from '../browser-launcher/BrowserLauncher.js';
 import { TestRunnerGroupConfig } from '../config/TestRunnerGroupConfig.js';
+import { generateEmptyReportsForUntouchedFiles } from '@web/test-runner-coverage-v8';
 
 interface EventMap {
   'test-run-started': { testRun: number };
@@ -205,7 +206,11 @@ export class TestRunner extends EventEmitter<EventMap> {
         let passedCoverage = true;
         let testCoverage: TestCoverage | undefined = undefined;
         if (this.config.coverage) {
-          testCoverage = getTestCoverage(this.sessions.all(), this.config.coverageConfig);
+          let allFilesCoverage;
+          if (this.config.coverageConfig?.all) {
+            allFilesCoverage = await generateEmptyReportsForUntouchedFiles(this.config, this.testFiles);
+          }
+          testCoverage = getTestCoverage(this.sessions.all(), this.config.coverageConfig, allFilesCoverage);
           passedCoverage = testCoverage.passed;
         }
 

packages/test-runner-coverage-v8/src/index.ts

@@ -1,13 +1,13 @@
-import { extname, join, isAbsolute, sep, posix } from 'path';
+import { extname, join, isAbsolute, sep, posix } from 'node:path';
 import { CoverageMapData } from 'istanbul-lib-coverage';
 import v8toIstanbulLib from 'v8-to-istanbul';
 import { TestRunnerCoreConfig, fetchSourceMap } from '@web/test-runner-core';
 import { Profiler } from 'inspector';
 import picoMatch from 'picomatch';
 import LruCache from 'lru-cache';
-import { readFile } from 'node:fs/promises';
-
-import { toFilePath } from './utils.js';
+import { readFile, readdir, stat } from 'node:fs/promises';
+import { Stats } from 'node:fs';
+import { toFilePath, toBrowserPath } from './utils.js';
 
 type V8Coverage = Profiler.ScriptCoverage;
 type Matcher = (test: string) => boolean;
@@ -32,11 +32,10 @@ function hasOriginalSource(source: IstanbulSource): boolean {
     typeof source.sourceMap.sourcemap === 'object' &&
     source.sourceMap.sourcemap !== null &&
     Array.isArray(source.sourceMap.sourcemap.sourcesContent) &&
-    source.sourceMap.sourcemap.sourcesContent.length > 0
-  );
+    source.sourceMap.sourcemap.sourcesContent.length > 0);
 }
 
-function getMatcher(patterns?: string[]) {
+function getMatcher(patterns?: string[]): picoMatch.Matcher {
   if (!patterns || patterns.length === 0) {
     return () => true;
   }
@@ -60,63 +59,154 @@ export async function v8ToIstanbul(
   testFiles: string[],
   coverage: V8Coverage[],
   userAgent?: string,
-) {
+): Promise<CoverageMapData> {
   const included = getMatcher(config?.coverageConfig?.include);
   const excluded = getMatcher(config?.coverageConfig?.exclude);
   const istanbulCoverage: CoverageMapData = {};
 
   for (const entry of coverage) {
-    const url = new URL(entry.url);
-    const path = url.pathname;
-    if (
-      // ignore non-http protocols (for exmaple webpack://)
-      url.protocol.startsWith('http') &&
-      // ignore external urls
-      url.hostname === config.hostname &&
-      url.port === `${config.port}` &&
-      // ignore non-files
-      !!extname(path) &&
-      // ignore virtual files
-      !path.startsWith('/__web-test-runner') &&
-      !path.startsWith('/__web-dev-server')
-    ) {
-      try {
+    try {
+      const url = new URL(entry.url);
+      const path = url.pathname;
+      if (
+        // ignore non-http protocols (for exmaple webpack://)
+        url.protocol.startsWith('http') &&
+        // ignore external urls
+        url.hostname === config.hostname &&
+        url.port === `${config.port}` &&
+        // ignore non-files
+        !!extname(path) &&
+        // ignore virtual files
+        !path.startsWith('/__web-test-runner') &&
+        !path.startsWith('/__web-dev-server')
+      ) {
         const filePath = join(config.rootDir, toFilePath(path));
-
         if (!testFiles.includes(filePath) && included(filePath) && !excluded(filePath)) {
           const browserUrl = `${url.pathname}${url.search}${url.hash}`;
-          const cachedSource = cachedSources.get(browserUrl);
-          const sources =
-            cachedSource ??
-            ((await fetchSourceMap({
-              protocol: config.protocol,
-              host: config.hostname,
-              port: config.port,
-              browserUrl,
-              userAgent,
-            })) as IstanbulSource);
-
-          if (!cachedSource) {
-            if (!hasOriginalSource(sources)) {
-              const contents = await readFile(filePath, 'utf8');
-              (sources as IstanbulSource & { originalSource: string }).originalSource = contents;
-            }
-            cachedSources.set(browserUrl, sources);
-          }
-
-          const converter = v8toIstanbulLib(filePath, 0, sources);
-          await converter.load();
-
-          converter.applyCoverage(entry.functions);
-          Object.assign(istanbulCoverage, converter.toIstanbul());
+          const sources = await getIstanbulSource(config, filePath, browserUrl, userAgent);
+          await addCoverageForFilePath(sources, filePath, entry, istanbulCoverage);
         }
-      } catch (error) {
-        console.error(`Error while generating code coverage for ${entry.url}.`);
-        console.error(error);
+      }
+    } catch (error) {
+      console.error(`Error while generating code coverage for ${entry.url}.`);
+      console.error(error);
+    }
+  }
+
+  return istanbulCoverage;
+}
+
+async function addCoverageForFilePath(
+  sources: IstanbulSource,
+  filePath: string,
+  entry: V8Coverage,
+  istanbulCoverage: CoverageMapData,
+): Promise<void> {
+  const converter = v8toIstanbulLib(filePath, 0, sources);
+  await converter.load();
+
+  converter.applyCoverage(entry.functions);
+  Object.assign(istanbulCoverage, converter.toIstanbul());
+}
+
+async function getIstanbulSource(
+  config: TestRunnerCoreConfig,
+  filePath: string,
+  browserUrl: string,
+  userAgent?: string,
+  doNotAddToCache?: boolean,
+): Promise<IstanbulSource> {
+  const cachedSource = cachedSources.get(browserUrl);
+  const sources =
+    cachedSource ??
+    ((await fetchSourceMap({
+      protocol: config.protocol,
+      host: config.hostname,
+      port: config.port,
+      browserUrl,
+      userAgent,
+    })) as IstanbulSource);
+
+  if (!cachedSource) {
+    if (!hasOriginalSource(sources)) {
+      const contents = await readFile(filePath, 'utf8');
+      (sources as IstanbulSource & { originalSource: string }).originalSource = contents;
+    }
+    !doNotAddToCache && cachedSources.set(browserUrl, sources);
+  }
+  return sources;
+}
+
+
+async function recursivelyAddEmptyReports(
+  config: TestRunnerCoreConfig,
+  testFiles: string[],
+  include: picoMatch.Matcher,
+  exclude: picoMatch.Matcher,
+  istanbulCoverage: CoverageMapData,
+  dir = '',
+): Promise<void> {
+  const contents = await readdir(join(coverageBaseDir, dir));
+  for (const file of contents) {
+    const filePath = join(coverageBaseDir, dir, file);
+    if (!exclude(filePath)) {
+      const stats = await stat(filePath);
+      const relativePath = join(dir, file);
+      if (stats.isDirectory()) {
+        await recursivelyAddEmptyReports(config, testFiles, include, exclude, istanbulCoverage, relativePath);
+      } else if (!testFiles.includes(filePath) && include(filePath)) {
+        await addEmptyReportIfFileUntouched(config, istanbulCoverage, filePath, stats, relativePath);
       }
     }
   }
+}
 
+async function addEmptyReportIfFileUntouched(
+  config: TestRunnerCoreConfig,
+  istanbulCoverage: CoverageMapData,
+  filePath: string,
+  stats: Stats,
+  relativePath: string,
+): Promise<void> {
+  try {
+    const browserUrl = toBrowserPath(relativePath);
+    const fileHasBeenTouched = cachedSources.find((_, key) => {
+      return key === browserUrl || key.startsWith(browserUrl+'?') || key.startsWith(browserUrl+'#');
+    });
+    if (fileHasBeenTouched) {
+      return;
+    }
+    const sources = await getIstanbulSource(config, filePath, browserUrl, undefined, true);
+    const entry = {
+      scriptId: browserUrl,
+      url: browserUrl,
+      functions: [{
+        functionName: '(empty-report)',
+        isBlockCoverage: true,
+        ranges: [{
+          startOffset: 0,
+          endOffset: stats.size,
+          count: 0
+        }]
+      }]
+    } as V8Coverage;
+    await addCoverageForFilePath(sources, filePath, entry, istanbulCoverage);
+  } catch (error) {
+    console.error(`Error while generating empty code coverage for ${filePath}.`);
+    console.error(error);
+  }
+}
+
+export async function generateEmptyReportsForUntouchedFiles(
+  config: TestRunnerCoreConfig,
+  testFiles: string[],
+): Promise<CoverageMapData> {
+  const istanbulCoverage: CoverageMapData = {};
+  if (config?.coverageConfig) {
+    const include = getMatcher(config.coverageConfig.include);
+    const exclude = getMatcher(config.coverageConfig.exclude);
+    await recursivelyAddEmptyReports(config, testFiles, include, exclude, istanbulCoverage);
+  }
   return istanbulCoverage;
 }
 

packages/test-runner-coverage-v8/src/utils.ts

@@ -1,7 +1,16 @@
-import path from 'path';
+import path from 'node:path';
 
 const REGEXP_TO_FILE_PATH = new RegExp('/', 'g');
+const REGEXP_TO_BROWSER_PATH = new RegExp('\\\\', 'g');
 
-export function toFilePath(browserPath: string) {
+export function toFilePath(browserPath: string): string {
   return browserPath.replace(REGEXP_TO_FILE_PATH, path.sep);
 }
+
+export function toBrowserPath(filePath: string): string {
+  const replaced = filePath.replace(REGEXP_TO_BROWSER_PATH, '/');
+  if (replaced[0] !== '/') {
+    return '/' + replaced;
+  }
+  return replaced;
+}