fix(cli): JUnit formatter respects fail-severity (#1375)

* fix(cli): list format choices

* fix(cli): junit respects fail-severity

Author: P0lip

docs/guides/2-cli.md

@@ -24,13 +24,14 @@ Other options include:
   --version                    Show version number                                          [boolean]
   --help                       Show help                                                    [boolean]
   --encoding, -e               text encoding to use                        [string] [default: "utf8"]
-  --format, -f                 formatter to use for outputting results  [string] [default: "stylish"]
+  --format, -f                 formatter to use for outputting results
+      [string] [choices: "json", "stylish", "junit", "html", "text", "teamcity"] [default: "stylish"]
   --output, -o                 output to a file instead of stdout                            [string]
   --resolver                   path to custom json-ref-resolver instance                     [string]
   --ruleset, -r                path/URL to a ruleset file                                    [string]
   --skip-rule, -s              ignore certain rules if they are causing trouble              [string]
   --fail-severity, -F          results of this level or above will trigger a failure exit code
-                                [string] [choices: "error", "warn", "info", "hint"] [default: "warn"]
+                               [string] [choices: "error", "warn", "info", "hint"] [default: "error"]
   --display-only-failures, -D  only output results equal to or greater than --fail-severity
                                                                            [boolean] [default: false]
   --ignore-unknown-format      do not warn about unmatched formats         [boolean] [default: false]

package.json

@@ -43,10 +43,10 @@
     "inline-version": "node ./scripts/inline-version.js",
     "prelint": "tsc --noEmit",
     "lint.fix": "yarn lint.prettier --write && yarn lint.eslint --fix",
-    "lint.eslint":" eslint --cache --cache-location .cache/ --ext=.js,.ts src test-harness",
+    "lint.eslint": "eslint --cache --cache-location .cache/ --ext=.js,.ts src test-harness",
     "lint.prettier": "prettier --ignore-path .eslintignore --check {src,test-harness}/**/*.{json,yaml} docs/**/*.md README.md",
     "lint": "yarn lint.prettier && yarn lint.eslint",
-    "copy.html-templates": "copyfiles -u 1 \"./src/formatters/html/*.html\" \"./dist/\"",
+    "copy.html-templates": "copyfiles -u 1 \"./src/cli/formatters/html/*.html\" \"./dist/\"",
     "postbuild.functions": "copyfiles -u 1 \"dist/rulesets/{oas,asyncapi}/functions/*.js\" ./",
     "postbuild": "yarn build.functions && yarn generate-assets",
     "prebuild": "yarn build.clean && copyfiles -u 1 \"src/rulesets/{oas,asyncapi}/**/*.json\" dist && copyfiles -u 1 \"src/rulesets/{oas,asyncapi}/**/*.json\" ./ && yarn copy.html-templates",

src/cli/commands/__tests__/lint.test.ts

@@ -1,6 +1,6 @@
 import * as yargs from 'yargs';
 
-import { DiagnosticSeverity } from '@stoplight/types/dist';
+import { DiagnosticSeverity } from '@stoplight/types';
 import { IRuleResult } from '../../../types';
 import { lint } from '../../services/linter';
 import { formatOutput, writeOutput } from '../../services/output';
@@ -150,7 +150,7 @@ describe('lint', () => {
     // needed by Node 8 (different ticking?) - can be simplified once we drop support for version 8
     await new Promise(resolve => {
       setImmediate(() => {
-        expect(formatOutput).toBeCalledWith(results, format);
+        expect(formatOutput).toBeCalledWith(results, format, { failSeverity: DiagnosticSeverity.Error });
         resolve();
       });
     });

src/cli/commands/lint.ts

@@ -62,7 +62,7 @@ const lintCommand: CommandModule = {
         format: {
           alias: 'f',
           description: 'formatter to use for outputting results',
-          options: formatOptions,
+          choices: formatOptions,
           default: OutputFormat.STYLISH,
           type: 'string',
         },
@@ -161,7 +161,7 @@ const lintCommand: CommandModule = {
         } else if (!config.quiet) {
           console.log(`No results with a severity of '${failSeverity}' or higher found!`);
         }
-        const formattedOutput = formatOutput(results, format);
+        const formattedOutput = formatOutput(results, format, { failSeverity: getDiagnosticSeverity(failSeverity) });
         return writeOutput(formattedOutput, output);
       })
       .catch(fail);

src/formatters/__tests__/__fixtures__/mixed-errors-with-different-paths.json renamed to src/cli/formatters/__tests__/__fixtures__/mixed-errors-with-different-paths.json

@@ -1,12 +1,13 @@
+import { DiagnosticSeverity } from '@stoplight/types';
 import { HTMLElement, parse } from 'node-html-parser';
-import { sortResults } from '../../utils';
+import { sortResults } from '../../../utils';
 import { html } from '../html';
 
 const mixedErrors = sortResults(require('./__fixtures__/mixed-errors.json'));
 
 describe('HTML formatter', () => {
   test('should display proper severity levels', () => {
-    const result = parse(html(mixedErrors)) as HTMLElement;
+    const result = parse(html(mixedErrors, { failSeverity: DiagnosticSeverity.Error })) as HTMLElement;
     const table = result.querySelector('table tbody');
     expect(table.innerHTML.trim()).toEqual(`<tr class="bg-error" data-group="f-0">
     <th colspan="4">

src/formatters/__tests__/__fixtures__/mixed-errors.json renamed to src/cli/formatters/__tests__/__fixtures__/mixed-errors.json

@@ -1,5 +1,6 @@
-import { IRuleResult } from '../../types';
-import { sortResults } from '../../utils';
+import { DiagnosticSeverity } from '@stoplight/types';
+import { IRuleResult } from '../../../types';
+import { sortResults } from '../../../utils';
 import { json } from '../json';
 
 const results: IRuleResult[] = sortResults([
@@ -41,7 +42,7 @@ const results: IRuleResult[] = sortResults([
 
 describe('JSON formatter', () => {
   test('should include ranges', () => {
-    expect(JSON.parse(json(results))).toEqual([
+    expect(JSON.parse(json(results, { failSeverity: DiagnosticSeverity.Error }))).toEqual([
       expect.objectContaining({
         range: {
           start: {
@@ -70,7 +71,7 @@ describe('JSON formatter', () => {
   });
 
   test('should include message', () => {
-    expect(JSON.parse(json(results))).toEqual([
+    expect(JSON.parse(json(results, { failSeverity: DiagnosticSeverity.Error }))).toEqual([
       expect.objectContaining({
         message: 'paths./pets.get.description is not truthy',
       }),

src/formatters/__tests__/__fixtures__/oas3-schema-errors.json renamed to src/cli/formatters/__tests__/__fixtures__/oas3-schema-errors.json

@@ -1,6 +1,7 @@
 import { Parser } from 'xml2js';
-import { sortResults } from '../../utils';
+import { sortResults } from '../../../utils';
 import { junit } from '../junit';
+import { DiagnosticSeverity } from '@stoplight/types';
 
 const oas3SchemaErrors = sortResults(require('./__fixtures__/oas3-schema-errors.json'));
 const mixedErrors = sortResults(require('./__fixtures__/mixed-errors.json'));
@@ -14,7 +15,7 @@ describe('JUnit formatter', () => {
   });
 
   test('should produce valid report', async () => {
-    const result = await parse(junit(oas3SchemaErrors));
+    const result = await parse(junit(oas3SchemaErrors, { failSeverity: DiagnosticSeverity.Error }));
     expect(result).toEqual({
       testsuites: {
         testsuite: [
@@ -83,8 +84,8 @@ describe('JUnit formatter', () => {
     });
   });
 
-  test('should filter out non-error validation results', async () => {
-    const result = await parse(junit(mixedErrors));
+  test('given failSeverity set to error, should filter out non-error validation results', async () => {
+    const result = await parse(junit(mixedErrors, { failSeverity: DiagnosticSeverity.Error }));
     expect(result).toEqual({
       testsuites: {
         testsuite: [
@@ -119,4 +120,57 @@ describe('JUnit formatter', () => {
       },
     });
   });
+
+  test('given failSeverity set to other value than error, should filter treat all validation results matching the severity as errors', async () => {
+    const result = await parse(junit(mixedErrors, { failSeverity: DiagnosticSeverity.Warning }));
+    expect(result).toEqual({
+      testsuites: {
+        testsuite: [
+          {
+            $: {
+              errors: '0',
+              failures: '2',
+              name: '/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3.json',
+              package: 'org.spectral',
+              tests: '2',
+              time: '0',
+            },
+            testcase: [
+              {
+                $: {
+                  classname: '/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3',
+                  name: 'org.spectral.info-description',
+                  time: '0',
+                },
+                failure: [
+                  {
+                    $: {
+                      message: 'OpenAPI object info `description` must be present and non-empty string.',
+                    },
+                    _:
+                      'line 3, col 10, OpenAPI object info `description` must be present and non-empty string. (info-description) at path #/info',
+                  },
+                ],
+              },
+              {
+                $: {
+                  classname: '/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3',
+                  name: 'org.spectral.info-matches-stoplight',
+                  time: '0',
+                },
+                failure: [
+                  {
+                    $: {
+                      message: 'Info must contain Stoplight',
+                    },
+                    _: 'line 5, col 14, Info must contain Stoplight (info-matches-stoplight) at path #/info/title',
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      },
+    });
+  });
 });