Skip to content

Commit d2f8071

Browse files
committed
feat(ibm-valid-schema-example): introduce new validation rule
Introduce a new rule for confirming that schema examples are valid instances of the schemas they are defined on. This replaces the Spectral rule 'oas3-valid-schema-example', which has reported a number of false positives across IBM APIs. Signed-off-by: Dustin Popp <[email protected]>
1 parent 1beb965 commit d2f8071

File tree

10 files changed

+654
-10
lines changed

10 files changed

+654
-10
lines changed

docs/ibm-cloud-rules.md

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ which is delivered in the `@ibm-cloud/openapi-ruleset` NPM package.
116116
* [ibm-unique-parameter-request-property-names](#ibm-unique-parameter-request-property-names)
117117
* [ibm-use-date-based-format](#ibm-use-date-based-format)
118118
* [ibm-valid-path-segments](#ibm-valid-path-segments)
119+
* [ibm-valid-schema-example](#ibm-valid-schema-example)
119120
* [ibm-well-defined-dictionaries](#ibm-well-defined-dictionaries)
120121

121122
<!-- tocstop -->
@@ -688,6 +689,12 @@ specific "allow-listed" keywords.</td>
688689
<td>oas3</td>
689690
</tr>
690691
<tr>
692+
<td><a href="#ibm-valid-schema-example">ibm-valid-schema-example</a></td>
693+
<td>warning</td>
694+
<td>Checks each individual schema example to ensure compliance with the schema.</td>
695+
<td>oas3</td>
696+
</tr>
697+
<tr>
691698
<td><a href="#ibm-well-defined-dictionaries">ibm-well-defined-dictionaries</a></td>
692699
<td>warn</td>
693700
<td>Dictionaries must be well defined and all values must share a single type.</td>
@@ -7316,6 +7323,54 @@ paths:
73167323
</tr>
73177324
</table>
73187325

7326+
7327+
### ibm-valid-schema-example
7328+
<table>
7329+
<tr>
7330+
<td><b>Rule id:</b></td>
7331+
<td><b>ibm-valid-schema-example</b></td>
7332+
</tr>
7333+
<tr>
7334+
<td valign=top><b>Description:</b></td>
7335+
<td>This rule validates each unique schema and ensures that any example(s) defined
7336+
is a valid instance of that schema.
7337+
</td>
7338+
</tr>
7339+
<tr>
7340+
<td><b>Severity:</b></td>
7341+
<td>warning</td>
7342+
</tr>
7343+
<tr>
7344+
<td><b>OAS Versions:</b></td>
7345+
<td>oas3</td>
7346+
</tr>
7347+
<tr>
7348+
<td valign=top><b>Non-compliant example:<b></td>
7349+
<td>
7350+
<pre>
7351+
components:
7352+
schemas:
7353+
Foo:
7354+
type: string
7355+
example: 42
7356+
</pre>
7357+
</td>
7358+
</tr>
7359+
<tr>
7360+
<td valign=top><b>Compliant example:</b></td>
7361+
<td>
7362+
<pre>
7363+
components:
7364+
schemas:
7365+
Foo:
7366+
type: string
7367+
example: 'value'
7368+
</pre>
7369+
</td>
7370+
</tr>
7371+
</table>
7372+
7373+
73197374
### ibm-well-defined-dictionaries
73207375
<table>
73217376
<tr>

package-lock.json

Lines changed: 16 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/ruleset/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
"@stoplight/spectral-functions": "^1.9.1",
2727
"@stoplight/spectral-rulesets": "^1.21.1",
2828
"chalk": "^4.1.2",
29+
"jsonschema": "^1.5.0",
2930
"lodash": "^4.17.21",
3031
"loglevel": "^1.9.2",
3132
"loglevel-plugin-prefix": "0.8.4",

packages/ruleset/src/functions/index.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,5 +80,6 @@ module.exports = {
8080
unusedTags: require('./unused-tags'),
8181
useDateBasedFormat: require('./use-date-based-format'),
8282
validatePathSegments: require('./valid-path-segments'),
83+
validSchemaExample: require('./valid-schema-example'),
8384
wellDefinedDictionaries: require('./well-defined-dictionaries'),
8485
};
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/**
2+
* Copyright 2025 IBM Corporation.
3+
* SPDX-License-Identifier: Apache2.0
4+
*/
5+
6+
const { validate } = require('jsonschema');
7+
const { validateSubschemas } = require('@ibm-cloud/openapi-ruleset-utilities');
8+
const { LoggerFactory } = require('../utils');
9+
10+
let ruleId;
11+
let logger;
12+
13+
module.exports = function (schema, _opts, context) {
14+
if (!logger) {
15+
ruleId = context.rule.name;
16+
logger = LoggerFactory.getInstance().getLogger(ruleId);
17+
}
18+
19+
return validateSubschemas(schema, context.path, checkSchemaExamples);
20+
};
21+
22+
function checkSchemaExamples(schema, path) {
23+
if (!isDefined(schema.example) && !definesElements(schema.examples)) {
24+
return [];
25+
}
26+
27+
const examplesToCheck = [];
28+
29+
if (definesElements(schema.examples)) {
30+
schema.examples.forEach((example, i) => {
31+
examplesToCheck.push({
32+
schema,
33+
example,
34+
path: [...path, 'examples', i],
35+
});
36+
});
37+
}
38+
39+
if (isDefined(schema.example)) {
40+
examplesToCheck.push({
41+
schema,
42+
example: schema.example,
43+
path: [...path, 'example'],
44+
});
45+
}
46+
47+
return validateExamples(examplesToCheck);
48+
}
49+
50+
function validateExamples(examples) {
51+
return examples
52+
.map(({ schema, example, path }) => {
53+
// Setting required: true prevents undefined values from passing validation.
54+
const { valid, errors } = validate(example, schema, { required: true });
55+
if (!valid) {
56+
const message = getMessage(errors, example, schema);
57+
return {
58+
message: `Schema example is not valid: ${message}`,
59+
path,
60+
};
61+
}
62+
})
63+
.filter(e => isDefined(e));
64+
}
65+
66+
function isDefined(x) {
67+
return x !== undefined;
68+
}
69+
70+
function definesElements(arr) {
71+
return Array.isArray(arr) && arr.length;
72+
}
73+
74+
function getMessage(errors, example, schema) {
75+
let message = getPrimaryErrorMessage(errors);
76+
77+
const primaryError = errors[0];
78+
if (Array.isArray(schema.oneOf) || Array.isArray(schema.anyOf)) {
79+
// If a schema has a oneOf or anyOf, jsonschema will supress nested validation
80+
// error messages by default. If this is the case, compute those messages and
81+
// append them to the primary message (on their own, they don't include context
82+
// and thus wouldn't be very helpful).
83+
const { errors } = validate(example, schema, { nestedErrors: true });
84+
message = appendMessage(message, getPrimaryErrorMessage(errors));
85+
} else if (Array.isArray(primaryError.argument?.valid?.errors)) {
86+
// Sometimes, jsonschema buries additional error info in the 'argument'
87+
// field of the validation result. If so, extract and include it.
88+
message = appendMessage(
89+
message,
90+
getPrimaryErrorMessage(primaryError.argument.valid.errors)
91+
);
92+
}
93+
94+
return message;
95+
}
96+
97+
function getPrimaryErrorMessage(errors) {
98+
const { path } = errors[0];
99+
let { message } = errors[0];
100+
101+
// If the violation is nested within an object or array, this field will hold
102+
// the path segments to the violation, which is necessary context for the user.
103+
if (path.length) {
104+
message = `${path.join('.')} ${message}`;
105+
}
106+
107+
// Sometimes, jsonschema appends a confusing message to an error - remove it.
108+
return message.replace(/ with \d+ error\[s\]:$/, '');
109+
}
110+
111+
function appendMessage(msg, app) {
112+
return `${msg} (${app})`;
113+
}

packages/ruleset/src/ibm-oas.js

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/**
2-
* Copyright 2017 - 2024 IBM Corporation.
2+
* Copyright 2017 - 2025 IBM Corporation.
33
* SPDX-License-Identifier: Apache2.0
44
*/
55

@@ -95,8 +95,8 @@ module.exports = {
9595
'oas3-server-trailing-slash': true,
9696
// Enable with warn severity
9797
'oas3-valid-media-example': 'warn',
98-
// Enable with warn severity
99-
'oas3-valid-schema-example': 'warn',
98+
// Disable - replaced with ibm-valid-schema-example.
99+
'oas3-valid-schema-example': 'off',
100100
// Enable with same severity as Spectral
101101
'oas3-schema': true,
102102
// Turn off - duplicates non-configurable validation in base validator
@@ -200,6 +200,7 @@ module.exports = {
200200
ibmRules.uniqueParameterRequestPropertyNames,
201201
'ibm-use-date-based-format': ibmRules.useDateBasedFormat,
202202
'ibm-valid-path-segments': ibmRules.validPathSegments,
203+
'ibm-valid-schema-example': ibmRules.validSchemaExample,
203204
'ibm-well-defined-dictionaries': ibmRules.wellDefinedDictionaries,
204205
},
205206
};

packages/ruleset/src/rules/index.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/**
2-
* Copyright 2017 - 2024 IBM Corporation.
2+
* Copyright 2017 - 2025 IBM Corporation.
33
* SPDX-License-Identifier: Apache2.0
44
*/
55

@@ -93,5 +93,6 @@ module.exports = {
9393
uniqueParameterRequestPropertyNames: require('./unique-parameter-request-property-names'),
9494
useDateBasedFormat: require('./use-date-based-format'),
9595
validPathSegments: require('./valid-path-segments'),
96+
validSchemaExample: require('./valid-schema-example'),
9697
wellDefinedDictionaries: require('./well-defined-dictionaries'),
9798
};
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/**
2+
* Copyright 2025 IBM Corporation.
3+
* SPDX-License-Identifier: Apache2.0
4+
*/
5+
6+
const {
7+
schemas,
8+
} = require('@ibm-cloud/openapi-ruleset-utilities/src/collections');
9+
const { oas3 } = require('@stoplight/spectral-formats');
10+
const { validSchemaExample } = require('../functions');
11+
12+
module.exports = {
13+
description:
14+
'Schema examples should validate against the schema they are defined for',
15+
message: '{{error}}',
16+
given: schemas,
17+
severity: 'warn',
18+
formats: [oas3],
19+
resolved: true,
20+
then: {
21+
function: validSchemaExample,
22+
},
23+
};

0 commit comments

Comments
 (0)