From cec7f5c0cb40b6a4b4d0c154d0e2d2573a5dbf23 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 30 Aug 2024 18:11:28 -0400 Subject: [PATCH 1/5] Add delimiter property to base field schema --- .../src/metaSchema/dictionarySchemas.ts | 1 + .../test/metaSchema/dictionarySchemas.spec.ts | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/packages/dictionary/src/metaSchema/dictionarySchemas.ts b/packages/dictionary/src/metaSchema/dictionarySchemas.ts index 63bb073..76fd376 100644 --- a/packages/dictionary/src/metaSchema/dictionarySchemas.ts +++ b/packages/dictionary/src/metaSchema/dictionarySchemas.ts @@ -181,6 +181,7 @@ export const SchemaFieldBase = zod .object({ name: NameValue, description: zod.string().optional(), + delimiter: zod.string().trim().min(1).optional(), isArray: zod.boolean().optional(), meta: DictionaryMeta.optional(), unique: zod.boolean().optional(), diff --git a/packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts b/packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts index 460365e..3da17a5 100644 --- a/packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts +++ b/packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts @@ -151,6 +151,35 @@ describe('Dictionary Schemas', () => { }; expect(SchemaField.safeParse(fieldBoolean).success, 'Boolean field invalid.').true; }); + describe('Delimiter', () => { + it('Field can have delimiter', () => { + const field: SchemaField = { + name: 'some-name', + valueType: 'string', + isArray: true, + delimiter: '|', + }; + expect(SchemaField.safeParse(field).success).true; + }); + it('Delimiter values must have minimum length 1', () => { + const field: SchemaField = { + name: 'some-name', + valueType: 'string', + isArray: true, + delimiter: '', + }; + expect(SchemaField.safeParse(field).success).false; + }); + it('Delimiter value can have multiple characters', () => { + const field: SchemaField = { + name: 'some-name', + valueType: 'string', + isArray: true, + delimiter: '-/-', + }; + expect(SchemaField.safeParse(field).success).true; + }); + }); }); describe('Schema', () => { it("Can't have repeated field names", () => { From 4e7c59cb43721ef10394241734fe7f9cd0b9cabe Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 30 Aug 2024 18:21:42 -0400 Subject: [PATCH 2/5] Use field delimiter value to split arrays --- .../validation/src/parseValues/parseValues.ts | 2 +- .../test/parseValues/parseField.spec.ts | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/validation/src/parseValues/parseValues.ts b/packages/validation/src/parseValues/parseValues.ts index 66af9a4..d6c829f 100644 --- a/packages/validation/src/parseValues/parseValues.ts +++ b/packages/validation/src/parseValues/parseValues.ts @@ -138,7 +138,7 @@ const convertArrayValue = (value: string, fieldDefinition: SchemaField): Result< /* === Start of convertArrayValue logic === */ const { valueType } = fieldDefinition; - const delimiter = DEFAULT_DELIMITER; + const delimiter = fieldDefinition.delimiter !== undefined ? fieldDefinition.delimiter : DEFAULT_DELIMITER; const normalizedValue = normalizeValue(value); if (normalizedValue === '') { diff --git a/packages/validation/test/parseValues/parseField.spec.ts b/packages/validation/test/parseValues/parseField.spec.ts index 74e4fc3..f12ab9d 100644 --- a/packages/validation/test/parseValues/parseField.spec.ts +++ b/packages/validation/test/parseValues/parseField.spec.ts @@ -300,5 +300,24 @@ describe('Parse Values - parseFieldValue', () => { it('Boolean array field, rejects array where value is missing (two delimiters are adjacent)'); expect(parseFieldValue(',true,false,TRUE', fieldBooleanArrayRequired).success).false; }); + describe('Custom delimiter', () => { + it('Uses a `,` as the delimiter when none is defined', () => { + const result = parseFieldValue(':,_,|,/', fieldStringArrayRequired); + expect(result.success).true; + expect(result.data).deep.equal([':', '_', '|', '/']); + }); + it('Splits array on the delimiter when defined', () => { + const customDelimiterField = { ...fieldStringArrayRequired, delimiter: '|' }; + const result = parseFieldValue(':,_,|,/', customDelimiterField); + expect(result.success).true; + expect(result.data).deep.equal([':,_,', ',/']); + }); + it('Splits arrays with delimiters with more than 1 character', () => { + const customDelimiterField = { ...fieldStringArrayRequired, delimiter: '-/-' }; + const result = parseFieldValue('a-/-b-/-c-/-d', customDelimiterField); + expect(result.success).true; + expect(result.data).deep.equal(['a', 'b', 'c', 'd']); + }); + }); }); }); From 0942c3c77c76d608fae6155a8e1734f7bc361567 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 30 Aug 2024 18:34:30 -0400 Subject: [PATCH 3/5] Update dictionary-reference for array delimiter to match implementation --- docs/dictionary-reference.md | 50 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/dictionary-reference.md b/docs/dictionary-reference.md index fe1f14c..aac8889 100644 --- a/docs/dictionary-reference.md +++ b/docs/dictionary-reference.md @@ -67,27 +67,27 @@ In addition to schemas, a Lectern Dictionary can contain reference values that c > } > ``` -| Property | Required | Default | Type | Description | Example | -| ---------------- | -------- | ---------------------- | --------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | -| `name` | Required | | NameString (no whitespace or `.`) | Name of the field. This will be used as the header in TSV files in this field's schema, and in any paths referencing this field. | `"example_field` | -| `valueType` | Required | | [Field Data Type](#field-data-types) | Type of value stored in this field | `"string"` | -| `arrayDelimiter` | Optional | `\|` | `string` | Character or string that will be used to split multiple values into an array. The default delimiter is the `\|` character. | | -| `description` | Optional | `""` No value | `string` | Free text description of the field, for use as a reference for users of the schema. This description is not used in dictionary validation. | `"Shows a string field with a required restriction"` | -| `meta` | Optional | Empty object, no value | [`MetaData`](#meta-data-structure) object | Schema implementor defined fields to capture any additional properties not defined in standard Lectern fields. | `{ "displayName": "Example Field" }` | -| `isArray` | Optional | `false` | `boolean` | Type of value stored in this field | | -| `restrictions` | Optional | No Restrictions | `RestrictionsObject` or `Array` | An object containing all validation rules for this field. This can be a single object containing all [restrictions](#field-restrictions) applied to this field or a list of objects whose restrictions will be combined. [Conditional restrictions](#conditional-restrictions) can also be used to apply validation rules based on values of other fields in the record. | `{ "required": true }` | -| `unique` | Optional | `false` | `boolean` | Indicates that every record in this schema should have a unique value for this field. This rule is applied when a collection of records are validated together, ensuring that no two records in that collection repeat a value. | `true` | +| Property | Required | Default | Type | Description | Example | +| -------------- | -------- | ---------------------- | --------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | +| `name` | Required | | NameString (no whitespace or `.`) | Name of the field. This will be used as the header in TSV files in this field's schema, and in any paths referencing this field. | `"example_field` | +| `valueType` | Required | | [Field Data Type](#field-data-types) | Type of value stored in this field | `"string"` | +| `delimiter` | Optional | `,` | `string` | Character or string that will be used to split multiple values into an array. The default delimiter is a comma `,`. Any characters can be used as a delimiter. The delimiter value can be one or more characters long, but cannot be an empty string. | `"\|"` | +| `description` | Optional | `""` No value | `string` | Free text description of the field, for use as a reference for users of the schema. This description is not used in dictionary validation. | `"Shows a string field with a required restriction"` | +| `meta` | Optional | Empty object, no value | [`MetaData`](#meta-data-structure) object | Schema implementor defined fields to capture any additional properties not defined in standard Lectern fields. | `{ "displayName": "Example Field" }` | +| `isArray` | Optional | `false` | `boolean` | Type of value stored in this field | | +| `restrictions` | Optional | No Restrictions | `RestrictionsObject` or `Array` | An object containing all validation rules for this field. This can be a single object containing all [restrictions](#field-restrictions) applied to this field or a list of objects whose restrictions will be combined. [Conditional restrictions](#conditional-restrictions) can also be used to apply validation rules based on values of other fields in the record. | `{ "required": true }` | +| `unique` | Optional | `false` | `boolean` | Indicates that every record in this schema should have a unique value for this field. This rule is applied when a collection of records are validated together, ensuring that no two records in that collection repeat a value. | `true` | #### Field Data Types -| valueType | Description | Examples | -| --------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | -| `boolean` | Boolean value, either `true` or `false`. Accepts values with any letter casing, for example `true`, `True`, and `TRUE` will all be interpretted as `true` | `true`, `false` | -| `integer` | Numeric integer value. Will accept positive and negative values (ex. `21` or `-8`) but will reject any decimals (ex. `1.23`) | `21`, `-8` | -| `number` | Numeric value. Will accept any numeric value, including those with decimals. | `1.23`, `-4.567` | -| `string` | String fields. Value can have any length and use any character other than the file delimiter (by default `tab`) or the array delimiter for an array field (by default ` \| `) | `asdf`, `Hello World`, `Another longer example of a string` | +| valueType | Description | Examples | +| --------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | +| `boolean` | Boolean value, either `true` or `false`. Accepts values with any letter casing, for example `true`, `True`, and `TRUE` will all be interpretted as `true` | `true`, `false` | +| `integer` | Numeric integer value. Will accept positive and negative values (ex. `21` or `-8`) but will reject any decimals (ex. `1.23`) | `21`, `-8` | +| `number` | Numeric value. Will accept any numeric value, including those with decimals. | `1.23`, `-4.567` | +| `string` | String fields. Value can have any length and use any character, other than the array delimiter for an array field (by default ` \| `) | `asdf`, `Hello World`, `Another longer example of a string` | #### Field Restrictions @@ -97,15 +97,15 @@ The restrictions property of a field can have a value that is either a single re The full list of available restrictions are: -| Restriction | Used with Field Types | Type | Description | Examples | -| ----------- | ----------------------------- | --------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `codeList` | `integer`, `number`, `string` | Array of type of the field | An array of values of the type matching this field. Data provided for this field must have one of the values in this list. | `["Weak", "Average", "Strong"]` | -| `compare` | all | [ComparedFieldsRule](#comparedfieldsrule-data-structure) object | Enforces that this field has a value based on the provided value in another field. Examples would be to ensure that the two values are not equal, or for numeric values ensure one is greater than the other. | `{ "fields": ["age_at_diagnosis"], "relation": "greaterThanOrEqual" }` Ensure that a field such as `age_at_death` is greater than the provided `age_at_diagnosis` | -| `count` | Array fields of all types | `integer` or [`RangeRule`](#rangerule-data-structure) object | Enfroces the number of entries in an array. Can specify an exact array size, or provide range rules that set maximum and minimum counts. | `7` or `{"min": 5, "max": 10}` | -| `empty` | all | | Requires that no value is provided. This is useful when used on a [conditional restriction](#conditional-restrictions) in order to prevent a value from being given when the condition is `true`. For an array field with this restriction, an empty array is a valid value for this restriction. | n/a | -| `range` | `integer`, `number` | | Uses a [RangeRule](#rangerule-data-structure) object to define minimum and/or maximum values for this field | `{"min": 5}`, `{"exclusiveMax": 50}`, `{"exclusiveMin": 5, "max": 50}` | -| `regex` | `string` | | A regular expression that all values must match. | `^[a-z0-9]+$` | -| `required` | all | | A value must be provided, missing/undefined values will fail validation. Empty strings will not be accepted, though `0` (for `number` and `int` fields) and `false` (for `boolean` fields) are accepted. An array field with this restriction must have at least one entry. | `true`, `false` | +| Restriction | Used with Field Types | Type | Description | Examples | +| ----------- | ----------------------------- | --------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `codeList` | `integer`, `number`, `string` | Array of type of the field | An array of values of the type matching this field. Data provided for this field must have one of the values in this list. | `["Weak", "Average", "Strong"]` | +| `compare` | all | [ComparedFieldsRule](#comparedfieldsrule-data-structure) object | Enforces that this field has a value based on the provided value in another field. Examples would be to ensure that the two values are not equal, or for numeric values ensure one is greater than the other. | `{ "fields": ["age_at_diagnosis"], "relation": "greaterThanOrEqual" }` Ensure that a field such as `age_at_death` is greater than the provided `age_at_diagnosis` | +| `count` | Array fields of all types | `integer` or [`RangeRule`](#rangerule-data-structure) object | Enfroces the number of entries in an array. Can specify an exact array size, or provide range rules that set maximum and minimum counts. | `7` or `{"min": 5, "max": 10}` | +| `empty` | all | | Requires that no value is provided. This is useful when used on a [conditional restriction](#conditional-restrictions) in order to prevent a value from being given when the condition is `true`. For an array field with this restriction, an empty array is a valid value for this restriction. | n/a | +| `range` | `integer`, `number` | | Uses a [RangeRule](#rangerule-data-structure) object to define minimum and/or maximum values for this field | `{"min": 5}`, `{"exclusiveMax": 50}`, `{"exclusiveMin": 5, "max": 50}` | +| `regex` | `string` | | A regular expression that all values must match. | `^[a-z0-9]+$` | +| `required` | all | | A value must be provided, missing/undefined values will fail validation. Empty strings will not be accepted, though `0` (for `number` and `int` fields) and `false` (for `boolean` fields) are accepted. An array field with this restriction must have at least one entry. | `true`, `false` | #### Conditional Restrictions From 7c787dc8e0a7b66263ffacefe8833fe346c3f575 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 30 Aug 2024 18:46:03 -0400 Subject: [PATCH 4/5] Field delimiter allows whitespace characters --- .../dictionary/src/metaSchema/dictionarySchemas.ts | 2 +- .../test/metaSchema/dictionarySchemas.spec.ts | 9 +++++++++ .../validation/test/parseValues/parseField.spec.ts | 10 ++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/dictionary/src/metaSchema/dictionarySchemas.ts b/packages/dictionary/src/metaSchema/dictionarySchemas.ts index 76fd376..c53dc60 100644 --- a/packages/dictionary/src/metaSchema/dictionarySchemas.ts +++ b/packages/dictionary/src/metaSchema/dictionarySchemas.ts @@ -181,7 +181,7 @@ export const SchemaFieldBase = zod .object({ name: NameValue, description: zod.string().optional(), - delimiter: zod.string().trim().min(1).optional(), + delimiter: zod.string().min(1).optional(), isArray: zod.boolean().optional(), meta: DictionaryMeta.optional(), unique: zod.boolean().optional(), diff --git a/packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts b/packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts index 3da17a5..c428e8a 100644 --- a/packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts +++ b/packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts @@ -179,6 +179,15 @@ describe('Dictionary Schemas', () => { }; expect(SchemaField.safeParse(field).success).true; }); + it('Delimiter value can be whitespace', () => { + const field: SchemaField = { + name: 'some-name', + valueType: 'string', + isArray: true, + delimiter: ' ', + }; + expect(SchemaField.safeParse(field).success).true; + }); }); }); describe('Schema', () => { diff --git a/packages/validation/test/parseValues/parseField.spec.ts b/packages/validation/test/parseValues/parseField.spec.ts index f12ab9d..d89101f 100644 --- a/packages/validation/test/parseValues/parseField.spec.ts +++ b/packages/validation/test/parseValues/parseField.spec.ts @@ -311,6 +311,10 @@ describe('Parse Values - parseFieldValue', () => { const result = parseFieldValue(':,_,|,/', customDelimiterField); expect(result.success).true; expect(result.data).deep.equal([':,_,', ',/']); + + const result2 = parseFieldValue('abc|def|ghi', customDelimiterField); + expect(result2.success).true; + expect(result2.data).deep.equal(['abc', 'def', 'ghi']); }); it('Splits arrays with delimiters with more than 1 character', () => { const customDelimiterField = { ...fieldStringArrayRequired, delimiter: '-/-' }; @@ -318,6 +322,12 @@ describe('Parse Values - parseFieldValue', () => { expect(result.success).true; expect(result.data).deep.equal(['a', 'b', 'c', 'd']); }); + it('Splits arrays with delimiters that are entirely whitespace', () => { + const customDelimiterField = { ...fieldStringArrayRequired, delimiter: ' ' }; + const result = parseFieldValue('a b c d', customDelimiterField); + expect(result.success).true; + expect(result.data).deep.equal(['a', 'b', 'c', 'd']); + }); }); }); }); From 1c32ea4b15233050f2e2383f7b107e4477874cb8 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 30 Aug 2024 18:53:42 -0400 Subject: [PATCH 5/5] Update delimiter description --- docs/dictionary-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dictionary-reference.md b/docs/dictionary-reference.md index aac8889..f192cdf 100644 --- a/docs/dictionary-reference.md +++ b/docs/dictionary-reference.md @@ -71,7 +71,7 @@ In addition to schemas, a Lectern Dictionary can contain reference values that c | -------------- | -------- | ---------------------- | --------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | | `name` | Required | | NameString (no whitespace or `.`) | Name of the field. This will be used as the header in TSV files in this field's schema, and in any paths referencing this field. | `"example_field` | | `valueType` | Required | | [Field Data Type](#field-data-types) | Type of value stored in this field | `"string"` | -| `delimiter` | Optional | `,` | `string` | Character or string that will be used to split multiple values into an array. The default delimiter is a comma `,`. Any characters can be used as a delimiter. The delimiter value can be one or more characters long, but cannot be an empty string. | `"\|"` | +| `delimiter` | Optional | `,` | `string` | Character or string that will be used to split multiple values into an array. The default delimiter is a comma `,`. Any characters can be used as a delimiter. The delimiter value can be one or more characters long, but cannot be an empty string. Note: This property has no effect unless the field has `isArray: true`. | `"\|"` | | `description` | Optional | `""` No value | `string` | Free text description of the field, for use as a reference for users of the schema. This description is not used in dictionary validation. | `"Shows a string field with a required restriction"` | | `meta` | Optional | Empty object, no value | [`MetaData`](#meta-data-structure) object | Schema implementor defined fields to capture any additional properties not defined in standard Lectern fields. | `{ "displayName": "Example Field" }` | | `isArray` | Optional | `false` | `boolean` | Type of value stored in this field | |