Merge pull request #249 from mrodrig/feat/add-key-expansion-functionality

Add key expansion capability

Author: mrodrig

README.md

@@ -128,9 +128,19 @@ Returns the CSV `string` or rejects with an `Error` if there was an issue.
     * Note: This may result in CSV output that does not map back exactly to the original JSON. See #102 for more information.
   * `keys` - Array - Specify the keys that should be converted.
     * Default: These will be auto-detected from your data by default.
-    * Keys can either be specified as a String representing the key path that should be converted, or as an Object with the `field` property specifying the path. When specifying keys as an Object, you can also optionally specify a `title` which will be used for that column in the header. The list specified can contain a combination of Objects and Strings. 
-      * `[ 'key1', 'key2', ... ]`
-      * `[ { field: 'key1', title: 'Key 1' }, { field: 'key2' }, 'key3', ... ]`
+    * Keys can either be specified as a String representing the key path that should be converted, or as an Object of the following format:
+    ```javascript
+    {
+      "field": "string", // required
+      "title": "string", // optional
+      "wildcardMatch": false, // optional - default: false
+    }
+    ```
+      * When specifying keys as an Object, the `field` property specifies the key path, while `title` specifies a more human readable field heading. Additionally, the `wildcardMatch` option allows you to optionally specify that all auto-detected fields with the specified field prefix should be included in the CSV. The list specified can contain a combination of Objects and Strings.
+      * Examples:
+        * `[ 'key1', 'key2', ... ]`
+        * `[ 'key1', { field: 'key2', wildcardMatch: true }]`
+        * `[ { field: 'key1', title: 'Key 1' }, { field: 'key2' }, 'key3', ... ]`
     * Key Paths - If you are converting a nested object (ie. {info : {name: 'Mike'}}), then set this to ['info.name']
   * `parseValue` - Function - Specify how values should be converted into CSV format. This function is provided a single field value at a time and must return a `String`. The built-in parsing method is provided as the second argument for cases where default parsing is preferred.
     * Default: A built-in method is used to parse out a variety of different value types to well-known formats.

src/json2csv.ts

@@ -172,26 +172,60 @@ export const Json2Csv = function(options: FullJson2CsvOptions) {
         });
     }
 
+    function extractWildcardMatchKeys() {
+        if (!options.keys) return [];
+
+        return options.keys.flatMap(item => {
+            if (typeof item === 'string') {
+                // Exclude plain strings that were passed in options.keys
+                return [];
+            } else if (item?.wildcardMatch) {
+                // Return "field" value for objects with wildcardMatch: true
+                return item.field;
+            } 
+            // Exclude other objects
+            return [];
+        });
+    }
+
     /**
      * Retrieve the headings for all documents and return it.
      * This checks that all documents have the same schema.
      */
     function retrieveHeaderFields(data: object[]) {
+        const wildcardMatchKeys = extractWildcardMatchKeys();
         const keyStrings = convertKeysToHeaderFields();
+        const fieldNames = getFieldNameList(data);
+        const processed = processSchemas(fieldNames);
 
         if (options.keys) {
             options.keys = keyStrings;
+            
+            const matchedKeys = keyStrings.flatMap((userProvidedKey) => {
+                // If this is not a wildcard matched key, then just return and include it in the resulting key list
+                if (!wildcardMatchKeys.includes(userProvidedKey)) {
+                    return userProvidedKey;
+                }
 
-            if (!options.unwindArrays) {
-                const filtered = filterExcludedKeys(keyStrings);
+                // Otherwise, identify all detected keys that match with the provided wildcard key:
+                const matches = [];
+                const regex = new RegExp(`^${userProvidedKey}`);
+
+                for (const detectedKey of processed) {
+                    if (userProvidedKey === detectedKey || detectedKey.match(regex)) {
+                        matches.push(detectedKey);
+                    }
+                }
+                return matches;
+            });
 
+            if (!options.unwindArrays) {
+                const filtered = filterExcludedKeys(matchedKeys);
 
                 return sortHeaderFields(filtered);
             }
         }
 
-        const fieldNames = getFieldNameList(data);
-        const processed = processSchemas(fieldNames);
         const filtered = filterExcludedKeys(processed);
 
         return sortHeaderFields(filtered);

src/types.ts

@@ -12,6 +12,7 @@ interface DelimiterOptions {
 export type KeysList = (string | {
   field: string;
   title?: string;
+  wildcardMatch?: boolean;
 })[];
 
 interface SharedConverterOptions {

test/config/testCsvFilesList.ts

@@ -51,6 +51,7 @@ const csvFileConfig = [
     {key: 'nestedNotUnwoundObjects', file: '../data/csv/nestedNotUnwoundObjects.csv'},
     {key: 'newlineWithWrapDelimiters', file: '../data/csv/newlineWithWrapDelimiters.csv'},
     {key: 'excludeKeyPattern', file: '../data/csv/excludeKeyPattern.csv'},
+    {key: 'wildcardMatch', file: '../data/csv/wildcardMatch.csv'},
 ];
 
 function readCsvFile(filePath: string) {

test/config/testJsonFilesList.ts

@@ -44,4 +44,5 @@ export default {
     falsyValues: require('../data/json/falsyValues.json'),
     newlineWithWrapDelimiters: require('../data/json/newlineWithWrapDelimiters'),
     excludeKeyPattern: require('../data/json/excludeKeyPattern'),
+    wildcardMatch: require('../data/json/wildcardMatch.json'),
 };

test/data/csv/wildcardMatch.csv

@@ -0,0 +1,2 @@
+foo,bar,baz.a,baz.array
+foo,bar,a,c

test/data/json/wildcardMatch.json

@@ -0,0 +1,11 @@
+[
+    {
+        "foo": "foo",
+        "bar": "bar",
+        "baz": {
+            "a": "a",
+            "b": "b",
+            "array": "c"
+        }
+    }
+]

test/json2csv.ts

@@ -394,13 +394,15 @@ export function runTests() {
             });
 
             it('should allow titles to be specified for certain keys, but not others when not unwinding arrays', () => {
+                console.log('START EXEC----');
                 const csv = json2csv(jsonTestData.unwind, {
                     unwindArrays: false,
                     keys: [
                         {field: 'data.category', title: 'Category'},
                         'data.options.name'
                     ]
                 });
+                console.log('END EXEC----');
                 assert.equal(csv, csvTestData.withSpecifiedKeys.replace('data.category,data.options.name', 'Category,data.options.name'));
             });
 
@@ -416,13 +418,16 @@ export function runTests() {
             });
 
             it('should allow titles to be specified', () => {
+                console.log('START EXEC----');
+
                 const csv = json2csv(jsonTestData.unwind, {
                     unwindArrays: false,
                     keys: [
                         {field: 'data.category', title: 'Category'},
                         {field: 'data.options.name', title: 'Option Name'}
                     ]
                 });
+                console.log('END EXEC----');
                 assert.equal(csv, csvTestData.withSpecifiedKeys.replace('data.category,data.options.name', 'Category,Option Name'));
             });
 
@@ -535,6 +540,44 @@ export function runTests() {
                 assert.equal(csv, csvTestData.nestedDotKeys.replace(/\\\./g, '.'));
             });
 
+            // Test case for #247
+            it('should not escape nested dots in keys with nested dots in them if turned off via the option', () => {
+                const csv = json2csv(jsonTestData.wildcardMatch, {
+                    keys: ['foo', 'bar', 'baz.a', 'baz.array'],
+                });
+                assert.equal(csv, csvTestData.wildcardMatch);
+            });
+
+            // Test case for #247
+            it('should not escape nested dots in keys with nested dots in them if turned off via the option', () => {
+                const csv = json2csv(jsonTestData.wildcardMatch, {
+                    keys: ['foo', 'bar', { field: 'baz.a', wildcardMatch: true }],
+                });
+                assert.equal(csv, csvTestData.wildcardMatch);
+            });
+
+            // Test case for #247
+            it('should not escape nested dots in keys with nested dots in them if turned off via the option', () => {
+                const updatedCsv = csvTestData.wildcardMatch.replace('baz.a,baz.array', 'baz.a,baz.b,baz.array')
+                    .replace('a,c', 'a,b,c');
+                
+                const csv = json2csv(jsonTestData.wildcardMatch, {
+                    keys: ['foo', 'bar', { field: 'baz', wildcardMatch: true }],
+                });
+                assert.equal(csv, updatedCsv);
+            });
+
+            // Test case for #247
+            it('should not escape nested dots in keys with nested dots in them if turned off via the option', () => {
+                const updatedCsv = csvTestData.wildcardMatch.replace('foo,bar,baz.a,baz.array', 'foo,baz.a,baz.array,bar')
+                    .replace('foo,bar,a,c', 'foo,a,c,bar');
+                
+                const csv = json2csv(jsonTestData.wildcardMatch, {
+                    keys: ['foo', { field: 'baz.a', wildcardMatch: true }, 'bar'],
+                });
+                assert.equal(csv, updatedCsv);
+            });
+
             it('should use a custom value parser function when provided', () => {
                 const updatedCsv = csvTestData.trimmedFields.split('\n');
                 const textRow = 'Parsed Value,Parsed Value,Parsed Value,Parsed Value,Parsed Value';