Skip to content

Commit d8804ae

Browse files
mrodrigmrodrig
authored
Headerless CSV support (#229)
* feat: Add option to support headerless CSV fixes #227 * test/doc: add tests and docs for headerFields option * docs: document headerFields option * ci: remove unused coveralls file * ci: add github actions to run tests * chore(rel): 3.19.0
1 parent 9fbe36c commit d8804ae

File tree

10 files changed

+176
-21
lines changed

10 files changed

+176
-21
lines changed

‎.coveralls.yml

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎.github/workflows/automated-tests-workflow.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Automated Node Version Tests
2+
3+
on: [push, pull_request]
4+
5+
jobs:
6+
test:
7+
runs-on: ${{ matrix.os }}
8+
9+
strategy:
10+
matrix:
11+
os: [ubuntu-latest]
12+
node: [16, 18, 19]
13+
14+
steps:
15+
- uses: actions/checkout@v3
16+
- name: Use Node ${{ matrix.node }}
17+
uses: actions/setup-node@v3
18+
with:
19+
node-version: ${{ matrix.node }}
20+
- name: Install NPM Modules
21+
run: npm ci
22+
- name: Lint
23+
run: npm run lint
24+
- name: Test
25+
run: |
26+
npm run coverage
27+
- name: Coveralls Parallel
28+
uses: coverallsapp/github-action@master
29+
with:
30+
github-token: ${{ secrets.github_token }}
31+
flag-name: node-${{ runner.os }}-${{ matrix.node }}
32+
parallel: true
33+
34+
coveralls-finished:
35+
needs: test
36+
runs-on: ubuntu-latest
37+
steps:
38+
- name: Coveralls Finished
39+
uses: coverallsapp/github-action@master
40+
with:
41+
github-token: ${{ secrets.github_token }}
42+
parallel-finished: true

‎README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,10 @@ Available in version `2.2.0`, this functionality makes use of promises from the
164164
* Default: `\n`
165165
* `excelBOM` - Boolean - Does the CSV contain a unicode character prepended in order to allow Excel to open a UTF-8 encoded file with non-ASCII characters present?
166166
* Default: `false`
167+
* `headerFields` - Array - Specify the field names (as strings) in place of a header line in the CSV itself.
168+
* Default: Parses the header fields directly from the CSV string
169+
* If you want to generate a nested object (ie. `{info : {name: 'Mike'}}`), then use `.` characters in the string to denote a nested field, like ['info.name']
170+
* If your CSV has a header line included, then don't specify the option to utilize the default values that will be parsed from the CSV.
167171
* `keys` - Array - Specify the keys (as strings) that should be converted.
168172
* Default: `null`
169173
* If you have a nested object (ie. `{info : {name: 'Mike'}}`), then set this to `['info.name']`

‎lib/converter.d.ts

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
export interface ISharedOptions {
1+
interface SharedConverterOptions {
22
/**
33
* Specifies the different types of delimiters
44
*/
@@ -56,7 +56,17 @@ export interface ISharedOptions {
5656
preventCsvInjection?: boolean;
5757
}
5858

59-
export interface IFullOptions extends ISharedOptions {
59+
export interface Csv2JsonOptions extends SharedConverterOptions {
60+
/**
61+
* Specify the header fields in the event that the CSV does not container a header line
62+
*
63+
* If you want to generate a nested object (ie. {info : {name: 'Mike'}}), then use `.` characters in the string to denote a nested field, like ['info.name']
64+
* If your CSV has a header line included, then don't specify the option to utilize the default values that will be parsed from the CSV.
65+
*/
66+
headerFields?: string[];
67+
}
68+
69+
export interface Json2CsvOptions extends SharedConverterOptions {
6070
/**
6171
* Should all documents have the same schema?
6272
* @default false
@@ -121,11 +131,11 @@ export interface IFullOptions extends ISharedOptions {
121131
}
122132

123133
export function json2csv(data: object[],
124-
callback: (err?: Error, csv?: string) => void, options?: IFullOptions): void;
134+
callback: (err?: Error, csv?: string) => void, options?: Json2CsvOptions): void;
125135

126-
export function json2csvAsync(data: object[], options?: IFullOptions): Promise<string>;
136+
export function json2csvAsync(data: object[], options?: Json2CsvOptions): Promise<string>;
127137

128138
export function csv2json(csv: string,
129-
callback: (err?: Error, data?: any[]) => void, options?: ISharedOptions): void;
139+
callback: (err?: Error, data?: any[]) => void, options?: Csv2JsonOptions): void;
130140

131-
export function csv2jsonAsync(csv: string, options?: ISharedOptions): Promise<any[]>;
141+
export function csv2jsonAsync(csv: string, options?: Csv2JsonOptions): Promise<any[]>;

‎lib/csv2json.js

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,25 @@ const Csv2Json = function(options) {
3030
* @returns {*}
3131
*/
3232
function retrieveHeading(lines) {
33-
let params = {lines},
33+
let params = {lines};
34+
35+
if (options.headerFields) {
36+
params.headerFields = options.headerFields.map((headerField, index) => ({
37+
value: processHeaderKey(headerField),
38+
index
39+
}));
40+
} else {
3441
// Generate and return the heading keys
35-
headerRow = params.lines[0];
36-
params.headerFields = headerRow.map((headerKey, index) => ({
37-
value: processHeaderKey(headerKey),
38-
index: index
39-
}));
40-
41-
// If the user provided keys, filter the generated keys to just the user provided keys so we also have the key index
42-
if (options.keys) {
43-
params.headerFields = params.headerFields.filter((headerKey) => options.keys.includes(headerKey.value));
42+
let headerRow = params.lines[0];
43+
params.headerFields = headerRow.map((headerKey, index) => ({
44+
value: processHeaderKey(headerKey),
45+
index
46+
}));
47+
48+
// If the user provided keys, filter the generated keys to just the user provided keys so we also have the key index
49+
if (options.keys) {
50+
params.headerFields = params.headerFields.filter((headerKey) => options.keys.includes(headerKey.value));
51+
}
4452
}
4553

4654
return params;
@@ -227,7 +235,11 @@ const Csv2Json = function(options) {
227235
* @returns {*}
228236
*/
229237
function retrieveRecordLines(params) {
230-
params.recordLines = params.lines.splice(1); // All lines except for the header line
238+
if (options.headerFields) { // This option is passed for instances where the CSV has no header line
239+
params.recordLines = params.lines;
240+
} else { // All lines except for the header line
241+
params.recordLines = params.lines.splice(1);
242+
}
231243

232244
return params;
233245
}

‎package-lock.json

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
},
66
"name": "json-2-csv",
77
"description": "A JSON to CSV and CSV to JSON converter that natively supports sub-documents and auto-generates the CSV heading.",
8-
"version": "3.18.1",
8+
"version": "3.19.0",
99
"homepage": "https://mrodrig.github.io/json-2-csv",
1010
"repository": {
1111
"type": "git",

‎test/config/testJsonFilesList.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ module.exports = {
2525
csvEmptyLastValue: require('../data/json/csvEmptyLastValue'),
2626
unwind: require('../data/json/unwind'),
2727
unwindEmptyArray: require('../data/json/unwindEmptyArray'),
28+
unwindWithSpecifiedKeys: require('../data/json/unwindWithSpecifiedKeys'),
2829
localeFormat: require('../data/json/localeFormat'),
2930
invalidParsedValues: require('../data/json/invalidParsedValues'),
3031
firstColumnWrapCRLF: require('../data/json/firstColumnWrapCRLF.json'),

‎test/csv2json.js

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,59 @@ function runTests(jsonTestData, csvTestData) {
449449
parseValue: () => 'Parsed Value'
450450
});
451451
});
452+
453+
it('should be able to parse CSV with no header line provided', (done) => {
454+
const headerlessCsv = csvTestData.unwindWithSpecifiedKeys.split(options.delimiter.eol)
455+
.splice(1)
456+
.join(options.delimiter.eol);
457+
458+
converter.csv2json(headerlessCsv, (err, json) => {
459+
if (err) done(err);
460+
json.should.deepEqual(jsonTestData.unwindWithSpecifiedKeys);
461+
done();
462+
}, {
463+
headerFields: ['data.category', 'data.options.name']
464+
});
465+
});
466+
467+
it('should be able to parse CSV with no header line provided and different header fields', (done) => {
468+
const headerlessCsv = csvTestData.unwindWithSpecifiedKeys.split(options.delimiter.eol)
469+
.splice(1)
470+
.join(options.delimiter.eol),
471+
expectedJson = jsonTestData.unwindWithSpecifiedKeys.map((doc) => {
472+
doc.category = doc.data.category;
473+
doc.product = doc.data.options.name;
474+
delete doc.data;
475+
return doc;
476+
});
477+
478+
converter.csv2json(headerlessCsv, (err, json) => {
479+
if (err) done(err);
480+
json.should.deepEqual(expectedJson);
481+
done();
482+
}, {
483+
headerFields: ['category', 'product']
484+
});
485+
});
486+
487+
it('should be able to parse CSV with no header line provided and only some of the fields', (done) => {
488+
const headerlessCsv = csvTestData.unwindWithSpecifiedKeys.split(options.delimiter.eol)
489+
.splice(1)
490+
.join(options.delimiter.eol),
491+
expectedJson = jsonTestData.unwindWithSpecifiedKeys.map((doc) => {
492+
doc.category = doc.data.category;
493+
delete doc.data;
494+
return doc;
495+
});
496+
497+
converter.csv2json(headerlessCsv, (err, json) => {
498+
if (err) done(err);
499+
json.should.deepEqual(expectedJson);
500+
done();
501+
}, {
502+
headerFields: ['category']
503+
});
504+
});
452505
});
453506
});
454507

‎test/data/json/unwindWithSpecifiedKeys.json

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
[
2+
{
3+
"data": {
4+
"category": "Computers",
5+
"options": {
6+
"name": "MacBook Pro 15"
7+
}
8+
}
9+
},
10+
{
11+
"data": {
12+
"category": "Computers",
13+
"options": {
14+
"name": "MacBook Air 13"
15+
}
16+
}
17+
},
18+
{
19+
"data": {
20+
"category": "Cars",
21+
"options": {
22+
"name": "Supercharger"
23+
}
24+
}
25+
},
26+
{
27+
"data": {
28+
"category": "Cars",
29+
"options": {
30+
"name": "Turbocharger"
31+
}
32+
}
33+
}
34+
]

0 commit comments

Comments
 (0)