Remove diacritics from identifiers

createInternetArchiveCsv.js

@@ -8,6 +8,7 @@ const { argv } = require('process');
 
 const fields = require('./fields');
 const airtableFields = require('./airtableFields');
+const {normalize} = require('./helpers');
 
 const year = argv[2];
 const month = argv[3];
@@ -59,11 +60,11 @@ function getCsvRow (record) {
   }, {});
 }
 
-const stagedDirectories = new Set(fs.readdirSync(process.env.IA_STAGING));
+const stagedDirectories = new Set(fs.readdirSync(process.env.IA_STAGING).map(normalize));
 
 getRecords().then(airtableRecords => {
   const rows = airtableRecords
-    .filter(record => stagedDirectories.has(record.get('Identifier')))
+    .filter(record => stagedDirectories.has(normalize(record.get('Identifier'))))
     .map(getCsvRow);
 
   const header = fields.map(field => ({id: field.name, title: field.name}));

fields.js

@@ -1,4 +1,5 @@
 const Field = require('./field');
+const {normalize} = require('./helpers');
 
 const fileMapper = record => {
   const id = record.get('Identifier');
@@ -45,7 +46,7 @@ const creatorMapper = record => {
 
 module.exports = [
   // reserved IA metadata fields
-  new Field('Identifier', 'identifier', record => `${record.get('Identifier').replace('+', '-')}`),
+  new Field('Identifier', 'identifier', record => `${normalize(record.get('Identifier')).replace('+', '-')}`),
   new Field(null, 'file', fileMapper),
   new Field('Title', 'title'),
   new Field(null, 'creator', creatorMapper),

helpers.js

@@ -0,0 +1,3 @@
+// Remove diacritics
+// https://stackoverflow.com/a/37511463
+exports.normalize = str => str.normalize("NFD").replace(/[\u0300-\u036f]/g, "");