Simple Node.js Dedup CSV

Published on:

Here is a simple way to de-duplicate a CSV file on a single field in Node.js.

Not gonna help you too much without the pipedream library. But . . .

#!/usr/bin/env node

'use strict';

const dups = {};
const field = process.argv[2];
if (!field) {
  console.error('Field to dedup on required.');
  process.exit(13);
}
require('pipedream').run({
  transform(i, cb) {
    if (dups[i[field]]) {
      // skip it

      cb();
    } else {
      dups[i[field]] = true;
      cb(null, i);
    }
  }
});