Add readCsvFile helper

build/cog.js

@@ -1,17 +1,11 @@
-const {createReadStream} = require('fs')
-const parse = require('csv-parser')
-const pumpify = require('pumpify').obj
-const getStream = require('get-stream').array
+const {readCsvFile} = require('./util')
 
 function parseTypeLiaison(TNCC) {
   return Number.parseInt(TNCC, 10)
 }
 
 async function extractDepartements(path) {
-  const rows = await getStream(pumpify(
-    createReadStream(path),
-    parse({separator: ',', strict: true})
-  ))
+  const rows = await readCsvFile(path)
 
   return rows.map(row => {
     return {
@@ -25,10 +19,7 @@ async function extractDepartements(path) {
 }
 
 async function extractRegions(path) {
-  const rows = await getStream(pumpify(
-    createReadStream(path),
-    parse({separator: ',', strict: true})
-  ))
+  const rows = await readCsvFile(path)
 
   return rows.map(row => {
     return {
@@ -41,10 +32,7 @@ async function extractRegions(path) {
 }
 
 async function extractArrondissements(path) {
-  const rows = await getStream(pumpify(
-    createReadStream(path),
-    parse({separator: ',', strict: true})
-  ))
+  const rows = await readCsvFile(path)
 
   return rows.map(row => {
     return {

build/collectivites-outremer.js

@@ -1,13 +1,7 @@
-const {createReadStream} = require('fs')
-const parse = require('csv-parser')
-const pumpify = require('pumpify').obj
-const getStream = require('get-stream').array
+const {readCsvFile} = require('./util')
 
 async function extractCommunesCOM(path) {
-  const rows = await getStream(pumpify(
-    createReadStream(path),
-    parse({separator: ',', strict: true})
-  ))
+  const rows = await readCsvFile(path)
 
   return rows.map(row => {
     const commune = {

build/util.js

@@ -1,6 +1,8 @@
 const {join} = require('path')
 const {promisify} = require('util')
 const zlib = require('zlib')
+const {createReadStream} = require('fs')
+const {createGunzip} = require('gunzip-stream')
 const getStream = require('get-stream').array
 const parse = require('csv-parser')
 const decompress = require('decompress')
@@ -37,6 +39,14 @@ async function readSheets(filePath) {
   return xlsx.parse(fileContent, {cellDates: true})
 }
 
+function readCsvFile(filePath, options = {}) {
+  return getStream(pumpify(
+    createReadStream(filePath),
+    createGunzip(),
+    parse({separator: ',', strict: true, ...options})
+  ))
+}
+
 async function writeJsonArray(path, data) {
   const jsonData = '[\n' + data.map(JSON.stringify).join(',\n') + '\n]\n'
   await outputFile(path, jsonData)
@@ -46,4 +56,4 @@ async function writeData(name, data) {
   await writeJsonArray(join(__dirname, '..', 'data', `${name}.json`), data)
 }
 
-module.exports = {writeData, extractDataFromSource, getSourceFilePath, readSheets}
+module.exports = {writeData, extractDataFromSource, getSourceFilePath, readSheets, readCsvFile}

package.json

@@ -26,6 +26,7 @@
     "decompress": "^4.2.0",
     "fs-extra": "^7.0.1",
     "get-stream": "^4.1.0",
+    "gunzip-stream": "^1.0.1",
     "iconv-lite": "^0.4.24",
     "into-stream": "^4.0.0",
     "lodash": "^4.17.11",

yarn.lock

@@ -1388,6 +1388,13 @@ graceful-fs@^4.1.6:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725"
 
+gunzip-stream@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/gunzip-stream/-/gunzip-stream-1.0.1.tgz#50db2d9a2a362265633b136dbaad504277401516"
+  integrity sha512-Hkz9Vd0pAf5JqjpGmNX3R9ZYzxLfFyuaq2f5X867Ki0gcZeryI+njWzZlm6UdbFUvQePgXWPHdweSpV9D7UR4Q==
+  dependencies:
+    is-gzip "^2.0.0"
+
 has-flag@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51"
@@ -1644,6 +1651,11 @@ is-glob@^4.0.0:
   dependencies:
     is-extglob "^2.1.1"
 
+is-gzip@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/is-gzip/-/is-gzip-2.0.0.tgz#f4fed2bbd9f96bf2cb39e19262797fdb15aad933"
+  integrity sha512-jtO4Njg6q58zDo/Pu4027beSZ0VdsZlt8/5Moco6yAg+DIxb5BK/xUYqYG2+MD4+piKldXJNHxRkhEYI2fvrxA==
+
 is-installed-globally@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.1.0.tgz#0dfd98f5a9111716dd535dda6492f67bf3d25a80"