tooling for generating upstream PRs (including exportify)

README.md

@@ -14,15 +14,15 @@ Ideally we'd like to only depend on utilities which come standard on most *nix s
 
 ```bash
 # check dependencies
-which cat find node git bzip2 gzip pv bsdtar
+which cat find node git bzip2 gzip pv bsdtar docker
 ```
 
 ```bash
 # ubuntu
-sudo apt-get install git bzip2 gzip pv libarchive-tools
+sudo apt-get install git bzip2 gzip pv libarchive-tools docker
 
 # mac OSX
-brew install git bzip2 gzip pv libarchive
+brew install git bzip2 gzip pv libarchive docker
 ```
 
 ## CLI

bin/cmd/feature/exportify.js

@@ -0,0 +1,21 @@
+const exportify = require('../../../whosonfirst/exportify')
+const stream = {
+  json: require('../../../stream/json'),
+  miss: require('../../../stream/miss')
+}
+
+module.exports = {
+  command: 'exportify',
+  describe: 'run WOF exportify tool on features',
+  handler: () => {
+    process.stdin
+      .pipe(stream.json.parse())
+      .pipe(stream.miss.through.obj((json, _, next) => {
+        // run the exportify command in a docker container
+        // note: unfortunately it doesn't support multiple lines
+        // of json, so we need to execute the container once per feature.
+        next(null, exportify(json))
+      }))
+      .pipe(process.stdout)
+  }
+}

bin/cmd/fs/import.js

@@ -0,0 +1,44 @@
+const fs = require('fs')
+const path = require('path')
+const filePath = require('../../../whosonfirst/file').path
+const exportify = require('../../../whosonfirst/exportify')
+const stream = {
+  json: require('../../../stream/json'),
+  miss: require('../../../stream/miss')
+}
+
+module.exports = {
+  command: 'import <path>',
+  describe: 'import (write) features to a filesystem path',
+  builder: (yargs) => {
+    // mandatory params
+    yargs.positional('path', {
+      type: 'string',
+      describe: 'Location of file/directory.'
+    })
+
+    // optional params
+    yargs.option('exportify', {
+      type: 'boolean',
+      default: false,
+      describe: 'Run WOF exportify tool before writing files to disk.'
+    })
+  },
+  handler: (argv) => {
+    process.stdin
+      .pipe(stream.json.parse())
+      .pipe(stream.miss.through.obj((feat, enc, next) => {
+        // ensure path exists
+        const fullpath = path.join(argv.path, filePath.fromFeature(feat))
+        fs.mkdirSync(path.dirname(fullpath), { recursive: true })
+
+        // optionally 'exportify' the record
+        if (argv.exportify) { feat = exportify(feat) }
+
+        if (argv.verbose) { console.error(`write ${fullpath}`) }
+        fs.writeFileSync(fullpath, feat)
+
+        next()
+      }))
+  }
+}

bin/cmd/sqlite/fix.js

@@ -12,9 +12,19 @@ module.exports = {
     })
 
     // optional params
+    yargs.option('dryrun', {
+      type: 'boolean',
+      default: false,
+      describe: 'Do not modify database.'
+    })
+    yargs.option('cat', {
+      type: 'boolean',
+      default: false,
+      describe: 'Print updated records to stdout.'
+    })
     yargs.option('hierarchies', {
       type: 'boolean',
-      default: true,
+      default: false,
       describe: 'Fix broken hierarchies.'
     })
   },
@@ -26,7 +36,7 @@ module.exports = {
     // hierarchies
     if (argv.hierarchies) {
       if (argv.verbose) { console.error('fixing orphaned hierarchies') }
-      sqlite.fix.hierarchies(db)
+      sqlite.fix.hierarchies(db, argv)
     }
   }
 }

sqlite/fix.js

@@ -73,7 +73,7 @@ function findIdsToFix (db, superseded) {
 }
 
 // fix orphaned hierarchies
-module.exports.hierarchies = (db) => {
+module.exports.hierarchies = (db, options) => {
   const fetchOne = db.prepare(`
     SELECT body
     FROM geojson
@@ -97,7 +97,7 @@ module.exports.hierarchies = (db) => {
     const parentID = feature.getParentId(feat)
     if (superseded.has(parentID)) {
       const replacement = superseded.get(parentID)
-      console.info(`${id} has an incorrect parent_id, replacing ${parentID} with ${replacement}`)
+      console.error(`${id} has an incorrect parent_id, replacing ${parentID} with ${replacement}`)
       _.set(feat, 'properties.wof:parent_id', replacement)
       reindex = true
     }
@@ -109,7 +109,7 @@ module.exports.hierarchies = (db) => {
         if (hierarchyId === id) { return } // do not update self-references
         if (superseded.has(hierarchyId)) {
           const replacement = superseded.get(hierarchyId)
-          console.info(`${id} has an incorrect ${key}, replacing ${hierarchyId} with ${replacement}`)
+          console.error(`${id} has an incorrect ${key}, replacing ${hierarchyId} with ${replacement}`)
           _.set(feat, `properties.wof:hierarchy[${branch}][${key}]`, replacement)
           reindex = true
         }
@@ -118,14 +118,22 @@ module.exports.hierarchies = (db) => {
 
     // delete record and reimport it
     if (reindex) {
-      db.prepare('DELETE FROM geojson WHERE is_alt = 0 AND id = :id').run({ id })
-      table.geojson.insert(db)(feat)
-
-      db.prepare('DELETE FROM ancestors WHERE id = :id').run({ id })
-      table.ancestors.insert(db)(feat)
-
-      db.prepare('DELETE FROM spr WHERE id = :id').run({ id })
-      table.spr.insert(db)(feat)
+      // honour the 'dryrun' flag (disables saving to the db)
+      if (_.get(options, 'dryrun') !== true) {
+        db.prepare('DELETE FROM geojson WHERE is_alt = 0 AND id = :id').run({ id })
+        table.geojson.insert(db)(feat)
+
+        db.prepare('DELETE FROM ancestors WHERE id = :id').run({ id })
+        table.ancestors.insert(db)(feat)
+
+        db.prepare('DELETE FROM spr WHERE id = :id').run({ id })
+        table.spr.insert(db)(feat)
+      }
+
+      // honour the 'cat' flag (prints updated docs to stdout)
+      if (_.get(options, 'cat') === true) {
+        console.log(JSON.stringify(feat))
+      }
     }
   })
 }

whosonfirst/exportify.js

@@ -0,0 +1,20 @@
+const _ = require('lodash')
+const child = require('child_process')
+
+const cmd = [
+  'docker', 'run', '-i', '--rm',
+  'whosonfirst-exportify',
+  '/usr/local/bin/wof-exportify',
+  '--exporter=stdout',
+  '--stdin'
+].join(' ')
+
+function exportify (feat) {
+  // serialize objects to JSON strings
+  if (_.isObject(feat)) { feat = JSON.stringify(feat) }
+
+  // run the exportify command in a docker container
+  return child.execSync(cmd, { input: feat }).toString('utf8')
+}
+
+module.exports = exportify