From f004a3d175e114cfe2a41f782e9f6328f33efa4d Mon Sep 17 00:00:00 2001 From: missinglink Date: Wed, 8 Jul 2020 11:40:16 +0200 Subject: [PATCH 1/2] feat(sqlite): add "dryrun" and "cat" options to fix command. require specifying which fixes to apply manually eg "--hierarchies" --- bin/cmd/sqlite/fix.js | 14 ++++++++++++-- sqlite/fix.js | 30 +++++++++++++++++++----------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/bin/cmd/sqlite/fix.js b/bin/cmd/sqlite/fix.js index 473d94f..58a610a 100644 --- a/bin/cmd/sqlite/fix.js +++ b/bin/cmd/sqlite/fix.js @@ -12,9 +12,19 @@ module.exports = { }) // optional params + yargs.option('dryrun', { + type: 'boolean', + default: false, + describe: 'Do not modify database.' + }) + yargs.option('cat', { + type: 'boolean', + default: false, + describe: 'Print updated records to stdout.' + }) yargs.option('hierarchies', { type: 'boolean', - default: true, + default: false, describe: 'Fix broken hierarchies.' }) }, @@ -26,7 +36,7 @@ module.exports = { // hierarchies if (argv.hierarchies) { if (argv.verbose) { console.error('fixing orphaned hierarchies') } - sqlite.fix.hierarchies(db) + sqlite.fix.hierarchies(db, argv) } } } diff --git a/sqlite/fix.js b/sqlite/fix.js index bf27420..7041c4d 100644 --- a/sqlite/fix.js +++ b/sqlite/fix.js @@ -73,7 +73,7 @@ function findIdsToFix (db, superseded) { } // fix orphaned hierarchies -module.exports.hierarchies = (db) => { +module.exports.hierarchies = (db, options) => { const fetchOne = db.prepare(` SELECT body FROM geojson @@ -97,7 +97,7 @@ module.exports.hierarchies = (db) => { const parentID = feature.getParentId(feat) if (superseded.has(parentID)) { const replacement = superseded.get(parentID) - console.info(`${id} has an incorrect parent_id, replacing ${parentID} with ${replacement}`) + console.error(`${id} has an incorrect parent_id, replacing ${parentID} with ${replacement}`) _.set(feat, 'properties.wof:parent_id', replacement) reindex = true } @@ -109,7 +109,7 @@ module.exports.hierarchies = (db) => { if (hierarchyId === id) { return } // do not update self-references if (superseded.has(hierarchyId)) { const replacement = superseded.get(hierarchyId) - console.info(`${id} has an incorrect ${key}, replacing ${hierarchyId} with ${replacement}`) + console.error(`${id} has an incorrect ${key}, replacing ${hierarchyId} with ${replacement}`) _.set(feat, `properties.wof:hierarchy[${branch}][${key}]`, replacement) reindex = true } @@ -118,14 +118,22 @@ module.exports.hierarchies = (db) => { // delete record and reimport it if (reindex) { - db.prepare('DELETE FROM geojson WHERE is_alt = 0 AND id = :id').run({ id }) - table.geojson.insert(db)(feat) - - db.prepare('DELETE FROM ancestors WHERE id = :id').run({ id }) - table.ancestors.insert(db)(feat) - - db.prepare('DELETE FROM spr WHERE id = :id').run({ id }) - table.spr.insert(db)(feat) + // honour the 'dryrun' flag (disables saving to the db) + if (_.get(options, 'dryrun') !== true) { + db.prepare('DELETE FROM geojson WHERE is_alt = 0 AND id = :id').run({ id }) + table.geojson.insert(db)(feat) + + db.prepare('DELETE FROM ancestors WHERE id = :id').run({ id }) + table.ancestors.insert(db)(feat) + + db.prepare('DELETE FROM spr WHERE id = :id').run({ id }) + table.spr.insert(db)(feat) + } + + // honour the 'cat' flag (prints upddated docs to stdout) + if (_.get(options, 'cat') === true) { + console.log(JSON.stringify(feat)) + } } }) } From c0e1169dc89524f8f6946f4cc7203c69f6574b21 Mon Sep 17 00:00:00 2001 From: missinglink Date: Wed, 8 Jul 2020 11:44:32 +0200 Subject: [PATCH 2/2] feat(exportify): add support for the WOF exportify command. add "fs import" command to write files a stream of features to the filesystem --- README.md | 6 ++--- bin/cmd/feature/exportify.js | 21 +++++++++++++++++ bin/cmd/fs/import.js | 44 ++++++++++++++++++++++++++++++++++++ sqlite/fix.js | 2 +- whosonfirst/exportify.js | 20 ++++++++++++++++ 5 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 bin/cmd/feature/exportify.js create mode 100644 bin/cmd/fs/import.js create mode 100644 whosonfirst/exportify.js diff --git a/README.md b/README.md index f7f0631..4153648 100644 --- a/README.md +++ b/README.md @@ -14,15 +14,15 @@ Ideally we'd like to only depend on utilities which come standard on most *nix s ```bash # check dependencies -which cat find node git bzip2 gzip pv bsdtar +which cat find node git bzip2 gzip pv bsdtar docker ``` ```bash # ubuntu -sudo apt-get install git bzip2 gzip pv libarchive-tools +sudo apt-get install git bzip2 gzip pv libarchive-tools docker # mac OSX -brew install git bzip2 gzip pv libarchive +brew install git bzip2 gzip pv libarchive docker ``` ## CLI diff --git a/bin/cmd/feature/exportify.js b/bin/cmd/feature/exportify.js new file mode 100644 index 0000000..5bbd666 --- /dev/null +++ b/bin/cmd/feature/exportify.js @@ -0,0 +1,21 @@ +const exportify = require('../../../whosonfirst/exportify') +const stream = { + json: require('../../../stream/json'), + miss: require('../../../stream/miss') +} + +module.exports = { + command: 'exportify', + describe: 'run WOF exportify tool on features', + handler: () => { + process.stdin + .pipe(stream.json.parse()) + .pipe(stream.miss.through.obj((json, _, next) => { + // run the exportify command in a docker container + // note: unfortunately it doesn't support multiple lines + // of json, so we need to execute the container once per feature. + next(null, exportify(json)) + })) + .pipe(process.stdout) + } +} diff --git a/bin/cmd/fs/import.js b/bin/cmd/fs/import.js new file mode 100644 index 0000000..148476a --- /dev/null +++ b/bin/cmd/fs/import.js @@ -0,0 +1,44 @@ +const fs = require('fs') +const path = require('path') +const filePath = require('../../../whosonfirst/file').path +const exportify = require('../../../whosonfirst/exportify') +const stream = { + json: require('../../../stream/json'), + miss: require('../../../stream/miss') +} + +module.exports = { + command: 'import ', + describe: 'import (write) features to a filesystem path', + builder: (yargs) => { + // mandatory params + yargs.positional('path', { + type: 'string', + describe: 'Location of file/directory.' + }) + + // optional params + yargs.option('exportify', { + type: 'boolean', + default: false, + describe: 'Run WOF exportify tool before writing files to disk.' + }) + }, + handler: (argv) => { + process.stdin + .pipe(stream.json.parse()) + .pipe(stream.miss.through.obj((feat, enc, next) => { + // ensure path exists + const fullpath = path.join(argv.path, filePath.fromFeature(feat)) + fs.mkdirSync(path.dirname(fullpath), { recursive: true }) + + // optionally 'exportify' the record + if (argv.exportify) { feat = exportify(feat) } + + if (argv.verbose) { console.error(`write ${fullpath}`) } + fs.writeFileSync(fullpath, feat) + + next() + })) + } +} diff --git a/sqlite/fix.js b/sqlite/fix.js index 7041c4d..5481e0f 100644 --- a/sqlite/fix.js +++ b/sqlite/fix.js @@ -130,7 +130,7 @@ module.exports.hierarchies = (db, options) => { table.spr.insert(db)(feat) } - // honour the 'cat' flag (prints upddated docs to stdout) + // honour the 'cat' flag (prints updated docs to stdout) if (_.get(options, 'cat') === true) { console.log(JSON.stringify(feat)) } diff --git a/whosonfirst/exportify.js b/whosonfirst/exportify.js new file mode 100644 index 0000000..a8ee40b --- /dev/null +++ b/whosonfirst/exportify.js @@ -0,0 +1,20 @@ +const _ = require('lodash') +const child = require('child_process') + +const cmd = [ + 'docker', 'run', '-i', '--rm', + 'whosonfirst-exportify', + '/usr/local/bin/wof-exportify', + '--exporter=stdout', + '--stdin' +].join(' ') + +function exportify (feat) { + // serialize objects to JSON strings + if (_.isObject(feat)) { feat = JSON.stringify(feat) } + + // run the exportify command in a docker container + return child.execSync(cmd, { input: feat }).toString('utf8') +} + +module.exports = exportify