pull data from official epsg.org API

build/fetch.js

@@ -0,0 +1,60 @@
+'use strict'
+
+const {stringify} = require('qs')
+const fetch = require('node-fetch')
+const createThrottle = require('p-throttle')
+const {ok, strictEqual} = require('assert')
+
+const BASE_URL = 'https://apps.epsg.org/api/v1/'
+const USER_AGENT = 'https://github.com/derhuerst/epsg-index'
+
+const rawFetchFromEpsgOrg = async (endpoint, query = {}) => {
+	const url = BASE_URL + endpoint + '?' + stringify(query)
+	const res = await fetch(url, {
+		mode: 'cors', redirect: 'follow',
+		headers: {
+			'User-Agent': USER_AGENT,
+			'Accept': 'application/json',
+		},
+	})
+	if (!res.ok) {
+		const err = new Error(res.statusText)
+		err.query = query
+		err.statusCode = res.status
+		err.res = res
+		throw err
+	}
+	return await res.json()
+}
+
+const throttle = createThrottle({
+	limit: 50, // find proper limit
+	interval: 60 * 1000,
+})
+const throttledFetchFromEpsgOrg = throttle(rawFetchFromEpsgOrg)
+
+const fetchEpsgOrgPage = async (page, includeDeprecated = true, includeWorld = true) => {
+	ok(Number.isInteger(page), 'page must be an integer')
+
+	return await throttledFetchFromEpsgOrg('/GeodeticCoordRefSystem/', {
+		includeDeprecated,
+		includeWorld,
+		sortField: 'Code',
+		pageSize: 100,
+		page,
+	})
+}
+
+const fetchEpsgOrgSystem = async (code) => {
+	strictEqual(typeof code, 'string', 'code must be a string')
+	ok(code, 'code must not be empty')
+
+	return await throttledFetchFromEpsgOrg(`/GeodeticCoordRefSystem/${code}`)
+}
+
+module.exports = {
+	rawFetchFromEpsgOrg,
+	fetchFromEpsgOrg: throttledFetchFromEpsgOrg,
+	fetchPage: fetchEpsgOrgPage,
+	fetchSystem: fetchEpsgOrgSystem,
+}

build/index.js

@@ -1,124 +1,97 @@
 'use strict'
 
-const createQueue = require('queue')
-const pick = require('lodash.pick')
-const path = require('path')
-const fs = require('fs')
-
-const req = require('./request')
-
-const showError = (err) => {
-	console.error(err)
-	process.exit(1)
+const {join: pathJoin} = require('path')
+const {writeFile} = require('fs/promises')
+
+const {
+	fetchPage: _fetchPage,
+	fetchSystem: _fetchSystem,
+} = require('./fetch')
+
+const BASE_DIR = pathJoin(__dirname, '..')
+const SYSTEMS_DIR = pathJoin(BASE_DIR, 's')
+
+const fetchSystem = async (code) => {
+	const s = await _fetchSystem(code)
+	const selfLink = Array.isArray(s.Links) ? s.Links.find(l => l.rel === 'self') || null : null
+
+	// todo: validate using schema?
+	return {
+		'@id': selfLink && selfLink.href || null,
+		code: s.Code ? s.Code + '' : null,
+		name: s.Name || null,
+		kind: s.Kind || null,
+
+		remark: s.Remark || null,
+		dataSource: s.DataSource || null,
+		informationSource: s.informationSource || null,
+		deprecated: s.Deprecated,
+		revisionDate: s.RevisionDate || null,
+		// todo: s.{GeoidModels,Usage,Deformations,Alias}
+		// todo: s.{Changes,Deprecations,Supersessions}
+
+		datum: s.Datum ? {
+			'@id': s.Datum.href || null,
+			code: s.Datum.Code ? s.Datum.Code + '' : null,
+			name: s.Datum.Name || null,
+		} : null,
+		baseCoordRefSystem: s.BaseCoordRefSystem ? {
+			'@id': s.BaseCoordRefSystem.href || null,
+			code: s.BaseCoordRefSystem.Code ? s.BaseCoordRefSystem.Code + '' : null,
+			name: s.BaseCoordRefSystem.Name || null,
+		} : null,
+		conversion: s.Conversion ? {
+			'@id': s.Conversion.href || null,
+			code: s.Conversion.Code ? s.Conversion.Code + '' : null,
+			name: s.Conversion.Name || null,
+		} : null,
+
+		// todo: wkt, proj4, bbox, unit, accuracy
+		// from https://apps.epsg.org/api/v1/CoordRefSystem/{id}/export/?format=wkt ?
+	}
 }
 
-const getNrOfPages = () => {
-	return req({q: ''})
-	.then(data => Math.ceil(data.number_result / data.results.length))
-}
-
-const fetchAll = (nrOfPages) => {
-	return new Promise((yay, nay) => {
-		const queue = createQueue({concurrency: 8, autostart: true})
-		let results = []
-
-		const fetch = (i) => {
-			const job = (cb) => {
-				req({q: '', page: i})
-				.then((data) => {
-					results = results.concat(data.results)
-					cb()
-				})
-				.catch(cb)
-			}
-
-			job.title = i + ''
-			return job
-		}
-
-		queue.once('error', (err) => {
-			queue.stop()
-			nay(err)
-		})
-		queue.once('end', (err) => {
-			if (!err) yay(results)
-		})
-		queue.on('success', (_, job) => {
-			console.error(job.title + '/' + nrOfPages)
-		})
-
-		for (let i = 0; i <= nrOfPages; i++) {
-		// for (let i = 0; i <= 10; i++) { // todo
-			queue.push(fetch(i))
-		}
-	})
-}
+const fetchPage = async (pageIdx) => {
+	const page = await _fetchPage(pageIdx)
 
-const parseResult = (res) => {
-	return Object.assign(pick(res, [
-		'code', 'kind', 'name'
-	]), {
-		wkt: res.wkt || null,
-		proj4: res.proj4 || null,
-		bbox: res.bbox || null,
-		unit: res.unit || null,
-		area: res.area || null,
-		accuracy: res.accuracy !== 'unknown' ? (res.accuracy || null) : null
-	})
+	const systemCodes = page.Results.map(system => system.Code + '')
+	return {
+		systemCodes,
+		totalSystems: page.TotalResults,
+		pageSize: page.PageSize,
+	}
 }
 
-const dir = path.join(__dirname, '..', 's')
-
-const storeIndividuals = (index) => {
-	return new Promise((yay, nay) => {
-		const queue = createQueue({concurrency: 8, autostart: true})
-
-		const store = (result) => {
-			const job = (cb) => {
-				const dest = path.join(dir, result.code + '.json')
-				fs.writeFile(dest, JSON.stringify(result), cb)
-			}
-
-			job.title = result.code
-			return job
-		}
-
-		queue.once('error', (err) => {
-			queue.stop()
-			nay(err)
-		})
-		queue.once('end', (err) => {
-			if (!err) yay()
-		})
-
-		for (let result of index) {
-			queue.push(store(result))
-		}
-	})
-}
-
-const storeAll = (index) => {
-	return new Promise((yay, nay) => {
-		const all = index.reduce((all, result) => {
-			all[result.code] = result
-			return all
-		}, {})
-
-		const dest = path.join(dir, '..', 'all.json')
-		fs.writeFile(dest, JSON.stringify(all), (err) => {
-			if (err) nay(err)
-			else yay()
-		})
-	})
-}
-
-getNrOfPages()
-.then(fetchAll)
-.then((results) => {
-	const index = results.map(parseResult)
-	return Promise.all([
-		storeAll(index),
-		storeIndividuals(index)
-	])
+;(async () => {
+	const page0 = await fetchPage(0)
+	const {totalSystems, pageSize} = page0
+
+	const allSystemCodes = new Set(page0.systemCodes)
+	for (let pageIdx = 1; pageIdx < Math.ceil(totalSystems / pageSize); pageIdx++) {
+		const {systemCodes} = await fetchPage(pageIdx)
+		console.info(`fetched page ${pageIdx}`)
+
+		for (const code of systemCodes) allSystemCodes.add(code)
+	}
+
+	let fetchedSystems = 0
+	const _systems = await Promise.all(Array.from(allSystemCodes).map((async (code) => {
+		const system = await fetchSystem(code)
+		fetchedSystems++
+		console.info(`fetched system ${code} – ${fetchedSystems}/${totalSystems} systems`)
+
+		const dest = pathJoin(SYSTEMS_DIR, system.code + '.json')
+		await writeFile(dest, JSON.stringify(system))
+		return system
+	})))
+
+	// todo: build all.json
+	const byCode = {}
+	for (const s of _systems) byCode[s.code] = s
+	const allDest = pathJoin(BASE_DIR, 'all.json')
+	await writeFile(allDest, JSON.stringify(byCode))
+})()
+.catch((err) => {
+	console.error(err)
+	process.exit(1)
 })
-.catch(console.error)

license.data.md

@@ -0,0 +1,10 @@
+https://epsg.org/terms-of-use.html
+
+> The data may be used, copied and distributed subject to the following conditions:
+> - Whilst every effort has been made to ensure the accuracy of the information contained in the EPSG Facilities, neither the IOGP nor any of its members past present or future warrants their accuracy or will, regardless of its or their negligence, assume liability for any foreseeable or unforeseeable use made thereof, which liability is hereby excluded. Consequently, such use is at your own risk. You are obliged to inform anyone to whom you provide the EPSG Facilities of these Terms of Use.
+> - DATA AND INFORMATION PROVIDED IN THE EPSG FACILITIES ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
+> - The data may be included in any commercial package provided that any commerciality is based on value added by the provider and not on a value ascribed to the EPSG Dataset which is made available at no charge.
+> - Ownership of the EPSG Dataset by IOGP must be acknowledged in any publication or transmission (by whatever means) thereof (including permitted modifications).
+> - Subsets of information may be extracted from the dataset. Users are advised that coordinate reference system and coordinate transformation descriptions are incomplete unless all elements detailed as essential in IOGP Surveying and Positioning Guidance Note 7-1 Annex A are included.
+> - Essential elements should preferably be reproduced as described in the dataset. Modification of parameter values is permitted as described in the table below to allow change to the content of the information provided that numeric equivalence is achieved. Numeric equivalence refers to the results of geodetic calculations in which the parameters are used, for example (i) conversion of ellipsoid defining parameters, or (ii) conversion of parameters between one and two standard parallel projection methods, or (iii) conversion of parameters between 7-parameter geocentric transformation methods.
+> - No data that has been modified other than as permitted in these Terms of Use shall be attributed to the EPSG Dataset.

package.json

@@ -6,7 +6,8 @@
 	"files": [
 		"index.js",
 		"s",
-		"all.json"
+		"all.json",
+		"license.data.md"
 	],
 	"keywords": [
 		"epsg",
@@ -25,13 +26,11 @@
 	"engines": {
 		"node": ">=8"
 	},
-	"dependencies": {},
+	"depedencies": {},
 	"devDependencies": {
-		"fetch-ponyfill": "^6.0.0",
-		"lodash.pick": "^4.4.0",
-		"pinkie-promise": "^2.0.1",
+		"node-fetch": "^2.6.7",
+		"p-throttle": "^4.1.1",
 		"qs": "^6.5.0",
-		"queue": "^6.0.0",
 		"tap-min": "^2.0.0",
 		"tape": "^5.0.0"
 	},

readme.md

@@ -7,6 +7,8 @@
 [![support me via GitHub Sponsors](https://img.shields.io/badge/support%20me-donate-fa7664.svg)](https://github.com/sponsors/derhuerst)
 [![chat with me on Twitter](https://img.shields.io/badge/chat%20with%20me-on%20Twitter-1da1f2.svg)](https://twitter.com/derhuerst)
 
+*Note:* The data is licensed according to [EPSG's apparently proprietary license](license.data.md).
+
 
 ## Installing
 
@@ -25,15 +27,29 @@ console.log(epsg4326)
 
 ```js
 {
+	'@id': 'https://apps.epsg.org/api/v1/GeodeticCoordRefSystem/4326',
 	code: '4326',
-	kind: 'CRS-GEOGCRS',
 	name: 'WGS 84',
-	wkt: 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]',
-	proj4: '+proj=longlat +datum=WGS84 +no_defs',
-	bbox: [90, -180, -90, 180],
-	unit: 'degree (supplier to define representation)',
-	area: 'World.',
-	accuracy: null
+	kind: 'geographic 2D',
+	remark: null,
+	dataSource: 'EPSG',
+	informationSource: null,
+	revisionDate: '2020-03-14T00:00:00',
+	datum: {
+		'@id': 'https://apps.epsg.org/api/v1/Datum/6326',
+		code: '6326',
+		name: 'World Geodetic System 1984 ensemble',
+	},
+	baseCoordRefSystem: {
+		'@id': 'https://apps.epsg.org/api/v1/GeodeticCoordRefSystem/4979',
+		code: '4979',
+		name: 'WGS 84',
+	},
+	conversion: {
+		'@id': 'https://apps.epsg.org/api/v1/Conversion/15593',
+		code: '15593',
+		name: 'geographic3D to geographic2D',
+	},
 }
 ```