diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7ec0fa7..98ec74f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,7 @@ jobs: - run: npm ci - run: npm run format:check - run: npm run lint + - run: npm run validate:data test: runs-on: ubuntu-latest diff --git a/DATA_REQUIREMENTS.md b/DATA_REQUIREMENTS.md index 37b4cc3..589a367 100644 --- a/DATA_REQUIREMENTS.md +++ b/DATA_REQUIREMENTS.md @@ -56,6 +56,7 @@ Additional format notes: ## 5) Testing - Add test cases to `test/tests.js` under `countriesPostalCodes`: include the country ISO/name, a representative zip, expected region code, and set `usingGoogle: false`. +- Run `npm run validate:data` to verify data-file consistency and static country/geocode allowlists. - Run `npm test` to execute the Node.js test runner. - Optional: verify pretty-name lookups via `getNameFromCountryAndRegion` using the `regionNames` file. diff --git a/README.md b/README.md index 5e09c7f..af9df1d 100644 --- a/README.md +++ b/README.md @@ -93,9 +93,10 @@ console.log(name); // Thüringen npm test npm run lint npm run format:check +npm run validate:data ``` -`npm run validate:data` checks consistency between `country/*.json`, `regions/*.json`, `regionNames/*.json`, and the statically supported countries in `lib/region.js`. +`npm run validate:data` checks consistency between `country/*.json`, `regions/*.json`, `regionNames/*.json`, `geocode/*.json`, and the statically supported countries in `lib/region.js` and `lib/geocode.js`. See [DATA_REQUIREMENTS.md](./DATA_REQUIREMENTS.md) for the country-data format and checklist. diff --git a/lib/geocode.js b/lib/geocode.js index 82029cb..1c40725 100644 --- a/lib/geocode.js +++ b/lib/geocode.js @@ -85,3 +85,4 @@ function coordinates(country, zipCode) { } module.exports = coordinates; +module.exports.availableCountries = Object.freeze([...availableCountries]); diff --git a/lib/region.js b/lib/region.js index e0a83b8..cbf64e0 100644 --- a/lib/region.js +++ b/lib/region.js @@ -324,3 +324,4 @@ class RegionIdentifier { } module.exports = RegionIdentifier; +module.exports.availableCountries = Object.freeze([...availableCountries]); diff --git a/package.json b/package.json index 7eadcbc..0fcc7de 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "scripts": { "test": "node --test", "lint": "eslint .", + "validate:data": "node scripts/validate-data.js", "format": "prettier --write .", "format:check": "prettier --check ." }, diff --git a/regionNames/BEL.json b/regionNames/BEL.json index c05b034..58a35b3 100644 --- a/regionNames/BEL.json +++ b/regionNames/BEL.json @@ -1,5 +1,6 @@ { "BE-BRU": "Brussels Hoofdstedelijk Gewest", "BE-VGL": "Vlaams Gewest", + "BE-VLG": "Vlaams Gewest", "BE-WAL": "Waals Gewest" -} \ No newline at end of file +} diff --git a/regionNames/USA.json b/regionNames/USA.json index fa963ef..4302c10 100644 --- a/regionNames/USA.json +++ b/regionNames/USA.json @@ -9,6 +9,7 @@ "US-CT": "Connecticut", "US-DE": "Delaware", "US-DC": "District of Columbia", + "US-FM": "Federated States of Micronesia", "US-FL": "Florida", "US-GA": "Georgia", "US-GU": "Guam", @@ -22,6 +23,7 @@ "US-LA": "Louisiana", "US-ME": "Maine", "US-MD": "Maryland", + "US-MH": "Marshall Islands", "US-MA": "Massachusetts", "US-MI": "Michigan", "US-MN": "Minnesota", diff --git a/regions/RUS.json b/regions/RUS.json index 8557421..ccc3fea 100644 --- a/regions/RUS.json +++ b/regions/RUS.json @@ -43723,6 +43723,6 @@ }, { "region": "RU-YAN", - "list": [ ] + "list": [] } ] diff --git a/scripts/validate-data.js b/scripts/validate-data.js new file mode 100644 index 0000000..037962b --- /dev/null +++ b/scripts/validate-data.js @@ -0,0 +1,417 @@ +const fs = require('node:fs'); +const path = require('node:path'); + +const Country = require('countryjs'); + +const coordinates = require('../lib/geocode'); +const RegionIdentifier = require('../lib/region'); + +const ISO3_PATTERN = /^[A-Z]{3}$/; +const NUMERIC_VALUE_PATTERN = /^\d+$/; +const ZIP_CODE_FORMATS = new Set(['numeric', 'alpha']); +const KNOWN_UNUSED_REGION_NAMES = new Map([ + ['AUT', new Map([['AT-8', 'legacy region-name entry not currently returned by mappings']])], + ['BEL', new Map([['BE-VGL', 'legacy compatibility alias; mappings return BE-VLG']])], +]); + +const rootDirectory = path.join(__dirname, '..'); +const errors = []; +const warnings = []; + +function addError(message) { + errors.push(message); +} + +function addWarning(message) { + warnings.push(message); +} +const isPlainObject = (value) => + value !== null && typeof value === 'object' && !Array.isArray(value); +const isPositiveInteger = (value) => Number.isSafeInteger(value) && value > 0; +const isNumericValue = (value) => + (typeof value === 'number' && Number.isFinite(value)) || + (typeof value === 'string' && NUMERIC_VALUE_PATTERN.test(value)); +const isCoordinateValue = (value) => + (typeof value === 'number' && Number.isFinite(value)) || typeof value === 'string'; +const sortedValues = (values) => + [...values].toSorted((firstValue, secondValue) => firstValue.localeCompare(secondValue)); + +/** Reads JSON and records parse/read failures instead of stopping validation. */ +function readJson(relativePath) { + try { + return JSON.parse(fs.readFileSync(path.join(rootDirectory, relativePath), 'utf8')); + } catch (err) { + addError(`${relativePath}: invalid JSON (${err.message})`); + return null; + } +} + +/** Returns ISO3 codes from JSON file names in a data directory. */ +function jsonIsoCodes(directoryName) { + return new Set( + fs + .readdirSync(path.join(rootDirectory, directoryName)) + .filter((fileName) => fileName.endsWith('.json')) + .map((fileName) => path.basename(fileName, '.json')), + ); +} + +/** Uses countryjs to reject codes that merely look like ISO3 but are not real countries. */ +function isKnownIso3(isoCode) { + try { + return Boolean(Country.ISOcodes(isoCode, 'ISO3')); + } catch { + return false; + } +} + +/** Validates ISO3 codes from file names and availableCountries sets. */ +function validateIsoSources(sources) { + for (const [sourceName, isoCodes] of Object.entries(sources)) { + for (const isoCode of isoCodes) { + // Catch typos such as lowercase codes, ISO2 codes, or arbitrary labels. + if (!ISO3_PATTERN.test(isoCode)) { + addError(`${sourceName}: invalid ISO3 code "${isoCode}"`); + } else if (!isKnownIso3(isoCode)) { + // Catch values that look like ISO3 but are not known country codes, e.g. "ZZZ". + addError(`${sourceName}: unknown ISO3 code "${isoCode}"`); + } + } + } +} + +/** Ensures one set is fully covered by another, with caller-specific error text. */ +function ensureSubset(requiredIsoCodes, availableIsoCodes, describeMissing) { + for (const isoCode of requiredIsoCodes) { + if (!availableIsoCodes.has(isoCode)) { + addError(describeMissing(isoCode)); + } + } +} + +/** Validates per-country postal-code config used by static region lookup. */ +function validateCountryConfig(isoCode, config) { + // country/*.json is read as a config object by static region lookup. + if (!isPlainObject(config)) { + addError(`country/${isoCode}.json: expected an object`); + return; + } + + // zipCodeFormat controls whether region ranges/lists are interpreted as numbers or strings. + if (!ZIP_CODE_FORMATS.has(config.zipCodeFormat)) { + addError(`country/${isoCode}.json: zipCodeFormat must be "numeric" or "alpha"`); + } + + // Numeric countries need zipCodeLength for zero-padding prefix matches. + if (config.zipCodeFormat === 'numeric' && !isPositiveInteger(config.zipCodeLength)) { + addError( + `country/${isoCode}.json: numeric countries must define positive integer zipCodeLength`, + ); + } +} + +/** Validates display names for region codes returned by static mappings. */ +function validateRegionNames(isoCode, regionNames) { + // regionNames/*.json maps returned ISO 3166-2-like codes to display names. + if (!isPlainObject(regionNames)) { + addError(`regionNames/${isoCode}.json: expected an object`); + return; + } + + // Empty name maps make getNameFromCountryAndRegion() useless for that country. + if (Object.keys(regionNames).length === 0) { + addError(`regionNames/${isoCode}.json: expected at least one region name`); + } + + for (const [regionCode, regionName] of Object.entries(regionNames)) { + // Region names are keyed by values such as "DE-TH" or "US-CA". + if (typeof regionCode !== 'string' || !regionCode.includes('-')) { + addError(`regionNames/${isoCode}.json: invalid region code "${regionCode}"`); + } + + // A code without a readable name would still resolve, but display-name lookup would be empty. + if (typeof regionName !== 'string' || regionName.length === 0) { + addError(`regionNames/${isoCode}.json: ${regionCode} must have a non-empty name`); + } + } +} + +/** Warns about display names that are not returned by current region mappings. */ +function warnAboutUnusedRegionNames(isoCode, regionNames, regionMappings) { + const mappedRegionCodes = new Set( + regionMappings + .filter((entry) => isPlainObject(entry) && typeof entry.region === 'string') + .map((entry) => entry.region), + ); + const knownUnusedRegionNames = KNOWN_UNUSED_REGION_NAMES.get(isoCode) ?? new Map(); + + const sortedRegionCodes = sortedValues(Object.keys(regionNames)); + + for (const regionCode of sortedRegionCodes) { + if (mappedRegionCodes.has(regionCode)) { + continue; + } + + const knownReason = knownUnusedRegionNames.get(regionCode); + const message = knownReason + ? `regionNames/${isoCode}.json: known unused region name "${regionCode}" (${knownReason})` + : `regionNames/${isoCode}.json: unused region name "${regionCode}" is not returned by regions/${isoCode}.json`; + + addWarning(message); + } +} + +/** Validates one regions/*.json entry; empty lists are allowed placeholders. */ +function validateRegionMappingEntry({ isoCode, entry, index, regionNames, countryConfig }) { + const location = `regions/${isoCode}.json[${index}]`; + + // Each regions/*.json item must describe one lookup rule. + if (!isPlainObject(entry)) { + addError(`${location}: expected an object`); + return; + } + + const hasList = Object.hasOwn(entry, 'list'); + const hasLow = Object.hasOwn(entry, 'low'); + const hasHigh = Object.hasOwn(entry, 'high'); + + // The region returned by lookup must also be resolvable to a display name. + if (typeof entry.region !== 'string') { + addError(`${location}: region must be a string`); + } else if (!Object.hasOwn(regionNames, entry.region)) { + addError(`${location}: region "${entry.region}" is missing in regionNames/${isoCode}.json`); + } + + // list is an explicit set of supported zip values; empty lists are allowed placeholders. + if (hasList && !Array.isArray(entry.list)) { + addError(`${location}: list must be an array`); + } + + // A mapping without list or low cannot match any zip code. + if (!hasList && !hasLow) { + addError(`${location}: expected either list or low`); + } + + // high only makes sense as the upper bound of a low/high range. + if (hasHigh && !hasLow) { + addError(`${location}: high requires low`); + } + + // Exact low-only mappings are compared against the cleaned zip string with strict equality. + if (hasLow && !hasHigh && typeof entry.low !== 'string') { + addError(`${location}: low must be a string when high is omitted`); + } + + // Range mappings compare parseInt(zip) to low/high, regardless of zipCodeFormat. + if (hasHigh) { + for (const key of ['low', 'high']) { + if (!isNumericValue(entry[key])) { + addError(`${location}: ${key} must be numeric for range mappings`); + } + } + + // Reversed ranges would never match correctly. + if (isNumericValue(entry.low) && isNumericValue(entry.high)) { + const low = Number(entry.low); + const high = Number(entry.high); + + if (low > high) { + addError(`${location}: low must not be greater than high`); + } + } + } + + if (!Array.isArray(entry.list)) { + return; + } + + if (countryConfig?.zipCodeFormat === 'numeric') { + // Numeric lists are used for exact matches and prefix checks. + for (const [listIndex, value] of entry.list.entries()) { + if (!isNumericValue(value)) { + addError( + `${location}.list[${listIndex}]: value must be numeric for numeric country config`, + ); + } + } + } else { + // Non-numeric lists are matched against the cleaned zip string. + for (const [listIndex, value] of entry.list.entries()) { + if (typeof value !== 'string' || value.length === 0) { + addError( + `${location}.list[${listIndex}]: value must be a non-empty string for non-numeric country config`, + ); + } + } + } +} + +/** Checks that a JSON file is an array, optionally requiring at least one entry. */ +function validateArrayFile(relativePath, value, { allowEmpty = true } = {}) { + // region and geocode data files are consumed as arrays by the runtime code. + if (!Array.isArray(value)) { + addError(`${relativePath}: expected an array`); + return false; + } + + // Top-level regions/*.json files must contain mappings; entry-level empty lists are still OK. + if (!allowEmpty && value.length === 0) { + addError(`${relativePath}: expected at least one mapping`); + } + + return true; +} + +/** Validates one geocode/*.json entry without judging legacy coordinate quality. */ +function validateGeocodeMappingEntry({ isoCode, entry, index }) { + const location = `geocode/${isoCode}.json[${index}]`; + + if (!isPlainObject(entry)) { + addError(`${location}: expected an object`); + return; + } + + if (typeof entry.zip !== 'string' || entry.zip.length === 0) { + addError(`${location}: zip must be a non-empty string`); + } + + for (const key of ['latitude', 'longitude']) { + if (!isCoordinateValue(entry[key])) { + addError(`${location}: ${key} must be a string or finite number`); + } + } +} + +/** Validates all static region lookup data for one country. */ +function validateRegionData(isoCode) { + const countryConfig = readJson(`country/${isoCode}.json`); + const regionNames = readJson(`regionNames/${isoCode}.json`); + const regionMappings = readJson(`regions/${isoCode}.json`); + + validateCountryConfig(isoCode, countryConfig); + validateRegionNames(isoCode, regionNames); + + if (!countryConfig || !regionNames) { + return; + } + + if (validateArrayFile(`regions/${isoCode}.json`, regionMappings, { allowEmpty: false })) { + regionMappings.forEach((entry, index) => { + validateRegionMappingEntry({ isoCode, entry, index, regionNames, countryConfig }); + }); + warnAboutUnusedRegionNames(isoCode, regionNames, regionMappings); + } +} + +/** Validates geocode file shape without judging legacy coordinate quality. */ +function validateGeocodeData(isoCode) { + const geocodeMappings = readJson(`geocode/${isoCode}.json`); + + if (validateArrayFile(`geocode/${isoCode}.json`, geocodeMappings)) { + geocodeMappings.forEach((entry, index) => { + validateGeocodeMappingEntry({ isoCode, entry, index }); + }); + } +} + +/** Prints all collected errors, warnings, or a short success summary. */ +function printResult(dataIsoCodes) { + if (warnings.length > 0) { + console.warn(`Data validation completed with ${warnings.length} warning(s):`); + + for (const warning of warnings) { + console.warn(`- ${warning}`); + } + } + + if (errors.length === 0) { + console.log(`Validated region data for ${dataIsoCodes.regions.size} countries.`); + console.log(`Validated geocode data for ${dataIsoCodes.geocode.size} countries.`); + return; + } + + console.error(`Data validation failed with ${errors.length} error(s):`); + + for (const error of errors) { + console.error(`- ${error}`); + } + + process.exitCode = 1; +} + +const dataIsoCodes = { + country: jsonIsoCodes('country'), + regions: jsonIsoCodes('regions'), + regionNames: jsonIsoCodes('regionNames'), + geocode: jsonIsoCodes('geocode'), +}; +const staticIsoCodes = new Set(RegionIdentifier.availableCountries); +const geocodeIsoCodes = new Set(coordinates.availableCountries); + +// Validate country codes from both data file names and runtime allowlists. +validateIsoSources({ + 'country': dataIsoCodes.country, + 'regions': dataIsoCodes.regions, + 'regionNames': dataIsoCodes.regionNames, + 'geocode': dataIsoCodes.geocode, + 'lib/region.js availableCountries': staticIsoCodes, + 'lib/geocode.js availableCountries': geocodeIsoCodes, +}); + +// Every country supported by lib/region.js needs zip config, region mappings, and names. +ensureSubset( + staticIsoCodes, + dataIsoCodes.country, + (isoCode) => `country/${isoCode}.json: missing file`, +); +ensureSubset( + staticIsoCodes, + dataIsoCodes.regions, + (isoCode) => `regions/${isoCode}.json: missing file`, +); +ensureSubset( + staticIsoCodes, + dataIsoCodes.regionNames, + (isoCode) => `regionNames/${isoCode}.json: missing file`, +); + +// Every regions/*.json file must have the config and names needed to interpret it. +ensureSubset( + dataIsoCodes.regions, + dataIsoCodes.country, + (isoCode) => `country/${isoCode}.json: missing file`, +); +ensureSubset( + dataIsoCodes.regions, + dataIsoCodes.regionNames, + (isoCode) => `regionNames/${isoCode}.json: missing file`, +); +ensureSubset( + dataIsoCodes.regions, + staticIsoCodes, + (isoCode) => `lib/region.js availableCountries: missing "${isoCode}"`, +); + +// lib/geocode.js and geocode/*.json must stay in sync in both directions. +ensureSubset( + geocodeIsoCodes, + dataIsoCodes.geocode, + (isoCode) => `geocode/${isoCode}.json: missing file`, +); +ensureSubset( + dataIsoCodes.geocode, + geocodeIsoCodes, + (isoCode) => `lib/geocode.js availableCountries: missing "${isoCode}"`, +); + +// Deep-validate the contents only after the cross-file existence checks are registered. +for (const isoCode of sortedValues(dataIsoCodes.regions)) { + validateRegionData(isoCode); +} + +// Geocode content validation is intentionally shallow because legacy coordinates are noisy. +for (const isoCode of sortedValues(dataIsoCodes.geocode)) { + validateGeocodeData(isoCode); +} + +printResult(dataIsoCodes);