diff --git a/package-lock.json b/package-lock.json index 4c2e019..dc20805 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,6 @@ "get-stdin": "^9.0.0", "json5": "^2.2.0", "mri": "^1.2.0", - "node-fetch": "^3.0.0", "semver": "^7.3.5", "tar-stream": "^2.2.0", "tmp": "^0.2.1", @@ -32,7 +31,7 @@ "check-code-coverage": "^1.10.0" }, "engines": { - "node": "16" + "node": ">=16" } }, "node_modules/@babel/code-frame": { @@ -625,9 +624,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1318,14 +1317,6 @@ "node": ">=0.10.0" } }, - "node_modules/data-uri-to-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", - "integrity": "sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==", - "engines": { - "node": ">= 6" - } - }, "node_modules/date-time": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/date-time/-/date-time-3.1.0.tgz", @@ -1634,27 +1625,6 @@ "reusify": "^1.0.4" } }, - "node_modules/fetch-blob": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.2.tgz", - "integrity": "sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "dependencies": { - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -2786,22 +2756,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, - "node_modules/node-fetch": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0.tgz", - "integrity": "sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==", - "dependencies": { - "data-uri-to-buffer": "^3.0.1", - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, "node_modules/normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -4052,14 +4006,6 @@ "defaults": "^1.0.3" } }, - "node_modules/web-streams-polyfill": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.1.tgz", - "integrity": "sha512-Czi3fG883e96T4DLEPRvufrF2ydhOOW1+1a6c3gNjH2aIh50DNFBdfwh2AKoOf1rXvpvavAoA11Qdq9+BKjE0Q==", - "engines": { - "node": ">= 8" - } - }, "node_modules/well-known-symbols": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", @@ -4679,9 +4625,9 @@ } }, "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5179,11 +5125,6 @@ "array-find-index": "^1.0.1" } }, - "data-uri-to-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", - "integrity": "sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==" - }, "date-time": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/date-time/-/date-time-3.1.0.tgz", @@ -5416,14 +5357,6 @@ "reusify": "^1.0.4" } }, - "fetch-blob": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.2.tgz", - "integrity": "sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==", - "requires": { - "web-streams-polyfill": "^3.0.3" - } - }, "figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -6196,15 +6129,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, - "node-fetch": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0.tgz", - "integrity": "sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==", - "requires": { - "data-uri-to-buffer": "^3.0.1", - "fetch-blob": "^3.1.2" - } - }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -7095,11 +7019,6 @@ "defaults": "^1.0.3" } }, - "web-streams-polyfill": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.1.tgz", - "integrity": "sha512-Czi3fG883e96T4DLEPRvufrF2ydhOOW1+1a6c3gNjH2aIh50DNFBdfwh2AKoOf1rXvpvavAoA11Qdq9+BKjE0Q==" - }, "well-known-symbols": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", diff --git a/package.json b/package.json index 108c3e0..3fb7039 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,6 @@ "get-stdin": "^9.0.0", "json5": "^2.2.0", "mri": "^1.2.0", - "node-fetch": "^3.0.0", "semver": "^7.3.5", "tar-stream": "^2.2.0", "tmp": "^0.2.1", diff --git a/tools/lib/chrome-source.js b/tools/lib/chrome-source.js index f26439a..80c34cf 100644 --- a/tools/lib/chrome-source.js +++ b/tools/lib/chrome-source.js @@ -25,7 +25,6 @@ import tar from 'tar-stream'; import * as stream from 'stream'; import * as path from 'path'; import { promises as fsPromises } from 'fs'; -import fetch from 'node-fetch'; /** @@ -54,15 +53,22 @@ export async function fetchAllTo(targetPath, chromePaths, revision) { // have (because the compressed version of a higher folder was fetched). chromePaths.sort(); - return Promise.all(chromePaths.map(async (chromePath, i) => { + const requests = []; + + for (const [i, chromePath] of chromePaths.entries()) { const isSatisfied = chromePaths.slice(0, i).some(previous => { return chromePath.startsWith(previous + '/'); }); if (isSatisfied) { - return null; // we'll already have something for this + continue; // we'll already have something for this } - return fetchTo(targetPath, chromePath, revision); - })); + requests.push(fetchTo(targetPath, chromePath, revision)); + + // Small wait to attempt to avoid rate limiting. + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + return Promise.all(requests); } diff --git a/tools/lib/known-namespaces.txt b/tools/lib/known-namespaces.txt new file mode 100644 index 0000000..3873120 --- /dev/null +++ b/tools/lib/known-namespaces.txt @@ -0,0 +1,126 @@ +accessibilityFeatures +action +alarms +app +app.runtime +app.window +appviewTag +audio +bluetooth +bluetoothLowEnergy +bluetoothSocket +bookmarks +browser +browserAction +browsingData +certificateProvider +chrome_url_overrides +clipboard +commands +contentScripts +contentSettings +contextMenus +cookies +crossOriginIsolation +debugger +declarativeContent +declarativeNetRequest +declarativeWebRequest +desktopCapture +devtools.inspectedWindow +devtools.network +devtools.panels +devtools.performance +devtools.recorder +diagnostics +dns +documentScan +dom +downloads +enterprise.deviceAttributes +enterprise.hardwarePlatform +enterprise.kioskInput +enterprise.login +enterprise.networkingAttributes +enterprise.platformKeys +enterprise.remoteApps +events +extension +extensionsManifestTypes +extensionTypes +fileBrowserHandler +fileHandlers +fileSystem +fileSystemProvider +fontSettings +gcm +hid +history +i18n +identity +idle +incognito +input.ime +instanceID +login +loginState +management +manifestTypes +mdns +mediaGalleries +networking.onc +notifications +oauth2 +offscreen +omnibox +pageAction +pageCapture +permissions +platformKeys +power +printerProvider +printing +printingMetrics +privacy +processes +protocolHandlers +proxy +readingList +runtime +scripting +search +serial +sessions +sharedModule +sidePanel +socket +sockets.tcp +sockets.tcpServer +sockets.udp +storage +syncFileSystem +system.cpu +system.display +system.memory +system.network +system.storage +systemLog +tabCapture +tabGroups +tabs +topSites +tts +ttsEngine +types +usb +userScripts +virtualKeyboard +vpnProvider +wallpaper +webAccessibleResources +webAuthenticationProxy +webNavigation +webRequest +webViewRequest +webviewTag +windows diff --git a/tools/lib/render-context.js b/tools/lib/render-context.js index 61eee82..d8fe13d 100644 --- a/tools/lib/render-context.js +++ b/tools/lib/render-context.js @@ -20,12 +20,17 @@ import * as overrideTypes from '../../types/override.js'; import { RenderBuffer } from './buffer.js'; import { isValidToken } from './js-internals.js'; import { last, TraverseContext } from './traverse.js'; +import { readFileSync } from 'fs'; +import log from 'fancy-log'; export class RenderContext { #override; #t; + /** @type {string[]} */ + #knownNamespaces; + /** @type {chromeTypes.SpecCallback[]} */ #callbacks = []; @@ -53,6 +58,7 @@ export class RenderContext { const isVisible = override.isVisible.bind(override); this.#t = new TraverseContext(isVisible); + this.#knownNamespaces = readFileSync("tools/lib/known-namespaces.txt").toString("utf8").split("\n"); } /** @@ -97,6 +103,11 @@ export class RenderContext { return ''; } + if (!this.#knownNamespaces.includes(namespace.namespace)) { + log.warn('Skipping unknown namespace:', namespace.namespace); + return ''; + } + const content = this.renderInnerNamespace(namespace, toplevel); if (!content) { return ''; diff --git a/tools/prepare.js b/tools/prepare.js index e34ff57..a708db4 100755 --- a/tools/prepare.js +++ b/tools/prepare.js @@ -80,15 +80,15 @@ async function prepareInTemp({ majorChrome, workPath, headRevision, definitionsR ? "c279767649d882b71a14697fe9935d3c890a1ec7" : headRevision; - const defitionsPromise = fetchAllTo(workPath, definitionPaths, definitionsRevision); - const toolsPromise = fetchAllTo( + const defitionsResult = await fetchAllTo(workPath, definitionPaths, definitionsRevision); + const toolsResult = await fetchAllTo( workPath, // In newer versions of the tool, generators is vendored into the `json_schema_compiler` directory. useOldToolRevision ? [...toolsPaths, 'ppapi/generators'] : toolsPaths, toolRevision ); - const definitionsFiles = (await defitionsPromise).flatMap(cand => cand ?? []); - const toolsFiles = (await toolsPromise).flatMap(cand => cand ?? []); + const definitionsFiles = defitionsResult.flatMap(cand => cand ?? []); + const toolsFiles = toolsResult.flatMap(cand => cand ?? []); log.warn(`Fetched ${chalk.blue(definitionsFiles.length)} definitions files (IDL/JSON)`); log.warn(`Fetched ${chalk.blue(toolsFiles.length)} tools files to convert IDL => JSON`);