diff --git a/.gitignore b/.gitignore index 081bef8..1d03d10 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ node_modules/ +dist/ .DS_Store .vscode/ +*.log diff --git a/.npmignore b/.npmignore index a549746..ecb515f 100644 --- a/.npmignore +++ b/.npmignore @@ -1,4 +1,11 @@ -.git/ .vscode/ node_modules/ .DS_Store +tests/ +docs/ +*.md +!README.md +eslint.config.mjs +.prettierrc +.prettierignore +tsconfig.json diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..82bde52 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,4 @@ +node_modules +dist +rules +bun.lock diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..c159b89 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,8 @@ +{ + "semi": true, + "singleQuote": false, + "tabWidth": 2, + "trailingComma": "all", + "printWidth": 100, + "endOfLine": "lf" +} diff --git a/README.md b/README.md index 74531b5..911625e 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,82 @@ # WhatsWeb -Identifies websites. +网站指纹识别与安全扫描工具。 / Identifies websites. ## Installation -``` bash -$ npm i -g whatsweb -# Or -$ yarn global add whatsweb +```bash +# 使用 Bun (推荐) +bun install -g whatsweb + +# 使用 npm +npm install -g whatsweb ``` ## Usage -``` bash -$ whatsweb URLs [options] +```bash +whatsweb [options] URLs ``` ### Options ``` - -V, --version output the version number - -f Targets file path - -c, --concurrency Start specified NUMBER of concurrency (default: 10) - --network Scan all Target/MASK hosts - --timeout Max scan minutes for request (default: 10000) - --user-agent Custom User-Agent (default: Mozilla/5.0 whatsweb/0.1.0) - -o, --output Output file path - --show-error Show error message - -h, --help output usage information +-V, --version 输出版本号 +-f 从文件读取目标列表 +-c, --concurrency 并发数 (默认: 50) +--network 扫描 Target/MASK 网段内所有主机 +--timeout 请求超时毫秒数 (默认: 10000) +--user-agent 自定义 User-Agent +-o, --output 输出文件路径 (JSONL 格式) +--show-error 显示错误信息 +-h, --help 显示帮助 ``` ### Example -[![example](https://asciinema.org/a/QlO4vqhnPEF0Hwf5zqCFjKgT2.png)](https://asciinema.org/a/QlO4vqhnPEF0Hwf5zqCFjKgT2) \ No newline at end of file +```bash +# 扫描单个目标 +whatsweb http://example.com + +# 扫描多个目标 +whatsweb http://a.com http://b.com http://c.com + +# 从文件读取 + 并发 20 + 输出到文件 +whatsweb -f targets.txt -c 20 -o results.jsonl + +# 扫描整个 /24 网段 +whatsweb http://192.168.1.1 --network 24 +``` + +## Tech Stack + +- **Runtime**: Bun +- **Language**: TypeScript +- **HTTP**: Bun native fetch +- **Utilities**: es-toolkit, p-limit, zod +- **Fingerprinting**: wappalyzer@6.10.66 +- **GeoIP**: geoip-lite + +## Architecture + +插件化架构：核心引擎 + 可扩展的插件系统。 + +内置插件： +- **Base Information** — HTTP 状态、标题、重定向、x-* 头 +- **Email** — HTML 中的邮箱地址提取 +- **Geolocation** — IP 地理位置 +- **Wappalyzer** — 技术栈识别 +- **BBScan** — 敏感路径扫描 + +## Development + +```bash +bun install # 安装依赖 +bun test # 运行测试 +bun run lint # Lint 检查 +bun run dev # 开发运行 +``` + +## License + +MIT diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..4d84a37 --- /dev/null +++ b/bun.lock @@ -0,0 +1,433 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "whatsweb", + "dependencies": { + "cli-progress": "^3.12.0", + "commander": "^12.0.0", + "es-toolkit": "^1.0.0", + "geoip-lite": "^1.4.0", + "p-limit": "^7.0.0", + "picocolors": "^1.0.0", + "wappalyzer": "6.10.66", + "zod": "^3.24.0", + }, + "devDependencies": { + "@eslint/js": "^10.0.1", + "@types/bun": "latest", + "@types/cli-progress": "^3.11.0", + "@types/geoip-lite": "^1.4.4", + "eslint": "^10.1.0", + "eslint-config-prettier": "^10.1.8", + "prettier": "^3.8.1", + "typescript": "^5.5.0", + "typescript-eslint": "^8.58.0", + }, + }, + }, + "packages": { + "@babel/code-frame": ["@babel/code-frame@7.29.0", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw=="], + + "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="], + + "@eslint-community/eslint-utils": ["@eslint-community/eslint-utils@4.9.1", "", { "dependencies": { "eslint-visitor-keys": "^3.4.3" }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ=="], + + "@eslint-community/regexpp": ["@eslint-community/regexpp@4.12.2", "", {}, "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew=="], + + "@eslint/config-array": ["@eslint/config-array@0.23.3", "", { "dependencies": { "@eslint/object-schema": "^3.0.3", "debug": "^4.3.1", "minimatch": "^10.2.4" } }, "sha512-j+eEWmB6YYLwcNOdlwQ6L2OsptI/LO6lNBuLIqe5R7RetD658HLoF+Mn7LzYmAWWNNzdC6cqP+L6r8ujeYXWLw=="], + + "@eslint/config-helpers": ["@eslint/config-helpers@0.5.3", "", { "dependencies": { "@eslint/core": "^1.1.1" } }, "sha512-lzGN0onllOZCGroKJmRwY6QcEHxbjBw1gwB8SgRSqK8YbbtEXMvKynsXc3553ckIEBxsbMBU7oOZXKIPGZNeZw=="], + + "@eslint/core": ["@eslint/core@1.1.1", "", { "dependencies": { "@types/json-schema": "^7.0.15" } }, "sha512-QUPblTtE51/7/Zhfv8BDwO0qkkzQL7P/aWWbqcf4xWLEYn1oKjdO0gglQBB4GAsu7u6wjijbCmzsUTy6mnk6oQ=="], + + "@eslint/js": ["@eslint/js@10.0.1", "", { "peerDependencies": { "eslint": "^10.0.0" }, "optionalPeers": ["eslint"] }, "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA=="], + + "@eslint/object-schema": ["@eslint/object-schema@3.0.3", "", {}, "sha512-iM869Pugn9Nsxbh/YHRqYiqd23AmIbxJOcpUMOuWCVNdoQJ5ZtwL6h3t0bcZzJUlC3Dq9jCFCESBZnX0GTv7iQ=="], + + "@eslint/plugin-kit": ["@eslint/plugin-kit@0.6.1", "", { "dependencies": { "@eslint/core": "^1.1.1", "levn": "^0.4.1" } }, "sha512-iH1B076HoAshH1mLpHMgwdGeTs0CYwL0SPMkGuSebZrwBp16v415e9NZXg2jtrqPVQjf6IANe2Vtlr5KswtcZQ=="], + + "@humanfs/core": ["@humanfs/core@0.19.1", "", {}, "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA=="], + + "@humanfs/node": ["@humanfs/node@0.16.7", "", { "dependencies": { "@humanfs/core": "^0.19.1", "@humanwhocodes/retry": "^0.4.0" } }, "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ=="], + + "@humanwhocodes/module-importer": ["@humanwhocodes/module-importer@1.0.1", "", {}, "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA=="], + + "@humanwhocodes/retry": ["@humanwhocodes/retry@0.4.3", "", {}, "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ=="], + + "@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="], + + "@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="], + + "@types/esrecurse": ["@types/esrecurse@4.3.1", "", {}, "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw=="], + + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], + + "@types/geoip-lite": ["@types/geoip-lite@1.4.4", "", {}, "sha512-2uVfn+C6bX/H356H6mjxsWUA5u8LO8dJgSBIRO/NFlpMe4DESzacutD/rKYrTDKm1Ugv78b4Wz1KvpHrlv3jSw=="], + + "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="], + + "@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], + + "@types/yauzl": ["@types/yauzl@2.10.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q=="], + + "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.58.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.12.2", "@typescript-eslint/scope-manager": "8.58.0", "@typescript-eslint/type-utils": "8.58.0", "@typescript-eslint/utils": "8.58.0", "@typescript-eslint/visitor-keys": "8.58.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.58.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-RLkVSiNuUP1C2ROIWfqX+YcUfLaSnxGE/8M+Y57lopVwg9VTYYfhuz15Yf1IzCKgZj6/rIbYTmJCUSqr76r0Wg=="], + + "@typescript-eslint/parser": ["@typescript-eslint/parser@8.58.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.58.0", "@typescript-eslint/types": "8.58.0", "@typescript-eslint/typescript-estree": "8.58.0", "@typescript-eslint/visitor-keys": "8.58.0", "debug": "^4.4.3" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-rLoGZIf9afaRBYsPUMtvkDWykwXwUPL60HebR4JgTI8mxfFe2cQTu3AGitANp4b9B2QlVru6WzjgB2IzJKiCSA=="], + + "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.58.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.58.0", "@typescript-eslint/types": "^8.58.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-8Q/wBPWLQP1j16NxoPNIKpDZFMaxl7yWIoqXWYeWO+Bbd2mjgvoF0dxP2jKZg5+x49rgKdf7Ck473M8PC3V9lg=="], + + "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.58.0", "", { "dependencies": { "@typescript-eslint/types": "8.58.0", "@typescript-eslint/visitor-keys": "8.58.0" } }, "sha512-W1Lur1oF50FxSnNdGp3Vs6P+yBRSmZiw4IIjEeYxd8UQJwhUF0gDgDD/W/Tgmh73mxgEU3qX0Bzdl/NGuSPEpQ=="], + + "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.58.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-doNSZEVJsWEu4htiVC+PR6NpM+pa+a4ClH9INRWOWCUzMst/VA9c4gXq92F8GUD1rwhNvRLkgjfYtFXegXQF7A=="], + + "@typescript-eslint/type-utils": ["@typescript-eslint/type-utils@8.58.0", "", { "dependencies": { "@typescript-eslint/types": "8.58.0", "@typescript-eslint/typescript-estree": "8.58.0", "@typescript-eslint/utils": "8.58.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-aGsCQImkDIqMyx1u4PrVlbi/krmDsQUs4zAcCV6M7yPcPev+RqVlndsJy9kJ8TLihW9TZ0kbDAzctpLn5o+lOg=="], + + "@typescript-eslint/types": ["@typescript-eslint/types@8.58.0", "", {}, "sha512-O9CjxypDT89fbHxRfETNoAnHj/i6IpRK0CvbVN3qibxlLdo5p5hcLmUuCCrHMpxiWSwKyI8mCP7qRNYuOJ0Uww=="], + + "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.58.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.58.0", "@typescript-eslint/tsconfig-utils": "8.58.0", "@typescript-eslint/types": "8.58.0", "@typescript-eslint/visitor-keys": "8.58.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-7vv5UWbHqew/dvs+D3e1RvLv1v2eeZ9txRHPnEEBUgSNLx5ghdzjHa0sgLWYVKssH+lYmV0JaWdoubo0ncGYLA=="], + + "@typescript-eslint/utils": ["@typescript-eslint/utils@8.58.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.58.0", "@typescript-eslint/types": "8.58.0", "@typescript-eslint/typescript-estree": "8.58.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-RfeSqcFeHMHlAWzt4TBjWOAtoW9lnsAGiP3GbaX9uVgTYYrMbVnGONEfUCiSss+xMHFl+eHZiipmA8WkQ7FuNA=="], + + "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.58.0", "", { "dependencies": { "@typescript-eslint/types": "8.58.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-XJ9UD9+bbDo4a4epraTwG3TsNPeiB9aShrUneAVXy8q4LuwowN+qu89/6ByLMINqvIMeI9H9hOHQtg/ijrYXzQ=="], + + "acorn": ["acorn@8.16.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="], + + "acorn-jsx": ["acorn-jsx@5.3.2", "", { "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ=="], + + "agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], + + "ajv": ["ajv@6.14.0", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw=="], + + "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + + "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + + "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], + + "async": ["async@2.6.4", "", { "dependencies": { "lodash": "^4.17.14" } }, "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA=="], + + "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], + + "bl": ["bl@4.1.0", "", { "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w=="], + + "brace-expansion": ["brace-expansion@5.0.5", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ=="], + + "buffer": ["buffer@5.7.1", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" } }, "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ=="], + + "buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="], + + "bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="], + + "callsites": ["callsites@3.1.0", "", {}, "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ=="], + + "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + + "chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="], + + "chromium-bidi": ["chromium-bidi@0.4.5", "", { "dependencies": { "mitt": "3.0.0" }, "peerDependencies": { "devtools-protocol": "*" } }, "sha512-rkav9YzRfAshSTG3wNXF7P7yNiI29QAo1xBXElPoCoSQR5n20q3cOyVhDv6S7+GlF/CJ/emUxlQiR0xOPurkGg=="], + + "cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="], + + "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], + + "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], + + "commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="], + + "concat-map": ["concat-map@0.0.1", "", {}, "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="], + + "cosmiconfig": ["cosmiconfig@8.1.0", "", { "dependencies": { "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", "parse-json": "^5.0.0", "path-type": "^4.0.0" } }, "sha512-0tLZ9URlPGU7JsKq0DQOQ3FoRsYX8xDZ7xMiATQfaiGMz7EHowNkbU9u1coAOmnh9p/1ySpm0RB3JNWRXM5GCg=="], + + "cross-fetch": ["cross-fetch@3.1.5", "", { "dependencies": { "node-fetch": "2.6.7" } }, "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw=="], + + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "deep-is": ["deep-is@0.1.4", "", {}, "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="], + + "devtools-protocol": ["devtools-protocol@0.0.1094867", "", {}, "sha512-pmMDBKiRVjh0uKK6CT1WqZmM3hBVSgD+N2MrgyV1uNizAZMw4tx6i/RTc+/uCsKSCmg0xXx7arCP/OFcIwTsiQ=="], + + "emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], + + "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], + + "error-ex": ["error-ex@1.3.4", "", { "dependencies": { "is-arrayish": "^0.2.1" } }, "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ=="], + + "es-toolkit": ["es-toolkit@1.45.1", "", {}, "sha512-/jhoOj/Fx+A+IIyDNOvO3TItGmlMKhtX8ISAHKE90c4b/k1tqaqEZ+uUqfpU8DMnW5cgNJv606zS55jGvza0Xw=="], + + "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], + + "eslint": ["eslint@10.1.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", "@eslint/config-array": "^0.23.3", "@eslint/config-helpers": "^0.5.3", "@eslint/core": "^1.1.1", "@eslint/plugin-kit": "^0.6.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", "ajv": "^6.14.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "espree": "^11.2.0", "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", "find-up": "^5.0.0", "glob-parent": "^6.0.2", "ignore": "^5.2.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", "minimatch": "^10.2.4", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, "peerDependencies": { "jiti": "*" }, "optionalPeers": ["jiti"], "bin": { "eslint": "bin/eslint.js" } }, "sha512-S9jlY/ELKEUwwQnqWDO+f+m6sercqOPSqXM5Go94l7DOmxHVDgmSFGWEzeE/gwgTAr0W103BWt0QLe/7mabIvA=="], + + "eslint-config-prettier": ["eslint-config-prettier@10.1.8", "", { "peerDependencies": { "eslint": ">=7.0.0" }, "bin": { "eslint-config-prettier": "bin/cli.js" } }, "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w=="], + + "eslint-scope": ["eslint-scope@9.1.2", "", { "dependencies": { "@types/esrecurse": "^4.3.1", "@types/estree": "^1.0.8", "esrecurse": "^4.3.0", "estraverse": "^5.2.0" } }, "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ=="], + + "eslint-visitor-keys": ["eslint-visitor-keys@5.0.1", "", {}, "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA=="], + + "espree": ["espree@11.2.0", "", { "dependencies": { "acorn": "^8.16.0", "acorn-jsx": "^5.3.2", "eslint-visitor-keys": "^5.0.1" } }, "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw=="], + + "esquery": ["esquery@1.7.0", "", { "dependencies": { "estraverse": "^5.1.0" } }, "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g=="], + + "esrecurse": ["esrecurse@4.3.0", "", { "dependencies": { "estraverse": "^5.2.0" } }, "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag=="], + + "estraverse": ["estraverse@5.3.0", "", {}, "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA=="], + + "esutils": ["esutils@2.0.3", "", {}, "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g=="], + + "extract-zip": ["extract-zip@2.0.1", "", { "dependencies": { "debug": "^4.1.1", "get-stream": "^5.1.0", "yauzl": "^2.10.0" }, "optionalDependencies": { "@types/yauzl": "^2.9.1" }, "bin": { "extract-zip": "cli.js" } }, "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg=="], + + "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], + + "fast-json-stable-stringify": ["fast-json-stable-stringify@2.1.0", "", {}, "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="], + + "fast-levenshtein": ["fast-levenshtein@2.0.6", "", {}, "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="], + + "fd-slicer": ["fd-slicer@1.1.0", "", { "dependencies": { "pend": "~1.2.0" } }, "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g=="], + + "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], + + "file-entry-cache": ["file-entry-cache@8.0.0", "", { "dependencies": { "flat-cache": "^4.0.0" } }, "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ=="], + + "find-up": ["find-up@5.0.0", "", { "dependencies": { "locate-path": "^6.0.0", "path-exists": "^4.0.0" } }, "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng=="], + + "flat-cache": ["flat-cache@4.0.1", "", { "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.4" } }, "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw=="], + + "flatted": ["flatted@3.4.2", "", {}, "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA=="], + + "fs-constants": ["fs-constants@1.0.0", "", {}, "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="], + + "fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="], + + "geoip-lite": ["geoip-lite@1.4.10", "", { "dependencies": { "async": "2.1 - 2.6.4", "chalk": "4.1 - 4.1.2", "iconv-lite": "0.4.13 - 0.6.3", "ip-address": "5.8.9 - 5.9.4", "lazy": "1.0.11", "rimraf": "2.5.2 - 2.7.1", "yauzl": "2.9.2 - 2.10.0" } }, "sha512-4N69uhpS3KFd97m00wiFEefwa+L+HT5xZbzPhwu+sDawStg6UN/dPwWtUfkQuZkGIY1Cj7wDVp80IsqNtGMi2w=="], + + "get-stream": ["get-stream@5.2.0", "", { "dependencies": { "pump": "^3.0.0" } }, "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA=="], + + "glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="], + + "glob-parent": ["glob-parent@6.0.2", "", { "dependencies": { "is-glob": "^4.0.3" } }, "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A=="], + + "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + + "https-proxy-agent": ["https-proxy-agent@5.0.1", "", { "dependencies": { "agent-base": "6", "debug": "4" } }, "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA=="], + + "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], + + "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], + + "ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], + + "import-fresh": ["import-fresh@3.3.1", "", { "dependencies": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" } }, "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ=="], + + "imurmurhash": ["imurmurhash@0.1.4", "", {}, "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA=="], + + "inflight": ["inflight@1.0.6", "", { "dependencies": { "once": "^1.3.0", "wrappy": "1" } }, "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA=="], + + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + + "ip-address": ["ip-address@5.9.4", "", { "dependencies": { "jsbn": "1.1.0", "lodash": "^4.17.15", "sprintf-js": "1.1.2" } }, "sha512-dHkI3/YNJq4b/qQaz+c8LuarD3pY24JqZWfjB8aZx1gtpc2MDILu9L9jpZe1sHpzo/yWFweQVn+U//FhazUxmw=="], + + "is-arrayish": ["is-arrayish@0.2.1", "", {}, "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg=="], + + "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], + + "is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], + + "is-glob": ["is-glob@4.0.3", "", { "dependencies": { "is-extglob": "^2.1.1" } }, "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg=="], + + "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + + "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + + "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + + "jsbn": ["jsbn@1.1.0", "", {}, "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A=="], + + "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], + + "json-parse-even-better-errors": ["json-parse-even-better-errors@2.3.1", "", {}, "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w=="], + + "json-schema-traverse": ["json-schema-traverse@0.4.1", "", {}, "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="], + + "json-stable-stringify-without-jsonify": ["json-stable-stringify-without-jsonify@1.0.1", "", {}, "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw=="], + + "keyv": ["keyv@4.5.4", "", { "dependencies": { "json-buffer": "3.0.1" } }, "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw=="], + + "lazy": ["lazy@1.0.11", "", {}, "sha512-Y+CjUfLmIpoUCCRl0ub4smrYtGGr5AOa2AKOaWelGHOGz33X/Y/KizefGqbkwfz44+cnq/+9habclf8vOmu2LA=="], + + "levn": ["levn@0.4.1", "", { "dependencies": { "prelude-ls": "^1.2.1", "type-check": "~0.4.0" } }, "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ=="], + + "lines-and-columns": ["lines-and-columns@1.2.4", "", {}, "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="], + + "locate-path": ["locate-path@6.0.0", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="], + + "lodash": ["lodash@4.17.21", "", {}, "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="], + + "lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], + + "minimatch": ["minimatch@10.2.5", "", { "dependencies": { "brace-expansion": "^5.0.5" } }, "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg=="], + + "minipass": ["minipass@4.2.8", "", {}, "sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ=="], + + "mitt": ["mitt@3.0.0", "", {}, "sha512-7dX2/10ITVyqh4aOSVI9gdape+t9l2/8QxHrFmUXu4EEUpdlxl6RudZUPZoc+zuY2hk1j7XxVroIVIan/pD/SQ=="], + + "mkdirp-classic": ["mkdirp-classic@0.5.3", "", {}, "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "natural-compare": ["natural-compare@1.4.0", "", {}, "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="], + + "node-fetch": ["node-fetch@2.6.7", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ=="], + + "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], + + "optionator": ["optionator@0.9.4", "", { "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" } }, "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g=="], + + "p-limit": ["p-limit@7.3.0", "", { "dependencies": { "yocto-queue": "^1.2.1" } }, "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw=="], + + "p-locate": ["p-locate@5.0.0", "", { "dependencies": { "p-limit": "^3.0.2" } }, "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw=="], + + "parent-module": ["parent-module@1.0.1", "", { "dependencies": { "callsites": "^3.0.0" } }, "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g=="], + + "parse-json": ["parse-json@5.2.0", "", { "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", "json-parse-even-better-errors": "^2.3.0", "lines-and-columns": "^1.1.6" } }, "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg=="], + + "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], + + "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], + + "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], + + "path-scurry": ["path-scurry@1.11.1", "", { "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" } }, "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA=="], + + "path-type": ["path-type@4.0.0", "", {}, "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw=="], + + "pend": ["pend@1.2.0", "", {}, "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg=="], + + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], + + "picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="], + + "prelude-ls": ["prelude-ls@1.2.1", "", {}, "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g=="], + + "prettier": ["prettier@3.8.1", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg=="], + + "progress": ["progress@2.0.3", "", {}, "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="], + + "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], + + "pump": ["pump@3.0.4", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA=="], + + "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], + + "puppeteer": ["puppeteer@19.7.5", "", { "dependencies": { "cosmiconfig": "8.1.0", "https-proxy-agent": "5.0.1", "progress": "2.0.3", "proxy-from-env": "1.1.0", "puppeteer-core": "19.7.5" } }, "sha512-UqD8K+yaZa6/hwzP54AATCiHrEYGGxzQcse9cZzrtsVGd8wT0llCdYhsBp8n+zvnb1ofY0YFgI3TYZ/MiX5uXQ=="], + + "puppeteer-core": ["puppeteer-core@19.7.5", "", { "dependencies": { "chromium-bidi": "0.4.5", "cross-fetch": "3.1.5", "debug": "4.3.4", "devtools-protocol": "0.0.1094867", "extract-zip": "2.0.1", "https-proxy-agent": "5.0.1", "proxy-from-env": "1.1.0", "rimraf": "4.4.0", "tar-fs": "2.1.1", "unbzip2-stream": "1.4.3", "ws": "8.12.1" }, "peerDependencies": { "typescript": ">= 4.7.4" }, "optionalPeers": ["typescript"] }, "sha512-EJuNha+SxPfaYFbkoWU80H3Wb1SiQH5fFyb2xdbWda0ziax5mhV63UMlqNfPeTDIWarwtR4OIcq/9VqY8HPOsg=="], + + "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + + "resolve-from": ["resolve-from@4.0.0", "", {}, "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g=="], + + "rimraf": ["rimraf@2.7.1", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w=="], + + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + + "semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], + + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], + + "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], + + "sprintf-js": ["sprintf-js@1.1.2", "", {}, "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug=="], + + "string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], + + "strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + + "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + + "tar-fs": ["tar-fs@2.1.1", "", { "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.1.4" } }, "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng=="], + + "tar-stream": ["tar-stream@2.2.0", "", { "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" } }, "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ=="], + + "through": ["through@2.3.8", "", {}, "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg=="], + + "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], + + "tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], + + "ts-api-utils": ["ts-api-utils@2.5.0", "", { "peerDependencies": { "typescript": ">=4.8.4" } }, "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA=="], + + "type-check": ["type-check@0.4.0", "", { "dependencies": { "prelude-ls": "^1.2.1" } }, "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "typescript-eslint": ["typescript-eslint@8.58.0", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.58.0", "@typescript-eslint/parser": "8.58.0", "@typescript-eslint/typescript-estree": "8.58.0", "@typescript-eslint/utils": "8.58.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-e2TQzKfaI85fO+F3QywtX+tCTsu/D3WW5LVU6nz8hTFKFZ8yBJ6mSYRpXqdR3mFjPWmO0eWsTa5f+UpAOe/FMA=="], + + "unbzip2-stream": ["unbzip2-stream@1.4.3", "", { "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" } }, "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg=="], + + "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], + + "uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="], + + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + + "wappalyzer": ["wappalyzer@6.10.66", "", { "dependencies": { "puppeteer": "~19.7.0" }, "bin": { "wappalyzer": "cli.js" } }, "sha512-rPnZY1dxIJvPrL0h7AKLrwQQ5vuSCD/ALwIXdwHurlfgex1sxFEQKwG/YjLEnR4iR+HpQsLl47EKlMu87/kbow=="], + + "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], + + "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], + + "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + + "word-wrap": ["word-wrap@1.2.5", "", {}, "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA=="], + + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + + "ws": ["ws@8.12.1", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-1qo+M9Ba+xNhPB+YTWUlK6M17brTut5EXbcBaMRN5pH5dFrXz7lzz1ChFSUq3bOUl8yEvSenhHmYUNJxFzdJew=="], + + "yauzl": ["yauzl@2.10.0", "", { "dependencies": { "buffer-crc32": "~0.2.3", "fd-slicer": "~1.1.0" } }, "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g=="], + + "yocto-queue": ["yocto-queue@1.2.2", "", {}, "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ=="], + + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + + "@eslint-community/eslint-utils/eslint-visitor-keys": ["eslint-visitor-keys@3.4.3", "", {}, "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag=="], + + "@typescript-eslint/eslint-plugin/ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], + + "glob/minimatch": ["minimatch@3.1.5", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w=="], + + "p-locate/p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="], + + "path-scurry/minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="], + + "puppeteer-core/debug": ["debug@4.3.4", "", { "dependencies": { "ms": "2.1.2" } }, "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ=="], + + "puppeteer-core/rimraf": ["rimraf@4.4.0", "", { "dependencies": { "glob": "^9.2.0" }, "bin": { "rimraf": "dist/cjs/src/bin.js" } }, "sha512-X36S+qpCUR0HjXlkDe4NAOhS//aHH0Z+h8Ckf2auGJk3PTnx5rLmrHkwNdbVQuCSUhOyFrlRvFEllZOYE+yZGQ=="], + + "glob/minimatch/brace-expansion": ["brace-expansion@1.1.13", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w=="], + + "p-locate/p-limit/yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], + + "puppeteer-core/debug/ms": ["ms@2.1.2", "", {}, "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="], + + "puppeteer-core/rimraf/glob": ["glob@9.3.5", "", { "dependencies": { "fs.realpath": "^1.0.0", "minimatch": "^8.0.2", "minipass": "^4.2.4", "path-scurry": "^1.6.1" } }, "sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q=="], + + "glob/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + + "puppeteer-core/rimraf/glob/minimatch": ["minimatch@8.0.7", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-V+1uQNdzybxa14e/p00HZnQNNcTjnRJjDxg2V8wtkjFctq4M7hXFws4oekyTP0Jebeq7QYtpFyOeBAjc88zvYg=="], + + "puppeteer-core/rimraf/glob/minimatch/brace-expansion": ["brace-expansion@2.0.3", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA=="], + + "puppeteer-core/rimraf/glob/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + } +} diff --git a/docs/2026-04-01/REFACTORING_PLAN.md b/docs/2026-04-01/REFACTORING_PLAN.md new file mode 100644 index 0000000..46abb3f --- /dev/null +++ b/docs/2026-04-01/REFACTORING_PLAN.md @@ -0,0 +1,688 @@ +# WhatsWeb 重构计划：基于 Bun 的全面重构 + +> 本文档描述了将 WhatsWeb 从旧版 Node.js + CommonJS 项目，全面迁移至 **Bun 运行时 + TypeScript + ESM** 的重构方案。 +> +> **可执行的 SOP（标准操作流程）已拆分至 [`sop/`](./sop/README.md)**，AI Agent 可直接按 Phase 顺序执行重构。 + +--- + +## 目录 + +1. [重构背景与目标](#1-重构背景与目标) +2. [技术选型](#2-技术选型) +3. [项目结构设计](#3-项目结构设计) +4. [模块重构详细方案](#4-模块重构详细方案) + - 4.1 [入口与 CLI (`whatsweb.js` → `src/cli.ts`)](#41-入口与-cli) + - 4.2 [核心引擎 (`index.js` → `src/core/scanner.ts`)](#42-核心引擎) + - 4.3 [工具函数 (`utils.js` → `src/utils/`)](#43-工具函数) + - 4.4 [插件系统 (`plugins/` → `src/plugins/`)](#44-插件系统) + - 4.5 [BBScan 规则引擎](#45-bbscan-规则引擎) +5. [依赖替换方案](#5-依赖替换方案) +6. [类型系统设计](#6-类型系统设计) +7. [测试方案](#7-测试方案) +8. [构建与分发](#8-构建与分发) +9. [重构步骤与里程碑](#9-重构步骤与里程碑) +10. [风险与注意事项](#10-风险与注意事项) + +--- + +## 1. 重构背景与目标 + +### 1.1 当前状况 + +WhatsWeb 是一个基于 Node.js 的网站指纹识别与安全扫描 CLI 工具，当前存在以下问题： + +| 问题 | 说明 | +|------|------| +| **老旧的模块系统** | 使用 CommonJS (`require`/`module.exports`)，无法利用现代 ESM 的 tree-shaking 和静态分析优势 | +| **缺乏类型安全** | 纯 JavaScript，无 TypeScript 类型定义，插件接口全靠约定 | +| **过时的依赖** | `bluebird`（原生 Promise 已足够）、`superagent`（有更现代的替代）、`wappalyzer@5`（已弃用的旧版） | +| **无测试覆盖** | 没有任何自动化测试，无法保证重构后的正确性 | +| **插件系统过于简陋** | 依赖 `fs.readdir` 动态加载，没有类型约束、没有生命周期管理、没有错误隔离 | +| **无构建流程** | 没有 lint、format、CI/CD 配置 | + +### 1.2 重构目标 + +- **运行时迁移**：从 Node.js 迁移到 Bun，充分利用 Bun 的原生 TypeScript 支持、内置测试框架、更快的启动速度和 HTTP 性能 +- **语言升级**：全面使用 TypeScript，为插件系统和核心 API 提供严格的类型定义 +- **模块现代化**：全面使用 ESM (`import`/`export`) +- **依赖精简**：利用 Bun 和现代 Web API 替换过时的第三方依赖 +- **架构优化**：重新设计插件系统，引入更清晰的分层架构 +- **质量保障**：引入完整的测试套件、lint 和格式化配置 +- **功能保持**：确保重构后的功能与原版完全一致 + +--- + +## 2. 技术选型 + +| 类别 | 当前方案 | 重构方案 | 理由 | +|------|----------|----------|------| +| 运行时 | Node.js | **Bun** | 原生 TS 支持、内置测试、更快的性能 | +| 语言 | JavaScript (ES6) | **TypeScript 5.x** | 类型安全、更好的 IDE 支持 | +| 模块系统 | CommonJS | **ESM** | 现代标准、tree-shaking | +| HTTP 客户端 | superagent | **Bun 原生 fetch** | 零依赖、Web 标准 API | +| 工具函数库 | 无 | **es-toolkit** | 替代 lodash — 2-3x 更快、97% 更小、原生 TS、完美 tree-shaking | +| 并发管理 | bluebird | **p-limit@7** | 成熟（~100M 周下载）、ESM 原生、API 简洁 | +| 数据验证 | 无 | **zod** | CLI 参数验证、插件结果验证、运行时 schema + 类型推断 | +| CLI 框架 | commander@2 | **commander@12** | 成熟稳定、ESM 支持 | +| 终端着色 | chalk@2 | **picocolors** | 零依赖、极小体积 | +| 进度条 | progress | **cli-progress** | 更现代的进度条库 | +| 文件匹配 | globby@7 | **Bun.Glob (内置)** | Bun 内置的 Glob API，零依赖 | +| IP 处理 | ip@1 | **自定义实现** | ip 包存在已知安全漏洞 (CVE-2023-42282) | +| GeoIP | geoip-lite | **geoip-lite** (保留) | 无更优替代，且功能稳定 | +| 技术栈识别 | wappalyzer@5 | **wappalyzer@6.10.66** | 使用 puppeteer 内核，支持 init/open/analyze/destroy 生命周期 | +| 测试框架 | 无 | **Bun 内置测试 (`bun:test`)** | 零配置、原生支持 | +| Lint | 无 | **ESLint 9** (flat config) + **typescript-eslint** | 代码质量检查 | +| Format | 无 | **Prettier** | 代码格式化 | + +--- + +## 3. 项目结构设计 + +``` +whatsweb/ +├── src/ +│ ├── cli.ts # CLI 入口 +│ ├── index.ts # 库入口（对外导出） +│ ├── core/ +│ │ ├── scanner.ts # 核心扫描引擎（原 index.js 中的 WhatsWeb 类） +│ │ ├── plugin-loader.ts # 插件加载器 +│ │ └── reporter.ts # 结果输出/报告格式化（原 whatsweb.js 中的 report 逻辑） +│ ├── plugins/ +│ │ ├── types.ts # 插件接口类型定义 +│ │ ├── base-info.ts # 基础信息插件 +│ │ ├── email.ts # 邮箱提取插件 +│ │ ├── geoip.ts # 地理位置插件 +│ │ ├── wappalyzer.ts # 技术栈识别插件 +│ │ └── bbscan/ +│ │ ├── index.ts # BBScan 插件入口 +│ │ ├── rule-parser.ts # 规则解析器 +│ │ └── scanner.ts # 路径扫描器 +│ └── utils/ +│ ├── url.ts # URL 规范化 +│ ├── dns.ts # DNS 解析 +│ ├── ip.ts # IP/CIDR 处理 +│ ├── http.ts # HTTP 请求封装 +│ └── concurrency.ts # 并发控制工具 +├── rules/ # 规则文件（保持原样） +│ ├── *.txt +│ ├── white.list +│ └── black.list +├── tests/ +│ ├── core/ +│ │ ├── scanner.test.ts +│ │ └── plugin-loader.test.ts +│ ├── plugins/ +│ │ ├── base-info.test.ts +│ │ ├── email.test.ts +│ │ ├── geoip.test.ts +│ │ └── bbscan/ +│ │ └── rule-parser.test.ts +│ └── utils/ +│ ├── url.test.ts +│ ├── ip.test.ts +│ └── concurrency.test.ts +├── package.json +├── tsconfig.json +├── eslint.config.mjs # ESLint 9 flat config +├── .prettierrc # Prettier 配置 +├── bunfig.toml # Bun 配置 +├── README.md +└── REFACTORING_PLAN.md # 本文档 +``` + +--- + +## 4. 模块重构详细方案 + +### 4.1 入口与 CLI + +**原文件**: `whatsweb.js` (134 行) +**新文件**: `src/cli.ts` + +#### 当前实现分析 + +- 使用 `commander@2` 解析 CLI 参数 +- 手动从文件读取目标列表 (`fs.readFileSync`) +- 使用 `bluebird.Promise.map` 进行并发控制 +- `report()` 函数直接内联了所有输出格式化逻辑 +- 进度条使用 `progress` 库 + +#### 重构方案 + +```typescript +// src/cli.ts — 伪代码概要 +import { Command } from "commander"; +import { Scanner } from "./core/scanner"; +import { Reporter } from "./core/reporter"; +import { expandCIDR } from "./utils/ip"; +import { version } from "../package.json"; + +const program = new Command() + .name("whatsweb") + .version(version) + .description("网站指纹识别与安全扫描工具") + .argument("[urls...]", "目标 URL 列表") + .option("-f ", "从文件读取目标列表") + .option("-c, --concurrency ", "并发数", parseInt, 50) + .option("--network ", "扫描 Target/MASK 网段内所有主机") + .option("--timeout ", "请求超时时间 (毫秒)", parseInt, 10000) + .option("--user-agent ", "自定义 User-Agent") + .option("-o, --output ", "输出文件路径") + .option("--show-error", "显示错误信息") + .action(async (urls, options) => { + // 1. 构建目标列表（文件 + 参数 + CIDR 展开 + 去重） + // 2. 创建 Scanner 和 Reporter 实例 + // 3. 并发扫描并报告结果 + }); +``` + +**关键变更点**: + +1. 将 `report()` 逻辑抽取到独立的 `Reporter` 类 +2. 用自定义并发控制替代 `bluebird.Promise.map` +3. 文件读取使用 `Bun.file().text()` 替代 `fs.readFileSync` +4. 输出文件写入使用 `Bun.write()` 替代 `fs.appendFileSync` + +### 4.2 核心引擎 + +**原文件**: `index.js` (54 行) +**新文件**: `src/core/scanner.ts` + +#### 当前实现分析 + +- `WhatsWeb` 类通过 `fs.readdir` 动态发现并 `require()` 加载 `plugins/` 目录下的所有 `.js` 文件 +- 使用 `superagent` 发送初始 GET 请求 +- 按顺序执行所有插件，收集非空结果 +- 错误处理：catch 所有异常并返回 error 对象（而非 throw） + +#### 重构方案 + +```typescript +// src/core/scanner.ts — 伪代码概要 +export class Scanner { + private url: string; + private timeout: number; + private userAgent: string; + private plugins: Plugin[]; + + constructor(options: ScannerOptions) { /* ... */ } + + async analyse(): Promise { + const response = await this.fetchTarget(); + const context: PluginContext = { url: this.url, timeout: this.timeout, userAgent: this.userAgent, response }; + const results: ScanResult[] = []; + for (const plugin of this.plugins) { + try { + const result = await plugin.execute(context); + if (result && Object.keys(result).length > 0) { + results.push({ name: plugin.name, result }); + } + } catch { + // 单个插件失败不影响其他插件 + } + } + return results; + } +} +``` + +**关键变更点**: + +1. 使用原生 `fetch` 替代 `superagent`，需要封装一层以处理重定向跟踪、响应体文本提取等 +2. 插件加载从 `readdir + require` 改为显式注册（内置插件）+ 可选的动态导入（外部插件） +3. 增加单个插件的错误隔离（try/catch 包裹每个插件执行） +4. 方法返回类型化的 `ScanResult[]` 而非裸的 array/error 混合类型 + +### 4.3 工具函数 + +**原文件**: `utils.js` (39 行) +**新文件**: `src/utils/url.ts`, `src/utils/dns.ts`, `src/utils/ip.ts`, `src/utils/http.ts`, `src/utils/concurrency.ts` + +#### 4.3.1 URL 工具 (`src/utils/url.ts`) + +```typescript +// 替代原 normalUrl，使用标准 URL API 增强验证 +export function normalizeUrl(input: string): string { + if (!input) throw new Error(`Invalid url: ${input}`); + const withProtocol = /^https?:\/\//i.test(input) ? input : `http://${input}`; + new URL(withProtocol); // 验证合法性 + return withProtocol; +} +``` + +#### 4.3.2 DNS 工具 (`src/utils/dns.ts`) + +```typescript +// 替代原 resolve4，使用 Bun 兼容的 dns 模块 +import { resolve4 as dnsResolve4 } from "node:dns/promises"; +import { isIPv4 } from "node:net"; + +export async function resolve4(domain: string): Promise { + if (isIPv4(domain)) return domain; + const addresses = await dnsResolve4(domain); + if (!addresses.length) throw new Error("DNS resolve failed"); + return addresses[0]; +} +``` + +#### 4.3.3 IP/CIDR 工具 (`src/utils/ip.ts`) + +替换 `ip` 包（存在已知安全漏洞），自行实现或使用更安全的 `ipaddr.js`： + +```typescript +export function* expandCIDR(cidr: string): Generator { + // 解析 CIDR 表示法，生成范围内的所有 IP + // 使用生成器避免一次性占用大量内存 +} +``` + +**关键改进**: 使用 `Generator` 而非预分配数组，避免 `/16` 等大网段导致内存问题。 + +#### 4.3.4 HTTP 封装 (`src/utils/http.ts`) + +```typescript +export interface HttpResponse { + status: number; + statusText: string; + headers: Record; + text: string; + url: string; // 最终 URL（经过重定向后） +} + +export async function httpGet(url: string, options: RequestOptions): Promise { + // 基于原生 fetch 封装，处理超时、UA、重定向跟踪 +} + +export async function httpHead(url: string, options: RequestOptions): Promise { + // HEAD 请求封装，用于 BBScan +} +``` + +#### 4.3.5 并发控制 (`src/utils/concurrency.ts`) + +替代 `bluebird.Promise.map`： + +```typescript +export async function mapConcurrent( + items: T[], + fn: (item: T) => Promise, + concurrency: number +): Promise { + // 基于信号量的并发控制池 +} +``` + +### 4.4 插件系统 + +**原文件**: `plugins/*.js` (5 个插件) +**新文件**: `src/plugins/types.ts` + `src/plugins/*.ts` + +#### 当前实现分析 + +当前插件系统非常简陋： +- 约定 `exports.register` 为 async 函数 +- 约定 `exports.register.attributes.name` 为插件名 +- 传入参数为 `{ url, timeout, userAgent, response }` +- 无类型约束、无生命周期、无错误隔离 + +#### 重构方案：类型化插件接口 + +```typescript +// src/plugins/types.ts + +export interface PluginContext { + url: string; + timeout: number; + userAgent: string; + response: HttpResponse; +} + +export interface PluginMeta { + name: string; + description?: string; + version?: string; +} + +export interface Plugin { + meta: PluginMeta; + execute(context: PluginContext): Promise | null>; +} + +export type PluginFactory = () => Plugin; +``` + +#### 各插件重构要点 + +**base-info.ts** (原 `plugins/base-info.js`) +- 状态码、标题、重定向、x-* 响应头提取 +- 改用标准 `Response` API 替代 `response.res` 的 Node.js 特有结构 +- 标题提取正则 `/([^<]+)<\/title>/i` 保持不变 + +**email.ts** (原 `plugins/email.js`) +- 邮箱正则 `/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}/gi` 保持不变 +- 保持 "至少 2 个唯一匹配才返回" 的逻辑 + +**geoip.ts** (原 `plugins/geoip.js`) +- `geoip-lite` 依赖保留 +- 使用新的 `dns.ts` 工具替代原 `utils.resolve4` +- 使用 `new URL()` 替代 `Url.parse()` + +**wappalyzer.ts** (原 `plugins/wappalyzer.js`) +- 需评估 wappalyzer 新版本的 API 变化 +- 如果新版不兼容，考虑使用 `wappalyzer-core` 或其他替代方案（如 `webanalyze`） +- 保持按 category 分组的输出格式 + +**bbscan/** (原 `plugins/bbscan.js`) +- 拆分为三个文件：`index.ts`（插件入口）、`rule-parser.ts`（规则解析）、`scanner.ts`（扫描逻辑） +- `rule-parser.ts` 专注解析 `rules/*.txt`、`white.list`、`black.list` +- `scanner.ts` 处理 404 检测、路径探测、结果过滤 +- 使用 `Bun.Glob` 替代 `globby` 来匹配规则文件 +- 使用原生 `fetch` 替代 `superagent.head` +- 并发控制从 `bluebird.Promise.map` 改为自定义并发池 + +### 4.5 BBScan 规则引擎 + +**原文件**: `rules/` 目录 (12 个 .txt 文件 + white.list + black.list) + +#### 保持不变 + +规则文件（`rules/*.txt`、`white.list`、`black.list`）**完全保留原样**，不做任何修改。这些是数据文件而非代码，向后兼容。 + +#### 规则解析器重构 + +将 `bbscan.js` 中的规则解析逻辑提取为独立模块 `rule-parser.ts`： + +```typescript +// src/plugins/bbscan/rule-parser.ts + +export interface ScanRule { + uri: string; + tag: string; + status: number | null; + contentType: string; + contentTypeNo: string; + rootOnly: boolean; +} + +export interface ListRule { + texts: string[]; + regexes: RegExp[]; +} + +export async function parseRuleFiles(rulesDir: string): Promise<ScanRule[]> { /* ... */ } +export async function parseWhiteList(filePath: string): Promise<ListRule> { /* ... */ } +export async function parseBlackList(filePath: string): Promise<ListRule> { /* ... */ } +``` + +--- + +## 5. 依赖替换方案 + +| 原依赖 | 状态 | 替换方案 | 说明 | +|--------|------|----------|------| +| `bluebird@^3.5.1` | **移除** | **p-limit@7** | 成熟的并发控制（~100M 周下载）、ESM 原生 | +| `superagent@^3.8.2` | **移除** | 原生 `fetch` + `src/utils/http.ts` | Bun 内置高性能 fetch | +| `chalk@^2.3.0` | **替换** | **picocolors** | 零依赖、极小体积 | +| `commander@^2.13.0` | **升级** | **commander@12** | 成熟稳定、ESM 支持 | +| `progress@^2.0.0` | **替换** | **cli-progress** | 更现代的进度条库 | +| `globby@^7.1.1` | **移除** | **Bun.Glob** (内置) | 零依赖 | +| `ip@^1.1.5` | **移除** | 自定义实现 (`src/utils/ip.ts`) | ip 包存在 SSRF 安全漏洞 (CVE-2023-42282) | +| `geoip-lite@^1.2.1` | **保留** | **geoip-lite** (最新版) | 无更优替代 | +| `wappalyzer@^5.2.2` | **升级** | **wappalyzer@6.10.66** | 最后功能版本，API: init/open/analyze/destroy | + +### 新增依赖 + +| 依赖 | 用途 | +|------|------| +| **es-toolkit** | 通用工具函数 — 替代 lodash，2-3x 更快、97% 更小、原生 TS | +| **p-limit@7** | 并发控制 — 替代 bluebird.Promise.map | +| **zod** | 运行时数据验证 + TypeScript 类型推断（CLI 参数、插件 meta、规则文件等） | +| `picocolors` | 终端文本着色（零依赖、高性能） | +| `commander@12` | CLI 参数解析 | +| `cli-progress` | 终端进度条 | + +### 新增开发依赖 + +| 依赖 | 用途 | +|------|------| +| `typescript` | TypeScript 编译器（Bun 内置 TS 执行，但需要 tsc 做类型检查） | +| `eslint` + `typescript-eslint` + `eslint-config-prettier` | Lint | +| `prettier` | Format | +| `@types/bun` | Bun 类型定义 | +| `@types/cli-progress` | cli-progress 类型定义 | +| `@types/geoip-lite` | geoip-lite 类型定义 | + +--- + +## 6. 类型系统设计 + +### 6.1 核心类型 + +```typescript +// src/types.ts + +/** 扫描器选项 */ +export interface ScannerOptions { + target: string; + timeout?: number; + userAgent?: string; + plugins?: Plugin[]; +} + +/** 单个插件的扫描结果 */ +export interface ScanResult { + name: string; + result: Record<string, unknown>; +} + +/** 扫描报告（成功时） */ +export interface ScanReport { + target: string; + plugins: ScanResult[]; +} + +/** 扫描报告（失败时） */ +export interface ScanError { + target: string; + error: string; +} + +/** CLI 选项 */ +export interface CliOptions { + concurrency: number; + network?: string; + timeout: number; + userAgent: string; + output?: string; + showError: boolean; + file?: string; +} +``` + +### 6.2 插件类型 + +见 [4.4 插件系统](#44-插件系统) 中的 `Plugin`、`PluginContext`、`PluginMeta` 定义。 + +### 6.3 HTTP 类型 + +见 [4.3.4 HTTP 封装](#434-http-封装-srcutilshttpts) 中的 `HttpResponse`、`RequestOptions` 定义。 + +--- + +## 7. 测试方案 + +使用 **Bun 内置测试框架** (`bun:test`)，无需额外安装测试运行器。 + +### 7.1 单元测试 + +| 测试文件 | 覆盖模块 | 测试内容 | +|----------|----------|----------| +| `tests/utils/url.test.ts` | `src/utils/url.ts` | URL 规范化：无协议补全、已有协议保持、空值抛错、特殊字符处理 | +| `tests/utils/ip.test.ts` | `src/utils/ip.ts` | CIDR 展开：/24 → 254 个地址、/32 → 1 个地址、边界值、非法输入 | +| `tests/utils/concurrency.test.ts` | `src/utils/concurrency.ts` | 并发池：并发上限、错误传播、空列表、单项列表 | +| `tests/plugins/base-info.test.ts` | `src/plugins/base-info.ts` | 标题提取、状态码、重定向检测、x-* 头过滤 | +| `tests/plugins/email.test.ts` | `src/plugins/email.ts` | 邮箱正则匹配、去重、不足 2 个时返回空 | +| `tests/plugins/bbscan/rule-parser.test.ts` | `src/plugins/bbscan/rule-parser.ts` | 规则文件解析、白名单/黑名单解析、注释行跳过、各字段提取 | +| `tests/core/scanner.test.ts` | `src/core/scanner.ts` | mock HTTP + mock 插件测试完整扫描流程 | +| `tests/core/plugin-loader.test.ts` | `src/core/plugin-loader.ts` | 插件发现、加载、注册验证 | + +### 7.2 集成测试 + +```typescript +// tests/integration/scan.test.ts +// 使用 Bun 的内置 HTTP server 创建 mock 服务器进行端到端测试 +import { describe, test, expect } from "bun:test"; + +describe("完整扫描流程", () => { + test("应正确识别基础信息", async () => { + // 启动 mock HTTP server（Bun.serve） + // 运行 Scanner + // 验证输出包含 status、title 等 + }); +}); +``` + +### 7.3 测试命令 + +```bash +bun test # 运行所有测试 +bun test --watch # 监视模式 +bun test tests/utils/ # 运行指定目录 +bun test --coverage # 覆盖率报告 +``` + +--- + +## 8. 构建与分发 + +### 8.1 package.json 配置 + +```jsonc +{ + "name": "whatsweb", + "version": "1.0.0", + "type": "module", + "main": "src/index.ts", + "bin": { + "whatsweb": "src/cli.ts" + }, + "scripts": { + "dev": "bun run src/cli.ts", + "test": "bun test", + "lint": "bunx eslint src/ tests/", + "lint:fix": "bunx eslint src/ tests/ --fix", + "format": "bunx prettier --write src/ tests/", + "format:check": "bunx prettier --check src/ tests/", + "typecheck": "tsc --noEmit", + "build": "bun build src/cli.ts --outdir dist --target node", + "prepublishOnly": "bun run typecheck && bun run lint && bun run test" + } +} +``` + +### 8.2 tsconfig.json + +```jsonc +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "types": ["bun-types"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "resolveJsonModule": true + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "tests"] +} +``` + +### 8.3 分发方式 + +1. **Bun 用户**: 直接 `bun install -g whatsweb` 运行 TypeScript 源码 +2. **Node.js 兼容**: 通过 `bun build` 编译为单文件 JavaScript，兼容 Node.js 用户 +3. **可选编译为可执行文件**: `bun build --compile src/cli.ts --outfile whatsweb` 生成无需运行时的独立二进制 + +--- + +## 9. 重构步骤与里程碑 + +> **详细的可执行 SOP 已拆分至 [`sop/`](./sop/README.md)**，以下为概要索引。 + +| Phase | SOP | 说明 | 状态 | +|-------|-----|------|------| +| 1 | [phase-01-infrastructure.md](./sop/phase-01-infrastructure.md) | 基础设施：Bun 初始化、TS/ESLint/Prettier 配置、目录骨架、基础类型 | ⬜ | +| 2 | [phase-02-utils.md](./sop/phase-02-utils.md) | 工具层 TDD：url、dns、ip、http + p-limit 封装 + Zod schema | ⬜ | +| 3 | [phase-03-plugin-system.md](./sop/phase-03-plugin-system.md) | 插件系统：类型定义（Zod）、PluginLoader | ⬜ | +| 4 | [phase-04-basic-plugins.md](./sop/phase-04-basic-plugins.md) | 基础插件 TDD：base-info、email、geoip + es-toolkit | ⬜ | +| 5 | [phase-05-wappalyzer.md](./sop/phase-05-wappalyzer.md) | Wappalyzer 插件：wappalyzer@6.10.66 集成 | ⬜ | +| 6 | [phase-06-bbscan.md](./sop/phase-06-bbscan.md) | BBScan 插件：规则解析器（Zod）+ 路径扫描器（p-limit） | ⬜ | +| 7 | [phase-07-scanner.md](./sop/phase-07-scanner.md) | 核心引擎：Scanner 类、Zod 参数验证 | ⬜ | +| 8 | [phase-08-cli.md](./sop/phase-08-cli.md) | CLI + Reporter：commander/picocolors/cli-progress/p-limit | ⬜ | +| 9 | [phase-09-integration.md](./sop/phase-09-integration.md) | 集成测试、旧文件清理、README 更新 | ⬜ | + +--- + +## 10. 风险与注意事项 + +### 10.1 Wappalyzer 兼容性 + +**风险**: `wappalyzer@6.10.66` 依赖 `puppeteer@~19.7.0`，需要 Chromium 运行环境。 + +**应对**: +- 已确定使用 `wappalyzer@6.10.66`（v6 最后功能版本），API 为 `init → open → analyze → destroy` +- v6 结果结构使用 `technologies[]`（不是 v5 的 `applications[]`），需注意字段差异 +- CI/部署环境需安装 Chromium 系统依赖（libX11、libatk 等），或使用 `--no-sandbox` 参数 +- 插件实现需确保 `finally` 块中调用 `wappalyzer.destroy()` 清理浏览器进程 + +### 10.2 Bun 的 Node.js 兼容性 + +**风险**: `geoip-lite` 等依赖可能使用 Node.js 特有的 API（如 `Buffer`、`fs` 原生绑定），在 Bun 中行为不一致。 + +**应对**: +- Bun 对 Node.js API 的兼容性已经很高，大部分包可直接使用 +- 在里程碑 2 完成后尽早运行依赖兼容性测试 +- 如有问题，考虑 polyfill 或替换依赖 + +### 10.3 fetch 与 superagent 行为差异 + +**风险**: 原代码大量依赖 `superagent` 的特定行为（如 `.ok(() => true)` 允许非 2xx 状态码、`response.res` 内部结构等）。 + +**应对**: +- 在 `src/utils/http.ts` 中封装一层，模拟原有行为 +- `fetch` 默认不抛异常（非 2xx 不 throw），这一点与 `.ok(() => true)` 行为一致 +- 需要注意 `fetch` 的重定向处理方式（默认 follow），需与原行为对齐 + +### 10.4 BBScan HEAD 请求行为 + +**风险**: 原 `bbscan.js` 使用 `superagent.head()` 但又检查 `response.res.text`，而 HEAD 请求通常不返回 body。 + +**应对**: +- 这可能是原代码的 bug（HEAD 请求不应有 body） +- 重构时需明确：哪些规则需要检查 body 内容（应改为 GET），哪些仅需状态码/头信息（可保持 HEAD） +- 建议：对含有 `{tag=...}` 的规则使用 GET 请求，其他使用 HEAD 请求 + +### 10.5 规则文件路径解析 + +**风险**: 原代码使用 `__dirname` 相对路径解析 `rules/` 目录。ESM 中 `__dirname` 不可用。 + +**应对**: +- 使用 `import.meta.dir`（Bun 原生支持）或 `import.meta.url` + `fileURLToPath` +- 在代码中封装为常量：`const RULES_DIR = path.join(import.meta.dir, "../../rules")` + +### 10.6 向后兼容性 + +重构后的 CLI 应确保以下兼容性： +- 所有原有命令行选项 (`-f`, `-c`, `--network`, `--timeout`, `--user-agent`, `-o`, `--show-error`) 保持一致 +- 输出格式（JSON lines 文件输出、控制台彩色输出）与原版行为一致 +- 规则文件格式完全不变 + +--- + +*文档版本: 1.0 | 最后更新: 2026-04-01* diff --git a/docs/2026-04-01/sop/README.md b/docs/2026-04-01/sop/README.md new file mode 100644 index 0000000..decb3e4 --- /dev/null +++ b/docs/2026-04-01/sop/README.md @@ -0,0 +1,109 @@ +# WhatsWeb Bun 重构 — SOP 总览 + +> 本目录包含将 WhatsWeb 从 Node.js + CommonJS 全面迁移至 **Bun + TypeScript + ESM** 的标准操作流程 (SOP)。 +> 每个 Phase 是一个**独立可执行的任务单元**，AI Agent 可直接按顺序执行。 + +--- + +## 执行原则 + +1. **测试驱动 (TDD)**：每个模块的开发顺序为 ① 写类型/接口 → ② 写测试 → ③ 写实现 → ④ 测试通过 +2. **渐进式交付**：每个 Phase 完成后提交代码、运行测试、更新本文档进度 +3. **向后兼容**：`rules/` 目录的数据文件不做任何修改 +4. **单一职责**：每个 SOP 文件对应一个可独立完成和验证的功能区块 + +--- + +## 技术栈 + +| 类别 | 选型 | 说明 | +|------|------|------| +| 运行时 | Bun | 原生 TS、内置测试、高性能 | +| 语言 | TypeScript (strict mode) | 全面类型安全 | +| 模块系统 | ESM (`import`/`export`) | 现代标准 | +| HTTP 客户端 | Bun 原生 `fetch` | 零依赖、Web 标准 | +| CLI 框架 | commander@12 | 成熟稳定、ESM 支持 | +| 终端着色 | picocolors | 零依赖、极小体积 | +| 进度条 | cli-progress | 现代终端进度条 | +| 文件匹配 | Bun.Glob (内置) | 零依赖 | +| 工具函数库 | **es-toolkit** | 替代 lodash — 2-3x 更快、97% 更小、原生 TS | +| 并发管理 | **p-limit@7** | 成熟的并发控制、~100M 周下载、ESM 原生 | +| 数据验证 | **zod** | 运行时 schema 验证 + 类型推断 | +| IP/DNS | 自定义实现 | 替换有漏洞的 ip 包 | +| GeoIP | geoip-lite | 保留，无更优替代 | +| 技术栈识别 | **wappalyzer@6.10.66** | 依赖 puppeteer@~19.7.0 | +| 测试 | bun:test (内置) | 零配置 | +| Lint | **ESLint 9** (flat config) + **typescript-eslint** | 代码质量检查 | +| Format | **Prettier** | 代码格式化 | + +--- + +## Phase 进度 + +| Phase | SOP 文件 | 说明 | 状态 | +|-------|----------|------|------| +| 1 | [phase-01-infrastructure.md](./phase-01-infrastructure.md) | 基础设施搭建：初始化 Bun 项目、配置 TS/ESLint/Prettier、建目录骨架 | ✅ 已完成 (2026-04-01) | +| 2 | [phase-02-utils.md](./phase-02-utils.md) | 工具层 TDD：url、dns、ip、http 四个工具模块 + p-limit 封装 | ✅ 已完成 (2026-04-01) | +| 3 | [phase-03-plugin-system.md](./phase-03-plugin-system.md) | 插件系统：类型定义、插件加载器 | ✅ 已完成 (2026-04-01) | +| 4 | [phase-04-basic-plugins.md](./phase-04-basic-plugins.md) | 基础插件 TDD：base-info、email、geoip 三个插件 | ✅ 已完成 (2026-04-01) | +| 5 | [phase-05-wappalyzer.md](./phase-05-wappalyzer.md) | Wappalyzer 插件：基于 wappalyzer@6.10.66 的技术栈识别 | ✅ 已完成 (2026-04-01) | +| 6 | [phase-06-bbscan.md](./phase-06-bbscan.md) | BBScan 插件：规则解析器 + 路径扫描器 | ✅ 已完成 (2026-04-01) | +| 7 | [phase-07-scanner.md](./phase-07-scanner.md) | 核心引擎：Scanner 类（调度插件、聚合结果） | ✅ 已完成 (2026-04-01) | +| 8 | [phase-08-cli.md](./phase-08-cli.md) | CLI 与 Reporter：命令行入口、结果格式化与输出 | ✅ 已完成 (2026-04-01) | +| 9 | [phase-09-integration.md](./phase-09-integration.md) | 集成测试、旧文件清理、README 更新、发布 | ✅ 已完成 (2026-04-01) | + +--- + +## 目标目录结构 + +``` +whatsweb/ +├── src/ +│ ├── cli.ts +│ ├── index.ts +│ ├── core/ +│ │ ├── scanner.ts +│ │ ├── plugin-loader.ts +│ │ └── reporter.ts +│ ├── plugins/ +│ │ ├── types.ts +│ │ ├── base-info.ts +│ │ ├── email.ts +│ │ ├── geoip.ts +│ │ ├── wappalyzer.ts +│ │ └── bbscan/ +│ │ ├── index.ts +│ │ ├── rule-parser.ts +│ │ └── scanner.ts +│ ├── schemas/ +│ │ └── cli.ts # Zod CLI 参数 schema +│ └── utils/ +│ ├── url.ts +│ ├── dns.ts +│ ├── ip.ts +│ └── http.ts +├── rules/ # 保持原样不动 +├── tests/ +│ ├── utils/ +│ ├── plugins/ +│ └── core/ +├── docs/2026-04-01/ # 重构文档 (计划 + SOP) +├── package.json +├── tsconfig.json +├── eslint.config.mjs +├── .prettierrc +└── bunfig.toml +``` + +--- + +## 进度更新规范 + +每个 Phase 完成后，执行 Agent 必须： + +1. 运行 `bun test` 确认所有测试通过 +2. 运行 `bunx eslint src/ tests/` 确认无 lint 错误 +3. 运行 `bunx prettier --check src/ tests/` 确认格式一致 +3. 将上方表格中对应 Phase 的状态从 `⬜ 未开始` 更新为 `✅ 已完成` +4. 在表格状态列追加完成时间，格式：`✅ 已完成 (YYYY-MM-DD)` +5. `git add . && git commit && git push` diff --git a/docs/2026-04-01/sop/phase-01-infrastructure.md b/docs/2026-04-01/sop/phase-01-infrastructure.md new file mode 100644 index 0000000..461f593 --- /dev/null +++ b/docs/2026-04-01/sop/phase-01-infrastructure.md @@ -0,0 +1,239 @@ +# Phase 1：基础设施搭建 + +> **前置条件**：无 +> **产出物**：可运行 `bun test`（空测试通过）和 `bunx eslint`（无报错）的项目骨架 +> **预计涉及文件**：`package.json`, `tsconfig.json`, `eslint.config.mjs`, `.prettierrc`, 目录骨架, 基础类型定义 + +--- + +## 步骤 + +### Step 1.1 — 初始化 Bun 项目 + +```bash +# 在项目根目录执行（保留已有 .git） +bun init -y +``` + +手动编辑生成的 `package.json`，确保包含以下内容： + +```jsonc +{ + "name": "whatsweb", + "version": "1.0.0", + "type": "module", + "main": "src/index.ts", + "bin": { + "whatsweb": "src/cli.ts" + }, + "scripts": { + "dev": "bun run src/cli.ts", + "test": "bun test", + "lint": "bunx eslint src/ tests/", + "lint:fix": "bunx eslint src/ tests/ --fix", + "format": "bunx prettier --write src/ tests/", + "format:check": "bunx prettier --check src/ tests/", + "typecheck": "bunx tsc --noEmit" + }, + "dependencies": { + "commander": "^12.0.0", + "picocolors": "^1.0.0", + "cli-progress": "^3.12.0", + "es-toolkit": "^1.0.0", + "p-limit": "^7.0.0", + "zod": "^3.24.0", + "geoip-lite": "^1.4.0", + "wappalyzer": "6.10.66" + }, + "devDependencies": { + "eslint": "^10.1.0", + "@eslint/js": "^10.0.0", + "typescript-eslint": "^8.58.0", + "eslint-config-prettier": "^10.1.0", + "prettier": "^3.8.0", + "@types/bun": "latest", + "@types/cli-progress": "^3.11.0", + "@types/geoip-lite": "^1.4.4", + "typescript": "^5.5.0" + } +} +``` + +然后安装依赖： + +```bash +bun install +``` + +### Step 1.2 — 配置 TypeScript + +创建 `tsconfig.json`： + +```jsonc +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "types": ["bun-types"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": ".", + "declaration": true, + "resolveJsonModule": true, + "allowImportingTsExtensions": true, + "noEmit": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts"], + "exclude": ["node_modules", "dist"] +} +``` + +### Step 1.3 — 配置 ESLint + Prettier + +创建 `eslint.config.mjs`（ESLint 9 flat config）： + +```javascript +import eslint from "@eslint/js"; +import tseslint from "typescript-eslint"; +import eslintConfigPrettier from "eslint-config-prettier/flat"; + +export default tseslint.config( + { ignores: ["dist/", "node_modules/", "rules/"] }, + eslint.configs.recommended, + ...tseslint.configs.recommended, + { + rules: { + "@typescript-eslint/no-explicit-any": "warn", + "@typescript-eslint/no-unused-vars": ["error", { argsIgnorePattern: "^_", varsIgnorePattern: "^_" }], + }, + }, + eslintConfigPrettier, // 必须放最后，禁用与 Prettier 冲突的规则 +); +``` + +创建 `.prettierrc`： + +```json +{ + "semi": true, + "singleQuote": false, + "tabWidth": 2, + "trailingComma": "all", + "printWidth": 100, + "endOfLine": "lf" +} +``` + +创建 `.prettierignore`： + +``` +node_modules +dist +rules +bun.lock +``` + +### Step 1.4 — 创建目录骨架 + +```bash +mkdir -p src/core src/plugins/bbscan src/utils +mkdir -p tests/core tests/plugins/bbscan tests/utils +``` + +### Step 1.5 — 创建基础类型定义 + +创建 `src/plugins/types.ts`： + +```typescript +export interface PluginContext { + url: string; + timeout: number; + userAgent: string; + response: HttpResponse; +} + +export interface PluginMeta { + name: string; + description?: string; +} + +export interface Plugin { + meta: PluginMeta; + execute(context: PluginContext): Promise<Record<string, unknown> | null>; +} + +export interface HttpResponse { + status: number; + statusText: string; + headers: Record<string, string>; + text: string; + url: string; +} +``` + +创建 `src/index.ts`（库入口占位）： + +```typescript +export { Scanner } from "./core/scanner.ts"; +export type { Plugin, PluginContext, PluginMeta, HttpResponse } from "./plugins/types.ts"; +``` + +> 注意：此时 `Scanner` 尚未实现，此文件在后续 Phase 才能真正使用。可以先注释掉 Scanner 导出或创建空文件占位。 + +创建 `src/cli.ts`（占位）： + +```typescript +#!/usr/bin/env bun +console.log("whatsweb CLI — under construction"); +``` + +### Step 1.6 — 创建验证用的空测试 + +创建 `tests/smoke.test.ts`： + +```typescript +import { describe, test, expect } from "bun:test"; + +describe("项目骨架验证", () => { + test("bun:test 框架可运行", () => { + expect(1 + 1).toBe(2); + }); + + test("可导入类型定义", async () => { + const types = await import("../src/plugins/types.ts"); + expect(types).toBeDefined(); + }); +}); +``` + +### Step 1.7 — 验证 + +```bash +bun test # 预期：2 个测试通过 +bunx eslint src/ # 预期：无错误 +bunx prettier --check src/ # 预期：无格式问题 +bunx tsc --noEmit # 预期：无类型错误 +``` + +### Step 1.8 — 提交 + +```bash +git add . +git commit -m "chore: Phase 1 — 初始化 Bun 项目基础设施" +git push +``` + +--- + +## 完成标志 + +- [ ] `bun test` 通过 +- [ ] `bunx eslint src/` 无错误 +- [ ] `bunx prettier --check src/` 无格式问题 +- [ ] 目录结构已按规划建立 +- [ ] `src/plugins/types.ts` 包含核心接口定义 +- [ ] 代码已提交并推送 +- [ ] `docs/2026-04-01/sop/README.md` 中 Phase 1 状态已更新为 ✅ diff --git a/docs/2026-04-01/sop/phase-02-utils.md b/docs/2026-04-01/sop/phase-02-utils.md new file mode 100644 index 0000000..df98be7 --- /dev/null +++ b/docs/2026-04-01/sop/phase-02-utils.md @@ -0,0 +1,517 @@ +# Phase 2：工具层 TDD + +> **前置条件**：Phase 1 已完成 +> **产出物**：`src/utils/` 下 4 个工具模块 + Zod schema + p-limit 集成，及其对应测试，全部测试通过 +> **开发顺序**：每个模块按 ① 写测试 → ② 写实现 → ③ 运行测试 → ④ 通过 的 TDD 流程 + +--- + +## 第三方库说明 + +本 Phase 会用到以下新引入的第三方库： + +| 库 | 用途 | 选型理由 | +|----|------|----------| +| **es-toolkit** | 通用工具函数（`groupBy`、`uniq`、`pick` 等） | 替代 lodash — 2-3x 更快、97% 更小、原生 TypeScript、完美 tree-shaking | +| **p-limit@7** | 并发控制 | 替代自写并发池和 bluebird.Promise.map — 成熟（~100M 周下载）、ESM 原生、API 简洁 | +| **zod** | 运行时数据验证 + TypeScript 类型推断 | CLI 参数验证、插件结果验证、规则文件解析验证 | + +> **es-toolkit 使用原则**：优先使用 JS 原生方法（如 `Array.prototype.flat`、`structuredClone`、`Object.groupBy`），仅在原生不支持或 es-toolkit 明显更优时引入。Bun 对现代 JS API 支持完善，可放心使用原生方法。 + +--- + +## 模块概览 + +| 文件 | 测试文件 | 功能 | 对应原始代码 | +|------|----------|------|-------------| +| `src/utils/url.ts` | `tests/utils/url.test.ts` | URL 规范化 + Zod schema | `utils.js: normalUrl` | +| `src/utils/dns.ts` | `tests/utils/dns.test.ts` | DNS A 记录解析 | `utils.js: resolve4` | +| `src/utils/ip.ts` | `tests/utils/ip.test.ts` | CIDR IP 展开 | `utils.js: ipGenerator` | +| `src/utils/http.ts` | `tests/utils/http.test.ts` | 基于 fetch 的 HTTP 封装 | `superagent` 调用点 | + +--- + +## Step 2.1 — URL 工具 (`src/utils/url.ts`) + +### 2.1.1 先写测试 `tests/utils/url.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { normalizeUrl, urlSchema } from "../../src/utils/url.ts"; + +describe("normalizeUrl", () => { + test("为无协议的 URL 自动补全 http://", () => { + expect(normalizeUrl("example.com")).toBe("http://example.com"); + }); + + test("保留已有的 http:// 协议", () => { + expect(normalizeUrl("http://example.com")).toBe("http://example.com"); + }); + + test("保留已有的 https:// 协议", () => { + expect(normalizeUrl("https://example.com")).toBe("https://example.com"); + }); + + test("处理带端口的 URL", () => { + expect(normalizeUrl("example.com:8080")).toBe("http://example.com:8080"); + }); + + test("处理带路径的 URL", () => { + expect(normalizeUrl("example.com/path")).toBe("http://example.com/path"); + }); + + test("空字符串应抛出错误", () => { + expect(() => normalizeUrl("")).toThrow(); + }); + + test("null/undefined 应抛出错误", () => { + expect(() => normalizeUrl(null as unknown as string)).toThrow(); + expect(() => normalizeUrl(undefined as unknown as string)).toThrow(); + }); +}); + +describe("urlSchema (Zod)", () => { + test("合法 URL 通过验证", () => { + const result = urlSchema.safeParse("http://example.com"); + expect(result.success).toBe(true); + }); + + test("空字符串验证失败", () => { + const result = urlSchema.safeParse(""); + expect(result.success).toBe(false); + }); + + test("自动补全协议后通过验证（使用 transform）", () => { + const result = urlSchema.safeParse("example.com"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data).toBe("http://example.com"); + } + }); +}); +``` + +### 2.1.2 实现 `src/utils/url.ts` + +**功能需求**（参考原 `utils.js` 第 20-23 行）： +- `normalizeUrl(url: string): string` — 基础函数，保持简单逻辑 + - 若 `url` 为空值，抛出 `Error` + - 若 `url` 不以 `http://` 或 `https://` 开头，自动补全 `http://` 前缀 +- `urlSchema` — Zod schema，用 `z.string().min(1).transform(...)` 实现 URL 验证 + 协议补全 + - 使用 `z.string()` 作为基础 + - 通过 `.transform()` 在验证过程中自动调用 `normalizeUrl` + +### 2.1.3 验证 + +```bash +bun test tests/utils/url.test.ts +``` + +--- + +## Step 2.2 — DNS 工具 (`src/utils/dns.ts`) + +### 2.2.1 先写测试 `tests/utils/dns.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { resolve4 } from "../../src/utils/dns.ts"; + +describe("resolve4", () => { + test("IPv4 地址直接返回", async () => { + expect(await resolve4("1.2.3.4")).toBe("1.2.3.4"); + }); + + test("非 IPv4 字符串进行 DNS 解析", async () => { + const ip = await resolve4("dns.google"); + expect(ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/); + }); + + test("不存在的域名应抛出错误", async () => { + await expect(resolve4("this-domain-does-not-exist-xyz123.com")).rejects.toThrow(); + }); +}); +``` + +### 2.2.2 实现 `src/utils/dns.ts` + +**功能需求**（参考原 `utils.js` 第 5-18 行）： +- 使用 `node:net` 的 `isIPv4()` 判断输入是否已经是 IPv4 地址，如果是则直接返回 +- 否则使用 `node:dns/promises` 的 `resolve4()` 进行 DNS 查询，返回第一个 A 记录 +- 查询失败或无结果时抛出 Error + +### 2.2.3 验证 + +```bash +bun test tests/utils/dns.test.ts +``` + +--- + +## Step 2.3 — IP/CIDR 工具 (`src/utils/ip.ts`) + +### 2.3.1 先写测试 `tests/utils/ip.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { expandCIDR, ipToLong, longToIp } from "../../src/utils/ip.ts"; + +describe("ipToLong / longToIp", () => { + test("IP 与长整型互转", () => { + expect(ipToLong("192.168.1.1")).toBe(3232235777); + expect(longToIp(3232235777)).toBe("192.168.1.1"); + }); + + test("边界值: 0.0.0.0 和 255.255.255.255", () => { + expect(ipToLong("0.0.0.0")).toBe(0); + expect(ipToLong("255.255.255.255")).toBe(4294967295); + expect(longToIp(0)).toBe("0.0.0.0"); + expect(longToIp(4294967295)).toBe("255.255.255.255"); + }); +}); + +describe("expandCIDR", () => { + test("/30 网段应展开为 2 个可用地址", () => { + const ips = [...expandCIDR("192.168.1.0/30")]; + expect(ips).toEqual(["192.168.1.1", "192.168.1.2"]); + }); + + test("/32 网段应展开为 1 个地址", () => { + const ips = [...expandCIDR("192.168.1.100/32")]; + expect(ips).toEqual(["192.168.1.100"]); + }); + + test("/24 网段应展开为 254 个地址", () => { + const ips = [...expandCIDR("10.0.0.0/24")]; + expect(ips.length).toBe(254); + expect(ips[0]).toBe("10.0.0.1"); + expect(ips[253]).toBe("10.0.0.254"); + }); + + test("非法 CIDR 应抛出错误", () => { + expect(() => [...expandCIDR("invalid")]).toThrow(); + }); +}); +``` + +### 2.3.2 实现 `src/utils/ip.ts` + +**功能需求**（替代原 `utils.js` 第 25-39 行 + `ip` 包）： +- `ipToLong(ip: string): number` — IP 字符串转 32 位无符号整数 +- `longToIp(long: number): string` — 32 位无符号整数转 IP 字符串 +- `expandCIDR(cidr: string): Generator<string>` — 使用 Generator 展开 CIDR 范围内所有可用 IP + - 对于 /32，返回单个地址 + - 对于 /31 及更大范围，排除网络地址和广播地址 + - 非法输入抛出 Error + +**重要**：使用 Generator（`function*`）替代原代码中的数组预分配，避免大网段内存爆炸。 + +### 2.3.3 验证 + +```bash +bun test tests/utils/ip.test.ts +``` + +--- + +## Step 2.4 — HTTP 封装 (`src/utils/http.ts`) + +### 2.4.1 先写测试 `tests/utils/http.test.ts` + +```typescript +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { httpGet, httpHead } from "../../src/utils/http.ts"; +import type { HttpResponse } from "../../src/utils/http.ts"; + +let server: ReturnType<typeof Bun.serve>; +let baseUrl: string; + +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/ok") { + return new Response("<html><title>Test Page", { + headers: { "content-type": "text/html", "x-powered-by": "test" }, + }); + } + if (url.pathname === "/redirect") { + return Response.redirect(`http://localhost:${server.port}/ok`, 302); + } + if (url.pathname === "/slow") { + return new Promise((resolve) => setTimeout(() => resolve(new Response("slow")), 5000)); + } + return new Response("Not Found", { status: 404 }); + }, + }); + baseUrl = `http://localhost:${server.port}`; +}); + +afterAll(() => { + server.stop(); +}); + +describe("httpGet", () => { + test("正常 GET 请求", async () => { + const res = await httpGet(`${baseUrl}/ok`, { timeout: 5000, userAgent: "test" }); + expect(res.status).toBe(200); + expect(res.text).toContain("Test Page"); + expect(res.headers["content-type"]).toContain("text/html"); + expect(res.headers["x-powered-by"]).toBe("test"); + }); + + test("非 2xx 状态码不抛异常（对齐原 superagent .ok(() => true) 行为）", async () => { + const res = await httpGet(`${baseUrl}/not-exist`, { timeout: 5000, userAgent: "test" }); + expect(res.status).toBe(404); + }); + + test("跟踪重定向并记录最终 URL", async () => { + const res = await httpGet(`${baseUrl}/redirect`, { timeout: 5000, userAgent: "test" }); + expect(res.status).toBe(200); + expect(res.url).toContain("/ok"); + }); + + test("超时应抛出错误", async () => { + await expect(httpGet(`${baseUrl}/slow`, { timeout: 500, userAgent: "test" })).rejects.toThrow(); + }); +}); + +describe("httpHead", () => { + test("HEAD 请求返回头信息", async () => { + const res = await httpHead(`${baseUrl}/ok`, { timeout: 5000, userAgent: "test" }); + expect(res.status).toBe(200); + expect(res.headers["content-type"]).toContain("text/html"); + }); +}); +``` + +### 2.4.2 实现 `src/utils/http.ts` + +**功能需求**（替代 `superagent` 的所有使用点）： + +```typescript +import { z } from "zod"; + +export interface RequestOptions { + timeout: number; + userAgent: string; +} + +export interface HttpResponse { + status: number; + statusText: string; + headers: Record; + text: string; + url: string; // 重定向后的最终 URL +} + +// 可选：使用 Zod 验证 RequestOptions +export const requestOptionsSchema = z.object({ + timeout: z.number().positive(), + userAgent: z.string().min(1), +}); + +export async function httpGet(url: string, options: RequestOptions): Promise; +export async function httpHead(url: string, options: RequestOptions): Promise; +``` + +**实现要点**： +- 基于 Bun 原生 `fetch` +- 设置 `Accept: */*`、`Accept-Encoding: ""`（与原 superagent 配置一致）、自定义 `User-Agent` +- 超时控制使用 `AbortController` + `setTimeout` +- **不抛异常**（非 2xx 也正常返回），对齐原代码 `.ok(() => true)` 的行为 +- 跟踪重定向：`fetch` 默认 `redirect: "follow"`，通过 `response.url` 获取最终 URL +- 将所有响应头转为小写 key 的 `Record` + +**注意**：`HttpResponse` 类型在此处定义后，需同步更新 `src/plugins/types.ts` 中的 `HttpResponse` 为从此文件 re-export，保持类型来源唯一。 + +### 2.4.3 验证 + +```bash +bun test tests/utils/http.test.ts +``` + +--- + +## Step 2.5 — p-limit 并发控制封装 + +**不再自写并发池**。改为封装 `p-limit@7`，提供项目统一的并发 map 工具。 + +### 2.5.1 先写测试 `tests/utils/concurrency.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { mapConcurrent } from "../../src/utils/concurrency.ts"; + +describe("mapConcurrent (基于 p-limit)", () => { + test("所有任务按原始顺序返回结果", async () => { + const input = [1, 2, 3, 4, 5]; + const results = await mapConcurrent(input, async (n) => n * 2, { concurrency: 2 }); + expect(results).toEqual([2, 4, 6, 8, 10]); + }); + + test("空数组返回空数组", async () => { + const results = await mapConcurrent([], async (n: number) => n, { concurrency: 5 }); + expect(results).toEqual([]); + }); + + test("并发度生效（不超过指定上限）", async () => { + let running = 0; + let maxRunning = 0; + await mapConcurrent( + Array.from({ length: 20 }, (_, i) => i), + async (n) => { + running++; + maxRunning = Math.max(maxRunning, running); + await new Promise((r) => setTimeout(r, 50)); + running--; + return n; + }, + { concurrency: 5 }, + ); + expect(maxRunning).toBeLessThanOrEqual(5); + }); + + test("单个任务失败不影响其他任务（传入 onError 回调时）", async () => { + const errors: Error[] = []; + const results = await mapConcurrent( + [1, 2, 3], + async (n) => { + if (n === 2) throw new Error("fail"); + return n; + }, + { + concurrency: 2, + onError: (err) => errors.push(err as Error), + }, + ); + expect(results).toEqual([1, undefined, 3]); + expect(errors.length).toBe(1); + }); + + test("未提供 onError 时，错误直接抛出", async () => { + await expect( + mapConcurrent( + [1, 2, 3], + async (n) => { + if (n === 2) throw new Error("boom"); + return n; + }, + { concurrency: 1 }, + ), + ).rejects.toThrow("boom"); + }); +}); +``` + +### 2.5.2 实现 `src/utils/concurrency.ts` + +```typescript +import pLimit from "p-limit"; + +export interface MapConcurrentOptions { + concurrency: number; + onError?: (error: unknown, item: unknown, index: number) => void; +} + +/** + * 基于 p-limit 的并发 map 工具，替代 bluebird.Promise.map。 + * 结果数组按原始顺序排列。 + */ +export async function mapConcurrent( + items: T[], + fn: (item: T, index: number) => Promise, + options: MapConcurrentOptions, +): Promise<(R | undefined)[]> { + const limit = pLimit(options.concurrency); + + const tasks = items.map((item, index) => + limit(async () => { + try { + return await fn(item, index); + } catch (error) { + if (options.onError) { + options.onError(error, item, index); + return undefined; + } + throw error; + } + }), + ); + + return Promise.all(tasks); +} +``` + +> **为什么用 p-limit 而不是自己写**：p-limit@7 经过充分测试（周下载 ~100M），支持 `activeCount`、`pendingCount`、`clearQueue()` 等高级特性，后续可直接使用。自写并发池容易在边界情况（如任务取消、错误传播）上出 bug。 + +### 2.5.3 验证 + +```bash +bun test tests/utils/concurrency.test.ts +``` + +--- + +## Step 2.6 — es-toolkit 使用指南 + +**es-toolkit 不需要单独的工具文件**，而是在需要的地方直接 import 使用。以下是本项目中的典型用法供后续 Phase 参考： + +```typescript +// 去重（替代 lodash.uniq / Array.from(new Set(...))） +import { uniq } from "es-toolkit"; +const targets = uniq(rawTargets); + +// 按分类分组（替代手动 for 循环） +import { groupBy } from "es-toolkit"; +const grouped = groupBy(apps, (app) => app.category); + +// 选取对象部分字段 +import { pick } from "es-toolkit"; +const subset = pick(result, ["status", "title", "redirect"]); + +// 对象深拷贝（优先用原生 structuredClone，es-toolkit 提供 cloneDeep 作为 fallback） +const copy = structuredClone(data); + +// 判空 +import { isEmpty } from "es-toolkit"; +if (isEmpty(result)) return null; +``` + +> **原则**：不要 `import * from "es-toolkit"` — 始终具名导入，确保 tree-shaking 生效。 + +--- + +## 最终验证 + +```bash +bun test tests/utils/ + +bunx eslint src/utils/ +bunx prettier --check src/utils/ +``` + +--- + +## 提交 + +```bash +git add . +git commit -m "feat: Phase 2 — 工具层 TDD (url/dns/ip/http + p-limit + zod)" +git push +``` + +--- + +## 完成标志 + +- [ ] `tests/utils/url.test.ts` 全部通过（含 Zod schema 测试） +- [ ] `tests/utils/dns.test.ts` 全部通过 +- [ ] `tests/utils/ip.test.ts` 全部通过 +- [ ] `tests/utils/http.test.ts` 全部通过 +- [ ] `tests/utils/concurrency.test.ts` 全部通过（基于 p-limit） +- [ ] `bunx eslint src/utils/ +bunx prettier --check src/utils/` 无错误 +- [ ] 代码已提交并推送 +- [ ] `docs/2026-04-01/sop/README.md` 中 Phase 2 状态已更新为 ✅ diff --git a/docs/2026-04-01/sop/phase-03-plugin-system.md b/docs/2026-04-01/sop/phase-03-plugin-system.md new file mode 100644 index 0000000..ff21d66 --- /dev/null +++ b/docs/2026-04-01/sop/phase-03-plugin-system.md @@ -0,0 +1,192 @@ +# Phase 3：插件系统 — 类型定义与加载器 + +> **前置条件**：Phase 2 已完成（`src/utils/http.ts` 中的 `HttpResponse` 类型可用） +> **产出物**：`src/plugins/types.ts` 完善版 + `src/core/plugin-loader.ts` + 对应测试 +> **TDD 顺序**：类型定义 → Zod schema → 加载器测试 → 加载器实现 + +--- + +## Step 3.1 — 完善插件类型定义 (`src/plugins/types.ts`) + +此文件在 Phase 1 已创建骨架。现在需要完善，整合 Zod 验证和 `HttpResponse` re-export。 + +```typescript +// src/plugins/types.ts + +import { z } from "zod"; +import type { HttpResponse } from "../utils/http.ts"; + +// 从 http 工具模块 re-export，保持类型来源唯一 +export type { HttpResponse } from "../utils/http.ts"; + +/** 传递给每个插件的上下文对象 */ +export interface PluginContext { + url: string; + timeout: number; + userAgent: string; + response: HttpResponse; +} + +/** 插件元信息 */ +export interface PluginMeta { + name: string; + description?: string; +} + +/** 插件接口 — 所有插件必须实现此接口 */ +export interface Plugin { + meta: PluginMeta; + execute(context: PluginContext): Promise | null>; +} + +/** Zod schema：验证插件元信息 */ +export const pluginMetaSchema = z.object({ + name: z.string().min(1, "插件名称不能为空"), + description: z.string().optional(), +}); + +/** Zod schema：验证插件结果（非空的键值对象） */ +export const pluginResultSchema = z.record(z.string(), z.unknown()).nullable(); +``` + +--- + +## Step 3.2 — 插件加载器测试 (`tests/core/plugin-loader.test.ts`) + +```typescript +import { describe, test, expect } from "bun:test"; +import { PluginLoader } from "../../src/core/plugin-loader.ts"; +import type { Plugin, PluginContext } from "../../src/plugins/types.ts"; + +const createMockPlugin = (name: string, result: Record | null): Plugin => ({ + meta: { name }, + execute: async (_ctx: PluginContext) => result, +}); + +describe("PluginLoader", () => { + test("注册并获取所有插件", () => { + const loader = new PluginLoader(); + const p1 = createMockPlugin("plugin-a", { key: "value" }); + const p2 = createMockPlugin("plugin-b", null); + loader.register(p1); + loader.register(p2); + expect(loader.getAll()).toEqual([p1, p2]); + }); + + test("不允许注册同名插件", () => { + const loader = new PluginLoader(); + loader.register(createMockPlugin("dup", {})); + expect(() => loader.register(createMockPlugin("dup", {}))).toThrow(); + }); + + test("不允许注册空名称插件", () => { + const loader = new PluginLoader(); + expect(() => loader.register(createMockPlugin("", {}))).toThrow(); + }); + + test("按名称获取插件", () => { + const loader = new PluginLoader(); + const p1 = createMockPlugin("find-me", {}); + loader.register(p1); + expect(loader.get("find-me")).toBe(p1); + expect(loader.get("not-exist")).toBeUndefined(); + }); + + test("pluginCount 返回正确数量", () => { + const loader = new PluginLoader(); + expect(loader.pluginCount).toBe(0); + loader.register(createMockPlugin("a", {})); + loader.register(createMockPlugin("b", {})); + expect(loader.pluginCount).toBe(2); + }); + + test("加载所有内置插件", () => { + const loader = PluginLoader.withBuiltins(); + const plugins = loader.getAll(); + // 初始阶段为空数组，后续 Phase 逐步补充内置插件 + expect(plugins).toBeInstanceOf(Array); + }); +}); +``` + +--- + +## Step 3.3 — 插件加载器实现 (`src/core/plugin-loader.ts`) + +```typescript +import { pluginMetaSchema } from "../plugins/types.ts"; +import type { Plugin } from "../plugins/types.ts"; + +export class PluginLoader { + private plugins: Map = new Map(); + + /** 注册一个插件（使用 Zod 验证 meta） */ + register(plugin: Plugin): void { + const parsed = pluginMetaSchema.safeParse(plugin.meta); + if (!parsed.success) { + throw new Error(`Invalid plugin meta: ${parsed.error.message}`); + } + if (this.plugins.has(plugin.meta.name)) { + throw new Error(`Plugin "${plugin.meta.name}" is already registered`); + } + this.plugins.set(plugin.meta.name, plugin); + } + + /** 获取所有已注册的插件（按注册顺序） */ + getAll(): Plugin[] { + return [...this.plugins.values()]; + } + + /** 按名称获取插件 */ + get(name: string): Plugin | undefined { + return this.plugins.get(name); + } + + /** 已注册插件数量 */ + get pluginCount(): number { + return this.plugins.size; + } + + /** 创建一个预加载所有内置插件的 PluginLoader */ + static withBuiltins(): PluginLoader { + const loader = new PluginLoader(); + // 后续 Phase 实现插件后，在此处 import 并注册： + // loader.register(baseInfoPlugin); + // loader.register(emailPlugin); + // loader.register(geoipPlugin); + // loader.register(wappalyzerPlugin); + // loader.register(bbscanPlugin); + return loader; + } +} +``` + +--- + +## Step 3.4 — 验证 + +```bash +bun test tests/core/plugin-loader.test.ts +bunx eslint src/core/ src/plugins/types.ts +``` + +--- + +## Step 3.5 — 提交 + +```bash +git add . +git commit -m "feat: Phase 3 — 插件类型定义与加载器 (Zod 验证)" +git push +``` + +--- + +## 完成标志 + +- [ ] `tests/core/plugin-loader.test.ts` 全部通过 +- [ ] `src/plugins/types.ts` 包含类型定义 + Zod schema +- [ ] `src/core/plugin-loader.ts` 注册时使用 Zod 验证 meta +- [ ] lint 无错误 +- [ ] 代码已提交并推送 +- [ ] `docs/2026-04-01/sop/README.md` 中 Phase 3 状态已更新为 ✅ diff --git a/docs/2026-04-01/sop/phase-04-basic-plugins.md b/docs/2026-04-01/sop/phase-04-basic-plugins.md new file mode 100644 index 0000000..3eaf45a --- /dev/null +++ b/docs/2026-04-01/sop/phase-04-basic-plugins.md @@ -0,0 +1,381 @@ +# Phase 4：基础插件 TDD — base-info、email、geoip + +> **前置条件**：Phase 3 已完成（Plugin 接口、PluginLoader 可用） +> **产出物**：三个基础插件及其测试，全部通过 +> **TDD 顺序**：每个插件 ① 写测试 → ② 写实现 → ③ 运行测试 → ④ 在 PluginLoader.withBuiltins() 中注册 + +--- + +## 插件概览 + +| 插件 | 文件 | 测试文件 | 原始代码 | 功能 | +|------|------|----------|----------|------| +| Base Info | `src/plugins/base-info.ts` | `tests/plugins/base-info.test.ts` | `plugins/base-info.js` | 状态码、标题、重定向、x-* 头 | +| Email | `src/plugins/email.ts` | `tests/plugins/email.test.ts` | `plugins/email.js` | 从 HTML 中提取邮箱 | +| GeoIP | `src/plugins/geoip.ts` | `tests/plugins/geoip.test.ts` | `plugins/geoip.js` | IP 地理位置查询 | + +--- + +## Step 4.1 — Base Info 插件 + +### 4.1.1 先写测试 `tests/plugins/base-info.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { baseInfoPlugin } from "../../src/plugins/base-info.ts"; +import type { PluginContext } from "../../src/plugins/types.ts"; + +const makeContext = (overrides: Partial & { url?: string }): PluginContext => ({ + url: overrides.url ?? "http://example.com", + timeout: 5000, + userAgent: "test", + response: { + status: 200, + statusText: "OK", + headers: {}, + text: "", + url: overrides.url ?? "http://example.com", + ...overrides, + }, +}); + +describe("baseInfoPlugin", () => { + test("meta.name 正确", () => { + expect(baseInfoPlugin.meta.name).toBe("Base Information"); + }); + + test("提取状态码", async () => { + const result = await baseInfoPlugin.execute(makeContext({ status: 200, statusText: "OK" })); + expect(result?.status).toBe("200 OK"); + }); + + test("提取 标签", async () => { + const result = await baseInfoPlugin.execute( + makeContext({ text: "<html><title>Hello World" }), + ); + expect(result?.title).toBe("Hello World"); + }); + + test("标题中的换行符被转义", async () => { + const result = await baseInfoPlugin.execute( + makeContext({ text: "Line1\nLine2\rLine3" }), + ); + expect(result?.title).toBe("Line1\\nLine2\\rLine3"); + }); + + test("无 title 时结果中不含 title 字段", async () => { + const result = await baseInfoPlugin.execute(makeContext({ text: "No title" })); + expect(result?.title).toBeUndefined(); + }); + + test("检测重定向（最终 URL 与原始 URL 不同）", async () => { + const result = await baseInfoPlugin.execute( + makeContext({ + url: "http://example.com", + status: 200, + statusText: "OK", + text: "", + headers: {}, + }), + ); + // url 不同才有 redirect + const ctx = makeContext({ text: "" }); + ctx.url = "http://example.com"; + ctx.response.url = "http://example.com/redirected"; + const r2 = await baseInfoPlugin.execute(ctx); + expect(r2?.redirect).toBe("http://example.com/redirected"); + }); + + test("提取 x-* 响应头", async () => { + const result = await baseInfoPlugin.execute( + makeContext({ + text: "", + headers: { + "x-powered-by": "Express", + "x-request-id": "abc123", + "content-type": "text/html", + }, + }), + ); + expect(result?.["x-powered-by"]).toBe("Express"); + expect(result?.["x-request-id"]).toBe("abc123"); + expect(result?.["content-type"]).toBeUndefined(); // 非 x-* 头不提取 + }); +}); +``` + +### 4.1.2 实现 `src/plugins/base-info.ts` + +**功能需求**（参考原 `plugins/base-info.js`）： + +```typescript +import type { Plugin } from "./types.ts"; + +export const baseInfoPlugin: Plugin = { + meta: { name: "Base Information" }, + async execute(ctx) { + const { response, url } = ctx; + const result: Record = {}; + + // 1. 状态码: "200 OK" + result.status = `${response.status} ${response.statusText}`; + + // 2. 重定向检测: response.url !== url 时记录 + if (response.url !== url) { + result.redirect = response.url; + } + + // 3. 标题提取: 正则 /([^<]+)<\/title>/i + // 换行符转义: \n → \\n, \r → \\r + + // 4. x-* 响应头: 遍历 headers，提取所有以 "x-" 开头的键 + + // 结果为空（仅有 status）时仍返回 + return result; + }, +}; +``` + +**es-toolkit 用法**：可使用 `pickBy` 过滤 x-* 头： +```typescript +import { pickBy } from "es-toolkit"; +const xHeaders = pickBy(response.headers, (_v, k) => k.startsWith("x-")); +Object.assign(result, xHeaders); +``` + +### 4.1.3 验证 + +```bash +bun test tests/plugins/base-info.test.ts +``` + +--- + +## Step 4.2 — Email 插件 + +### 4.2.1 先写测试 `tests/plugins/email.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { emailPlugin } from "../../src/plugins/email.ts"; +import type { PluginContext } from "../../src/plugins/types.ts"; + +const makeContext = (html: string): PluginContext => ({ + url: "http://example.com", + timeout: 5000, + userAgent: "test", + response: { status: 200, statusText: "OK", headers: {}, text: html, url: "http://example.com" }, +}); + +describe("emailPlugin", () => { + test("meta.name 正确", () => { + expect(emailPlugin.meta.name).toBe("Email"); + }); + + test("从 HTML 中提取多个邮箱", async () => { + const html = "Contact: admin@test.com and info@test.com for details"; + const result = await emailPlugin.execute(makeContext(html)); + expect(result?.email).toEqual(expect.arrayContaining(["admin@test.com", "info@test.com"])); + }); + + test("邮箱去重", async () => { + const html = "admin@test.com admin@test.com admin@test.com info@test.com"; + const result = await emailPlugin.execute(makeContext(html)); + expect(result?.email).toEqual(["admin@test.com", "info@test.com"]); + }); + + test("少于 2 个唯一邮箱时返回 null", async () => { + const html = "Only one: admin@test.com"; + const result = await emailPlugin.execute(makeContext(html)); + expect(result).toBeNull(); + }); + + test("无邮箱时返回 null", async () => { + const result = await emailPlugin.execute(makeContext("<html>No emails here</html>")); + expect(result).toBeNull(); + }); +}); +``` + +### 4.2.2 实现 `src/plugins/email.ts` + +**功能需求**（参考原 `plugins/email.js`）： + +```typescript +import type { Plugin } from "./types.ts"; +import { uniq } from "es-toolkit"; + +export const emailPlugin: Plugin = { + meta: { name: "Email" }, + async execute(ctx) { + const { response } = ctx; + const matched = response.text.match(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}/gi); + if (!matched) return null; + + // 使用 es-toolkit 的 uniq 去重 + const emails = uniq(matched); + + // 原逻辑：至少 2 个唯一邮箱才返回 + if (emails.length < 2) return null; + return { email: emails }; + }, +}; +``` + +### 4.2.3 验证 + +```bash +bun test tests/plugins/email.test.ts +``` + +--- + +## Step 4.3 — GeoIP 插件 + +### 4.3.1 先写测试 `tests/plugins/geoip.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { geoipPlugin } from "../../src/plugins/geoip.ts"; +import type { PluginContext } from "../../src/plugins/types.ts"; + +const makeContext = (url: string): PluginContext => ({ + url, + timeout: 5000, + userAgent: "test", + response: { status: 200, statusText: "OK", headers: {}, text: "", url }, +}); + +describe("geoipPlugin", () => { + test("meta.name 正确", () => { + expect(geoipPlugin.meta.name).toBe("Geolocation"); + }); + + test("解析 IP 并返回地理信息", async () => { + // 使用已知的公共 DNS IP + const result = await geoipPlugin.execute(makeContext("http://8.8.8.8")); + expect(result?.ip).toBe("8.8.8.8"); + // geoip-lite 应能查到 Google DNS 的地理信息 + expect(result?.country).toBeDefined(); + }); + + test("域名会被解析为 IP", async () => { + const result = await geoipPlugin.execute(makeContext("http://dns.google")); + expect(result?.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/); + }); + + test("无法解析的域名应返回 null 或抛出错误", async () => { + try { + const result = await geoipPlugin.execute( + makeContext("http://this-domain-does-not-exist-xyz123.com"), + ); + // 如果不抛错，结果应为 null + expect(result).toBeNull(); + } catch { + // 抛错也是可接受的行为 + } + }); +}); +``` + +### 4.3.2 实现 `src/plugins/geoip.ts` + +**功能需求**（参考原 `plugins/geoip.js`）： + +```typescript +import type { Plugin } from "./types.ts"; +import { resolve4 } from "../utils/dns.ts"; +import geoip from "geoip-lite"; + +export const geoipPlugin: Plugin = { + meta: { name: "Geolocation" }, + async execute(ctx) { + const { url } = ctx; + const { host } = new URL(url); // 替代原 Url.parse() + if (!host) return null; + + const ip = await resolve4(host); + const result: Record<string, unknown> = { ip }; + + const geo = geoip.lookup(ip); + if (geo) { + const { country, city } = geo; + result.country = [city, country].filter(Boolean).join("/"); + } + + return result; + }, +}; +``` + +### 4.3.3 验证 + +```bash +bun test tests/plugins/geoip.test.ts +``` + +--- + +## Step 4.4 — 注册内置插件 + +在 `src/core/plugin-loader.ts` 的 `withBuiltins()` 中注册三个插件： + +```typescript +import { baseInfoPlugin } from "../plugins/base-info.ts"; +import { emailPlugin } from "../plugins/email.ts"; +import { geoipPlugin } from "../plugins/geoip.ts"; + +static withBuiltins(): PluginLoader { + const loader = new PluginLoader(); + loader.register(baseInfoPlugin); + loader.register(emailPlugin); + loader.register(geoipPlugin); + // wappalyzer 和 bbscan 在后续 Phase 添加 + return loader; +} +``` + +更新 `tests/core/plugin-loader.test.ts` 中 `withBuiltins` 的测试： + +```typescript +test("加载所有内置插件", () => { + const loader = PluginLoader.withBuiltins(); + const names = loader.getAll().map((p) => p.meta.name); + expect(names).toContain("Base Information"); + expect(names).toContain("Email"); + expect(names).toContain("Geolocation"); +}); +``` + +--- + +## 最终验证 + +```bash +bun test tests/plugins/base-info.test.ts tests/plugins/email.test.ts tests/plugins/geoip.test.ts +bun test tests/core/plugin-loader.test.ts +bunx eslint src/plugins/ src/core/ +``` + +--- + +## 提交 + +```bash +git add . +git commit -m "feat: Phase 4 — 基础插件 TDD (base-info/email/geoip + es-toolkit)" +git push +``` + +--- + +## 完成标志 + +- [ ] `tests/plugins/base-info.test.ts` 全部通过 +- [ ] `tests/plugins/email.test.ts` 全部通过 +- [ ] `tests/plugins/geoip.test.ts` 全部通过 +- [ ] 三个插件均已在 `PluginLoader.withBuiltins()` 注册 +- [ ] `tests/core/plugin-loader.test.ts` 更新后全部通过 +- [ ] lint 无错误 +- [ ] 代码已提交并推送 +- [ ] `docs/2026-04-01/sop/README.md` 中 Phase 4 状态已更新为 ✅ diff --git a/docs/2026-04-01/sop/phase-05-wappalyzer.md b/docs/2026-04-01/sop/phase-05-wappalyzer.md new file mode 100644 index 0000000..0249d8b --- /dev/null +++ b/docs/2026-04-01/sop/phase-05-wappalyzer.md @@ -0,0 +1,262 @@ +# Phase 5：Wappalyzer 插件 + +> **前置条件**：Phase 4 已完成 +> **产出物**：`src/plugins/wappalyzer.ts` + 测试 + 注册到 PluginLoader +> **特殊依赖**：`wappalyzer@6.10.66`（依赖 `puppeteer@~19.7.0`，内含 Chromium） + +--- + +## 背景 + +`wappalyzer@6.10.66` 是 wappalyzer npm 包被废弃前的最后一个功能版本。其 API 与旧版 `@5.x` 完全不同： + +### wappalyzer@6.10.66 API + +```typescript +const Wappalyzer = require("wappalyzer"); + +const wappalyzer = new Wappalyzer(options); +await wappalyzer.init(); // 初始化 (启动 puppeteer/chromium) +const site = await wappalyzer.open(url, headers); // 打开目标站点 +site.on("error", console.error); // 错误事件 +const results = await site.analyze(); // 分析 +await wappalyzer.destroy(); // 销毁 (关闭浏览器) +``` + +### Options + +```typescript +const options = { + debug: false, + delay: 500, + headers: {}, + maxDepth: 3, + maxUrls: 10, + maxWait: 5000, + recursive: true, + probe: true, + proxy: false, + userAgent: "Wappalyzer", + htmlMaxCols: 2000, + htmlMaxRows: 2000, + noScripts: false, + noRedirect: false, +}; +``` + +### Results 结构 + +```json +{ + "urls": { "http://example.com/": { "status": 200 } }, + "technologies": [ + { + "slug": "wordpress", + "name": "WordPress", + "confidence": 100, + "version": "5.9", + "icon": "WordPress.svg", + "website": "https://wordpress.org", + "cpe": "cpe:/a:wordpress:wordpress", + "categories": [ + { "id": 1, "slug": "cms", "name": "CMS" } + ] + } + ] +} +``` + +**注意**：v6 的结果结构是 `technologies[]`（不是 v5 的 `applications[]`），每个 technology 直接包含 `categories[]` 数组。 + +--- + +## Step 5.1 — 先写测试 `tests/plugins/wappalyzer.test.ts` + +```typescript +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { wappalyzerPlugin } from "../../src/plugins/wappalyzer.ts"; +import type { PluginContext } from "../../src/plugins/types.ts"; + +let server: ReturnType<typeof Bun.serve>; +let baseUrl: string; + +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + return new Response( + `<html> + <head> + <title>Test + + +

Test Page

+ `, + { headers: { "content-type": "text/html", "x-powered-by": "Express" } }, + ); + }, + }); + baseUrl = `http://localhost:${server.port}`; +}); + +afterAll(() => { + server.stop(); +}); + +const makeContext = (url: string): PluginContext => ({ + url, + timeout: 30000, + userAgent: "WhatsWeb/1.0", + response: { status: 200, statusText: "OK", headers: {}, text: "", url }, +}); + +describe("wappalyzerPlugin", () => { + test("meta.name 正确", () => { + expect(wappalyzerPlugin.meta.name).toBe("Wappalyzer"); + }); + + // 注意：这个测试需要 puppeteer/chromium 可用 + // 在 CI 环境中可能需要 skip + test("分析本地测试服务器", async () => { + const result = await wappalyzerPlugin.execute(makeContext(baseUrl)); + // 至少应该识别出 Express (通过 x-powered-by 头) + expect(result).toBeDefined(); + if (result) { + // 结果应该是 { category: [techName, ...] } 格式 + const values = Object.values(result); + expect(values.some((v) => Array.isArray(v))).toBe(true); + } + }, 60000); // wappalyzer 分析可能较慢 + + test("无法访问的 URL 应返回 null 或空对象", async () => { + const result = await wappalyzerPlugin.execute( + makeContext("http://localhost:1"), + ); + // 应优雅处理而不是 throw + expect(result === null || (result && Object.keys(result).length === 0)).toBe(true); + }, 30000); +}); +``` + +--- + +## Step 5.2 — 实现 `src/plugins/wappalyzer.ts` + +```typescript +import type { Plugin } from "./types.ts"; +import { groupBy } from "es-toolkit"; + +export const wappalyzerPlugin: Plugin = { + meta: { name: "Wappalyzer", description: "技术栈识别 (wappalyzer@6.10.66)" }, + + async execute(ctx) { + // wappalyzer 是 CJS 包，需要动态 import 或 require + const Wappalyzer = (await import("wappalyzer")).default; + + const options = { + debug: false, + delay: 500, + maxDepth: 3, + maxUrls: 10, + maxWait: Math.min(ctx.timeout, 5000), + recursive: true, + probe: true, + userAgent: ctx.userAgent, + noRedirect: false, + }; + + const wappalyzer = new Wappalyzer(options); + + try { + await wappalyzer.init(); + const site = await wappalyzer.open(ctx.url); + + // 静默错误事件，防止未处理的 rejection + site.on("error", () => {}); + + const json = await site.analyze(); + const result: Record = {}; + + // v6 结构: json.technologies[] + for (const tech of json.technologies ?? []) { + let item = tech.name; + if (tech.version) item += `[${tech.version}]`; + + for (const category of tech.categories ?? []) { + const catName = category.name; + if (!result[catName]) { + result[catName] = [item]; + } else { + (result[catName] as string[]).push(item); + } + } + } + + return Object.keys(result).length > 0 ? result : null; + } catch { + return null; + } finally { + try { await wappalyzer.destroy(); } catch { /* ignore */ } + } + }, +}; +``` + +**关键实现要点**： + +1. **`wappalyzer.init()` 会启动 Chromium**——必须在 finally 中 `wappalyzer.destroy()` 确保关闭 +2. **v6 的结果字段是 `technologies`** 而非 v5 的 `applications` +3. **v6 的 category 结构是 `{ id, slug, name }`**，使用 `category.name` 作为分组键 +4. **错误处理**：`site.on("error", ...)` 防止未处理 rejection；外层 try/catch 确保任何失败都返回 `null` +5. **CJS 兼容**：wappalyzer 是 CJS 包，在 Bun 中用 `import("wappalyzer")` 动态导入 + +--- + +## Step 5.3 — 注册到 PluginLoader + +在 `src/core/plugin-loader.ts` 的 `withBuiltins()` 中增加： + +```typescript +import { wappalyzerPlugin } from "../plugins/wappalyzer.ts"; +// ... +loader.register(wappalyzerPlugin); +``` + +更新 `tests/core/plugin-loader.test.ts`： + +```typescript +expect(names).toContain("Wappalyzer"); +``` + +--- + +## Step 5.4 — 验证 + +```bash +bun test tests/plugins/wappalyzer.test.ts +bun test tests/core/plugin-loader.test.ts +bunx eslint src/plugins/wappalyzer.ts +``` + +> **注意**：wappalyzer 测试依赖 puppeteer 能成功启动 Chromium。若 CI 环境缺少依赖（如 libX11、libatk 等），可能需要安装系统包或使用 `--no-sandbox` 参数。可在测试中增加 skip 条件。 + +--- + +## Step 5.5 — 提交 + +```bash +git add . +git commit -m "feat: Phase 5 — Wappalyzer 插件 (wappalyzer@6.10.66)" +git push +``` + +--- + +## 完成标志 + +- [ ] `tests/plugins/wappalyzer.test.ts` 全部通过（或在无 Chromium 环境中标记 skip） +- [ ] 插件已在 `PluginLoader.withBuiltins()` 注册 +- [ ] 使用 `init → open → analyze → destroy` 生命周期，finally 确保清理 +- [ ] lint 无错误 +- [ ] 代码已提交并推送 +- [ ] `docs/2026-04-01/sop/README.md` 中 Phase 5 状态已更新为 ✅ diff --git a/docs/2026-04-01/sop/phase-06-bbscan.md b/docs/2026-04-01/sop/phase-06-bbscan.md new file mode 100644 index 0000000..093c444 --- /dev/null +++ b/docs/2026-04-01/sop/phase-06-bbscan.md @@ -0,0 +1,376 @@ +# Phase 6：BBScan 插件 + +> **前置条件**：Phase 4 已完成（http 工具、p-limit 可用） +> **产出物**：`src/plugins/bbscan/` 三文件模块 + 测试 + 注册到 PluginLoader +> **TDD 顺序**：rule-parser（纯逻辑，先做）→ scanner → index（插件入口） + +--- + +## 模块拆分 + +原 `plugins/bbscan.js`（210 行）拆分为三个文件： + +| 文件 | 职责 | 测试文件 | +|------|------|----------| +| `src/plugins/bbscan/rule-parser.ts` | 解析 `rules/*.txt`、`white.list`、`black.list` | `tests/plugins/bbscan/rule-parser.test.ts` | +| `src/plugins/bbscan/scanner.ts` | 404 检测、路径探测、结果过滤 | `tests/plugins/bbscan/scanner.test.ts` | +| `src/plugins/bbscan/index.ts` | 插件入口，实现 `Plugin` 接口 | `tests/plugins/bbscan/index.test.ts` | + +--- + +## Step 6.1 — 规则解析器 TDD (`rule-parser.ts`) + +### 6.1.1 Zod Schema 定义 + +```typescript +import { z } from "zod"; + +/** 单条扫描规则 */ +export const scanRuleSchema = z.object({ + uri: z.string().startsWith("/"), + tag: z.string().default(""), + status: z.number().int().min(100).max(599).nullable().default(null), + contentType: z.string().default(""), + contentTypeNo: z.string().default(""), + rootOnly: z.boolean().default(false), +}); + +export type ScanRule = z.infer; + +/** 白名单/黑名单规则 */ +export interface ListRule { + texts: string[]; + regexes: RegExp[]; +} +``` + +### 6.1.2 先写测试 `tests/plugins/bbscan/rule-parser.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { parseRuleLine, parseRuleFiles, parseListFile } from "../../../src/plugins/bbscan/rule-parser.ts"; + +describe("parseRuleLine", () => { + test("解析完整规则行", () => { + const line = '/admin.php {status=200} {tag="type=\\"password\\""} {root_only}'; + const rule = parseRuleLine(line); + expect(rule).not.toBeNull(); + expect(rule?.uri).toBe("/admin.php"); + expect(rule?.status).toBe(200); + expect(rule?.rootOnly).toBe(true); + }); + + test("解析仅有 URI 和 status 的行", () => { + const line = "/core {status=200} {tag=\"ELF\"} {root_only}"; + const rule = parseRuleLine(line); + expect(rule?.uri).toBe("/core"); + expect(rule?.status).toBe(200); + expect(rule?.tag).toBe("ELF"); + expect(rule?.rootOnly).toBe(true); + }); + + test("解析含 content-type 的行", () => { + const line = '/debug.txt {status=200} {type="text/plain"} {root_only}'; + const rule = parseRuleLine(line); + expect(rule?.contentType).toBe("text/plain"); + expect(rule?.contentTypeNo).toBe(""); + }); + + test("解析含 type_no 的行", () => { + const line = '/config/database.yml {status=200} {type_no="html"} {tag="password"} {root_only}'; + const rule = parseRuleLine(line); + expect(rule?.contentTypeNo).toBe("html"); + expect(rule?.tag).toBe("password"); + }); + + test("跳过注释行", () => { + expect(parseRuleLine("# This is a comment")).toBeNull(); + }); + + test("跳过非 / 开头的行", () => { + expect(parseRuleLine("not a rule")).toBeNull(); + expect(parseRuleLine("")).toBeNull(); + }); +}); + +describe("parseListFile", () => { + test("解析白名单文件内容", () => { + const content = ` +# comment +{text="Index of"} +{text="<title>phpMyAdmin"} +{regex_text=".*后台.*"} + `.trim(); + const result = parseListFile(content); + expect(result.texts).toContain("Index of"); + expect(result.texts).toContain("<title>phpMyAdmin"); + expect(result.regexes.length).toBe(1); + expect(result.regexes[0].test("管理后台系统")).toBe(true); + }); + + test("跳过空行和注释", () => { + const content = "# comment\n\n{text=\"valid\"}"; + const result = parseListFile(content); + expect(result.texts).toEqual(["valid"]); + }); +}); + +describe("parseRuleFiles", () => { + test("从 rules/ 目录加载真实规则文件", async () => { + const rules = await parseRuleFiles(); + expect(rules.length).toBeGreaterThan(0); + // 每条规则应有 uri 字段 + for (const rule of rules) { + expect(rule.uri).toMatch(/^\//); + } + }); +}); +``` + +### 6.1.3 实现 `src/plugins/bbscan/rule-parser.ts` + +**功能需求**（参考原 `bbscan.js` 第 35-128 行）： + +- `parseRuleLine(line: string): ScanRule | null` — 解析单行规则 + - 正则提取 `{tag="..."}`, `{status=NNN}`, `{type="..."}`, `{type_no="..."}`, `{root_only}` + - 非 `/` 开头的行返回 `null` + - 使用 Zod `scanRuleSchema` 验证解析结果 +- `parseListFile(content: string): ListRule` — 解析白/黑名单文件内容 + - 提取 `{text="..."}` 和 `{regex_text="..."}` 条目 +- `parseRuleFiles(rulesDir?: string): Promise` — 使用 `Bun.Glob` 扫描 `rules/*.txt` 并解析 +- `loadWhiteList(rulesDir?: string): Promise` — 加载并解析 `white.list` +- `loadBlackList(rulesDir?: string): Promise` — 加载并解析 `black.list` + +**Bun API 使用**： +```typescript +// 使用 Bun.Glob 替代 globby +const glob = new Bun.Glob("*.txt"); +for await (const file of glob.scan({ cwd: rulesDir })) { + const content = await Bun.file(path.join(rulesDir, file)).text(); + // ... +} +``` + +**`import.meta.dir` 替代 `__dirname`**： +```typescript +const defaultRulesDir = path.join(import.meta.dir, "../../../rules"); +``` + +### 6.1.4 验证 + +```bash +bun test tests/plugins/bbscan/rule-parser.test.ts +``` + +--- + +## Step 6.2 — 路径扫描器 TDD (`scanner.ts`) + +### 6.2.1 先写测试 `tests/plugins/bbscan/scanner.test.ts` + +```typescript +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { BBScanner } from "../../../src/plugins/bbscan/scanner.ts"; + +let server: ReturnType; +let baseUrl: string; + +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + + // 404 检测端点 + if (url.pathname === "/WhatsWeb-404-existence-check") { + return new Response("Not Found", { status: 404 }); + } + + // 模拟敏感路径 + if (url.pathname === "/.git/config") { + return new Response("[core]\n\trepositoryformatversion = 0", { + status: 200, + headers: { "content-type": "text/plain" }, + }); + } + + if (url.pathname === "/.svn/entries") { + return new Response("12\n-props", { + status: 200, + headers: { "content-type": "text/plain" }, + }); + } + + return new Response("Not Found", { status: 404 }); + }, + }); + baseUrl = `http://localhost:${server.port}`; +}); + +afterAll(() => { + server.stop(); +}); + +describe("BBScanner", () => { + test("check404 — 服务器支持 404", async () => { + const scanner = new BBScanner({ url: baseUrl, userAgent: "test", timeout: 5000 }); + const has404 = await scanner.check404(); + expect(has404).toBe(true); + }); + + test("完整扫描流程 — 发现敏感路径", async () => { + const scanner = new BBScanner({ url: baseUrl, userAgent: "test", timeout: 5000 }); + const results = await scanner.run(); + expect(results).toBeInstanceOf(Array); + // 应能发现 .git/config + if (Array.isArray(results)) { + expect(results.some((r) => r.includes(".git"))).toBe(true); + } + }, 30000); +}); +``` + +### 6.2.2 实现 `src/plugins/bbscan/scanner.ts` + +**功能需求**（参考原 `bbscan.js` 第 9-193 行）： + +```typescript +import pLimit from "p-limit"; +import { httpHead, httpGet } from "../../utils/http.ts"; +import type { ScanRule, ListRule } from "./rule-parser.ts"; + +export interface BBScannerOptions { + url: string; + userAgent: string; + timeout: number; +} + +export class BBScanner { + private host: string; + private protocol: string; + private rules: ScanRule[] = []; + private whiteList: ListRule = { texts: [], regexes: [] }; + private blackList: ListRule = { texts: [], regexes: [] }; + + constructor(private opts: BBScannerOptions) { + const parsed = new URL(opts.url); + this.host = parsed.host; + this.protocol = parsed.protocol; + } + + /** 检测服务器是否正确返回 404 */ + async check404(): Promise; + + /** 加载规则 + 白/黑名单 */ + async loadRules(): Promise; + + /** 执行扫描 */ + async run(): Promise; +} +``` + +**实现要点**： +- `check404()`：HEAD 请求 `/WhatsWeb-404-existence-check`，期望 404 +- `loadRules()`：调用 rule-parser 的 `parseRuleFiles()`, `loadWhiteList()`, `loadBlackList()` +- `run()`： + 1. 先 `check404()`，不支持则返回 `false` + 2. 加载规则 + 3. 使用 **p-limit** 并发 50 扫描所有路径 + 4. 路径中替换 `{sub}`, `{hostname}`, `{hostname_or_folder}` 变量 + 5. 对含 `{tag=...}` 的规则使用 `httpGet`（需要 body），其他用 `httpHead` + 6. 匹配白名单/黑名单文本 + 7. 检查 status、content-type 等条件 + 8. 返回命中的路径数组 + +### 6.2.3 验证 + +```bash +bun test tests/plugins/bbscan/scanner.test.ts +``` + +--- + +## Step 6.3 — 插件入口 (`index.ts`) + +### 6.3.1 先写测试 `tests/plugins/bbscan/index.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { bbscanPlugin } from "../../../src/plugins/bbscan/index.ts"; + +describe("bbscanPlugin", () => { + test("meta.name 正确", () => { + expect(bbscanPlugin.meta.name).toBe("BBScan"); + }); + + test("实现了 Plugin 接口", () => { + expect(typeof bbscanPlugin.execute).toBe("function"); + }); +}); +``` + +### 6.3.2 实现 `src/plugins/bbscan/index.ts` + +```typescript +import type { Plugin } from "../types.ts"; +import { BBScanner } from "./scanner.ts"; + +export const bbscanPlugin: Plugin = { + meta: { name: "BBScan", description: "路径扫描 (基于规则文件)" }, + async execute(ctx) { + const scanner = new BBScanner({ + url: ctx.url, + userAgent: ctx.userAgent, + timeout: ctx.timeout, + }); + const paths = await scanner.run(); + if (!paths || paths.length === 0) return null; + return { paths }; + }, +}; +``` + +--- + +## Step 6.4 — 注册到 PluginLoader + +```typescript +import { bbscanPlugin } from "../plugins/bbscan/index.ts"; +loader.register(bbscanPlugin); +``` + +--- + +## 最终验证 + +```bash +bun test tests/plugins/bbscan/ +bun test tests/core/plugin-loader.test.ts +bunx eslint src/plugins/bbscan/ +``` + +--- + +## 提交 + +```bash +git add . +git commit -m "feat: Phase 6 — BBScan 插件 (rule-parser + scanner + p-limit + Zod)" +git push +``` + +--- + +## 完成标志 + +- [ ] `tests/plugins/bbscan/rule-parser.test.ts` 全部通过 +- [ ] `tests/plugins/bbscan/scanner.test.ts` 全部通过 +- [ ] `tests/plugins/bbscan/index.test.ts` 全部通过 +- [ ] 规则解析使用 Zod schema 验证 +- [ ] 并发扫描使用 p-limit +- [ ] 路径变量替换（`{sub}`, `{hostname}`, `{hostname_or_folder}`）正确 +- [ ] 使用 `Bun.Glob` + `Bun.file().text()` 读取规则文件 +- [ ] 插件已注册到 `PluginLoader.withBuiltins()` +- [ ] lint 无错误 +- [ ] 代码已提交并推送 +- [ ] `docs/2026-04-01/sop/README.md` 中 Phase 6 状态已更新为 ✅ diff --git a/docs/2026-04-01/sop/phase-07-scanner.md b/docs/2026-04-01/sop/phase-07-scanner.md new file mode 100644 index 0000000..1458446 --- /dev/null +++ b/docs/2026-04-01/sop/phase-07-scanner.md @@ -0,0 +1,308 @@ +# Phase 7：核心扫描引擎 + +> **前置条件**：Phase 6 已完成（所有插件就绪） +> **产出物**：`src/core/scanner.ts` + 测试 +> **TDD 顺序**：测试 → 实现 → 验证 + +--- + +## 功能概述 + +`Scanner` 是项目的核心调度器（对应原 `index.js` 的 `WhatsWeb` 类），负责： + +1. 接收目标 URL 和选项 +2. 对目标发起一次初始 HTTP GET 请求 +3. 按顺序执行所有已注册插件 +4. 聚合并返回结果 + +--- + +## Step 7.1 — Zod Schema 定义 + +```typescript +import { z } from "zod"; +import { urlSchema } from "../utils/url.ts"; + +export const scannerOptionsSchema = z.object({ + target: urlSchema, + timeout: z.number().positive().default(10000), + userAgent: z.string().min(1).default("Mozilla/5.0 whatsweb/1.0.0"), +}); + +export type ScannerOptions = z.infer; + +export const scanResultSchema = z.object({ + name: z.string(), + result: z.record(z.string(), z.unknown()), +}); + +export type ScanResult = z.infer; +``` + +--- + +## Step 7.2 — 先写测试 `tests/core/scanner.test.ts` + +```typescript +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { Scanner } from "../../src/core/scanner.ts"; +import { PluginLoader } from "../../src/core/plugin-loader.ts"; +import type { Plugin, PluginContext } from "../../src/plugins/types.ts"; + +let server: ReturnType; +let baseUrl: string; + +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/") { + return new Response( + "Test Siteadmin@test.com info@test.com", + { headers: { "content-type": "text/html", "x-powered-by": "TestServer" } }, + ); + } + if (url.pathname === "/WhatsWeb-404-existence-check") { + return new Response("Not Found", { status: 404 }); + } + return new Response("Not Found", { status: 404 }); + }, + }); + baseUrl = `http://localhost:${server.port}`; +}); + +afterAll(() => { + server.stop(); +}); + +// Mock 插件用于测试 +const successPlugin: Plugin = { + meta: { name: "success-plugin" }, + execute: async (ctx: PluginContext) => ({ found: true, url: ctx.url }), +}; + +const emptyPlugin: Plugin = { + meta: { name: "empty-plugin" }, + execute: async () => null, +}; + +const errorPlugin: Plugin = { + meta: { name: "error-plugin" }, + execute: async () => { throw new Error("plugin crashed"); }, +}; + +describe("Scanner", () => { + test("使用 mock 插件扫描", async () => { + const loader = new PluginLoader(); + loader.register(successPlugin); + loader.register(emptyPlugin); + + const scanner = new Scanner({ + target: baseUrl, + timeout: 5000, + userAgent: "test", + pluginLoader: loader, + }); + + const results = await scanner.analyse(); + expect(Array.isArray(results)).toBe(true); + + // success-plugin 有结果 + const successResult = results.find((r) => r.name === "success-plugin"); + expect(successResult).toBeDefined(); + expect(successResult?.result.found).toBe(true); + + // empty-plugin 被跳过（返回 null） + const emptyResult = results.find((r) => r.name === "empty-plugin"); + expect(emptyResult).toBeUndefined(); + }); + + test("单个插件失败不影响其他插件", async () => { + const loader = new PluginLoader(); + loader.register(successPlugin); + loader.register(errorPlugin); + + const scanner = new Scanner({ + target: baseUrl, + timeout: 5000, + userAgent: "test", + pluginLoader: loader, + }); + + const results = await scanner.analyse(); + // success-plugin 应仍有结果 + expect(results.find((r) => r.name === "success-plugin")).toBeDefined(); + // error-plugin 应被跳过 + expect(results.find((r) => r.name === "error-plugin")).toBeUndefined(); + }); + + test("target 不可达时返回错误", async () => { + const loader = new PluginLoader(); + loader.register(successPlugin); + + const scanner = new Scanner({ + target: "http://localhost:1", + timeout: 2000, + userAgent: "test", + pluginLoader: loader, + }); + + const result = await scanner.analyse(); + // 应返回 Error 对象或空数组 + expect(result instanceof Error || (Array.isArray(result) && result.length === 0)).toBe(true); + }); + + test("使用 Zod 验证选项", () => { + // 空 target 应被 Zod 拒绝 + expect(() => { + new Scanner({ target: "", timeout: 5000, userAgent: "test" }); + }).toThrow(); + }); + + test("使用内置插件扫描本地服务器", async () => { + const scanner = new Scanner({ + target: baseUrl, + timeout: 10000, + userAgent: "test", + }); + + const results = await scanner.analyse(); + expect(Array.isArray(results)).toBe(true); + + // 至少应有 Base Information 结果 + const baseInfo = results.find((r) => r.name === "Base Information"); + expect(baseInfo).toBeDefined(); + expect(baseInfo?.result.status).toContain("200"); + }, 30000); +}); +``` + +--- + +## Step 7.3 — 实现 `src/core/scanner.ts` + +```typescript +import { httpGet } from "../utils/http.ts"; +import { PluginLoader } from "./plugin-loader.ts"; +import { scannerOptionsSchema } from "./scanner-schema.ts"; +import type { ScanResult } from "./scanner-schema.ts"; +import type { PluginContext } from "../plugins/types.ts"; + +export interface ScannerInit { + target: string; + timeout?: number; + userAgent?: string; + pluginLoader?: PluginLoader; +} + +export class Scanner { + private url: string; + private timeout: number; + private userAgent: string; + private pluginLoader: PluginLoader; + + constructor(init: ScannerInit) { + // 使用 Zod 验证并填充默认值 + const opts = scannerOptionsSchema.parse({ + target: init.target, + timeout: init.timeout, + userAgent: init.userAgent, + }); + + this.url = opts.target; + this.timeout = opts.timeout; + this.userAgent = opts.userAgent; + this.pluginLoader = init.pluginLoader ?? PluginLoader.withBuiltins(); + } + + async analyse(): Promise { + try { + // 1. 初始 GET 请求 + const response = await httpGet(this.url, { + timeout: this.timeout, + userAgent: this.userAgent, + }); + + if (response.status < 200 || response.status > 599) { + return new Error(`Status code ${response.status} not in range`); + } + + // 2. 构建插件上下文 + const context: PluginContext = { + url: this.url, + timeout: this.timeout, + userAgent: this.userAgent, + response, + }; + + // 3. 按顺序执行所有插件，单个失败不影响其他 + const results: ScanResult[] = []; + for (const plugin of this.pluginLoader.getAll()) { + try { + const result = await plugin.execute(context); + if (result && Object.keys(result).length > 0) { + results.push({ name: plugin.meta.name, result }); + } + } catch { + // 单个插件失败，静默跳过 + } + } + + return results; + } catch (err) { + return err instanceof Error ? err : new Error(String(err)); + } + } +} +``` + +**关键要点**： +1. **Zod 验证**：构造函数中使用 `scannerOptionsSchema.parse()` 验证参数，非法参数直接抛 ZodError +2. **默认 PluginLoader**：未传入 `pluginLoader` 时使用 `PluginLoader.withBuiltins()` 加载所有内置插件 +3. **错误隔离**：每个插件执行包在 try/catch 中，单个插件 crash 不会影响其他插件 +4. **返回类型**：`ScanResult[] | Error`，与原代码行为一致（成功返回数组，失败返回 Error） + +--- + +## Step 7.4 — 更新库入口 (`src/index.ts`) + +```typescript +export { Scanner } from "./core/scanner.ts"; +export { PluginLoader } from "./core/plugin-loader.ts"; +export type { Plugin, PluginContext, PluginMeta, HttpResponse } from "./plugins/types.ts"; +export type { ScanResult, ScannerOptions } from "./core/scanner-schema.ts"; +``` + +--- + +## 验证 + +```bash +bun test tests/core/scanner.test.ts +bun test # 确保所有测试仍通过 +bunx eslint src/core/ +``` + +--- + +## 提交 + +```bash +git add . +git commit -m "feat: Phase 7 — 核心扫描引擎 Scanner (Zod 验证)" +git push +``` + +--- + +## 完成标志 + +- [ ] `tests/core/scanner.test.ts` 全部通过 +- [ ] `Scanner` 构造函数使用 Zod 验证参数 +- [ ] 支持自定义 `PluginLoader` 和默认内置插件 +- [ ] 单个插件失败不影响其他插件 +- [ ] `src/index.ts` 正确导出 Scanner 和相关类型 +- [ ] lint 无错误 +- [ ] 代码已提交并推送 +- [ ] `docs/2026-04-01/sop/README.md` 中 Phase 7 状态已更新为 ✅ diff --git a/docs/2026-04-01/sop/phase-08-cli.md b/docs/2026-04-01/sop/phase-08-cli.md new file mode 100644 index 0000000..18d7d90 --- /dev/null +++ b/docs/2026-04-01/sop/phase-08-cli.md @@ -0,0 +1,342 @@ +# Phase 8：CLI 与 Reporter + +> **前置条件**：Phase 7 已完成（Scanner 可用） +> **产出物**：`src/cli.ts`（CLI 入口）+ `src/core/reporter.ts`（结果格式化）+ `src/schemas/cli.ts`（CLI 参数 Zod schema）+ 测试 +> **TDD 顺序**：Zod schema → Reporter 测试/实现 → CLI 测试/实现 + +--- + +## 模块概览 + +| 文件 | 职责 | 对应原始代码 | +|------|------|-------------| +| `src/schemas/cli.ts` | CLI 参数 Zod schema + 类型 | `whatsweb.js` 中 commander 选项 | +| `src/core/reporter.ts` | 控制台彩色输出 + JSON 文件输出 | `whatsweb.js` 中 `report()` 函数 | +| `src/cli.ts` | CLI 入口，commander 参数解析 + 主流程 | `whatsweb.js` 整体 | + +--- + +## Step 8.1 — CLI 参数 Zod Schema (`src/schemas/cli.ts`) + +### 8.1.1 先写测试 `tests/schemas/cli.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { cliOptionsSchema } from "../../src/schemas/cli.ts"; + +describe("cliOptionsSchema", () => { + test("默认值填充", () => { + const result = cliOptionsSchema.parse({}); + expect(result.concurrency).toBe(50); + expect(result.timeout).toBe(10000); + expect(result.showError).toBe(false); + expect(result.userAgent).toContain("whatsweb"); + }); + + test("自定义值通过验证", () => { + const result = cliOptionsSchema.parse({ + concurrency: 10, + timeout: 5000, + userAgent: "custom", + output: "/tmp/out.json", + showError: true, + network: "24", + }); + expect(result.concurrency).toBe(10); + expect(result.output).toBe("/tmp/out.json"); + expect(result.network).toBe("24"); + }); + + test("并发数必须为正整数", () => { + const result = cliOptionsSchema.safeParse({ concurrency: -1 }); + expect(result.success).toBe(false); + }); + + test("超时必须为正数", () => { + const result = cliOptionsSchema.safeParse({ timeout: 0 }); + expect(result.success).toBe(false); + }); +}); +``` + +### 8.1.2 实现 `src/schemas/cli.ts` + +```typescript +import { z } from "zod"; + +export const cliOptionsSchema = z.object({ + concurrency: z.number().int().positive().default(50), + timeout: z.number().positive().default(10000), + userAgent: z.string().min(1).default("Mozilla/5.0 whatsweb/1.0.0"), + output: z.string().optional(), + showError: z.boolean().default(false), + network: z.string().optional(), + file: z.string().optional(), +}); + +export type CliOptions = z.infer; +``` + +--- + +## Step 8.2 — Reporter TDD (`src/core/reporter.ts`) + +### 8.2.1 先写测试 `tests/core/reporter.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; +import { Reporter } from "../../src/core/reporter.ts"; +import type { ScanResult } from "../../src/core/scanner.ts"; +import { existsSync, unlinkSync } from "node:fs"; + +describe("Reporter", () => { + test("formatResult 生成彩色字符串", () => { + const reporter = new Reporter({ showError: false }); + const results: ScanResult[] = [ + { name: "Base Information", result: { status: "200 OK", title: "Test Page" } }, + { name: "Email", result: { email: ["a@b.com", "c@d.com"] } }, + ]; + const output = reporter.formatResult("http://example.com", results); + expect(output).toContain("http://example.com"); + expect(output).toContain("Base Information"); + expect(output).toContain("200 OK"); + }); + + test("formatError 包含 URL 和错误信息", () => { + const reporter = new Reporter({ showError: true }); + const output = reporter.formatError("http://example.com", new Error("timeout")); + expect(output).toContain("http://example.com"); + expect(output).toContain("timeout"); + }); + + test("formatError 在 showError=false 时返回空", () => { + const reporter = new Reporter({ showError: false }); + const output = reporter.formatError("http://example.com", new Error("timeout")); + expect(output).toBe(""); + }); + + test("appendJsonLine 写入文件", async () => { + const tmpFile = `/tmp/whatsweb-test-${Date.now()}.jsonl`; + const reporter = new Reporter({ showError: false, outputFile: tmpFile }); + + await reporter.appendJsonLine("http://example.com", [ + { name: "test", result: { key: "value" } }, + ]); + + expect(existsSync(tmpFile)).toBe(true); + const content = await Bun.file(tmpFile).text(); + const parsed = JSON.parse(content.trim()); + expect(parsed.target).toBe("http://example.com"); + expect(parsed.plugins[0].name).toBe("test"); + + unlinkSync(tmpFile); + }); + + test("数组值以逗号连接显示", () => { + const reporter = new Reporter({ showError: false }); + const results: ScanResult[] = [ + { name: "Email", result: { email: ["a@b.com", "c@d.com"] } }, + ]; + const output = reporter.formatResult("http://example.com", results); + expect(output).toContain("a@b.com, c@d.com"); + }); +}); +``` + +### 8.2.2 实现 `src/core/reporter.ts` + +**功能需求**（参考原 `whatsweb.js` 第 77-124 行的 `report()` 函数）： + +```typescript +import pc from "picocolors"; +import type { ScanResult } from "./scanner.ts"; + +export interface ReporterOptions { + showError: boolean; + outputFile?: string; +} + +export class Reporter { + constructor(private opts: ReporterOptions) {} + + /** 格式化成功结果为彩色控制台字符串 */ + formatResult(url: string, results: ScanResult[]): string { + // 参考原代码的格式： + // [+] WhatsWeb report for + // [ Plugin Name ] key1: value1, key2: value2 + // ... + // + // 使用 picocolors 着色： + // - URL: pc.bold(pc.blue(url)) + // - 插件名: pc.bold(pc.cyan(name)) + // - key: pc.bold(pc.white(key)) + // - title 和 redirect 值: pc.bold(pc.yellow(value)) + // - 数组值: 用 ", " 连接 + } + + /** 格式化错误信息 */ + formatError(url: string, error: Error): string { + // showError=false 时返回空字符串 + // showError=true 时返回 "[-] Request failed. " + } + + /** 追加 JSON 行到输出文件 */ + async appendJsonLine(url: string, results: ScanResult[]): Promise { + // 使用 Bun.write (append mode) 写入 JSONL 格式 + // 格式: { target: url, plugins: results }\n + } +} +``` + +**es-toolkit 用法**： +```typescript +import { isEmpty } from "es-toolkit/predicate"; +// 检查结果是否为空 +if (isEmpty(result)) return; +``` + +--- + +## Step 8.3 — CLI 入口 (`src/cli.ts`) + +### 8.3.1 先写测试 `tests/cli.test.ts` + +```typescript +import { describe, test, expect } from "bun:test"; + +describe("CLI", () => { + test("无参数时显示帮助信息", async () => { + const proc = Bun.spawn(["bun", "run", "src/cli.ts"], { + stdout: "pipe", + stderr: "pipe", + }); + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + const output = stdout + stderr; + expect(output).toContain("Usage"); + expect(proc.exitCode).not.toBe(undefined); + }); + + test("--version 输出版本号", async () => { + const proc = Bun.spawn(["bun", "run", "src/cli.ts", "--version"], { + stdout: "pipe", + }); + const output = await new Response(proc.stdout).text(); + expect(output.trim()).toMatch(/^\d+\.\d+\.\d+$/); + }); + + test("--help 输出帮助信息", async () => { + const proc = Bun.spawn(["bun", "run", "src/cli.ts", "--help"], { + stdout: "pipe", + }); + const output = await new Response(proc.stdout).text(); + expect(output).toContain("-f"); + expect(output).toContain("--concurrency"); + expect(output).toContain("--network"); + expect(output).toContain("--timeout"); + expect(output).toContain("--user-agent"); + expect(output).toContain("--output"); + expect(output).toContain("--show-error"); + }); +}); +``` + +### 8.3.2 实现 `src/cli.ts` + +```typescript +#!/usr/bin/env bun + +import { Command } from "commander"; +import pc from "picocolors"; +import { SingleBar, Presets } from "cli-progress"; +import pLimit from "p-limit"; +import { uniq } from "es-toolkit"; + +import { Scanner } from "./core/scanner.ts"; +import { Reporter } from "./core/reporter.ts"; +import { normalizeUrl } from "./utils/url.ts"; +import { resolve4 } from "./utils/dns.ts"; +import { expandCIDR } from "./utils/ip.ts"; +import { cliOptionsSchema } from "./schemas/cli.ts"; +import { version, name } from "../package.json"; + +const program = new Command() + .name(name) + .version(version) + .usage("[options] URLs") + .description("Identifies websites.") + .argument("[urls...]", "目标 URL 列表") + .option("-f ", "Targets file path") + .option("-c, --concurrency ", "并发数", (v) => Math.abs(parseInt(v)), 50) + .option("--network ", "Scan all Target/MASK hosts") + .option("--timeout ", "请求超时 (毫秒)", (v) => Math.abs(parseInt(v)), 10000) + .option("--user-agent ", "Custom User-Agent", `Mozilla/5.0 ${name}/${version}`) + .option("-o, --output ", "Output file path") + .option("--show-error", "Show error message", false) + .action(async (urls: string[], opts) => { + // 主流程参考原 whatsweb.js: + // 1. 使用 cliOptionsSchema.parse(opts) 验证参数 + // 2. 构建目标列表（参数 URLs + 文件 + CIDR 展开） + // 3. 使用 es-toolkit 的 uniq() 去重 + // 4. 创建进度条 (cli-progress) + // 5. 使用 p-limit 控制并发 + // 6. 对每个目标创建 Scanner 并 analyse() + // 7. 使用 Reporter 格式化输出 + }); + +program.parse(); +``` + +**关键实现要点**： + +1. **Zod 验证 CLI 参数**：`cliOptionsSchema.parse(opts)` 确保运行时参数合法 +2. **es-toolkit `uniq()`**：替代 `Array.from(new Set(targets))` +3. **p-limit 并发控制**：替代 `bluebird.Promise.map({ concurrency })` +4. **picocolors**：替代 `chalk` 进行终端着色 +5. **cli-progress**：替代 `progress` 库 +6. **Bun.file().text()**：替代 `fs.readFileSync()` 读取目标文件 +7. **Bun.write()**：替代 `fs.appendFileSync()` 写入输出文件 + +**进度条格式保持兼容**： +``` +:valid Hits(:rate Targets/s) | :current/:total(:percent) scanned in :elapseds, :etas left +``` + +--- + +## Step 8.4 — 验证 + +```bash +bun test tests/schemas/cli.test.ts +bun test tests/core/reporter.test.ts +bun test tests/cli.test.ts +bun test # 全部测试 +bunx eslint src/ tests/ +bunx prettier --check src/ tests/ +``` + +--- + +## 提交 + +```bash +git add . +git commit -m "feat: Phase 8 — CLI + Reporter (commander/picocolors/cli-progress/p-limit/zod)" +git push +``` + +--- + +## 完成标志 + +- [ ] `tests/schemas/cli.test.ts` 全部通过 +- [ ] `tests/core/reporter.test.ts` 全部通过 +- [ ] `tests/cli.test.ts` 全部通过 +- [ ] CLI 所有原有选项（`-f`, `-c`, `--network`, `--timeout`, `--user-agent`, `-o`, `--show-error`）兼容 +- [ ] 输出格式与原版一致（彩色控制台 + JSONL 文件） +- [ ] CLI 参数使用 Zod 运行时验证 +- [ ] 去重使用 `es-toolkit/uniq` +- [ ] 并发控制使用 `p-limit` +- [ ] lint 无错误 +- [ ] 代码已提交并推送 +- [ ] `docs/2026-04-01/sop/README.md` 中 Phase 8 状态已更新为 ✅ diff --git a/docs/2026-04-01/sop/phase-09-integration.md b/docs/2026-04-01/sop/phase-09-integration.md new file mode 100644 index 0000000..3d8fd8b --- /dev/null +++ b/docs/2026-04-01/sop/phase-09-integration.md @@ -0,0 +1,384 @@ +# Phase 9：集成测试、清理与发布 + +> **前置条件**：Phase 8 已完成（所有模块就绪） +> **产出物**：集成测试通过、旧文件清理、README 更新、可发布状态 + +--- + +## Step 9.1 — 端到端集成测试 + +### 9.1.1 编写集成测试 `tests/integration/e2e.test.ts` + +```typescript +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { Scanner } from "../../src/core/scanner.ts"; +import { existsSync, unlinkSync } from "node:fs"; + +let server: ReturnType; +let baseUrl: string; + +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + + if (url.pathname === "/") { + return new Response( + ` + Integration Test Site + + Contact: admin@test.com, support@test.com + + + `, + { + headers: { + "content-type": "text/html", + "x-powered-by": "Express", + "x-request-id": "test-123", + }, + }, + ); + } + + if (url.pathname === "/WhatsWeb-404-existence-check") { + return new Response("Not Found", { status: 404 }); + } + + return new Response("Not Found", { status: 404 }); + }, + }); + baseUrl = `http://localhost:${server.port}`; +}); + +afterAll(() => { + server.stop(); +}); + +describe("端到端集成测试", () => { + test("完整扫描流程 — Scanner + 内置插件", async () => { + const scanner = new Scanner({ + target: baseUrl, + timeout: 10000, + userAgent: "WhatsWeb-Integration-Test", + }); + + const results = await scanner.analyse(); + + // 应成功返回数组 + expect(Array.isArray(results)).toBe(true); + if (!Array.isArray(results)) return; + + // Base Information 应存在 + const baseInfo = results.find((r) => r.name === "Base Information"); + expect(baseInfo).toBeDefined(); + expect(baseInfo?.result.status).toContain("200"); + expect(baseInfo?.result.title).toBe("Integration Test Site"); + expect(baseInfo?.result["x-powered-by"]).toBe("Express"); + expect(baseInfo?.result["x-request-id"]).toBe("test-123"); + + // Email 应存在 + const email = results.find((r) => r.name === "Email"); + expect(email).toBeDefined(); + expect(email?.result.email).toEqual( + expect.arrayContaining(["admin@test.com", "support@test.com"]), + ); + + // Geolocation — 本地地址可能无 geo 数据，但 ip 应存在 + const geo = results.find((r) => r.name === "Geolocation"); + expect(geo).toBeDefined(); + expect(geo?.result.ip).toBeDefined(); + }, 30000); +}); + +describe("CLI 端到端", () => { + test("扫描真实 URL 并输出到文件", async () => { + const tmpFile = `/tmp/whatsweb-e2e-${Date.now()}.jsonl`; + + const proc = Bun.spawn( + ["bun", "run", "src/cli.ts", baseUrl, "-o", tmpFile, "--timeout", "10000"], + { stdout: "pipe", stderr: "pipe" }, + ); + + await proc.exited; + + // 检查输出文件 + if (existsSync(tmpFile)) { + const content = await Bun.file(tmpFile).text(); + expect(content.trim().length).toBeGreaterThan(0); + const line = JSON.parse(content.trim().split("\n")[0]); + expect(line.target).toBe(baseUrl); + expect(line.plugins).toBeInstanceOf(Array); + unlinkSync(tmpFile); + } + }, 60000); +}); +``` + +### 9.1.2 运行全部测试 + +```bash +bun test +``` + +确保所有模块的测试全部通过。 + +--- + +## Step 9.2 — 全面 Lint 检查 + +```bash +bunx eslint src/ tests/ +bunx prettier --check src/ tests/ +bunx tsc --noEmit +``` + +修复所有 lint 错误、格式问题和类型错误。 + +--- + +## Step 9.3 — 清理旧文件 + +确认所有新代码可运行后，删除原始的 CJS 源文件： + +```bash +# 旧的入口文件 +git rm index.js +git rm whatsweb.js +git rm utils.js + +# 旧的插件目录 +git rm -r plugins/ + +# 保留 rules/ 目录（数据文件不变） +``` + +> **重要**：`rules/` 目录**不删除**，保持原样。 + +--- + +## Step 9.4 — 更新 `.gitignore` + +```gitignore +node_modules/ +dist/ +.DS_Store +.vscode/ +*.log +``` + +--- + +## Step 9.5 — 更新 README.md + +用中英双语更新 README，保持原有使用方式兼容： + +```markdown +# WhatsWeb + +网站指纹识别与安全扫描工具。 / Identifies websites. + +## Installation + +```bash +# 使用 Bun (推荐) +bun install -g whatsweb + +# 使用 npm +npm install -g whatsweb +``` + +## Usage + +```bash +whatsweb [options] URLs +``` + +### Options + +``` +-V, --version 输出版本号 +-f 从文件读取目标列表 +-c, --concurrency 并发数 (默认: 50) +--network 扫描 Target/MASK 网段内所有主机 +--timeout 请求超时毫秒数 (默认: 10000) +--user-agent 自定义 User-Agent +-o, --output 输出文件路径 (JSONL 格式) +--show-error 显示错误信息 +-h, --help 显示帮助 +``` + +### Example + +```bash +# 扫描单个目标 +whatsweb http://example.com + +# 扫描多个目标 +whatsweb http://a.com http://b.com http://c.com + +# 从文件读取 + 并发 20 + 输出到文件 +whatsweb -f targets.txt -c 20 -o results.jsonl + +# 扫描整个 /24 网段 +whatsweb http://192.168.1.1 --network 24 +``` + +## Tech Stack + +- **Runtime**: Bun +- **Language**: TypeScript +- **HTTP**: Bun native fetch +- **Utilities**: es-toolkit, p-limit, zod +- **Fingerprinting**: wappalyzer@6.10.66 +- **GeoIP**: geoip-lite + +## Architecture + +插件化架构：核心引擎 + 可扩展的插件系统。 + +内置插件： +- **Base Information** — HTTP 状态、标题、重定向、x-* 头 +- **Email** — HTML 中的邮箱地址提取 +- **Geolocation** — IP 地理位置 +- **Wappalyzer** — 技术栈识别 +- **BBScan** — 敏感路径扫描 + +## Development + +```bash +bun install # 安装依赖 +bun test # 运行测试 +bun run lint # Lint 检查 +bun run dev # 开发运行 +``` + +## License + +MIT +``` + +--- + +## Step 9.6 — 更新 package.json 最终版 + +确认 `package.json` 的最终状态： + +```jsonc +{ + "name": "whatsweb", + "version": "1.0.0", + "type": "module", + "description": "网站指纹识别与安全扫描工具 / Identifies websites", + "main": "src/index.ts", + "bin": { + "whatsweb": "src/cli.ts" + }, + "scripts": { + "dev": "bun run src/cli.ts", + "test": "bun test", + "lint": "bunx eslint src/ tests/", + "lint:fix": "bunx eslint src/ tests/ --fix", + "format": "bunx prettier --write src/ tests/", + "format:check": "bunx prettier --check src/ tests/", + "typecheck": "bunx tsc --noEmit", + "build": "bun build src/cli.ts --outdir dist --target node", + "prepublishOnly": "bun run typecheck && bun run lint && bun run test" + }, + "dependencies": { + "cli-progress": "^3.12.0", + "commander": "^12.0.0", + "es-toolkit": "^1.0.0", + "geoip-lite": "^1.4.0", + "p-limit": "^7.0.0", + "picocolors": "^1.0.0", + "wappalyzer": "6.10.66", + "zod": "^3.24.0" + }, + "devDependencies": { + "@eslint/js": "^10.0.0", + "@types/bun": "latest", + "@types/cli-progress": "^3.11.0", + "@types/geoip-lite": "^1.4.4", + "eslint": "^10.1.0", + "eslint-config-prettier": "^10.1.0", + "prettier": "^3.8.0", + "typescript": "^5.5.0", + "typescript-eslint": "^8.58.0" + }, + "author": "M4ster ", + "repository": { + "type": "git", + "url": "https://github.com/monkeym4ster/WhatsWeb.git" + }, + "license": "MIT", + "engines": { + "bun": ">=1.0.0" + } +} +``` + +--- + +## Step 9.7 — 更新 SOP 进度 + +更新 `docs/2026-04-01/sop/README.md` 中所有 Phase 的状态为 ✅。 + +--- + +## Step 9.8 — 最终验证清单 + +```bash +# 1. 全部测试通过 +bun test + +# 2. lint 无错误 +bunx eslint src/ tests/ + +# 2.5 格式一致 +bunx prettier --check src/ tests/ + +# 3. 类型检查通过 +bunx tsc --noEmit + +# 4. CLI 可运行 +bun run src/cli.ts --help +bun run src/cli.ts --version + +# 5. 旧文件已删除 +ls index.js 2>/dev/null && echo "ERROR: old file exists" || echo "OK: cleaned" +ls whatsweb.js 2>/dev/null && echo "ERROR: old file exists" || echo "OK: cleaned" +ls utils.js 2>/dev/null && echo "ERROR: old file exists" || echo "OK: cleaned" +ls plugins/ 2>/dev/null && echo "ERROR: old dir exists" || echo "OK: cleaned" + +# 6. rules/ 目录完好 +ls rules/*.txt | wc -l # 应大于 0 +``` + +--- + +## 提交 + +```bash +git add . +git commit -m "feat: Phase 9 — 集成测试、清理旧文件、更新文档" +git push +``` + +--- + +## 完成标志 + +- [ ] `tests/integration/e2e.test.ts` 全部通过 +- [ ] `bun test` 全部测试通过（所有 Phase 的测试） +- [ ] `bunx eslint src/ tests/` 无错误 +- [ ] `bunx prettier --check src/ tests/` 格式一致 +- [ ] `bunx tsc --noEmit` 无类型错误 +- [ ] 旧文件（`index.js`, `whatsweb.js`, `utils.js`, `plugins/`）已删除 +- [ ] `rules/` 目录完好 +- [ ] `README.md` 已更新 +- [ ] `package.json` 已更新为最终版 +- [ ] `.gitignore` 已更新 +- [ ] `docs/2026-04-01/sop/README.md` 中所有 Phase 状态已更新为 ✅ +- [ ] 代码已提交并推送 diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 0000000..a104a80 --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,19 @@ +import eslint from "@eslint/js"; +import tseslint from "typescript-eslint"; +import eslintConfigPrettier from "eslint-config-prettier/flat"; + +export default tseslint.config( + { ignores: ["dist/", "node_modules/", "rules/"] }, + eslint.configs.recommended, + ...tseslint.configs.recommended, + { + rules: { + "@typescript-eslint/no-explicit-any": "warn", + "@typescript-eslint/no-unused-vars": [ + "error", + { argsIgnorePattern: "^_", varsIgnorePattern: "^_" }, + ], + }, + }, + eslintConfigPrettier, +); diff --git a/index.js b/index.js deleted file mode 100644 index 130bd70..0000000 --- a/index.js +++ /dev/null @@ -1,53 +0,0 @@ -const fs = require('fs') -const path = require('path') -const chalk = require('chalk') -const Promise = require('bluebird') -const request = require('superagent') -const { normalUrl } = require('./utils') - -class WhatsWeb { - constructor (opts) { - const { target, timeout, userAgent } = opts - this.timeout = timeout || 10000 - this.url = normalUrl(target) - this.userAgent = userAgent - return this - } - - async analyse () { - try { - const { url, timeout, userAgent } = this - const basePath = path.resolve(__dirname) - const pluginsDir = path.join(basePath, 'plugins') - const files = await Promise.promisify(fs.readdir)(pluginsDir) - const plugins = files.map( file => path.join(pluginsDir, file)) - const response = await request.get(url).timeout(timeout).set('Accept', '*/*').set('Accept-Encoding', '').set('user-agent', userAgent).ok(() => true) - if (response.status < 200 || response.status > 599) throw new Error(`Status code ${response.status} not in range`) - this.response = response - - const results = [] - for (const _ of plugins) { - const plugin = require(_) - const name = plugin.register.attributes.name - const result = await plugin.register({ url, timeout, userAgent, response }) - // 跳过无效结果 - if (!result || !Object.keys(result).length) continue - results.push({ name, result }) - } - return results - } catch (err) { - return err - } - } -} - -module.exports = WhatsWeb - -if (require.main === module) { - const target = process.argv[2] || 'http://www.freebuf.com' - const whatsWeb = new WhatsWeb({ target, timeout: 5000, userAgent: 'Mozilla/5.0' }) - whatsWeb - .analyse() - .then(res => console.log(JSON.stringify(res, null, 4))) - .catch(err => console.log(err)) -} diff --git a/package.json b/package.json index a1f8a0a..1cf2374 100644 --- a/package.json +++ b/package.json @@ -1,29 +1,49 @@ { "name": "whatsweb", - "version": "0.1.3", - "description": "WhatsWeb", - "main": "index.js", + "version": "1.0.0", + "type": "module", + "description": "网站指纹识别与安全扫描工具 / Identifies websites", + "main": "src/index.ts", + "bin": { + "whatsweb": "src/cli.ts" + }, + "scripts": { + "dev": "bun run src/cli.ts", + "test": "bun test", + "lint": "bunx eslint src/ tests/", + "lint:fix": "bunx eslint src/ tests/ --fix", + "format": "bunx prettier --write src/ tests/", + "format:check": "bunx prettier --check src/ tests/", + "typecheck": "bunx tsc --noEmit" + }, + "dependencies": { + "cli-progress": "^3.12.0", + "commander": "^12.0.0", + "es-toolkit": "^1.0.0", + "geoip-lite": "^1.4.0", + "p-limit": "^7.0.0", + "picocolors": "^1.0.0", + "wappalyzer": "6.10.66", + "zod": "^3.24.0" + }, + "devDependencies": { + "@eslint/js": "^10.0.1", + "@types/bun": "latest", + "@types/cli-progress": "^3.11.0", + "@types/geoip-lite": "^1.4.4", + "eslint": "^10.1.0", + "eslint-config-prettier": "^10.1.8", + "prettier": "^3.8.1", + "typescript": "^5.5.0", + "typescript-eslint": "^8.58.0" + }, "author": "M4ster ", "repository": { "type": "git", "url": "https://github.com/monkeym4ster/WhatsWeb.git" }, - "bugs": { - "url": "https://github.com/monkeym4ster/WhatsWeb/issues" - }, - "bin": { - "whatsweb": "whatsweb.js" - }, "license": "MIT", - "dependencies": { - "bluebird": "^3.5.1", - "chalk": "^2.3.0", - "commander": "^2.13.0", - "geoip-lite": "^1.2.1", - "globby": "^7.1.1", - "ip": "^1.1.5", - "progress": "^2.0.0", - "superagent": "^3.8.2", - "wappalyzer": "^5.2.2" + "engines": { + "bun": ">=1.0.0" } } diff --git a/plugins/base-info.js b/plugins/base-info.js deleted file mode 100644 index e9145e8..0000000 --- a/plugins/base-info.js +++ /dev/null @@ -1,26 +0,0 @@ -const dns = require('dns') -const geoip = require('geoip-lite') - -const plugin = async (opts) => { - const { response, url } = opts - const result = {} - const _response = response.res || response.response - const { text, headers, statusCode, statusMessage } = _response - - // Status - result.status = [statusCode, statusMessage].join(' ') - const matched = text.match(/([^<]+)<\/title>/i) - // Redirect - const currentUrl = response.request.url - if (currentUrl !== url) result.redirect = currentUrl - // Title - if (matched && matched.length >= 2) result.title = matched[1].trim().replace(/\n/g, '\\n').replace(/\r/g, '\\r') - // x-headers - for (const i in headers) if (i.startsWith('x-')) result[i] = headers[i] - return result -} - -exports.register = plugin -exports.register.attributes = { - name: 'Base Infomation' -} diff --git a/plugins/bbscan.js b/plugins/bbscan.js deleted file mode 100644 index 613acc1..0000000 --- a/plugins/bbscan.js +++ /dev/null @@ -1,210 +0,0 @@ -const Url = require('url') -const net = require('net') -const path = require('path') -const fs = require('fs') -const request = require('superagent') -const globby = require('globby') -const Promise = require('bluebird') - -class BBScan { - constructor (opts) { - const { userAgent, url } = opts - const { host, protocol } = Url.parse(url) - - this.userAgent = userAgent - this.url = url - this.host = host - this.protocol = protocol - const timeout = 2000 - } - - async fetch (path) { - const { host, protocol, userAgent } = this - const url = `${protocol}//${host}${path}` - const timeout = this.timeout - const response = await request.head(url).timeout(timeout).set('Accept', '*/*').set('Accept-Encoding', '').set('user-agent', userAgent).ok(() => true) - return response - } - - async check404 () { - const response = await this.fetch(`/WhatsWeb-404-existence-check`) - if (response.status !== 404) return false - return true - } - - async loadRules () { - const rules = this.rules = [] // 规则 - const textToFind = this.textToFind = [] // 匹配文本 - const regexToFind = this.regexToFind = [] // 欲匹配的正则 - const textToExclude = this.textToExclude = [] // 欲排除文本 - const regexToExclude = this.regexToExclude = [] // 欲排除的正则 - - // 正则 - const regTag = /{tag="(.*?)"}/ - const regStatus = /{status=(\d{3})}/ - const regContentType = /{type="(.*?)"}/ - const regContentTypeNo = /{type_no="(.*?)"}/ - const regText = /{text="(.*)"}/ - const regRegexText = /{regex_text="(.*)"}/ - - // 遍历规则文件 - const ruleFiles = await globby(path.join(__dirname, '../', 'rules', '*.txt')) - // const ruleFiles = ['rules/3.phpinfo_and_test.txt'] - for (const ruleFile of ruleFiles) { - // 读取文件内容 - const fileData = await Promise.promisify(fs.readFile)(ruleFile, 'utf8') - const lines = fileData.split('\n') - for (const l of lines) { - const line = l.trim() - // 跳过非『/』开头的行 - if (!line.startsWith('/')) continue - // 匹配 tag - let _ = line.match(regTag) - const tag = _ && _.length > 1 ? _[1] : '' - // 匹配 status - _ = line.match(regStatus) - const status = _ && _.length > 1 ? parseInt(_[1]) : '' - // 匹配 content_type - _ = line.match(regContentType) - const contentType = _ && _.length > 1 ? _[1] : '' - // 匹配 content_type_no - _ = line.match(regContentTypeNo) - const contentTypeNo = _ && _.length > 1 ? _[1] : '' - // 匹配 root_only - const rootOnly = line.includes('{root_only}') - // 匹配 uri - const uri = line.split(/\s+/)[0] - const rule = [ uri, tag, status, contentType, contentTypeNo, rootOnly] - // 追加至 rules - rules.push(rule) - } - } - - // 白名单文件 - const whiteListFile = path.join(__dirname, '../', 'rules', 'white.list') - const whiteListFileData = await Promise.promisify(fs.readFile)(whiteListFile, 'utf8') - const whiteListLines = whiteListFileData.split(/\n/) - for (const l of whiteListLines) { - const line = l.trim() - if (!line || line.startsWith('#')) continue - // 匹配 text - let _ = line.match(regText) - const text = _ && _.length > 1 ? _[1] : '' - if (text) { - textToFind.push(text) - continue - } - // 匹配 regex_text - _ = line.match(regRegexText) - const regexText = _ && _.length > 1 ? _[1] : '' - if (regexText) { - regexToFind.push(new RegExp(regexText, 'i')) - continue - } - } - - // 黑名单文件 - const blackListFile = path.join(__dirname, '../', 'rules', 'black.list') - const blackListFileData = await Promise.promisify(fs.readFile)(blackListFile, 'utf8') - const blackListLines = blackListFileData.split(/\n/) - for (const l of blackListLines) { - const line = l.trim() - if (!line || line.startsWith('#')) continue - // 匹配 text - let _ = line.match(regText) - const text = _ && _.length > 1 ? _[1] : '' - if (text) { - textToExclude.push(text) - continue - } - // 匹配 regex_text - _ = line.match(regRegexText) - const regexText = _ && _.length > 1 ? _[1] : '' - if (regexText) { - regexToExclude.push(regexText) - continue - } - } - } - - findText (str) { - if (this.textToFind.includes(str)) return true - for (const regex of this.regexToFind) { - if (regex.test(str)) return true - } - return false - } - - findExcludeText (str) { - if (this.textToExclude.includes(str)) return true - for (const regex of this.regexToExclude) { - if (regex.test(str)) return true - } - return false - } - - async run() { - // 检测是否支持 404 - const has404 = await this.check404() - if (!has404) return false //not support 404 - // 加载规则 - await this.loadRules() - const result = await Promise.map(this.rules, async (rule) => { - try { - const [ uri, tag, status, contentType, contentTypeNo, rootOnly] = rule - let path = uri - const sub = /\d$/.test(this.host) ? this.host.split('.')[3] : this.host.split('.')[0] - if (path.includes('{sub}')) path = path.replace(/{sub}/g, sub) - if (path.includes('{hostname_or_folder}')) path = path.replace(/{hostname_or_folder}/g, this.host) - if (path.includes('{hostname}')) path = path.replace(/{hostname}/g, this.host) - const response = await this.fetch(path) - const _response = response.res || response.response - const { text, headers, statusCode, statusMessage } = _response - const curContentType = headers['content-type'] - // 跳过空内容 - if (['html', 'text'].includes(curContentType) && !text.length) throw new Error('Empty content') - // 跳过图片 - if (curContentType.includes('image/')) throw new Error('Ignore images') - // 命中文本 - if (this.findText(text)) return path - // 命中 json 文件，但 uri 不是以 .json 结尾 - if (curContentType.includes('application/json') && !path.endsWith('.json')) throw new Error('Invalid json file') - // 支持 404，并且当前状态码不是 404 - if (has404 || statusCode !== 404) { - // 和规则中的 status 不同，也不是 206 - if (statusCode !== status && statusCode !== 206) throw new Error('The status code is not the same, not 206') - if (status => 200 && status <= 206 && statusCode === 206) return path - if (status && statusCode !== status) throw new Error('The status code is not the same') - if (status !== 403 && statusCode === 403) throw new Error('Status code is 403') - } - - // 不支持 404，状态码在200-206之间，不是默认页，不存在tag标签，通过判断是不是等于 404 页面决定去留 - // if (!has404 && statusCode >= 200 && statusCode <= 206 && uri !== '/' && !tag) { - // const contentLen = text.length - // } - if (statusCode === 206 && tag === '' && !curContentType.includes('text') && !curContentType.includes('html')) path - return path - }catch (err) { } - }, { concurrency: 50 }).catch(function ignore (err) { - // console.log(err) - }) - return result.filter(_ => !!_) - } -} - - -const plugin = async (opts) => { - const bbScan = new BBScan(opts) - return bbScan.run() -} - -if (require.main === module) { - plugin({ url: 'http://101.231.136.48/', userAgent: 'WhatsWeb' }) - .then(res => console.log(JSON.stringify(res, null, 4))) - .catch(err => console.log(err)) -} - -exports.register = plugin -exports.register.attributes = { - name: 'BBScan' -} \ No newline at end of file diff --git a/plugins/email.js b/plugins/email.js deleted file mode 100644 index 98d6d50..0000000 --- a/plugins/email.js +++ /dev/null @@ -1,33 +0,0 @@ -const plugin = async (opts) => { - const { url, response } = opts - const result = {} - const _response = response.res || response.response - const { text } = _response - const matched = text.match(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}/gi) - // Detect valid email address in the page - if (matched && matched.length >= 2) result.email = Array.from(new Set(matched)) - // Detect any email address in a mailto: tag - return result -} - -if (require.main === module) { - const request = require('superagent') - const Promise = require('bluebird') - const url = 'http://www.freebuf.com' - request.get(url) - .set('Accept', '*/*') - .set('Accept-Encoding', '') - .set('user-agent', 'WhatsWeb/0.0.1') - .ok(() => true) - .then(response => { - plugin({ url, response }) - .then(res => console.log(JSON.stringify(res, null, 4))) - .catch(err => console.log(err)) - }) - .catch((err) => console.error('Request failed.', err)) -} - -exports.register = plugin -exports.register.attributes = { - name: 'Email' -} \ No newline at end of file diff --git a/plugins/geoip.js b/plugins/geoip.js deleted file mode 100644 index d4ebe92..0000000 --- a/plugins/geoip.js +++ /dev/null @@ -1,31 +0,0 @@ -const Url = require('url') -const { resolve4 } = require('../utils') - -const geoip = require('geoip-lite') - -const plugin = async (opts) => { - const { url } = opts - const { host } = Url.parse(url) - if (!host) throw new Error('Not valid host') - const resolve = await resolve4(host) - const result = {} - result.ip = resolve - const geo = geoip.lookup(result.ip) - // 可能查询结果为 null - if (geo) { - const { country, city } = geo - result.country = [city, country].join('/') - } - return result -} - -if (require.main === module) { - plugin({ url: 'http://www.freebuf.com/x-404' }) - .then(res => console.log(JSON.stringify(res, null, 4))) - .catch(err => console.log(err)) -} - -exports.register = plugin -exports.register.attributes = { - name: 'Geolocation' -} \ No newline at end of file diff --git a/plugins/wappalyzer.js b/plugins/wappalyzer.js deleted file mode 100644 index 461d2e6..0000000 --- a/plugins/wappalyzer.js +++ /dev/null @@ -1,41 +0,0 @@ -const Wappalyzer = require('wappalyzer') - -const plugin = async (opts) => { - const { url, userAgent } = opts - const result = {} - const options = { - debug: false, // 输出调试消息 - delay: 500, // 等待请求之间的毫秒数 - maxDepth: 3, // 不要分析比num深度更多的页面 - maxUrls: 10, // 当已经分析了num个URL时退出 - maxWait: 3000, // 等待页面资源加载时间不超过ms毫秒。 - recursive: true, // 跟随页面上的链接（爬虫） - userAgent: userAgent, // 设置用户 UA - } - const wappalyzer = new Wappalyzer(url, options) - const json = await wappalyzer.analyze() - for (const app of json.applications) { - const categories = app.categories.map(_ => Object.values(_)) - // 遍历所有种类，将 app 对象提交到对应的种类中 - categories.forEach(category => { - let item = app.name - if (app.version) item += `[${app.version}]` - // 未有此类，新建 - if (!result[category]) return result[category] = [ item ] - // 已有此类，追加 - result[category].push(item) - }) - } - return result -} - -if (require.main === module) { - plugin({ url: 'http://www.baidu.com/', userAgent: 'whatsweb' }) - .then(res => console.log(JSON.stringify(res, null, 4))) - .catch(err => console.log(err)) -} - -exports.register = plugin -exports.register.attributes = { - name: 'Wappalyzer' -} \ No newline at end of file diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..e86a987 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,166 @@ +#!/usr/bin/env bun + +import { readFileSync } from "node:fs"; +import { Presets, SingleBar } from "cli-progress"; +import { Command } from "commander"; +import { uniq } from "es-toolkit"; +import pLimit from "p-limit"; +import pc from "picocolors"; + +import { Reporter } from "./core/reporter.ts"; +import { Scanner } from "./core/scanner.ts"; +import { cliOptionsSchema } from "./schemas/cli.ts"; +import { resolve4 } from "./utils/dns.ts"; +import { expandCIDR } from "./utils/ip.ts"; +import { normalizeUrl } from "./utils/url.ts"; + +const pack = await import("../package.json"); + +const program = new Command() + .name(pack.name) + .version(pack.version) + .usage("[options] URLs") + .description("Identifies websites.") + .argument("[urls...]", "Target URLs") + .option("-f <file>", "Targets file path") + .option( + "-c, --concurrency <num>", + "Concurrency limit", + (v: string) => Math.abs(Number.parseInt(v)), + 50, + ) + .option("--network <mask>", "Scan all Target/MASK hosts") + .option( + "--timeout <ms>", + "Request timeout in ms", + (v: string) => Math.abs(Number.parseInt(v)), + 10000, + ) + .option("--user-agent <string>", "Custom User-Agent", `Mozilla/5.0 ${pack.name}/${pack.version}`) + .option("-o, --output <path>", "Output file path") + .option("--show-error", "Show error message", false) + .action(async (urls: string[], rawOpts) => { + const opts = cliOptionsSchema.parse({ + concurrency: rawOpts.concurrency, + timeout: rawOpts.timeout, + userAgent: rawOpts.userAgent, + output: rawOpts.output, + showError: rawOpts.showError, + network: rawOpts.network, + file: rawOpts.F, + }); + + let targets = [...urls]; + + if (process.argv.length < 3) { + program.outputHelp(); + process.exit(0); + } + + if (!targets.length && !opts.file) { + console.error(pc.red("[-] Error. Target is required.")); + process.exit(1); + } + + if (opts.file) { + const fileData = readFileSync(opts.file, "utf8").trim(); + if (!fileData.length) { + console.error(pc.red("[-] Error. File is empty.")); + process.exit(1); + } + targets = targets.concat( + fileData + .split("\n") + .map((l) => l.trim()) + .filter(Boolean), + ); + } + + if (opts.network) { + const expanded: string[] = []; + for (const target of targets) { + try { + const url = normalizeUrl(target); + const { host } = new URL(url); + if (!host) continue; + const ip = await resolve4(host); + for (const addr of expandCIDR(`${ip}/${opts.network}`)) { + expanded.push(addr); + } + } catch { + /* ignore expansion errors */ + } + } + targets = targets.concat(expanded); + } + + targets = uniq(targets); + + console.log( + pc.dim( + `[*] Options: ${JSON.stringify({ + outputFile: opts.output, + concurrency: opts.concurrency, + network: opts.network, + timeout: opts.timeout, + userAgent: opts.userAgent, + targetCount: targets.length, + showError: opts.showError, + })}`, + ), + ); + + const reporter = new Reporter({ showError: opts.showError, outputFile: opts.output }); + let valid = 0; + + const bar = new SingleBar( + { + format: + "{valid} Hits({rate} Targets/s) | {value}/{total}({percentage}%) scanned in {duration}s, ETA: {eta}s", + }, + Presets.shades_classic, + ); + bar.start(targets.length, 0, { valid: 0, rate: "N/A" }); + + const limit = pLimit(opts.concurrency); + const startTime = Date.now(); + + const tasks = targets.map((target) => + limit(async () => { + const scanner = new Scanner({ + target, + timeout: opts.timeout, + userAgent: opts.userAgent, + }); + + const data = await scanner.analyse(); + + const printAboveBar = (msg: string) => { + process.stdout.write(`\x1b[2K\x1b[0G${msg}\n`); + }; + + if (Array.isArray(data)) { + valid++; + reporter.appendJsonLine(normalizeUrl(target), data); + const output = reporter.formatResult(normalizeUrl(target), data); + if (output.includes("[+]")) { + printAboveBar(output); + } + const elapsed = (Date.now() - startTime) / 1000; + const rate = elapsed > 0 ? (valid / elapsed).toFixed(1) : "N/A"; + bar.increment(1, { valid, rate }); + } else { + bar.increment(1, { valid }); + if (opts.showError && data instanceof Error) { + printAboveBar(reporter.formatError(normalizeUrl(target), data)); + } + } + }), + ); + + await Promise.all(tasks); + bar.stop(); + process.exit(0); + }); + +program.parse(); diff --git a/src/core/plugin-loader.ts b/src/core/plugin-loader.ts new file mode 100644 index 0000000..c8037f7 --- /dev/null +++ b/src/core/plugin-loader.ts @@ -0,0 +1,44 @@ +import { baseInfoPlugin } from "../plugins/base-info.ts"; +import { bbscanPlugin } from "../plugins/bbscan/index.ts"; +import { emailPlugin } from "../plugins/email.ts"; +import { geoipPlugin } from "../plugins/geoip.ts"; +import { pluginMetaSchema } from "../plugins/types.ts"; +import type { Plugin } from "../plugins/types.ts"; +import { wappalyzerPlugin } from "../plugins/wappalyzer.ts"; + +export class PluginLoader { + private plugins: Map<string, Plugin> = new Map(); + + register(plugin: Plugin): void { + const parsed = pluginMetaSchema.safeParse(plugin.meta); + if (!parsed.success) { + throw new Error(`Invalid plugin meta: ${parsed.error.message}`); + } + if (this.plugins.has(plugin.meta.name)) { + throw new Error(`Plugin "${plugin.meta.name}" is already registered`); + } + this.plugins.set(plugin.meta.name, plugin); + } + + getAll(): Plugin[] { + return [...this.plugins.values()]; + } + + get(name: string): Plugin | undefined { + return this.plugins.get(name); + } + + get pluginCount(): number { + return this.plugins.size; + } + + static withBuiltins(): PluginLoader { + const loader = new PluginLoader(); + loader.register(baseInfoPlugin); + loader.register(emailPlugin); + loader.register(geoipPlugin); + loader.register(wappalyzerPlugin); + loader.register(bbscanPlugin); + return loader; + } +} diff --git a/src/core/reporter.ts b/src/core/reporter.ts new file mode 100644 index 0000000..d9d5cf6 --- /dev/null +++ b/src/core/reporter.ts @@ -0,0 +1,53 @@ +import { appendFileSync } from "node:fs"; +import { EOL } from "node:os"; +import pc from "picocolors"; +import type { ScanResult } from "./scanner.ts"; + +export interface ReporterOptions { + showError: boolean; + outputFile?: string; +} + +export class Reporter { + constructor(private opts: ReporterOptions) {} + + formatResult(url: string, results: ScanResult[]): string { + const lines: string[] = []; + lines.push(`[+] WhatsWeb report for ${pc.bold(pc.blue(url))}`); + + for (const item of results) { + const nameStr = `[ ${pc.bold(pc.cyan(item.name))} ]`; + const parts: string[] = []; + + for (const [key, value] of Object.entries(item.result)) { + const displayValue = Array.isArray(value) ? value.join(", ") : String(value); + let valueStr: string; + + if (key === "redirect") { + valueStr = pc.bold(pc.yellow(displayValue)); + } else if (key === "title") { + valueStr = pc.bold(pc.yellow(displayValue)); + } else { + valueStr = displayValue; + } + + parts.push(`${pc.bold(pc.white(key))}: ${valueStr}`); + } + + lines.push(`${nameStr} ${parts.join(", ")}`); + } + + return lines.join("\n"); + } + + formatError(url: string, error: Error): string { + if (!this.opts.showError) return ""; + return pc.gray(`[-] Request ${url} failed. ${error.message}`); + } + + appendJsonLine(url: string, results: ScanResult[]): void { + if (!this.opts.outputFile) return; + const data = JSON.stringify({ target: url, plugins: results }); + appendFileSync(this.opts.outputFile, data + EOL); + } +} diff --git a/src/core/scanner.ts b/src/core/scanner.ts new file mode 100644 index 0000000..fbca2d9 --- /dev/null +++ b/src/core/scanner.ts @@ -0,0 +1,84 @@ +import { z } from "zod"; +import type { PluginContext } from "../plugins/types.ts"; +import { httpGet } from "../utils/http.ts"; +import { normalizeUrl } from "../utils/url.ts"; +import { PluginLoader } from "./plugin-loader.ts"; + +export const scannerOptionsSchema = z.object({ + target: z + .string() + .min(1) + .transform((val) => normalizeUrl(val)), + timeout: z.number().positive().default(10000), + userAgent: z.string().min(1).default("Mozilla/5.0 whatsweb/1.0.0"), +}); + +export type ScannerOptions = z.infer<typeof scannerOptionsSchema>; + +export interface ScanResult { + name: string; + result: Record<string, unknown>; +} + +export interface ScannerInit { + target: string; + timeout?: number; + userAgent?: string; + pluginLoader?: PluginLoader; +} + +export class Scanner { + private url: string; + private timeout: number; + private userAgent: string; + private pluginLoader: PluginLoader; + + constructor(init: ScannerInit) { + const opts = scannerOptionsSchema.parse({ + target: init.target, + timeout: init.timeout, + userAgent: init.userAgent, + }); + + this.url = opts.target; + this.timeout = opts.timeout; + this.userAgent = opts.userAgent; + this.pluginLoader = init.pluginLoader ?? PluginLoader.withBuiltins(); + } + + async analyse(): Promise<ScanResult[] | Error> { + try { + const response = await httpGet(this.url, { + timeout: this.timeout, + userAgent: this.userAgent, + }); + + if (response.status < 200 || response.status > 599) { + return new Error(`Status code ${response.status} not in range`); + } + + const context: PluginContext = { + url: this.url, + timeout: this.timeout, + userAgent: this.userAgent, + response, + }; + + const results: ScanResult[] = []; + for (const plugin of this.pluginLoader.getAll()) { + try { + const result = await plugin.execute(context); + if (result && Object.keys(result).length > 0) { + results.push({ name: plugin.meta.name, result }); + } + } catch { + // single plugin failure doesn't affect others + } + } + + return results; + } catch (err) { + return err instanceof Error ? err : new Error(String(err)); + } + } +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..e0923ce --- /dev/null +++ b/src/index.ts @@ -0,0 +1,4 @@ +export { Scanner } from "./core/scanner.ts"; +export type { ScanResult, ScannerInit } from "./core/scanner.ts"; +export { PluginLoader } from "./core/plugin-loader.ts"; +export type { Plugin, PluginContext, PluginMeta, HttpResponse } from "./plugins/types.ts"; diff --git a/src/plugins/base-info.ts b/src/plugins/base-info.ts new file mode 100644 index 0000000..76408d0 --- /dev/null +++ b/src/plugins/base-info.ts @@ -0,0 +1,28 @@ +import type { Plugin } from "./types.ts"; + +export const baseInfoPlugin: Plugin = { + meta: { name: "Base Information" }, + async execute(ctx) { + const { response, url } = ctx; + const result: Record<string, unknown> = {}; + + result.status = `${response.status} ${response.statusText}`; + + if (response.url !== url) { + result.redirect = response.url; + } + + const matched = response.text.match(/<title>([^<]+)<\/title>/i); + if (matched?.[1]) { + result.title = matched[1].trim().replace(/\n/g, "\\n").replace(/\r/g, "\\r"); + } + + for (const [key, value] of Object.entries(response.headers)) { + if (key.startsWith("x-")) { + result[key] = value; + } + } + + return result; + }, +}; diff --git a/src/plugins/bbscan/index.ts b/src/plugins/bbscan/index.ts new file mode 100644 index 0000000..1b4d096 --- /dev/null +++ b/src/plugins/bbscan/index.ts @@ -0,0 +1,16 @@ +import type { Plugin } from "../types.ts"; +import { BBScanner } from "./scanner.ts"; + +export const bbscanPlugin: Plugin = { + meta: { name: "BBScan", description: "Path scanning based on rule files" }, + async execute(ctx) { + const scanner = new BBScanner({ + url: ctx.url, + userAgent: ctx.userAgent, + timeout: ctx.timeout, + }); + const paths = await scanner.run(); + if (!paths || paths.length === 0) return null; + return { paths }; + }, +}; diff --git a/src/plugins/bbscan/rule-parser.ts b/src/plugins/bbscan/rule-parser.ts new file mode 100644 index 0000000..915e35e --- /dev/null +++ b/src/plugins/bbscan/rule-parser.ts @@ -0,0 +1,111 @@ +import path from "node:path"; +import { z } from "zod"; + +export const scanRuleSchema = z.object({ + uri: z.string().startsWith("/"), + tag: z.string().default(""), + status: z.number().int().min(100).max(599).nullable().default(null), + contentType: z.string().default(""), + contentTypeNo: z.string().default(""), + rootOnly: z.boolean().default(false), +}); + +export type ScanRule = z.infer<typeof scanRuleSchema>; + +export interface ListRule { + texts: string[]; + regexes: RegExp[]; +} + +const regTag = /{tag="(.*?)"}/; +const regStatus = /{status=(\d{3})}/; +const regContentType = /{type="(.*?)"}/; +const regContentTypeNo = /{type_no="(.*?)"}/; +const regText = /{text="(.*)"}/; +const regRegexText = /{regex_text="(.*)"}/; + +export function parseRuleLine(line: string): ScanRule | null { + const trimmed = line.trim(); + if (!trimmed.startsWith("/")) return null; + + const uri = trimmed.split(/\s+/)[0]; + + let m = trimmed.match(regTag); + const tag = m?.[1] ?? ""; + + m = trimmed.match(regStatus); + const status = m?.[1] ? Number.parseInt(m[1], 10) : null; + + m = trimmed.match(regContentType); + const contentType = m?.[1] ?? ""; + + m = trimmed.match(regContentTypeNo); + const contentTypeNo = m?.[1] ?? ""; + + const rootOnly = trimmed.includes("{root_only}"); + + const parsed = scanRuleSchema.safeParse({ + uri, + tag, + status, + contentType, + contentTypeNo, + rootOnly, + }); + return parsed.success ? parsed.data : null; +} + +export function parseListFile(content: string): ListRule { + const texts: string[] = []; + const regexes: RegExp[] = []; + + for (const l of content.split("\n")) { + const line = l.trim(); + if (!line || line.startsWith("#")) continue; + + let m = line.match(regText); + if (m?.[1]) { + texts.push(m[1]); + continue; + } + + m = line.match(regRegexText); + if (m?.[1]) { + regexes.push(new RegExp(m[1], "i")); + } + } + + return { texts, regexes }; +} + +function getDefaultRulesDir(): string { + return path.join(import.meta.dir, "../../../rules"); +} + +export async function parseRuleFiles(rulesDir?: string): Promise<ScanRule[]> { + const dir = rulesDir ?? getDefaultRulesDir(); + const rules: ScanRule[] = []; + const glob = new Bun.Glob("*.txt"); + + for await (const file of glob.scan({ cwd: dir })) { + const content = await Bun.file(path.join(dir, file)).text(); + for (const line of content.split("\n")) { + const rule = parseRuleLine(line); + if (rule) rules.push(rule); + } + } + + return rules; +} + +export async function loadWhiteList(rulesDir?: string): Promise<ListRule> { + const dir = rulesDir ?? getDefaultRulesDir(); + const content = await Bun.file(path.join(dir, "white.list")).text(); + return parseListFile(content); +} + +export async function loadBlackList(rulesDir?: string): Promise<ListRule> { + const dir = rulesDir ?? getDefaultRulesDir(); + const content = await Bun.file(path.join(dir, "black.list")).text(); + return parseListFile(content); +} diff --git a/src/plugins/bbscan/scanner.ts b/src/plugins/bbscan/scanner.ts new file mode 100644 index 0000000..abb1798 --- /dev/null +++ b/src/plugins/bbscan/scanner.ts @@ -0,0 +1,104 @@ +import pLimit from "p-limit"; +import { httpGet, httpHead } from "../../utils/http.ts"; +import type { ListRule, ScanRule } from "./rule-parser.ts"; +import { loadBlackList, loadWhiteList, parseRuleFiles } from "./rule-parser.ts"; + +export interface BBScannerOptions { + url: string; + userAgent: string; + timeout: number; +} + +export class BBScanner { + private host: string; + private protocol: string; + private rules: ScanRule[] = []; + private whiteList: ListRule = { texts: [], regexes: [] }; + private blackList: ListRule = { texts: [], regexes: [] }; + + constructor(private opts: BBScannerOptions) { + const parsed = new URL(opts.url); + this.host = parsed.host; + this.protocol = parsed.protocol; + } + + async check404(): Promise<boolean> { + try { + const res = await httpHead(`${this.protocol}//${this.host}/WhatsWeb-404-existence-check`, { + timeout: this.opts.timeout, + userAgent: this.opts.userAgent, + }); + return res.status === 404; + } catch { + return false; + } + } + + private findInWhiteList(text: string): boolean { + if (this.whiteList.texts.some((t) => text.includes(t))) return true; + return this.whiteList.regexes.some((r) => r.test(text)); + } + + private findInBlackList(text: string): boolean { + if (this.blackList.texts.some((t) => text.includes(t))) return true; + return this.blackList.regexes.some((r) => r.test(text)); + } + + async run(): Promise<string[] | false> { + const has404 = await this.check404(); + if (!has404) return false; + + this.rules = await parseRuleFiles(); + this.whiteList = await loadWhiteList(); + this.blackList = await loadBlackList(); + + const limit = pLimit(50); + const sub = /\d$/.test(this.host) ? this.host.split(".")[3] : this.host.split(".")[0]; + + const tasks = this.rules.map((rule) => + limit(async (): Promise<string | null> => { + try { + let rulePath = rule.uri; + if (rulePath.includes("{sub}")) rulePath = rulePath.replace(/{sub}/g, sub); + if (rulePath.includes("{hostname_or_folder}")) + rulePath = rulePath.replace(/{hostname_or_folder}/g, this.host); + if (rulePath.includes("{hostname}")) + rulePath = rulePath.replace(/{hostname}/g, this.host); + + const needBody = !!rule.tag; + const fetchFn = needBody ? httpGet : httpHead; + const res = await fetchFn(`${this.protocol}//${this.host}${rulePath}`, { + timeout: this.opts.timeout, + userAgent: this.opts.userAgent, + }); + + const curContentType = res.headers["content-type"] ?? ""; + + if (needBody && ["html", "text"].includes(curContentType) && !res.text.length) + return null; + if (curContentType.includes("image/")) return null; + + if (needBody && this.findInWhiteList(res.text)) return rulePath; + if (needBody && this.findInBlackList(res.text)) return null; + + if (curContentType.includes("application/json") && !rulePath.endsWith(".json")) + return null; + + if (res.status === 404) return null; + if (rule.status && res.status !== rule.status && res.status !== 206) return null; + if (rule.status !== 403 && res.status === 403) return null; + if (rule.tag && needBody && !res.text.includes(rule.tag)) return null; + if (rule.contentType && !curContentType.includes(rule.contentType)) return null; + if (rule.contentTypeNo && curContentType.includes(rule.contentTypeNo)) return null; + + return rulePath; + } catch { + return null; + } + }), + ); + + const results = await Promise.all(tasks); + return results.filter((r): r is string => r !== null); + } +} diff --git a/src/plugins/email.ts b/src/plugins/email.ts new file mode 100644 index 0000000..fb232bb --- /dev/null +++ b/src/plugins/email.ts @@ -0,0 +1,15 @@ +import { uniq } from "es-toolkit"; +import type { Plugin } from "./types.ts"; + +export const emailPlugin: Plugin = { + meta: { name: "Email" }, + async execute(ctx) { + const matched = ctx.response.text.match(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}/gi); + if (!matched) return null; + + const emails = uniq(matched); + if (emails.length < 2) return null; + + return { email: emails }; + }, +}; diff --git a/src/plugins/geoip.ts b/src/plugins/geoip.ts new file mode 100644 index 0000000..c630899 --- /dev/null +++ b/src/plugins/geoip.ts @@ -0,0 +1,26 @@ +import geoip from "geoip-lite"; +import { resolve4 } from "../utils/dns.ts"; +import type { Plugin } from "./types.ts"; + +export const geoipPlugin: Plugin = { + meta: { name: "Geolocation" }, + async execute(ctx) { + try { + const { host } = new URL(ctx.url); + if (!host) return null; + + const ip = await resolve4(host); + const result: Record<string, unknown> = { ip }; + + const geo = geoip.lookup(ip); + if (geo) { + const { country, city } = geo; + result.country = [city, country].filter(Boolean).join("/"); + } + + return result; + } catch { + return null; + } + }, +}; diff --git a/src/plugins/types.ts b/src/plugins/types.ts new file mode 100644 index 0000000..0439073 --- /dev/null +++ b/src/plugins/types.ts @@ -0,0 +1,28 @@ +import { z } from "zod"; +import type { HttpResponse } from "../utils/http.ts"; + +export type { HttpResponse } from "../utils/http.ts"; + +export interface PluginContext { + url: string; + timeout: number; + userAgent: string; + response: HttpResponse; +} + +export interface PluginMeta { + name: string; + description?: string; +} + +export interface Plugin { + meta: PluginMeta; + execute(context: PluginContext): Promise<Record<string, unknown> | null>; +} + +export const pluginMetaSchema = z.object({ + name: z.string().min(1, "Plugin name cannot be empty"), + description: z.string().optional(), +}); + +export const pluginResultSchema = z.record(z.string(), z.unknown()).nullable(); diff --git a/src/plugins/wappalyzer.ts b/src/plugins/wappalyzer.ts new file mode 100644 index 0000000..59a69f4 --- /dev/null +++ b/src/plugins/wappalyzer.ts @@ -0,0 +1,58 @@ +import type { Plugin } from "./types.ts"; + +export const wappalyzerPlugin: Plugin = { + meta: { name: "Wappalyzer", description: "Technology fingerprinting (wappalyzer@6.10.66)" }, + + async execute(ctx) { + // biome-ignore lint/suspicious/noExplicitAny: wappalyzer has no type definitions + let wappalyzer: any = null; // eslint-disable-line + try { + const Wappalyzer = (await import("wappalyzer")).default; + + const options = { + debug: false, + delay: 500, + maxDepth: 3, + maxUrls: 10, + maxWait: Math.min(ctx.timeout, 5000), + recursive: true, + probe: true, + userAgent: ctx.userAgent, + noRedirect: false, + }; + + wappalyzer = new Wappalyzer(options); + await wappalyzer.init(); + + const site = await wappalyzer.open(ctx.url); + site.on("error", () => {}); + + const json = await site.analyze(); + const result: Record<string, unknown> = {}; + + for (const tech of json.technologies ?? []) { + let item = tech.name; + if (tech.version) item += `[${tech.version}]`; + + for (const category of tech.categories ?? []) { + const catName = category.name; + if (!result[catName]) { + result[catName] = [item]; + } else { + (result[catName] as string[]).push(item); + } + } + } + + return Object.keys(result).length > 0 ? result : null; + } catch { + return null; + } finally { + try { + await wappalyzer?.destroy(); + } catch { + /* ignore cleanup errors */ + } + } + }, +}; diff --git a/src/schemas/cli.ts b/src/schemas/cli.ts new file mode 100644 index 0000000..8ebf9ba --- /dev/null +++ b/src/schemas/cli.ts @@ -0,0 +1,13 @@ +import { z } from "zod"; + +export const cliOptionsSchema = z.object({ + concurrency: z.number().int().positive().default(50), + timeout: z.number().positive().default(10000), + userAgent: z.string().min(1).default("Mozilla/5.0 whatsweb/1.0.0"), + output: z.string().optional(), + showError: z.boolean().default(false), + network: z.string().optional(), + file: z.string().optional(), +}); + +export type CliOptions = z.infer<typeof cliOptionsSchema>; diff --git a/src/utils/concurrency.ts b/src/utils/concurrency.ts new file mode 100644 index 0000000..6b3a986 --- /dev/null +++ b/src/utils/concurrency.ts @@ -0,0 +1,30 @@ +import pLimit from "p-limit"; + +export interface MapConcurrentOptions { + concurrency: number; + onError?: (error: unknown, item: unknown, index: number) => void; +} + +export async function mapConcurrent<T, R>( + items: T[], + fn: (item: T, index: number) => Promise<R>, + options: MapConcurrentOptions, +): Promise<(R | undefined)[]> { + const limit = pLimit(options.concurrency); + + const tasks = items.map((item, index) => + limit(async () => { + try { + return await fn(item, index); + } catch (error) { + if (options.onError) { + options.onError(error, item, index); + return undefined; + } + throw error; + } + }), + ); + + return Promise.all(tasks); +} diff --git a/src/utils/dns.ts b/src/utils/dns.ts new file mode 100644 index 0000000..041a307 --- /dev/null +++ b/src/utils/dns.ts @@ -0,0 +1,9 @@ +import { resolve4 as dnsResolve4 } from "node:dns/promises"; +import { isIPv4 } from "node:net"; + +export async function resolve4(domain: string): Promise<string> { + if (isIPv4(domain)) return domain; + const addresses = await dnsResolve4(domain); + if (!addresses.length) throw new Error(`DNS resolve failed for ${domain}`); + return addresses[0]; +} diff --git a/src/utils/http.ts b/src/utils/http.ts new file mode 100644 index 0000000..1cd22df --- /dev/null +++ b/src/utils/http.ts @@ -0,0 +1,59 @@ +export interface RequestOptions { + timeout: number; + userAgent: string; +} + +export interface HttpResponse { + status: number; + statusText: string; + headers: Record<string, string>; + text: string; + url: string; +} + +async function doFetch( + url: string, + method: string, + options: RequestOptions, +): Promise<HttpResponse> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), options.timeout); + + try { + const res = await fetch(url, { + method, + signal: controller.signal, + redirect: "follow", + headers: { + Accept: "*/*", + "Accept-Encoding": "", + "User-Agent": options.userAgent, + }, + }); + + const headersRecord: Record<string, string> = {}; + res.headers.forEach((value, key) => { + headersRecord[key.toLowerCase()] = value; + }); + + const text = method === "HEAD" ? "" : await res.text(); + + return { + status: res.status, + statusText: res.statusText, + headers: headersRecord, + text, + url: res.url, + }; + } finally { + clearTimeout(timer); + } +} + +export async function httpGet(url: string, options: RequestOptions): Promise<HttpResponse> { + return doFetch(url, "GET", options); +} + +export async function httpHead(url: string, options: RequestOptions): Promise<HttpResponse> { + return doFetch(url, "HEAD", options); +} diff --git a/src/utils/ip.ts b/src/utils/ip.ts new file mode 100644 index 0000000..f3e1e6a --- /dev/null +++ b/src/utils/ip.ts @@ -0,0 +1,46 @@ +export function ipToLong(ip: string): number { + const parts = ip.split("."); + if (parts.length !== 4) throw new Error(`Invalid IP address: ${ip}`); + let result = 0; + for (const part of parts) { + const n = Number.parseInt(part, 10); + if (Number.isNaN(n) || n < 0 || n > 255) throw new Error(`Invalid IP address: ${ip}`); + result = (result >>> 0) * 256 + n; + } + return result >>> 0; +} + +export function longToIp(long: number): string { + return [(long >>> 24) & 0xff, (long >>> 16) & 0xff, (long >>> 8) & 0xff, long & 0xff].join("."); +} + +export function* expandCIDR(cidr: string): Generator<string> { + const parts = cidr.split("/"); + if (parts.length !== 2) throw new Error(`Invalid CIDR: ${cidr}`); + + const ip = parts[0]; + const mask = Number.parseInt(parts[1], 10); + if (Number.isNaN(mask) || mask < 0 || mask > 32) throw new Error(`Invalid CIDR mask: ${cidr}`); + + const ipLong = ipToLong(ip); + const hostBits = 32 - mask; + + if (mask === 32) { + yield longToIp(ipLong); + return; + } + + const networkAddr = (ipLong >>> hostBits) << hostBits; + const totalHosts = 1 << hostBits; + + if (mask === 31) { + for (let i = 0; i < totalHosts; i++) { + yield longToIp((networkAddr + i) >>> 0); + } + return; + } + + for (let i = 1; i < totalHosts - 1; i++) { + yield longToIp((networkAddr + i) >>> 0); + } +} diff --git a/src/utils/url.ts b/src/utils/url.ts new file mode 100644 index 0000000..d11eb02 --- /dev/null +++ b/src/utils/url.ts @@ -0,0 +1,11 @@ +import { z } from "zod"; + +export function normalizeUrl(url: string): string { + if (!url) throw new Error(`Invalid url: ${url}`); + return /^https?:\/\//i.test(url) ? url : `http://${url}`; +} + +export const urlSchema = z + .string() + .min(1, "URL cannot be empty") + .transform((val) => normalizeUrl(val)); diff --git a/tests/cli.test.ts b/tests/cli.test.ts new file mode 100644 index 0000000..f56189d --- /dev/null +++ b/tests/cli.test.ts @@ -0,0 +1,39 @@ +import { describe, test, expect } from "bun:test"; + +describe("CLI", () => { + test("无参数时显示帮助信息", async () => { + const proc = Bun.spawn(["bun", "run", "src/cli.ts"], { + stdout: "pipe", + stderr: "pipe", + }); + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + const output = stdout + stderr; + await proc.exited; + expect(output).toContain("Usage"); + }); + + test("--version 输出版本号", async () => { + const proc = Bun.spawn(["bun", "run", "src/cli.ts", "--version"], { + stdout: "pipe", + }); + const output = await new Response(proc.stdout).text(); + await proc.exited; + expect(output.trim()).toMatch(/^\d+\.\d+\.\d+$/); + }); + + test("--help 输出帮助信息", async () => { + const proc = Bun.spawn(["bun", "run", "src/cli.ts", "--help"], { + stdout: "pipe", + }); + const output = await new Response(proc.stdout).text(); + await proc.exited; + expect(output).toContain("-f"); + expect(output).toContain("--concurrency"); + expect(output).toContain("--network"); + expect(output).toContain("--timeout"); + expect(output).toContain("--user-agent"); + expect(output).toContain("--output"); + expect(output).toContain("--show-error"); + }); +}); diff --git a/tests/core/plugin-loader.test.ts b/tests/core/plugin-loader.test.ts new file mode 100644 index 0000000..31153af --- /dev/null +++ b/tests/core/plugin-loader.test.ts @@ -0,0 +1,52 @@ +import { describe, test, expect } from "bun:test"; +import { PluginLoader } from "../../src/core/plugin-loader.ts"; +import type { Plugin, PluginContext } from "../../src/plugins/types.ts"; + +const createMockPlugin = (name: string, result: Record<string, unknown> | null): Plugin => ({ + meta: { name }, + execute: async (_ctx: PluginContext) => result, +}); + +describe("PluginLoader", () => { + test("注册并获取所有插件", () => { + const loader = new PluginLoader(); + const p1 = createMockPlugin("plugin-a", { key: "value" }); + const p2 = createMockPlugin("plugin-b", null); + loader.register(p1); + loader.register(p2); + expect(loader.getAll()).toEqual([p1, p2]); + }); + + test("不允许注册同名插件", () => { + const loader = new PluginLoader(); + loader.register(createMockPlugin("dup", {})); + expect(() => loader.register(createMockPlugin("dup", {}))).toThrow(); + }); + + test("不允许注册空名称插件", () => { + const loader = new PluginLoader(); + expect(() => loader.register(createMockPlugin("", {}))).toThrow(); + }); + + test("按名称获取插件", () => { + const loader = new PluginLoader(); + const p1 = createMockPlugin("find-me", {}); + loader.register(p1); + expect(loader.get("find-me")).toBe(p1); + expect(loader.get("not-exist")).toBeUndefined(); + }); + + test("pluginCount 返回正确数量", () => { + const loader = new PluginLoader(); + expect(loader.pluginCount).toBe(0); + loader.register(createMockPlugin("a", {})); + loader.register(createMockPlugin("b", {})); + expect(loader.pluginCount).toBe(2); + }); + + test("加载所有内置插件", () => { + const loader = PluginLoader.withBuiltins(); + const plugins = loader.getAll(); + expect(plugins).toBeInstanceOf(Array); + }); +}); diff --git a/tests/core/reporter.test.ts b/tests/core/reporter.test.ts new file mode 100644 index 0000000..ad2cf50 --- /dev/null +++ b/tests/core/reporter.test.ts @@ -0,0 +1,51 @@ +import { describe, test, expect } from "bun:test"; +import { Reporter } from "../../src/core/reporter.ts"; +import type { ScanResult } from "../../src/core/scanner.ts"; +import { existsSync, unlinkSync } from "node:fs"; + +describe("Reporter", () => { + test("formatResult 生成包含关键信息的字符串", () => { + const reporter = new Reporter({ showError: false }); + const results: ScanResult[] = [ + { name: "Base Information", result: { status: "200 OK", title: "Test Page" } }, + { name: "Email", result: { email: ["a@b.com", "c@d.com"] } }, + ]; + const output = reporter.formatResult("http://example.com", results); + expect(output).toContain("http://example.com"); + expect(output).toContain("Base Information"); + expect(output).toContain("200 OK"); + }); + + test("formatError 包含 URL 和错误信息", () => { + const reporter = new Reporter({ showError: true }); + const output = reporter.formatError("http://example.com", new Error("timeout")); + expect(output).toContain("http://example.com"); + expect(output).toContain("timeout"); + }); + + test("formatError 在 showError=false 时返回空", () => { + const reporter = new Reporter({ showError: false }); + const output = reporter.formatError("http://example.com", new Error("timeout")); + expect(output).toBe(""); + }); + + test("appendJsonLine 写入文件", () => { + const tmpFile = `/tmp/whatsweb-test-${Date.now()}.jsonl`; + const reporter = new Reporter({ showError: false, outputFile: tmpFile }); + + reporter.appendJsonLine("http://example.com", [{ name: "test", result: { key: "value" } }]); + + expect(existsSync(tmpFile)).toBe(true); + const content = Bun.file(tmpFile); + expect(content.size).toBeGreaterThan(0); + + unlinkSync(tmpFile); + }); + + test("数组值以逗号连接显示", () => { + const reporter = new Reporter({ showError: false }); + const results: ScanResult[] = [{ name: "Email", result: { email: ["a@b.com", "c@d.com"] } }]; + const output = reporter.formatResult("http://example.com", results); + expect(output).toContain("a@b.com, c@d.com"); + }); +}); diff --git a/tests/core/scanner.test.ts b/tests/core/scanner.test.ts new file mode 100644 index 0000000..0ed83ff --- /dev/null +++ b/tests/core/scanner.test.ts @@ -0,0 +1,131 @@ +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { Scanner } from "../../src/core/scanner.ts"; +import { PluginLoader } from "../../src/core/plugin-loader.ts"; +import type { Plugin, PluginContext } from "../../src/plugins/types.ts"; + +let server: ReturnType<typeof Bun.serve>; +let baseUrl: string; + +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/") { + return new Response( + "<html><title>Test Siteadmin@test.com info@test.com", + { headers: { "content-type": "text/html", "x-powered-by": "TestServer" } }, + ); + } + if (url.pathname === "/WhatsWeb-404-existence-check") { + return new Response("Not Found", { status: 404 }); + } + return new Response("Not Found", { status: 404 }); + }, + }); + baseUrl = `http://localhost:${server.port}`; +}); + +afterAll(() => { + server.stop(); +}); + +const successPlugin: Plugin = { + meta: { name: "success-plugin" }, + execute: async (ctx: PluginContext) => ({ found: true, url: ctx.url }), +}; + +const emptyPlugin: Plugin = { + meta: { name: "empty-plugin" }, + execute: async () => null, +}; + +const errorPlugin: Plugin = { + meta: { name: "error-plugin" }, + execute: async () => { + throw new Error("plugin crashed"); + }, +}; + +describe("Scanner", () => { + test("使用 mock 插件扫描", async () => { + const loader = new PluginLoader(); + loader.register(successPlugin); + loader.register(emptyPlugin); + + const scanner = new Scanner({ + target: baseUrl, + timeout: 5000, + userAgent: "test", + pluginLoader: loader, + }); + + const results = await scanner.analyse(); + expect(Array.isArray(results)).toBe(true); + if (!Array.isArray(results)) return; + + const successResult = results.find((r) => r.name === "success-plugin"); + expect(successResult).toBeDefined(); + expect(successResult?.result.found).toBe(true); + + const emptyResult = results.find((r) => r.name === "empty-plugin"); + expect(emptyResult).toBeUndefined(); + }); + + test("单个插件失败不影响其他插件", async () => { + const loader = new PluginLoader(); + loader.register(successPlugin); + loader.register(errorPlugin); + + const scanner = new Scanner({ + target: baseUrl, + timeout: 5000, + userAgent: "test", + pluginLoader: loader, + }); + + const results = await scanner.analyse(); + expect(Array.isArray(results)).toBe(true); + if (!Array.isArray(results)) return; + + expect(results.find((r) => r.name === "success-plugin")).toBeDefined(); + expect(results.find((r) => r.name === "error-plugin")).toBeUndefined(); + }); + + test("target 不可达时返回错误", async () => { + const loader = new PluginLoader(); + loader.register(successPlugin); + + const scanner = new Scanner({ + target: "http://localhost:1", + timeout: 2000, + userAgent: "test", + pluginLoader: loader, + }); + + const result = await scanner.analyse(); + expect(result instanceof Error).toBe(true); + }); + + test("使用 Zod 验证选项 — 空 target 应抛错", () => { + expect(() => { + new Scanner({ target: "", timeout: 5000, userAgent: "test" }); + }).toThrow(); + }); + + test("使用内置插件扫描本地服务器", async () => { + const scanner = new Scanner({ + target: baseUrl, + timeout: 10000, + userAgent: "test", + }); + + const results = await scanner.analyse(); + expect(Array.isArray(results)).toBe(true); + if (!Array.isArray(results)) return; + + const baseInfo = results.find((r) => r.name === "Base Information"); + expect(baseInfo).toBeDefined(); + expect(String(baseInfo?.result.status)).toContain("200"); + }, 30000); +}); diff --git a/tests/integration/e2e.test.ts b/tests/integration/e2e.test.ts new file mode 100644 index 0000000..813f980 --- /dev/null +++ b/tests/integration/e2e.test.ts @@ -0,0 +1,103 @@ +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { Scanner } from "../../src/core/scanner.ts"; +import { existsSync, unlinkSync } from "node:fs"; + +let server: ReturnType; +let baseUrl: string; + +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + + if (url.pathname === "/") { + return new Response( + ` + Integration Test Site + + Contact: admin@test.com, support@test.com + + + `, + { + headers: { + "content-type": "text/html", + "x-powered-by": "Express", + "x-request-id": "test-123", + }, + }, + ); + } + + if (url.pathname === "/WhatsWeb-404-existence-check") { + return new Response("Not Found", { status: 404 }); + } + + return new Response("Not Found", { status: 404 }); + }, + }); + baseUrl = `http://localhost:${server.port}`; +}); + +afterAll(() => { + server.stop(); +}); + +describe("端到端集成测试", () => { + test("完整扫描流程 — Scanner + 内置插件", async () => { + const scanner = new Scanner({ + target: baseUrl, + timeout: 10000, + userAgent: "WhatsWeb-Integration-Test", + }); + + const results = await scanner.analyse(); + expect(Array.isArray(results)).toBe(true); + if (!Array.isArray(results)) return; + + const baseInfo = results.find((r) => r.name === "Base Information"); + expect(baseInfo).toBeDefined(); + expect(String(baseInfo?.result.status)).toContain("200"); + expect(baseInfo?.result.title).toBe("Integration Test Site"); + expect(baseInfo?.result["x-powered-by"]).toBe("Express"); + expect(baseInfo?.result["x-request-id"]).toBe("test-123"); + + const email = results.find((r) => r.name === "Email"); + expect(email).toBeDefined(); + expect(email?.result.email).toEqual( + expect.arrayContaining(["admin@test.com", "support@test.com"]), + ); + + // Geolocation may return null for localhost (127.0.0.1 has no geo data) + const geo = results.find((r) => r.name === "Geolocation"); + if (geo) { + expect(geo.result.ip).toBeDefined(); + } + }, 30000); +}); + +describe("CLI 端到端", () => { + test("扫描并输出到文件", async () => { + const tmpFile = `/tmp/whatsweb-e2e-${Date.now()}.jsonl`; + + const proc = Bun.spawn( + ["bun", "run", "src/cli.ts", baseUrl, "-o", tmpFile, "--timeout", "10000"], + { + stdout: "pipe", + stderr: "pipe", + }, + ); + + await proc.exited; + + if (existsSync(tmpFile)) { + const content = await Bun.file(tmpFile).text(); + expect(content.trim().length).toBeGreaterThan(0); + const line = JSON.parse(content.trim().split("\n")[0]); + expect(line.target).toBe(baseUrl); + expect(line.plugins).toBeInstanceOf(Array); + unlinkSync(tmpFile); + } + }, 60000); +}); diff --git a/tests/plugins/base-info.test.ts b/tests/plugins/base-info.test.ts new file mode 100644 index 0000000..be52bcb --- /dev/null +++ b/tests/plugins/base-info.test.ts @@ -0,0 +1,87 @@ +import { describe, test, expect } from "bun:test"; +import { baseInfoPlugin } from "../../src/plugins/base-info.ts"; +import type { PluginContext, HttpResponse } from "../../src/plugins/types.ts"; + +const makeContext = (overrides: Partial & { ctxUrl?: string }): PluginContext => ({ + url: overrides.ctxUrl ?? "http://example.com", + timeout: 5000, + userAgent: "test", + response: { + status: 200, + statusText: "OK", + headers: {}, + text: "", + url: overrides.ctxUrl ?? "http://example.com", + ...overrides, + }, +}); + +describe("baseInfoPlugin", () => { + test("meta.name 正确", () => { + expect(baseInfoPlugin.meta.name).toBe("Base Information"); + }); + + test("提取状态码", async () => { + const result = await baseInfoPlugin.execute(makeContext({ status: 200, statusText: "OK" })); + expect(result?.status).toBe("200 OK"); + }); + + test("提取 标签", async () => { + const result = await baseInfoPlugin.execute( + makeContext({ text: "<html><title>Hello World" }), + ); + expect(result?.title).toBe("Hello World"); + }); + + test("标题中的换行符被转义", async () => { + const result = await baseInfoPlugin.execute( + makeContext({ text: "Line1\nLine2\rLine3" }), + ); + expect(result?.title).toBe("Line1\\nLine2\\rLine3"); + }); + + test("无 title 时结果中不含 title 字段", async () => { + const result = await baseInfoPlugin.execute( + makeContext({ text: "No title" }), + ); + expect(result?.title).toBeUndefined(); + }); + + test("检测重定向", async () => { + const ctx: PluginContext = { + url: "http://example.com", + timeout: 5000, + userAgent: "test", + response: { + status: 200, + statusText: "OK", + headers: {}, + text: "", + url: "http://example.com/redirected", + }, + }; + const result = await baseInfoPlugin.execute(ctx); + expect(result?.redirect).toBe("http://example.com/redirected"); + }); + + test("无重定向时不含 redirect 字段", async () => { + const result = await baseInfoPlugin.execute(makeContext({ text: "" })); + expect(result?.redirect).toBeUndefined(); + }); + + test("提取 x-* 响应头", async () => { + const result = await baseInfoPlugin.execute( + makeContext({ + text: "", + headers: { + "x-powered-by": "Express", + "x-request-id": "abc123", + "content-type": "text/html", + }, + }), + ); + expect(result?.["x-powered-by"]).toBe("Express"); + expect(result?.["x-request-id"]).toBe("abc123"); + expect(result?.["content-type"]).toBeUndefined(); + }); +}); diff --git a/tests/plugins/bbscan/index.test.ts b/tests/plugins/bbscan/index.test.ts new file mode 100644 index 0000000..613917e --- /dev/null +++ b/tests/plugins/bbscan/index.test.ts @@ -0,0 +1,12 @@ +import { describe, test, expect } from "bun:test"; +import { bbscanPlugin } from "../../../src/plugins/bbscan/index.ts"; + +describe("bbscanPlugin", () => { + test("meta.name 正确", () => { + expect(bbscanPlugin.meta.name).toBe("BBScan"); + }); + + test("实现了 Plugin 接口", () => { + expect(typeof bbscanPlugin.execute).toBe("function"); + }); +}); diff --git a/tests/plugins/bbscan/rule-parser.test.ts b/tests/plugins/bbscan/rule-parser.test.ts new file mode 100644 index 0000000..7514b87 --- /dev/null +++ b/tests/plugins/bbscan/rule-parser.test.ts @@ -0,0 +1,82 @@ +import { describe, test, expect } from "bun:test"; +import { + parseRuleLine, + parseRuleFiles, + parseListFile, +} from "../../../src/plugins/bbscan/rule-parser.ts"; + +describe("parseRuleLine", () => { + test("解析完整规则行", () => { + const line = '/admin.php {status=200} {tag="type=\\"password\\""} {root_only}'; + const rule = parseRuleLine(line); + expect(rule).not.toBeNull(); + expect(rule?.uri).toBe("/admin.php"); + expect(rule?.status).toBe(200); + expect(rule?.rootOnly).toBe(true); + }); + + test("解析仅有 URI 和 status 的行", () => { + const line = '/core {status=200} {tag="ELF"} {root_only}'; + const rule = parseRuleLine(line); + expect(rule?.uri).toBe("/core"); + expect(rule?.status).toBe(200); + expect(rule?.tag).toBe("ELF"); + expect(rule?.rootOnly).toBe(true); + }); + + test("解析含 content-type 的行", () => { + const line = '/debug.txt {status=200} {type="text/plain"} {root_only}'; + const rule = parseRuleLine(line); + expect(rule?.contentType).toBe("text/plain"); + expect(rule?.contentTypeNo).toBe(""); + }); + + test("解析含 type_no 的行", () => { + const line = + '/config/database.yml {status=200} {type_no="html"} {tag="password"} {root_only}'; + const rule = parseRuleLine(line); + expect(rule?.contentTypeNo).toBe("html"); + expect(rule?.tag).toBe("password"); + }); + + test("跳过注释行", () => { + expect(parseRuleLine("# This is a comment")).toBeNull(); + }); + + test("跳过非 / 开头的行", () => { + expect(parseRuleLine("not a rule")).toBeNull(); + expect(parseRuleLine("")).toBeNull(); + }); +}); + +describe("parseListFile", () => { + test("解析白名单文件内容", () => { + const content = ` +# comment +{text="Index of"} +{text="<title>phpMyAdmin"} +{regex_text=".*后台.*"} + `.trim(); + const result = parseListFile(content); + expect(result.texts).toContain("Index of"); + expect(result.texts).toContain("<title>phpMyAdmin"); + expect(result.regexes.length).toBe(1); + expect(result.regexes[0].test("管理后台系统")).toBe(true); + }); + + test("跳过空行和注释", () => { + const content = '# comment\n\n{text="valid"}'; + const result = parseListFile(content); + expect(result.texts).toEqual(["valid"]); + }); +}); + +describe("parseRuleFiles", () => { + test("从 rules/ 目录加载真实规则文件", async () => { + const rules = await parseRuleFiles(); + expect(rules.length).toBeGreaterThan(0); + for (const rule of rules) { + expect(rule.uri).toMatch(/^\//); + } + }); +}); diff --git a/tests/plugins/bbscan/scanner.test.ts b/tests/plugins/bbscan/scanner.test.ts new file mode 100644 index 0000000..3e3e438 --- /dev/null +++ b/tests/plugins/bbscan/scanner.test.ts @@ -0,0 +1,56 @@ +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { BBScanner } from "../../../src/plugins/bbscan/scanner.ts"; + +let server: ReturnType; +let baseUrl: string; + +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + + if (url.pathname === "/WhatsWeb-404-existence-check") { + return new Response("Not Found", { status: 404 }); + } + + if (url.pathname === "/.git/config") { + return new Response("[core]\n\trepositoryformatversion = 0", { + status: 200, + headers: { "content-type": "text/plain" }, + }); + } + + if (url.pathname === "/.svn/entries") { + return new Response("12\n-props", { + status: 200, + headers: { "content-type": "text/plain" }, + }); + } + + return new Response("Not Found", { status: 404 }); + }, + }); + baseUrl = `http://localhost:${server.port}`; +}); + +afterAll(() => { + server.stop(); +}); + +describe("BBScanner", () => { + test("check404 — 服务器支持 404", async () => { + const scanner = new BBScanner({ url: baseUrl, userAgent: "test", timeout: 5000 }); + const has404 = await scanner.check404(); + expect(has404).toBe(true); + }); + + test("完整扫描流程 — 发现敏感路径", async () => { + const scanner = new BBScanner({ url: baseUrl, userAgent: "test", timeout: 5000 }); + const results = await scanner.run(); + expect(results).toBeInstanceOf(Array); + if (Array.isArray(results)) { + expect(results.some((r) => r.includes(".git"))).toBe(true); + } + }, 30000); +}); diff --git a/tests/plugins/email.test.ts b/tests/plugins/email.test.ts new file mode 100644 index 0000000..21ace1c --- /dev/null +++ b/tests/plugins/email.test.ts @@ -0,0 +1,39 @@ +import { describe, test, expect } from "bun:test"; +import { emailPlugin } from "../../src/plugins/email.ts"; +import type { PluginContext } from "../../src/plugins/types.ts"; + +const makeContext = (html: string): PluginContext => ({ + url: "http://example.com", + timeout: 5000, + userAgent: "test", + response: { status: 200, statusText: "OK", headers: {}, text: html, url: "http://example.com" }, +}); + +describe("emailPlugin", () => { + test("meta.name 正确", () => { + expect(emailPlugin.meta.name).toBe("Email"); + }); + + test("从 HTML 中提取多个邮箱", async () => { + const html = "Contact: admin@test.com and info@test.com for details"; + const result = await emailPlugin.execute(makeContext(html)); + expect(result?.email).toEqual(expect.arrayContaining(["admin@test.com", "info@test.com"])); + }); + + test("邮箱去重", async () => { + const html = "admin@test.com admin@test.com admin@test.com info@test.com"; + const result = await emailPlugin.execute(makeContext(html)); + expect(result?.email).toEqual(["admin@test.com", "info@test.com"]); + }); + + test("少于 2 个唯一邮箱时返回 null", async () => { + const html = "Only one: admin@test.com"; + const result = await emailPlugin.execute(makeContext(html)); + expect(result).toBeNull(); + }); + + test("无邮箱时返回 null", async () => { + const result = await emailPlugin.execute(makeContext("No emails here")); + expect(result).toBeNull(); + }); +}); diff --git a/tests/plugins/geoip.test.ts b/tests/plugins/geoip.test.ts new file mode 100644 index 0000000..da57680 --- /dev/null +++ b/tests/plugins/geoip.test.ts @@ -0,0 +1,34 @@ +import { describe, test, expect } from "bun:test"; +import { geoipPlugin } from "../../src/plugins/geoip.ts"; +import type { PluginContext } from "../../src/plugins/types.ts"; + +const makeContext = (url: string): PluginContext => ({ + url, + timeout: 5000, + userAgent: "test", + response: { status: 200, statusText: "OK", headers: {}, text: "", url }, +}); + +describe("geoipPlugin", () => { + test("meta.name 正确", () => { + expect(geoipPlugin.meta.name).toBe("Geolocation"); + }); + + test("解析 IP 并返回地理信息", async () => { + const result = await geoipPlugin.execute(makeContext("http://8.8.8.8")); + expect(result?.ip).toBe("8.8.8.8"); + expect(result?.country).toBeDefined(); + }); + + test("域名会被解析为 IP", async () => { + const result = await geoipPlugin.execute(makeContext("http://dns.google")); + expect(result?.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/); + }); + + test("无法解析的域名应返回 null", async () => { + const result = await geoipPlugin.execute( + makeContext("http://this-domain-does-not-exist-xyz123.invalid"), + ); + expect(result).toBeNull(); + }); +}); diff --git a/tests/plugins/wappalyzer.test.ts b/tests/plugins/wappalyzer.test.ts new file mode 100644 index 0000000..771f5be --- /dev/null +++ b/tests/plugins/wappalyzer.test.ts @@ -0,0 +1,23 @@ +import { describe, test, expect } from "bun:test"; +import { wappalyzerPlugin } from "../../src/plugins/wappalyzer.ts"; + +describe("wappalyzerPlugin", () => { + test("meta.name 正确", () => { + expect(wappalyzerPlugin.meta.name).toBe("Wappalyzer"); + }); + + test("实现了 Plugin 接口", () => { + expect(typeof wappalyzerPlugin.execute).toBe("function"); + }); + + test("无法访问的 URL 应返回 null", async () => { + const ctx = { + url: "http://localhost:1", + timeout: 5000, + userAgent: "test", + response: { status: 200, statusText: "OK", headers: {}, text: "", url: "http://localhost:1" }, + }; + const result = await wappalyzerPlugin.execute(ctx); + expect(result === null || (result && Object.keys(result).length === 0)).toBe(true); + }, 30000); +}); diff --git a/tests/schemas/cli.test.ts b/tests/schemas/cli.test.ts new file mode 100644 index 0000000..f31fbf3 --- /dev/null +++ b/tests/schemas/cli.test.ts @@ -0,0 +1,36 @@ +import { describe, test, expect } from "bun:test"; +import { cliOptionsSchema } from "../../src/schemas/cli.ts"; + +describe("cliOptionsSchema", () => { + test("默认值填充", () => { + const result = cliOptionsSchema.parse({}); + expect(result.concurrency).toBe(50); + expect(result.timeout).toBe(10000); + expect(result.showError).toBe(false); + expect(result.userAgent).toContain("whatsweb"); + }); + + test("自定义值通过验证", () => { + const result = cliOptionsSchema.parse({ + concurrency: 10, + timeout: 5000, + userAgent: "custom", + output: "/tmp/out.json", + showError: true, + network: "24", + }); + expect(result.concurrency).toBe(10); + expect(result.output).toBe("/tmp/out.json"); + expect(result.network).toBe("24"); + }); + + test("并发数必须为正整数", () => { + const result = cliOptionsSchema.safeParse({ concurrency: -1 }); + expect(result.success).toBe(false); + }); + + test("超时必须为正数", () => { + const result = cliOptionsSchema.safeParse({ timeout: 0 }); + expect(result.success).toBe(false); + }); +}); diff --git a/tests/smoke.test.ts b/tests/smoke.test.ts new file mode 100644 index 0000000..f5b3157 --- /dev/null +++ b/tests/smoke.test.ts @@ -0,0 +1,12 @@ +import { describe, test, expect } from "bun:test"; + +describe("项目骨架验证", () => { + test("bun:test 框架可运行", () => { + expect(1 + 1).toBe(2); + }); + + test("可导入类型定义", async () => { + const types = await import("../src/plugins/types.ts"); + expect(types).toBeDefined(); + }); +}); diff --git a/tests/utils/concurrency.test.ts b/tests/utils/concurrency.test.ts new file mode 100644 index 0000000..7f2ad73 --- /dev/null +++ b/tests/utils/concurrency.test.ts @@ -0,0 +1,62 @@ +import { describe, test, expect } from "bun:test"; +import { mapConcurrent } from "../../src/utils/concurrency.ts"; + +describe("mapConcurrent (基于 p-limit)", () => { + test("所有任务按原始顺序返回结果", async () => { + const input = [1, 2, 3, 4, 5]; + const results = await mapConcurrent(input, async (n) => n * 2, { concurrency: 2 }); + expect(results).toEqual([2, 4, 6, 8, 10]); + }); + + test("空数组返回空数组", async () => { + const results = await mapConcurrent([], async (n: number) => n, { concurrency: 5 }); + expect(results).toEqual([]); + }); + + test("并发度生效（不超过指定上限）", async () => { + let running = 0; + let maxRunning = 0; + await mapConcurrent( + Array.from({ length: 20 }, (_, i) => i), + async (n) => { + running++; + maxRunning = Math.max(maxRunning, running); + await new Promise((r) => setTimeout(r, 50)); + running--; + return n; + }, + { concurrency: 5 }, + ); + expect(maxRunning).toBeLessThanOrEqual(5); + }); + + test("单个任务失败不影响其他任务（传入 onError 回调时）", async () => { + const errors: Error[] = []; + const results = await mapConcurrent( + [1, 2, 3], + async (n) => { + if (n === 2) throw new Error("fail"); + return n; + }, + { + concurrency: 2, + onError: (err) => errors.push(err as Error), + }, + ); + expect(results).toEqual([1, undefined, 3]); + expect(errors.length).toBe(1); + }); + + test("未提供 onError 时，错误直接抛出", async () => { + await expect( + mapConcurrent( + [1, 2, 3], + async (n) => { + if (n === 2) throw new Error("boom"); + return n; + }, + { concurrency: 1 }, + ), + ).rejects.toThrow("boom"); + }); +}); diff --git a/tests/utils/dns.test.ts b/tests/utils/dns.test.ts new file mode 100644 index 0000000..385b136 --- /dev/null +++ b/tests/utils/dns.test.ts @@ -0,0 +1,17 @@ +import { describe, test, expect } from "bun:test"; +import { resolve4 } from "../../src/utils/dns.ts"; + +describe("resolve4", () => { + test("IPv4 地址直接返回", async () => { + expect(await resolve4("1.2.3.4")).toBe("1.2.3.4"); + }); + + test("非 IPv4 字符串进行 DNS 解析", async () => { + const ip = await resolve4("dns.google"); + expect(ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/); + }); + + test("不存在的域名应抛出错误", async () => { + await expect(resolve4("this-domain-does-not-exist-xyz123.invalid")).rejects.toThrow(); + }); +}); diff --git a/tests/utils/http.test.ts b/tests/utils/http.test.ts new file mode 100644 index 0000000..e87d83c --- /dev/null +++ b/tests/utils/http.test.ts @@ -0,0 +1,64 @@ +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import { httpGet, httpHead } from "../../src/utils/http.ts"; + +let server: ReturnType; +let baseUrl: string; + +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/ok") { + return new Response("Test Page", { + headers: { "content-type": "text/html", "x-powered-by": "test" }, + }); + } + if (url.pathname === "/redirect") { + return Response.redirect(`http://localhost:${server.port}/ok`, 302); + } + if (url.pathname === "/slow") { + return new Promise((resolve) => setTimeout(() => resolve(new Response("slow")), 5000)); + } + return new Response("Not Found", { status: 404 }); + }, + }); + baseUrl = `http://localhost:${server.port}`; +}); + +afterAll(() => { + server.stop(); +}); + +describe("httpGet", () => { + test("正常 GET 请求", async () => { + const res = await httpGet(`${baseUrl}/ok`, { timeout: 5000, userAgent: "test" }); + expect(res.status).toBe(200); + expect(res.text).toContain("Test Page"); + expect(res.headers["content-type"]).toContain("text/html"); + expect(res.headers["x-powered-by"]).toBe("test"); + }); + + test("非 2xx 状态码不抛异常", async () => { + const res = await httpGet(`${baseUrl}/not-exist`, { timeout: 5000, userAgent: "test" }); + expect(res.status).toBe(404); + }); + + test("跟踪重定向并记录最终 URL", async () => { + const res = await httpGet(`${baseUrl}/redirect`, { timeout: 5000, userAgent: "test" }); + expect(res.status).toBe(200); + expect(res.url).toContain("/ok"); + }); + + test("超时应抛出错误", async () => { + await expect(httpGet(`${baseUrl}/slow`, { timeout: 500, userAgent: "test" })).rejects.toThrow(); + }); +}); + +describe("httpHead", () => { + test("HEAD 请求返回头信息", async () => { + const res = await httpHead(`${baseUrl}/ok`, { timeout: 5000, userAgent: "test" }); + expect(res.status).toBe(200); + expect(res.headers["content-type"]).toContain("text/html"); + }); +}); diff --git a/tests/utils/ip.test.ts b/tests/utils/ip.test.ts new file mode 100644 index 0000000..4a7add7 --- /dev/null +++ b/tests/utils/ip.test.ts @@ -0,0 +1,39 @@ +import { describe, test, expect } from "bun:test"; +import { expandCIDR, ipToLong, longToIp } from "../../src/utils/ip.ts"; + +describe("ipToLong / longToIp", () => { + test("IP 与长整型互转", () => { + expect(ipToLong("192.168.1.1")).toBe(3232235777); + expect(longToIp(3232235777)).toBe("192.168.1.1"); + }); + + test("边界值: 0.0.0.0 和 255.255.255.255", () => { + expect(ipToLong("0.0.0.0")).toBe(0); + expect(ipToLong("255.255.255.255")).toBe(4294967295); + expect(longToIp(0)).toBe("0.0.0.0"); + expect(longToIp(4294967295)).toBe("255.255.255.255"); + }); +}); + +describe("expandCIDR", () => { + test("/30 网段应展开为 2 个可用地址", () => { + const ips = [...expandCIDR("192.168.1.0/30")]; + expect(ips).toEqual(["192.168.1.1", "192.168.1.2"]); + }); + + test("/32 网段应展开为 1 个地址", () => { + const ips = [...expandCIDR("192.168.1.100/32")]; + expect(ips).toEqual(["192.168.1.100"]); + }); + + test("/24 网段应展开为 254 个地址", () => { + const ips = [...expandCIDR("10.0.0.0/24")]; + expect(ips.length).toBe(254); + expect(ips[0]).toBe("10.0.0.1"); + expect(ips[253]).toBe("10.0.0.254"); + }); + + test("非法 CIDR 应抛出错误", () => { + expect(() => [...expandCIDR("invalid")]).toThrow(); + }); +}); diff --git a/tests/utils/url.test.ts b/tests/utils/url.test.ts new file mode 100644 index 0000000..5f4dd63 --- /dev/null +++ b/tests/utils/url.test.ts @@ -0,0 +1,53 @@ +import { describe, test, expect } from "bun:test"; +import { normalizeUrl, urlSchema } from "../../src/utils/url.ts"; + +describe("normalizeUrl", () => { + test("为无协议的 URL 自动补全 http://", () => { + expect(normalizeUrl("example.com")).toBe("http://example.com"); + }); + + test("保留已有的 http:// 协议", () => { + expect(normalizeUrl("http://example.com")).toBe("http://example.com"); + }); + + test("保留已有的 https:// 协议", () => { + expect(normalizeUrl("https://example.com")).toBe("https://example.com"); + }); + + test("处理带端口的 URL", () => { + expect(normalizeUrl("example.com:8080")).toBe("http://example.com:8080"); + }); + + test("处理带路径的 URL", () => { + expect(normalizeUrl("example.com/path")).toBe("http://example.com/path"); + }); + + test("空字符串应抛出错误", () => { + expect(() => normalizeUrl("")).toThrow(); + }); + + test("null/undefined 应抛出错误", () => { + expect(() => normalizeUrl(null as unknown as string)).toThrow(); + expect(() => normalizeUrl(undefined as unknown as string)).toThrow(); + }); +}); + +describe("urlSchema (Zod)", () => { + test("合法 URL 通过验证", () => { + const result = urlSchema.safeParse("http://example.com"); + expect(result.success).toBe(true); + }); + + test("空字符串验证失败", () => { + const result = urlSchema.safeParse(""); + expect(result.success).toBe(false); + }); + + test("自动补全协议后通过验证（使用 transform）", () => { + const result = urlSchema.safeParse("example.com"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data).toBe("http://example.com"); + } + }); +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..4d8935e --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "types": ["bun-types"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": ".", + "declaration": true, + "resolveJsonModule": true, + "allowImportingTsExtensions": true, + "noEmit": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/utils.js b/utils.js deleted file mode 100644 index f42dc22..0000000 --- a/utils.js +++ /dev/null @@ -1,39 +0,0 @@ -const ip = require('ip') -const dns = require('dns') -const Url = require('url') - -const resolve4 = exports.resolve4 = (domain) => { - return new Promise((resolve, reject) => { - // IPV4，直接返回 IP 地址 - if (ip.isV4Format(domain)) return resolve(domain) - // 发送 DNS 查询请求 - dns.resolve4(domain, (err, res) => { - // DNS 解析出错 - if (err) return reject(err) - // 仅返回有效结果 - if (res && res.length) return resolve(res[0]) - reject(new Error('Not valid data')) - }) - }) -} - -const normalUrl = exports.normalUrl = url => { - if (!url) throw new Error(`Invalid url ${url}`) - return /^https?:/.test(url) ? url : `http://${url}` -} - -const ipGenerator = exports.ipGenerator = async (target, network) => { - const url = normalUrl(target) - const { host } = Url.parse(url) - if (!host) throw new Error('Invalid host') - const resolve = await resolve4(host) - const cidr = ip.cidrSubnet(`${resolve}/${network}`) - const { firstAddress, lastAddress } = cidr - const start = ip.toLong(firstAddress) - const stop = ip.toLong(lastAddress) - const ipList = [] - for (let i = start; i <= stop; i++) { - ipList.push(ip.fromLong(i)) - } - return ipList -} diff --git a/whatsweb.js b/whatsweb.js deleted file mode 100755 index e6a1935..0000000 --- a/whatsweb.js +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env node - -const fs = require('fs') -const os = require('os') -const Path = require('path') -const Program = require('commander') -const ProgressBar = require('progress') -const chalk = require('chalk') -const Promise = require('bluebird') -const Pack = require('./package') -const WhatsWeb = require('./index') -const { ipGenerator } = require('./utils') - -const main = async () => { - const opts = Program - .version(Pack.version) - .usage('[options] URLs') - .description('Identifies websites.') - .option('-f ', 'Targets file path') - .option('-c, --concurrency ', 'Start specified NUMBER of concurrency', Math.abs, 50) - .option('--network ', 'Scan all Target/MASK hosts') - .option('--timeout ', 'Max scan minutes for request', Math.abs, 10000) - .option('--user-agent ', 'Custom User-Agent', `Mozilla/5.0 ${Pack.name}/${Pack.version}`) - .option('-o, --output ', 'Output file path') - .option('--show-error', 'Show error message', false) - .parse(process.argv) - - const targetFile = opts.F - const concurrency = opts.concurrency - const network = opts.network - const timeout = opts.timeout - const userAgent = opts.userAgent - const outputFile = opts.output - const showError = opts.showError - let targets = opts.args - - // 没带任何参数 - if (process.argv.length < 3) { - opts.outputHelp() - process.exit(0) - } - - // 没指定URL列表，也没有指定目标文件 - if (!targets.length && !targetFile) throw new Error('Target is required.') - - // 从目标文件中读取URL列表 - if (targetFile) { - const fileData = fs.readFileSync(targetFile, 'utf8').trim() - if (!fileData.length) throw new Error('File is empty.') - targets = fileData.split('\n') - } - - // 根据 network 生成 IP 段 - if (network) { - for (const target of targets) { - try{ - const ipList = await ipGenerator(target, network) - targets = targets.concat(ipList) - } catch (err) {} - } - } - - // 数据去重 - targets = Array.from(new Set(targets)) - - console.log(chalk.dim(`[*] Options: ${JSON.stringify({ outputFile, concurrency, targetFile, network, timeout, userAgent, targetCount: targets.length, showError })}`)) - - // 有效数量 - let valid = 0 - - // 进度条 - const bar = new ProgressBar(':valid Hits(:rate Targets/s) | :current/:total(:percent) scanned in :elapseds, :etas left', { total: targets.length }) - - Promise.map(targets, (target) => { - const whatsWeb = new WhatsWeb({ target, timeout, userAgent }) - const highlight = (str) => chalk.bold(chalk.yellow(str)) - const report = (data) => { - // 有效数 +1 - if(Array.isArray(data)) valid += 1 - // 进度条 +1 - bar.tick({ valid }) - // 使用了 showError 选项时，记录错误信息。 - if (!Array.isArray(data)) { - // 记录错误 - if (showError) bar.interrupt(chalk.gray(`[-] Request ${whatsWeb.url} failed. ${data.message}`)) - return false - } - - // 记录到文件 - if (outputFile) fs.appendFileSync(outputFile, JSON.stringify({target: whatsWeb.url, plugins: data }) + os.EOL) - - // 每一个 item 是一个插件的执行结果，lines 是所有插件执行结果的汇合 - const _lines = [] - for (const item of data) { - const { result, name } = item - const nameStr = `[ ${chalk.bold(chalk.cyan(name))} ]` - const _line = [] - for (const _ in result) { - const key = _ - const value = result[_] - // 插件执行结果拼接的字符串 - let pluginResult = '' - // name - pluginResult += chalk.bold(chalk.white(key)) + ': ' - // value - const _value = Array.isArray(value) ? value.join(', ') : value - let str = '' - // 高亮跳转之后的URL - if (key === 'redirect') { - if (_value !== whatsWeb.url) str = highlight(_value) - } else { - // 对于特殊键的值，进行高亮显示 - str = ['title'].includes(key) ? highlight(_value) : _value - } - pluginResult += str - _line.push(pluginResult) - } - const line = _line.join(', ') - _lines.push(`${nameStr} ${line}`) - } - const lines = _lines.join('\n') - // 记录数据 - bar.interrupt(`[+] WhatsWeb report for ${chalk.bold(chalk.blue(whatsWeb.url))}\n${lines}\n`) - } - // 记录结果，忽略错误 - return whatsWeb.analyse().then(report).catch(function ignore(err) { }) - }, { concurrency }) - .then(() => process.exit(0)) -} - -if (process.mainModule) main().catch((e) => { - console.error(chalk.red(`[-] Error. ${e.message}`)) - process.exit(-1) -}) \ No newline at end of file