url-fetch.mjs

import http from "node:http";
import https from "node:https";
import { lookup } from "node:dns/promises";
import { isIP } from "node:net";

const maxPageBytes = 1_500_000;
const redirectStatuses = new Set([301, 302, 303, 307, 308]);

const blockedIpv4Ranges = [
  ["0.0.0.0", 8],
  ["10.0.0.0", 8],
  ["100.64.0.0", 10],
  ["127.0.0.0", 8],
  ["169.254.0.0", 16],
  ["172.16.0.0", 12],
  ["192.0.0.0", 24],
  ["192.0.2.0", 24],
  ["192.88.99.0", 24],
  ["192.168.0.0", 16],
  ["198.18.0.0", 15],
  ["198.51.100.0", 24],
  ["203.0.113.0", 24],
  ["224.0.0.0", 4],
  ["240.0.0.0", 4]
];

const blockedIpv6Ranges = [
  ["64:ff9b:1::", 48],
  ["100::", 64],
  ["2001:2::", 48],
  ["2001:10::", 28],
  ["2001:db8::", 32],
  ["fc00::", 7],
  ["fe80::", 10],
  ["fec0::", 10],
  ["ff00::", 8]
];

function ipv4ToBigInt(address) {
  const parts = address.split(".").map(Number);
  if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return null;
  return parts.reduce((value, part) => (value << 8n) + BigInt(part), 0n);
}

function ipv6ToBigInt(address) {
  let normalized = address.toLowerCase().replace(/^\[|\]$/g, "").split("%")[0];
  if (normalized.includes(".")) {
    const separator = normalized.lastIndexOf(":");
    const ipv4 = ipv4ToBigInt(normalized.slice(separator + 1));
    if (separator < 0 || ipv4 === null) return null;
    normalized = `${normalized.slice(0, separator)}:${(ipv4 >> 16n).toString(16)}:${(ipv4 & 0xffffn).toString(16)}`;
  }

  const halves = normalized.split("::");
  if (halves.length > 2) return null;
  const left = halves[0] ? halves[0].split(":") : [];
  const right = halves[1] ? halves[1].split(":") : [];
  const missing = 8 - left.length - right.length;
  if ((halves.length === 1 && missing !== 0) || missing < 0) return null;
  const parts = [...left, ...Array(missing).fill("0"), ...right];
  if (parts.length !== 8 || parts.some((part) => !/^[0-9a-f]{1,4}$/.test(part))) return null;
  return parts.reduce((value, part) => (value << 16n) + BigInt(`0x${part}`), 0n);
}

function isInRange(value, network, prefixLength, bitLength) {
  const shift = BigInt(bitLength - prefixLength);
  return value >> shift === network >> shift;
}

function isBlockedIpv4(address) {
  const value = ipv4ToBigInt(address);
  return value !== null && blockedIpv4Ranges.some(([network, prefix]) => (
    isInRange(value, ipv4ToBigInt(network), prefix, 32)
  ));
}

function embeddedIpv4(value, prefix, prefixLength) {
  const network = ipv6ToBigInt(prefix);
  if (!isInRange(value, network, prefixLength, 128)) return false;
  const ipv4 = Number(value & 0xffffffffn);
  return isBlockedIpv4([24, 16, 8, 0].map((shift) => (ipv4 >>> shift) & 255).join("."));
}

export function isBlockedAddress(address) {
  const normalized = address.toLowerCase().replace(/^\[|\]$/g, "").split("%")[0];
  const version = isIP(normalized);
  if (version === 4) return isBlockedIpv4(normalized);
  if (version !== 6) return true;

  const value = ipv6ToBigInt(normalized);
  if (value === null) return true;
  if (embeddedIpv4(value, "::", 96) || embeddedIpv4(value, "::ffff:0:0", 96) || embeddedIpv4(value, "64:ff9b::", 96)) return true;
  return blockedIpv6Ranges.some(([network, prefix]) => isInRange(value, ipv6ToBigInt(network), prefix, 128));
}

function isBlockedHost(hostname) {
  const host = hostname.toLowerCase().replace(/^\[|\]$/g, "");
  if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) return true;
  return isIP(host) > 0 && isBlockedAddress(host);
}

export async function resolvePublicUrl(target, resolver = lookup) {
  const url = new URL(target);
  if (!["http:", "https:"].includes(url.protocol) || isBlockedHost(url.hostname)) {
    throw new Error("只支持公开的 HTTP 或 HTTPS 网页");
  }

  const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, "");
  const addresses = await resolver(hostname, { all: true, verbatim: true });
  if (!addresses.length || addresses.some(({ address }) => isBlockedAddress(address))) {
    throw new Error("不能读取本机、内网或特殊用途地址");
  }
  return { url, addresses };
}

export function createPinnedLookup(addresses) {
  const approved = addresses.map(({ address, family }) => ({ address, family: Number(family) || isIP(address) }));
  if (!approved.length || approved.some(({ address, family }) => !family || isBlockedAddress(address))) {
    throw new Error("没有可用的公开地址");
  }

  return (_hostname, options, callback) => {
    const settings = typeof options === "number" ? { family: options } : (options || {});
    const requestedFamily = Number(settings.family || 0);
    const candidates = requestedFamily ? approved.filter(({ family }) => family === requestedFamily) : approved;
    if (!candidates.length) {
      const error = new Error("没有符合协议族的公开地址");
      error.code = "ENOTFOUND";
      queueMicrotask(() => callback(error));
      return;
    }
    if (settings.all) {
      queueMicrotask(() => callback(null, candidates));
      return;
    }
    queueMicrotask(() => callback(null, candidates[0].address, candidates[0].family));
  };
}

async function requestPage(url, addresses, signal, requesters = {}) {
  const requester = url.protocol === "https:" ? (requesters.https || https.request) : (requesters.http || http.request);
  return new Promise((resolve, reject) => {
    const request = requester(url, {
      method: "GET",
      signal,
      lookup: createPinnedLookup(addresses),
      headers: { "User-Agent": "QuickLearn/1.0 (local learning assistant)", Accept: "text/html, text/plain" }
    }, (response) => {
      const remoteAddress = response.socket?.remoteAddress;
      if (!remoteAddress || isBlockedAddress(remoteAddress)) {
        response.destroy();
        reject(new Error("连接到的地址不是公开地址"));
        return;
      }
      resolve(response);
    });
    request.on("error", reject);
    request.end();
  });
}

async function readPageBody(response) {
  const chunks = [];
  let size = 0;
  for await (const chunk of response) {
    size += chunk.byteLength;
    if (size > maxPageBytes) {
      response.destroy();
      throw new Error("网页内容过大");
    }
    chunks.push(chunk);
  }
  return new TextDecoder().decode(Buffer.concat(chunks));
}

export async function fetchPage(target, options = {}) {
  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), 9000);
  const resolver = options.resolver || lookup;
  try {
    let resolved = await resolvePublicUrl(target, resolver);
    for (let redirects = 0; redirects <= 4; redirects += 1) {
      const response = await requestPage(resolved.url, resolved.addresses, controller.signal, options.requesters);
      const status = response.statusCode || 0;
      if (redirectStatuses.has(status)) {
        const location = response.headers.location;
        response.resume();
        if (!location || redirects === 4) throw new Error("网页跳转次数过多");
        resolved = await resolvePublicUrl(new URL(location, resolved.url).href, resolver);
        continue;
      }
      if (status < 200 || status >= 300) {
        response.resume();
        throw new Error(`网页返回 ${status}`);
      }
      const contentType = String(response.headers["content-type"] || "");
      if (!contentType.includes("text/html") && !contentType.includes("text/plain")) {
        response.resume();
        throw new Error("暂不支持这种页面格式");
      }
      return { html: await readPageBody(response), finalUrl: resolved.url.href };
    }
    throw new Error("网页跳转次数过多");
  } finally {
    clearTimeout(timeout);
  }
}