实现抓取节点令牌的管理

docs/api.md

@@ -70,6 +70,7 @@
 | | `GET`、`POST /admin/colleges`，`GET`、`PATCH /admin/colleges/{id}` | 管理员查询、新增、修改学院 |
 | | `GET`、`POST /admin/sources`，`GET`、`PATCH /admin/sources/{id}` | 管理员查询、接入、配置来源，暂停或恢复抓取 |
 | | `POST /admin/users/{id}/revoke-edits` | 撤销某人的全部修改，收回这个人在鼠钟的修改权限 |
+| | `GET`、`POST /admin/crawler-tokens`，`POST /admin/crawler-tokens/{id}/revoke` | 站长查询、生成、作废抓取节点令牌：生成时返回的令牌只出现这一次，作废马上生效；生成和作废记进操作日志 |
 | | `/admin/tags`、`/admin/users`、`/admin/settings` | 标签预设、修改权限、系统设置 |
 
 ## 4. MCP

web/app/components/site-header.tsx

@@ -18,7 +18,7 @@ type Me = {
   display_name: string;
   avatar_url: string | null;
   frozen: boolean;
-  can: { edit: boolean; admin: boolean };
+  can: { edit: boolean; admin: boolean; owner: boolean };
 };
 
 function navLink({ isActive }: { isActive: boolean }) {
@@ -67,6 +67,11 @@ export function SiteHeader({ me }: { me: Me | null }) {
               来源管理
             </NavLink>
           )}
+          {me?.can.owner && (
+            <NavLink to="/admin/settings" className={navLink}>
+              系统设置
+            </NavLink>
+          )}
         </nav>
         <div className="ml-auto flex shrink-0 items-center gap-1.5">
           {me ? (

web/app/routes.ts

@@ -11,4 +11,5 @@ export default [
   route("logout", "routes/logout.ts"),
   route("admin/colleges", "routes/admin-colleges.tsx"),
   route("admin/sources", "routes/admin-sources.tsx"),
+  route("admin/settings", "routes/admin-settings.tsx"),
 ] satisfies RouteConfig;

web/app/routes/admin-settings.tsx

@@ -0,0 +1,156 @@
+import type { InferResponseType } from "hono/client";
+import { data, Form, Link, redirect, useNavigation } from "react-router";
+import { formText } from "../admin.server.ts";
+import { apiContext, readApi, type createApi } from "../api.server.ts";
+import { CrawlTime, Field, FormMessage, inputClass, panelClass } from "../components/admin.tsx";
+import { Tag } from "../components/tags.tsx";
+import { Button, buttonVariants } from "../components/ui/button.tsx";
+import type { Route } from "./+types/admin-settings.ts";
+
+type Api = ReturnType<typeof createApi>;
+type Tokens = InferResponseType<Api["admin"]["crawler-tokens"]["$get"], 200>;
+
+export function meta() {
+  return [{ title: "系统设置 · 鼠钟" }];
+}
+
+export async function loader({ request, context }: Route.LoaderArgs) {
+  const params = new URL(request.url).searchParams;
+  const tokens = await context
+    .get(apiContext)
+    .admin["crawler-tokens"].$get()
+    .then((res) => readApi<Tokens>(res));
+  return { tokens, revoking: Number(params.get("revoke")), saved: params.has("saved") };
+}
+
+export async function action({ request, context }: Route.ActionArgs) {
+  const api = context.get(apiContext).admin["crawler-tokens"];
+  const form = await request.formData();
+  const intent = formText(form, "intent");
+  if (intent === "create") {
+    const name = formText(form, "name");
+    const res = await api.$post({ json: { name } });
+    if (res.status !== 201) {
+      return data({ error: (await res.json()).error.message, name }, { status: res.status });
+    }
+    return { token: (await res.json()).token, name };
+  }
+  if (intent === "revoke") {
+    const res = await api[":id"].revoke.$post({ param: { id: formText(form, "id") } });
+    if (res.status !== 204) {
+      return data({ error: (await res.json()).error.message, name: "" }, { status: res.status });
+    }
+    return redirect("/admin/settings?saved=1");
+  }
+  throw data("未知操作", { status: 400 });
+}
+
+export default function Settings({ loaderData, actionData }: Route.ComponentProps) {
+  const { tokens, revoking, saved } = loaderData;
+  const busy = useNavigation().state !== "idle";
+  const made = actionData && "token" in actionData ? actionData : null;
+  const error = actionData && "error" in actionData ? actionData.error : undefined;
+  return (
+    <main className="mx-auto max-w-4xl space-y-6 px-3 py-6 sm:px-5">
+      <header>
+        <p className="mb-2 text-sm font-medium text-brand-strong">站长</p>
+        <h1 className="text-2xl font-semibold">系统设置</h1>
+      </header>
+
+      <section className={`${panelClass} space-y-4`}>
+        <div>
+          <h2 className="text-lg font-semibold">抓取节点令牌</h2>
+          <p className="mt-1 text-sm text-muted-foreground">
+            抓取节点用它调网站的接口（接口
+            §6）。每个抓取节点一个，换机器或者泄露了就作废，重新生成。
+          </p>
+        </div>
+        <FormMessage error={error} saved={saved && !made} />
+        {made && (
+          <div className="space-y-2 rounded-lg border border-success/30 bg-success/5 p-3 text-sm">
+            <p className="text-success">
+              「{made.name}
+              」的令牌生成了。它只显示这一次，现在复制下来填进抓取节点的配置；忘了就只能作废、重新生成。
+            </p>
+            <input
+              readOnly
+              value={made.token}
+              aria-label="新令牌"
+              onFocus={(event) => event.currentTarget.select()}
+              className={`${inputClass} font-mono`}
+            />
+          </div>
+        )}
+        {/* A new key once a token is made, so the name typed for it is cleared. */}
+        <Form key={made?.token} method="post" className="flex flex-wrap items-end gap-2">
+          <input type="hidden" name="intent" value="create" />
+          <div className="min-w-48 flex-1">
+            <Field label="名字" hint="用在哪个抓取节点，比如「上海」">
+              <input
+                name="name"
+                required
+                maxLength={100}
+                defaultValue={made ? "" : actionData?.name}
+                className={inputClass}
+              />
+            </Field>
+          </div>
+          <Button disabled={busy} className="mb-6">
+            生成令牌
+          </Button>
+        </Form>
+
+        {tokens.length === 0 ? (
+          <p className="text-sm text-muted-foreground">还没有令牌。</p>
+        ) : (
+          <ul className="divide-y divide-border text-sm">
+            {tokens.map((token) => (
+              <li key={token.id} className="flex flex-wrap items-center gap-x-3 gap-y-1 py-3">
+                <span className="font-medium">{token.name}</span>
+                {token.revoked_at !== null && <Tag color="red">已作废</Tag>}
+                <span className="text-xs text-muted-foreground">
+                  生成于 <CrawlTime value={token.created_at} /> · 上次来领活{" "}
+                  {token.last_seen_at === null ? (
+                    "还没有"
+                  ) : (
+                    <CrawlTime value={token.last_seen_at} />
+                  )}
+                  {token.revoked_at !== null && (
+                    <>
+                      {" "}
+                      · 作废于 <CrawlTime value={token.revoked_at} />
+                    </>
+                  )}
+                </span>
+                {token.revoked_at === null &&
+                  (revoking === token.id ? (
+                    <Form method="post" className="ml-auto flex flex-wrap items-center gap-2">
+                      <input type="hidden" name="intent" value="revoke" />
+                      <input type="hidden" name="id" value={token.id} />
+                      <span className="text-destructive">作废后用它的抓取节点马上就连不上了。</span>
+                      <Button size="sm" variant="destructive" disabled={busy}>
+                        确认作废
+                      </Button>
+                      <Link
+                        to="/admin/settings"
+                        className={buttonVariants({ variant: "ghost", size: "sm" })}
+                      >
+                        取消
+                      </Link>
+                    </Form>
+                  ) : (
+                    <Link
+                      to={`?revoke=${token.id}`}
+                      className={`${buttonVariants({ variant: "ghost", size: "sm" })} ml-auto`}
+                    >
+                      作废
+                    </Link>
+                  ))}
+              </li>
+            ))}
+          </ul>
+        )}
+      </section>
+    </main>
+  );
+}

web/scripts/crawler-token.ts

@@ -2,7 +2,7 @@ import { createCrawlerToken } from "../server/modules/ingest/service.ts";
 import { db } from "../server/platform/db.ts";
 
 // For running a crawler against a local site; on the server the owner makes tokens in the
-// settings (design §6.2). The token is shown once: only its SHA-256 is kept.
+// settings (design §6.2).
 const name = process.argv[2];
 if (!name) throw new Error("Usage: pnpm crawler-token <name>");
 console.log(await createCrawlerToken(name));

web/server/app.ts

@@ -4,6 +4,7 @@ import { sql } from "kysely";
 import { accountsRoutes, authenticate, autoLogin, loginRoutes } from "./modules/accounts/routes.ts";
 import { dashboardRoutes } from "./modules/ingest/dashboard-routes.ts";
 import { reviewItemsRoutes } from "./modules/ingest/review-routes.ts";
+import { tokenRoutes } from "./modules/ingest/token-routes.ts";
 import { ingestRoutes } from "./modules/ingest/routes.ts";
 import { programsRoutes } from "./modules/programs/routes.ts";
 import { reviewRoutes } from "./modules/review/routes.ts";
@@ -18,6 +19,7 @@ const api = createRouter()
   .route("/", ingestRoutes)
   .route("/", dashboardRoutes)
   .route("/", reviewItemsRoutes)
+  .route("/", tokenRoutes)
   .route("/", programsRoutes)
   .route("/", reviewRoutes)
   .route("/", sourcesRoutes);

web/server/modules/ingest/service.ts

@@ -2,6 +2,7 @@ import { sql, type Kysely, type Transaction } from "kysely";
 import type { DB, Json } from "../../../db/types.ts";
 import { db } from "../../platform/db.ts";
 import { AppError } from "../../platform/errors.ts";
+import { unix } from "../../platform/http.ts";
 import { hashToken, newToken } from "../../platform/tokens.ts";
 import type { ExtractedFields, Kind, Role } from "../programs/schemas.ts";
 import {
@@ -13,6 +14,7 @@ import {
 } from "../programs/service.ts";
 import { deadlineOf, startOf } from "../programs/time.ts";
 import { finishProbe, finishSourceCrawl, leaseProbes, leaseSources } from "../sources/service.ts";
+import { recordAudit } from "../review/service.ts";
 
 // How long the crawler has to report on work before it is handed out again (api.md §6).
 const LEASE_MS = 2 * 3600_000;
@@ -27,22 +29,79 @@ const UNDATED_RECHECK_MS = 90 * 24 * 3600_000;
 const MAX_SOURCES = 50;
 const MAX_PROBES = 5;
 
+type Owner = { user_id: number; via: "web" };
+
 export type Failure =
   "timeout" | "unreachable" | "too_many_redirects" | "http_status" | "no_items" | "error";
 export type Usage = { model: string; input_tokens: number; output_tokens: number };
 export const PURPOSES = ["screening", "extraction", "probe"] as const;
 export type Purpose = (typeof PURPOSES)[number];
 export type ListedItem = { url: string; title: string; published_on?: string };
 
-export async function createCrawlerToken(name: string) {
+// Made by the owner in the system settings (design §6.2), or for a local crawler with
+// `pnpm crawler-token`, which no one logs in for. The token is shown once: only its SHA-256 is
+// kept.
+export async function createCrawlerToken(name: string, owner?: Owner) {
   const token = newToken();
-  await db
-    .insertInto("crawler_tokens")
-    .values({ name, token_hash: hashToken(token) })
-    .execute();
+  await db.transaction().execute(async (trx) => {
+    const { id } = await trx
+      .insertInto("crawler_tokens")
+      .values({ name, token_hash: hashToken(token) })
+      .returning("id")
+      .executeTakeFirstOrThrow();
+    if (owner) {
+      await recordAudit(trx, {
+        user_id: owner.user_id,
+        action: "create",
+        target_type: "crawler_token",
+        target_id: id,
+        detail: { via: owner.via, name },
+      });
+    }
+  });
   return token;
 }
 
+export async function listCrawlerTokens() {
+  const rows = await db
+    .selectFrom("crawler_tokens")
+    .select(["id", "name", "created_at", "last_seen_at", "revoked_at"])
+    .orderBy("id", "desc")
+    .execute();
+  return rows.map((row) => ({
+    ...row,
+    created_at: unix(row.created_at),
+    last_seen_at: row.last_seen_at && unix(row.last_seen_at),
+    revoked_at: row.revoked_at && unix(row.revoked_at),
+  }));
+}
+
+// A revoked token stops working at once: every request looks it up (crawlerTokenId).
+export async function revokeCrawlerToken(owner: Owner, id: number) {
+  await db.transaction().execute(async (trx) => {
+    const token = await trx
+      .selectFrom("crawler_tokens")
+      .select(["name", "revoked_at"])
+      .where("id", "=", id)
+      .forUpdate()
+      .executeTakeFirst();
+    if (!token) throw new AppError(404, "not_found", "令牌不存在");
+    if (token.revoked_at) throw new AppError(409, "already_revoked", "这个令牌已经作废了");
+    await trx
+      .updateTable("crawler_tokens")
+      .set({ revoked_at: new Date() })
+      .where("id", "=", id)
+      .execute();
+    await recordAudit(trx, {
+      user_id: owner.user_id,
+      action: "revoke",
+      target_type: "crawler_token",
+      target_id: id,
+      detail: { via: owner.via, name: token.name },
+    });
+  });
+}
+
 export async function crawlerTokenId(token: string) {
   const row = await db
     .selectFrom("crawler_tokens")