diff --git a/docs/api.md b/docs/api.md index b216a4b..8b151c7 100644 --- a/docs/api.md +++ b/docs/api.md @@ -70,6 +70,7 @@ | | `GET`、`POST /admin/colleges`，`GET`、`PATCH /admin/colleges/{id}` | 管理员查询、新增、修改学院 | | | `GET`、`POST /admin/sources`，`GET`、`PATCH /admin/sources/{id}` | 管理员查询、接入、配置来源，暂停或恢复抓取 | | | `POST /admin/users/{id}/revoke-edits` | 撤销某人的全部修改，收回这个人在鼠钟的修改权限 | +| | `GET`、`POST /admin/crawler-tokens`，`POST /admin/crawler-tokens/{id}/revoke` | 站长查询、生成、作废抓取节点令牌：生成时返回的令牌只出现这一次，作废马上生效；生成和作废记进操作日志 | | | `/admin/tags`、`/admin/users`、`/admin/settings` | 标签预设、修改权限、系统设置 | ## 4. MCP diff --git a/web/app/components/site-header.tsx b/web/app/components/site-header.tsx index d2ef129..f911838 100644 --- a/web/app/components/site-header.tsx +++ b/web/app/components/site-header.tsx @@ -18,7 +18,7 @@ type Me = { display_name: string; avatar_url: string | null; frozen: boolean; - can: { edit: boolean; admin: boolean }; + can: { edit: boolean; admin: boolean; owner: boolean }; }; function navLink({ isActive }: { isActive: boolean }) { @@ -67,6 +67,11 @@ export function SiteHeader({ me }: { me: Me | null }) { 来源管理 )} + {me?.can.owner && ( + + 系统设置 + + )}
{me ? ( diff --git a/web/app/routes.ts b/web/app/routes.ts index 46359d8..5183ac0 100644 --- a/web/app/routes.ts +++ b/web/app/routes.ts @@ -11,4 +11,5 @@ export default [ route("logout", "routes/logout.ts"), route("admin/colleges", "routes/admin-colleges.tsx"), route("admin/sources", "routes/admin-sources.tsx"), + route("admin/settings", "routes/admin-settings.tsx"), ] satisfies RouteConfig; diff --git a/web/app/routes/admin-settings.tsx b/web/app/routes/admin-settings.tsx new file mode 100644 index 0000000..682d8f2 --- /dev/null +++ b/web/app/routes/admin-settings.tsx @@ -0,0 +1,156 @@ +import type { InferResponseType } from "hono/client"; +import { data, Form, Link, redirect, useNavigation } from "react-router"; +import { formText } from "../admin.server.ts"; +import { apiContext, readApi, type createApi } from "../api.server.ts"; +import { CrawlTime, Field, FormMessage, inputClass, panelClass } from "../components/admin.tsx"; +import { Tag } from "../components/tags.tsx"; +import { Button, buttonVariants } from "../components/ui/button.tsx"; +import type { Route } from "./+types/admin-settings.ts"; + +type Api = ReturnType; +type Tokens = InferResponseType; + +export function meta() { + return [{ title: "系统设置 · 鼠钟" }]; +} + +export async function loader({ request, context }: Route.LoaderArgs) { + const params = new URL(request.url).searchParams; + const tokens = await context + .get(apiContext) + .admin["crawler-tokens"].$get() + .then((res) => readApi(res)); + return { tokens, revoking: Number(params.get("revoke")), saved: params.has("saved") }; +} + +export async function action({ request, context }: Route.ActionArgs) { + const api = context.get(apiContext).admin["crawler-tokens"]; + const form = await request.formData(); + const intent = formText(form, "intent"); + if (intent === "create") { + const name = formText(form, "name"); + const res = await api.$post({ json: { name } }); + if (res.status !== 201) { + return data({ error: (await res.json()).error.message, name }, { status: res.status }); + } + return { token: (await res.json()).token, name }; + } + if (intent === "revoke") { + const res = await api[":id"].revoke.$post({ param: { id: formText(form, "id") } }); + if (res.status !== 204) { + return data({ error: (await res.json()).error.message, name: "" }, { status: res.status }); + } + return redirect("/admin/settings?saved=1"); + } + throw data("未知操作", { status: 400 }); +} + +export default function Settings({ loaderData, actionData }: Route.ComponentProps) { + const { tokens, revoking, saved } = loaderData; + const busy = useNavigation().state !== "idle"; + const made = actionData && "token" in actionData ? actionData : null; + const error = actionData && "error" in actionData ? actionData.error : undefined; + return ( +
+
+

站长

+

系统设置

+
+ +
+
+

抓取节点令牌

+

+ 抓取节点用它调网站的接口（接口 + §6）。每个抓取节点一个，换机器或者泄露了就作废，重新生成。 +

+
+ + {made && ( +
+

+ 「{made.name} + 」的令牌生成了。它只显示这一次，现在复制下来填进抓取节点的配置；忘了就只能作废、重新生成。 +

+ event.currentTarget.select()} + className={`${inputClass} font-mono`} + /> +
+ )} + {/* A new key once a token is made, so the name typed for it is cleared. */} +
+ +
+ + + +
+ +
+ + {tokens.length === 0 ? ( +

还没有令牌。

+ ) : ( +
    + {tokens.map((token) => ( +
  • + {token.name} + {token.revoked_at !== null && 已作废} + + 生成于 · 上次来领活{" "} + {token.last_seen_at === null ? ( + "还没有" + ) : ( + + )} + {token.revoked_at !== null && ( + <> + {" "} + · 作废于 + + )} + + {token.revoked_at === null && + (revoking === token.id ? ( +
    + + + 作废后用它的抓取节点马上就连不上了。 + + + 取消 + +
    + ) : ( + + 作废 + + ))} +
    • + ))} +
+ )} +
+
+ ); +} diff --git a/web/scripts/crawler-token.ts b/web/scripts/crawler-token.ts index 00f0ac9..6541984 100644 --- a/web/scripts/crawler-token.ts +++ b/web/scripts/crawler-token.ts @@ -2,7 +2,7 @@ import { createCrawlerToken } from "../server/modules/ingest/service.ts"; import { db } from "../server/platform/db.ts"; // For running a crawler against a local site; on the server the owner makes tokens in the -// settings (design §6.2). The token is shown once: only its SHA-256 is kept. +// settings (design §6.2). const name = process.argv[2]; if (!name) throw new Error("Usage: pnpm crawler-token "); console.log(await createCrawlerToken(name)); diff --git a/web/server/app.ts b/web/server/app.ts index c8aa506..28bd66e 100644 --- a/web/server/app.ts +++ b/web/server/app.ts @@ -4,6 +4,7 @@ import { sql } from "kysely"; import { accountsRoutes, authenticate, autoLogin, loginRoutes } from "./modules/accounts/routes.ts"; import { dashboardRoutes } from "./modules/ingest/dashboard-routes.ts"; import { reviewItemsRoutes } from "./modules/ingest/review-routes.ts"; +import { tokenRoutes } from "./modules/ingest/token-routes.ts"; import { ingestRoutes } from "./modules/ingest/routes.ts"; import { programsRoutes } from "./modules/programs/routes.ts"; import { reviewRoutes } from "./modules/review/routes.ts"; @@ -18,6 +19,7 @@ const api = createRouter() .route("/", ingestRoutes) .route("/", dashboardRoutes) .route("/", reviewItemsRoutes) + .route("/", tokenRoutes) .route("/", programsRoutes) .route("/", reviewRoutes) .route("/", sourcesRoutes); diff --git a/web/server/modules/ingest/service.ts b/web/server/modules/ingest/service.ts index 194cc25..e82ed37 100644 --- a/web/server/modules/ingest/service.ts +++ b/web/server/modules/ingest/service.ts @@ -2,6 +2,7 @@ import { sql, type Kysely, type Transaction } from "kysely"; import type { DB, Json } from "../../../db/types.ts"; import { db } from "../../platform/db.ts"; import { AppError } from "../../platform/errors.ts"; +import { unix } from "../../platform/http.ts"; import { hashToken, newToken } from "../../platform/tokens.ts"; import type { ExtractedFields, Kind, Role } from "../programs/schemas.ts"; import { @@ -13,6 +14,7 @@ import { } from "../programs/service.ts"; import { deadlineOf, startOf } from "../programs/time.ts"; import { finishProbe, finishSourceCrawl, leaseProbes, leaseSources } from "../sources/service.ts"; +import { recordAudit } from "../review/service.ts"; // How long the crawler has to report on work before it is handed out again (api.md §6). const LEASE_MS = 2 * 3600_000; @@ -27,6 +29,8 @@ const UNDATED_RECHECK_MS = 90 * 24 * 3600_000; const MAX_SOURCES = 50; const MAX_PROBES = 5; +type Owner = { user_id: number; via: "web" }; + export type Failure = "timeout" | "unreachable" | "too_many_redirects" | "http_status" | "no_items" | "error"; export type Usage = { model: string; input_tokens: number; output_tokens: number }; @@ -34,15 +38,70 @@ export const PURPOSES = ["screening", "extraction", "probe"] as const; export type Purpose = (typeof PURPOSES)[number]; export type ListedItem = { url: string; title: string; published_on?: string }; -export async function createCrawlerToken(name: string) { +// Made by the owner in the system settings (design §6.2), or for a local crawler with +// `pnpm crawler-token`, which no one logs in for. The token is shown once: only its SHA-256 is +// kept. +export async function createCrawlerToken(name: string, owner?: Owner) { const token = newToken(); - await db - .insertInto("crawler_tokens") - .values({ name, token_hash: hashToken(token) }) - .execute(); + await db.transaction().execute(async (trx) => { + const { id } = await trx + .insertInto("crawler_tokens") + .values({ name, token_hash: hashToken(token) }) + .returning("id") + .executeTakeFirstOrThrow(); + if (owner) { + await recordAudit(trx, { + user_id: owner.user_id, + action: "create", + target_type: "crawler_token", + target_id: id, + detail: { via: owner.via, name }, + }); + } + }); return token; } +export async function listCrawlerTokens() { + const rows = await db + .selectFrom("crawler_tokens") + .select(["id", "name", "created_at", "last_seen_at", "revoked_at"]) + .orderBy("id", "desc") + .execute(); + return rows.map((row) => ({ + ...row, + created_at: unix(row.created_at), + last_seen_at: row.last_seen_at && unix(row.last_seen_at), + revoked_at: row.revoked_at && unix(row.revoked_at), + })); +} + +// A revoked token stops working at once: every request looks it up (crawlerTokenId). +export async function revokeCrawlerToken(owner: Owner, id: number) { + await db.transaction().execute(async (trx) => { + const token = await trx + .selectFrom("crawler_tokens") + .select(["name", "revoked_at"]) + .where("id", "=", id) + .forUpdate() + .executeTakeFirst(); + if (!token) throw new AppError(404, "not_found", "令牌不存在"); + if (token.revoked_at) throw new AppError(409, "already_revoked", "这个令牌已经作废了"); + await trx + .updateTable("crawler_tokens") + .set({ revoked_at: new Date() }) + .where("id", "=", id) + .execute(); + await recordAudit(trx, { + user_id: owner.user_id, + action: "revoke", + target_type: "crawler_token", + target_id: id, + detail: { via: owner.via, name: token.name }, + }); + }); +} + export async function crawlerTokenId(token: string) { const row = await db .selectFrom("crawler_tokens") diff --git a/web/server/modules/ingest/token-routes.ts b/web/server/modules/ingest/token-routes.ts new file mode 100644 index 0000000..d30b243 --- /dev/null +++ b/web/server/modules/ingest/token-routes.ts @@ -0,0 +1,103 @@ +import { createRoute, z } from "@hono/zod-openapi"; +import { + authorize, + createRouter, + errorResponse, + IdParam, + Int, + named, + UnixSeconds, +} from "../../platform/http.ts"; +import { createCrawlerToken, listCrawlerTokens, revokeCrawlerToken } from "./service.ts"; + +// The owner's crawler tokens, in the system settings (design §6.2). + +function json(schema: T) { + return { "application/json": { schema } }; +} + +const CrawlerToken = z + .object({ + id: Int, + name: z.string(), + created_at: UnixSeconds, + last_seen_at: UnixSeconds.nullable().openapi({ description: "上次来领活；为空是还没来过" }), + revoked_at: UnixSeconds.nullable().openapi({ description: "为空是还在用" }), + }) + .openapi("CrawlerToken"); + +const errors = { + 401: errorResponse("没有登录"), + 403: errorResponse("只有站长能操作"), +}; + +export const tokenRoutes = createRouter() + .openapi( + createRoute({ + method: "get", + path: "/admin/crawler-tokens", + summary: "抓取节点令牌", + responses: { + 200: { description: "令牌，新的在前", content: json(z.array(CrawlerToken)) }, + ...errors, + }, + }), + async (c) => { + authorize(c, "owner"); + return c.json(await listCrawlerTokens(), 200); + }, + ) + .openapi( + createRoute({ + method: "post", + path: "/admin/crawler-tokens", + summary: "生成抓取节点令牌", + description: "返回的令牌只出现这一次，库里只存 SHA-256。", + request: { + body: { + required: true, + content: json( + z.strictObject({ + name: z + .string(named("名字")) + .trim() + .min(1) + .max(100) + .openapi({ description: "用在哪个抓取节点，比如「上海」" }), + }), + ), + }, + }, + responses: { + 201: { description: "新令牌", content: json(z.object({ token: z.string() })) }, + 400: errorResponse("请求格式有误"), + ...errors, + }, + }), + async (c) => { + const { id } = authorize(c, "owner"); + const token = await createCrawlerToken(c.req.valid("json").name, { user_id: id, via: "web" }); + return c.json({ token }, 201); + }, + ) + .openapi( + createRoute({ + method: "post", + path: "/admin/crawler-tokens/{id}/revoke", + summary: "作废抓取节点令牌", + description: "马上生效：用它的抓取节点之后的请求都返回 401。", + request: { params: IdParam }, + responses: { + 204: { description: "已作废" }, + 400: errorResponse("编号不对"), + ...errors, + 404: errorResponse("令牌不存在"), + 409: errorResponse("已经作废了"), + }, + }), + async (c) => { + const { id } = authorize(c, "owner"); + await revokeCrawlerToken({ user_id: id, via: "web" }, c.req.valid("param").id); + return c.body(null, 204); + }, + ); diff --git a/web/server/modules/ingest/tokens.test.ts b/web/server/modules/ingest/tokens.test.ts new file mode 100644 index 0000000..da1df84 --- /dev/null +++ b/web/server/modules/ingest/tokens.test.ts @@ -0,0 +1,60 @@ +import { testClient } from "hono/testing"; +import { beforeAll, expect, test } from "vitest"; +import { fakeUsers, type FakeUser } from "../../../test/fake-mousehole.ts"; +import { app } from "../../app.ts"; +import { db } from "../../platform/db.ts"; +import { env } from "../../platform/env.ts"; +import { login } from "../accounts/service.ts"; + +const v1 = testClient(app).api.v1; +const tokens = v1.admin["crawler-tokens"]; + +let admin = { headers: {} }; +let owner = admin; + +async function session(profile: FakeUser) { + const token = await login(profile, "fake", { ip: null, userAgent: "test" }); + return { headers: { cookie: `mouseclock_session=${token}`, origin: env.PUBLIC_URL } }; +} + +beforeAll(async () => { + [admin, owner] = await Promise.all([session(fakeUsers[2]), session(fakeUsers[3])]); +}); + +test("the owner makes a token the crawler can use, and revoking it stops it at once", async () => { + expect((await tokens.$post({ json: { name: " " } }, owner)).status).toBe(400); + const made = await tokens.$post({ json: { name: "上海" } }, owner); + if (made.status !== 201) throw new Error(`status ${made.status}`); + const { token } = await made.json(); + const crawler = { headers: { authorization: `Bearer ${token}` } }; + expect((await v1.crawler.work.$get({}, crawler)).status).toBe(200); + + const list = await tokens.$get({}, owner); + if (list.status !== 200) throw new Error(`status ${list.status}`); + const [listed] = await list.json(); + expect(listed).toMatchObject({ name: "上海", revoked_at: null }); + expect(listed.last_seen_at).not.toBeNull(); + + const revoke = (as: typeof owner) => + tokens[":id"].revoke.$post({ param: { id: String(listed.id) } }, as); + expect((await revoke(owner)).status).toBe(204); + expect((await v1.crawler.work.$get({}, crawler)).status).toBe(401); + expect((await revoke(owner)).status).toBe(409); + + const audit = await db + .selectFrom("audit_log") + .select("action") + .where("target_type", "=", "crawler_token") + .where("target_id", "=", listed.id) + .orderBy("id") + .execute(); + expect(audit.map((row) => row.action)).toEqual(["create", "revoke"]); +}); + +test("only the owner sees, makes and revokes tokens", async () => { + expect((await tokens.$get({}, admin)).status).toBe(403); + expect((await tokens.$post({ json: { name: "x" } }, admin)).status).toBe(403); + const revoke = tokens[":id"].revoke.$post({ param: { id: "1" } }, admin); + expect((await revoke).status).toBe(403); + expect((await tokens.$get({})).status).toBe(401); +}); diff --git a/web/server/modules/review/service.ts b/web/server/modules/review/service.ts index 7fe201a..8896237 100644 --- a/web/server/modules/review/service.ts +++ b/web/server/modules/review/service.ts @@ -22,7 +22,7 @@ export async function recordAudit( entry: { user_id: number; action: string; - target_type: "school" | "college" | "source" | "user" | "review_item"; + target_type: "school" | "college" | "source" | "user" | "review_item" | "crawler_token"; target_id: number; detail: Record; },