diff --git a/.github/workflows/gallery_check.yml b/.github/workflows/gallery_check.yml new file mode 100644 index 0000000..894634d --- /dev/null +++ b/.github/workflows/gallery_check.yml @@ -0,0 +1,24 @@ +name: Gallery link check + +on: + pull_request: + paths: + - "src/data/gallery/**" + - "scripts/check-gallery-links.mjs" + - "package.json" + - "yarn.lock" + - ".github/workflows/gallery_check.yml" + +permissions: + contents: read + +jobs: + gallery-links: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: "22" + - run: yarn + - run: yarn test:gallery-links diff --git a/.github/workflows/gallery_notify.yml b/.github/workflows/gallery_notify.yml new file mode 100644 index 0000000..6cc59a4 --- /dev/null +++ b/.github/workflows/gallery_notify.yml @@ -0,0 +1,70 @@ +name: Gallery link maintenance + +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + push: + branches: + - master + paths: + - "src/data/gallery/**" + - "scripts/check-gallery-links.mjs" + - "scripts/notify-gallery-links.mjs" + - ".github/workflows/gallery_check.yml" + - ".github/workflows/gallery_notify.yml" + - "package.json" + - "yarn.lock" + +permissions: + contents: read + +concurrency: + group: gallery-link-maintenance-${{ github.ref_name }} + cancel-in-progress: false + +env: + REPORT_PATH: /tmp/gallery-link-report.json + STATE_PATH: .cache/gallery-link-state.json + +jobs: + gallery-maintenance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: "22" + - run: yarn + - name: Prepare state directory + run: mkdir -p "$(dirname "$STATE_PATH")" + - name: Restore notification state + uses: actions/cache@v4 + with: + path: ${{ env.STATE_PATH }} + key: gallery-link-state-${{ github.repository }}-${{ github.ref_name }}-${{ github.run_id }} + restore-keys: | + gallery-link-state-${{ github.repository }}-${{ github.ref_name }}- + - name: Check gallery links + id: checker + continue-on-error: true + run: yarn test:gallery-links --report "$REPORT_PATH" + - name: Notify gallery link failures + id: notifier + if: ${{ always() }} + env: + DISCORD_GALLERY_WEBHOOK_URL: ${{ secrets.DISCORD_GALLERY_WEBHOOK_URL }} + WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: yarn notify-gallery-links --report "$REPORT_PATH" --state "$STATE_PATH" --workflow-url "$WORKFLOW_URL" + - name: Save notification state + if: ${{ always() && steps.notifier.outcome == 'success' }} + uses: actions/cache/save@v4 + with: + path: ${{ env.STATE_PATH }} + key: gallery-link-state-${{ github.repository }}-${{ github.ref_name }}-${{ github.run_id }} + - name: Preserve maintenance result + if: ${{ always() }} + run: | + if [ "${{ steps.checker.outcome }}" != "success" ] || [ "${{ steps.notifier.outcome }}" != "success" ]; then + exit 1 + fi diff --git a/README.md b/README.md index a72a63c..ea191da 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,17 @@ yarn prettier 目前大部分的照片都放在 GitHub 的 FCU-iOSClub/Website2022ImageBed 上。 +### Gallery Link Check + +掃描 `src/data/gallery` 中所有非空的 `gdrive_url`,以未登入、未使用任何 Google credential 的訪客身分檢查 Google Drive 資料夾是否可存取: + +```bash +yarn test:gallery-links +yarn test:gallery-links --url "https://drive.google.com/drive/folders/" +``` + +若結果確認需要權限、網址無效、發生網路錯誤或無法判定,指令會以 non-zero exit code 結束。 + ### 競賽得獎 在 `/src/data/contest` 中新增檔案,檔案名稱以年份命名。 @@ -151,6 +162,20 @@ yarn prettier } ``` +## Gallery link checker + +相簿連結檢查器會在 Pull Request 上執行檢查,並在 `master` 的相關更新時維護檢查狀態。PR gate 不需要 Discord secret,也**不會傳送 Discord 通知**;完整掃描可能因為目前儲存庫中既有連結受到限制而失敗。 + +### Discord 通知設定 + +維護工作流程若要傳送狀態變更通知,請在 GitHub 儲存庫中前往 **Settings → Secrets and variables → Actions → New repository secret**,建立名稱完全相同的 `DISCORD_GALLERY_WEBHOOK_URL` secret,並將 Discord webhook URL 填入 secret value。README、程式碼與 workflow 中都不要直接寫入 webhook value。 + +維護工作流程會在 `master` 的相關 push、每日 **UTC** 排程,以及手動 dispatch 時執行。Discord 僅通知兩種結果:`Permission denied` 與 `Invalid URL`。通知採 transition-only:第一次出現失敗或失敗集合改變時通知一次;相同失敗重複出現時保持靜默;恢復後通知一次。 + +檢查器使用快取保存上一輪的失敗狀態,因此快取是精簡、可重現且不含憑證的狀態;它不是完整歷史紀錄,也不保證跨工作流程執行永遠保留。GitHub Actions Cache 項目以 key 建立後不可覆寫,而目前 workflow 使用固定的 repository/ref key;後續執行可能恢復較舊的狀態,因此 transition-only 判斷不一定會以緊鄰上一輪執行為基準。PR gate 與 `master` 維護執行可能各自使用不同的快取內容,不能把快取當作連結目前一定可用的證明。 + +如果 webhook value 曾經暴露(包括提交到 Git、日誌或公開訊息),請立即在 Discord 撤銷該 webhook 並建立新的 webhook,再更新 GitHub Actions secret;不要繼續使用已暴露的 URL。 + ## Button ### Slider Button diff --git a/package.json b/package.json index c6808e8..7889d83 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,9 @@ "develop": "gatsby develop", "dev": "gatsby develop -H 0.0.0.0", "build": "gatsby build", - "prettier": "prettier --write ." + "prettier": "prettier --write .", + "test:gallery-links": "node scripts/check-gallery-links.mjs", + "notify-gallery-links": "node scripts/notify-gallery-links.mjs" }, "engines": { "node": ">=22", diff --git a/scripts/check-gallery-links.mjs b/scripts/check-gallery-links.mjs new file mode 100644 index 0000000..f0fbf3f --- /dev/null +++ b/scripts/check-gallery-links.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node + +import { readdir, readFile, rename, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const DEFAULT_DIRECTORY = join( + dirname(fileURLToPath(import.meta.url)), + "..", + "src", + "data", + "gallery", +); +const TIMEOUT_MS = 10_000; + +export function extractGoogleDriveFolderId(value) { + try { + const url = new URL(value); + if (url.protocol !== "https:") return null; + if (!["drive.google.com", "www.drive.google.com"].includes(url.hostname)) + return null; + const match = url.pathname.match(/^\/drive\/folders\/([^/]+)\/?$/); + return match?.[1] || null; + } catch { + return null; + } +} + +export function classifyGoogleDriveResponse(response, html = "") { + const finalUrl = response?.url || ""; + const lowerUrl = finalUrl.toLowerCase(); + const lowerHtml = String(html).toLowerCase(); + const permissionMarkers = [ + "request access", + "you need access", + "ask for access", + "sign in to continue", + "permission denied", + "access denied", + "drive.google.com/drive/u/0/my-drive", + ]; + + const finalUrlObject = (() => { + try { + return new URL(finalUrl); + } catch { + return null; + } + })(); + const isDriveLoginEndpoint = + finalUrlObject && + ["drive.google.com", "www.drive.google.com"].includes( + finalUrlObject.hostname, + ) && + /\/(?:servicelogin|signin|login)(?:[/?#]|$)/i.test( + finalUrlObject.pathname + finalUrlObject.search, + ); + + if (lowerUrl.includes("accounts.google.com") || isDriveLoginEndpoint) { + return { + status: "permission", + reason: "Google redirected to a sign-in page", + }; + } + if ( + permissionMarkers.some( + (marker) => lowerUrl.includes(marker) || lowerHtml.includes(marker), + ) + ) { + return { + status: "permission", + reason: "Google Drive reports that access is restricted", + }; + } + + if (response && response.ok === false) { + return { + status: "unknown", + reason: `Google Drive returned HTTP ${response.status}`, + }; + } + + const drivePage = lowerUrl.includes("drive.google.com/drive/folders/"); + const contentMarkers = [ + "drive-viewer-content", + "drive-viewer-list", + "drive.google.com/drive/folders/", + "application/vnd.google-apps.folder", + 'data-id="folder', + "folderview", + ]; + const hasFolderContent = contentMarkers.some((marker) => + lowerHtml.includes(marker), + ); + if (drivePage && hasFolderContent) return { status: "accessible" }; + return { + status: "unknown", + reason: + "The response did not contain recognizable Google Drive folder content", + }; +} + +function invalidResult(url, reason) { + return { + status: "invalid", + url, + finalUrl: url, + reason: `Invalid Google Drive URL: ${reason}`, + }; +} + +function isRetryableNetworkError(error) { + if (!error) return false; + if (error.name === "TimeoutError" || error.name === "AbortError") return true; + if (error.name !== "TypeError") + return [ + "ECONNRESET", + "ECONNREFUSED", + "ENOTFOUND", + "EAI_AGAIN", + "ETIMEDOUT", + ].includes(error.code); + const message = + `${error.message || ""} ${error.cause?.message || ""}`.toLowerCase(); + return /fetch failed|network|socket|econnreset|econnrefused|enotfound|eai_again|etimedout|timed out|dns/.test( + message, + ); +} + +export async function checkGoogleDriveFolder(url, options = {}) { + const originalUrl = String(url); + let parsed; + try { + parsed = new URL(originalUrl); + } catch { + return invalidResult(originalUrl, "URL could not be parsed"); + } + if (parsed.protocol !== "https:") + return invalidResult(originalUrl, "protocol must be HTTPS"); + if (!["drive.google.com", "www.drive.google.com"].includes(parsed.hostname)) { + return invalidResult(originalUrl, "host must be drive.google.com"); + } + if (!extractGoogleDriveFolderId(originalUrl)) { + return invalidResult( + originalUrl, + "path must match /drive/folders/", + ); + } + + const fetcher = options.fetch ?? fetch; + let lastError; + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + const response = await fetcher(originalUrl, { + redirect: "follow", + signal: AbortSignal.timeout(options.timeout ?? TIMEOUT_MS), + }); + let html; + try { + html = await response.text(); + } catch (error) { + lastError = error; + if (isRetryableNetworkError(error) && attempt === 0) continue; + return { + status: "network", + url: originalUrl, + finalUrl: response.url || originalUrl, + reason: "Failed to read Google Drive response body", + error: error?.message, + }; + } + const classification = classifyGoogleDriveResponse(response, html); + return { + ...classification, + url: originalUrl, + finalUrl: response.url || originalUrl, + }; + } catch (error) { + lastError = error; + const retryable = isRetryableNetworkError(error); + if (!retryable || attempt === 1) break; + } + } + const timedOut = + lastError?.name === "TimeoutError" || lastError?.name === "AbortError"; + return { + status: "network", + url: originalUrl, + finalUrl: originalUrl, + reason: timedOut + ? "Request timed out after two attempts" + : "Network request failed after two attempts", + error: lastError?.message, + }; +} + +export async function loadGalleryLinks(directory = DEFAULT_DIRECTORY) { + const entries = (await readdir(directory, { withFileTypes: true })) + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .sort((a, b) => a.name.localeCompare(b.name)); + const records = []; + for (const entry of entries) { + const filename = entry.name; + try { + const record = JSON.parse( + await readFile(join(directory, filename), "utf8"), + ); + if (!record || typeof record !== "object" || Array.isArray(record)) { + records.push({ + invalidRecord: true, + filename, + error: "Invalid gallery JSON: record must be an object", + }); + continue; + } + if ( + record.gdrive_url === undefined || + record.gdrive_url === null || + typeof record.gdrive_url === "string" + ) { + if (typeof record.gdrive_url === "string" && record.gdrive_url.trim()) { + records.push({ + url: record.gdrive_url.trim(), + name: record.name || filename, + date: record.date || "", + filename, + }); + } + } else { + records.push({ + invalidRecord: true, + filename, + error: "Invalid gallery JSON: gdrive_url must be a string or null", + }); + } + } catch (error) { + records.push({ + invalidRecord: true, + filename, + error: `Invalid gallery JSON: ${error.message}`, + }); + } + } + return records; +} + +function usage() { + return "Usage: yarn node scripts/check-gallery-links.mjs [--url ] [--report ]"; +} + +function reportKey(item) { + return createHash("sha256") + .update(`${item.filename || "--url"}\0${item.url}`) + .digest("hex"); +} + +function reportFailure(item, result) { + return { + key: reportKey(item), + name: item.name || item.filename || "Provided URL", + date: item.date || "", + filename: item.filename || "--url", + url: item.url || "", + status: result.status, + ...(result.reason ? { reason: result.reason } : {}), + }; +} + +async function writeReport(reportPath, items, results) { + const failures = results + .map((result, index) => ({ result, item: items[index] })) + .filter(({ result }) => result.status !== "accessible") + .map(({ item, result }) => reportFailure(item, result)) + .sort((a, b) => a.key.localeCompare(b.key)); + const report = { + checked: results.length, + passed: results.length - failures.length, + failed: failures.length, + failures, + }; + const destination = resolve(reportPath); + const temporary = `${destination}.tmp-${process.pid}-${Date.now()}`; + await writeFile(temporary, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + await rename(temporary, destination); +} + +function label(result) { + if (result.status === "accessible") return "✅ Accessible"; + if (result.status === "permission") return "❌ Permission denied"; + if (result.status === "invalid") return "❌ Invalid Google Drive URL"; + if (result.status === "network") + return result.reason?.toLowerCase().includes("timed out") + ? "❌ Timeout" + : "❌ Network error"; + return "⚠️ Unable to determine accessibility"; +} + +export async function main(argv = process.argv.slice(2)) { + if (argv.includes("--help") || argv.includes("-h")) { + console.log( + `${usage()}\n\nChecks Google Drive gallery folders anonymously.`, + ); + return 0; + } + const urlIndex = argv.indexOf("--url"); + const reportIndex = argv.indexOf("--report"); + if (reportIndex !== -1 && !argv[reportIndex + 1]) { + console.error("--report requires a path"); + return 1; + } + const reportPath = reportIndex === -1 ? null : argv[reportIndex + 1]; + let items; + if (urlIndex !== -1) { + if (!argv[urlIndex + 1]) { + console.error("--url requires a URL"); + return 1; + } + items = [{ url: argv[urlIndex + 1], name: "Provided URL", date: "" }]; + } else { + items = await loadGalleryLinks(); + } + + console.log("Google Drive Gallery Accessibility Check"); + const results = []; + for (const item of items) { + if (item.invalidRecord) { + console.log( + `${item.filename}: ❌ Invalid gallery record (${item.error})`, + ); + results.push({ status: "invalid", reason: "Invalid gallery record" }); + continue; + } + const result = await checkGoogleDriveFolder(item.url); + results.push(result); + const suffix = result.reason ? ` — ${result.reason}` : ""; + console.log( + `${item.name}${item.date ? ` (${item.date})` : ""}: ${label(result)}${suffix}`, + ); + if (result.finalUrl && result.finalUrl !== result.url) + console.log(` Final URL: ${result.finalUrl}`); + } + const passed = results.filter( + (result) => result.status === "accessible", + ).length; + const failed = results.length - passed; + console.log(`Checked: ${results.length}`); + console.log(`Passed: ${passed}`); + console.log(`Failed: ${failed}`); + if (reportPath) await writeReport(reportPath, items, results); + return failed ? 1 : 0; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main() + .then((code) => { + process.exitCode = code; + }) + .catch((error) => { + console.error(`Gallery link checker failed: ${error.message}`); + process.exitCode = 1; + }); +} diff --git a/scripts/notify-gallery-links.mjs b/scripts/notify-gallery-links.mjs new file mode 100644 index 0000000..d09468e --- /dev/null +++ b/scripts/notify-gallery-links.mjs @@ -0,0 +1,229 @@ +#!/usr/bin/env node +import { readFile, writeFile, rename } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { randomUUID } from "node:crypto"; + +const TIMEOUT_MS = 10_000; +const MAX_NAMES = 10; +const MAX_CONTENT = 2000; +const ALLOWED_REASONS = new Set(["permission", "invalid"]); + +function usage() { + throw new Error( + "Usage: node scripts/notify-gallery-links.mjs --report --state --workflow-url ", + ); +} + +function parseArgs(argv) { + const result = {}; + for (let i = 2; i < argv.length; i += 2) { + const flag = argv[i]; + const value = argv[i + 1]; + if (!flag?.startsWith("--") || !value || result[flag.slice(2)]) usage(); + result[flag.slice(2)] = value; + } + if ( + !result.report || + !result.state || + !result["workflow-url"] || + Object.keys(result).length !== 3 + ) + usage(); + return result; +} + +function validReport(report) { + if ( + !report || + typeof report !== "object" || + !Number.isInteger(report.checked) || + !Number.isInteger(report.passed) || + !Number.isInteger(report.failed) || + !Array.isArray(report.failures) || + report.checked < 0 || + report.passed < 0 || + report.failed < 0 || + report.checked !== report.passed + report.failed || + report.failures.length !== report.failed + ) + throw new Error("Malformed report"); + return true; +} + +function filteredFailures(report) { + validReport(report); + const candidates = report.failures + .map((failure) => { + if ( + !failure || + typeof failure !== "object" || + ["key", "name", "date", "filename", "url", "status", "reason"].some( + (field) => typeof failure[field] !== "string", + ) + ) { + throw new Error("Malformed report"); + } + return failure; + }) + .filter((failure) => ALLOWED_REASONS.has(failure.status)); + const byKey = new Map(); + for (const failure of candidates) { + if (!failure.key) throw new Error("Malformed report"); + if (!byKey.has(failure.key)) + byKey.set(failure.key, { + key: failure.key, + name: failure.name || failure.key, + reason: failure.reason, + status: failure.status, + }); + } + return [...byKey.values()]; +} + +function validState(state) { + return ( + state && + typeof state === "object" && + (state.status === "healthy" || state.status === "failing") && + Array.isArray(state.failureKeys) && + state.failureKeys.every((key) => typeof key === "string") && + Number.isInteger(state.permissionCount) && + Number.isInteger(state.invalidCount) + ); +} + +async function readJson(path) { + return JSON.parse(await readFile(path, "utf8")); +} + +async function saveState(path, state) { + const temporary = join( + dirname(path), + `.${stateFileName(path)}.${randomUUID()}.tmp`, + ); + await writeFile(temporary, `${JSON.stringify(state)}\n`, { mode: 0o600 }); + await rename(temporary, path); +} +function stateFileName(path) { + return path.split("/").pop() || "state"; +} + +function makePayload({ transition, failures, previous, workflowUrl }) { + const names = failures.slice(0, MAX_NAMES).map(({ name }) => name); + const omitted = failures.length - names.length; + let content; + if (transition === "recovery") { + content = `Gallery link checks recovered.\n\nPrevious failures: ${previous.permissionCount} permission, ${previous.invalidCount} invalid.`; + } else { + content = `${transition === "initial" ? "Gallery link checks failing" : "Gallery link failures changed"}: ${failures.length} affected.`; + if (names.length) { + const affected = names.map((name) => `- ${name}`).join("\n"); + content += `\n\nAffected galleries:\n${affected}`; + if (omitted) content += `\n- ...and ${omitted} more`; + } + } + content += `\n\nWorkflow: ${String(workflowUrl).slice(0, 500)}`; + return { + content: + content.length <= MAX_CONTENT + ? content + : `${content.slice(0, MAX_CONTENT - 1)}…`, + allowed_mentions: { parse: [] }, + }; +} + +async function send(payload, webhook, fetchImpl = globalThis.fetch) { + if (typeof fetchImpl !== "function") throw new Error("Fetch unavailable"); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), TIMEOUT_MS); + try { + const response = await fetchImpl(webhook, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + signal: controller.signal, + }); + if (!response || !response.ok) throw new Error("Discord request failed"); + } catch (error) { + if (error?.name === "AbortError") + throw new Error("Discord request timed out"); + throw error instanceof Error && error.message === "Discord request failed" + ? error + : new Error("Discord request failed"); + } finally { + clearTimeout(timer); + } +} + +export async function processNotification({ + report, + statePath, + workflowUrl, + webhook = process.env.DISCORD_GALLERY_WEBHOOK_URL, + fetchImpl = globalThis.fetch, +}) { + const failures = filteredFailures(report); + const keys = [...new Set(failures.map(({ key }) => key))].sort(); + let previous = null; + try { + previous = await readJson(statePath); + if (!validState(previous)) previous = null; + } catch { + /* corrupt/missing means no previous state */ + } + const status = keys.length ? "failing" : "healthy"; + const failureSignature = failures + .map(({ key, status }) => `${key}:${status}`) + .sort(); + const changed = + !previous || + previous.status !== status || + (status === "failing" && + JSON.stringify(previous.failureKeys) !== + JSON.stringify(failureSignature)); + const transition = + status === "healthy" + ? previous?.status === "failing" + ? "recovery" + : null + : changed + ? previous + ? "changed" + : "initial" + : null; + if (transition) { + if (!webhook) throw new Error("Missing Discord webhook"); + await send( + makePayload({ + transition, + failures, + previous: previous ?? { permissionCount: 0, invalidCount: 0 }, + workflowUrl, + }), + webhook, + fetchImpl, + ); + } + const next = { + status, + failureKeys: failureSignature, + permissionCount: failures.filter((f) => f.status === "permission").length, + invalidCount: failures.filter((f) => f.status === "invalid").length, + }; + await saveState(statePath, next); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + try { + const args = parseArgs(process.argv); + await processNotification({ + report: await readJson(args.report), + statePath: args.state, + workflowUrl: args["workflow-url"], + }); + process.exitCode = 0; + } catch (error) { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; + } +}