feat(deck): hydrate missing live charts from moenotes-assets

docs/deck-worker.md

@@ -86,9 +86,12 @@ against the message before using it. Then it:
 2. imports the glue, a wasm-bindgen `--target web` ES module, from a `blob:` URL of its verified bytes, and calls its
    default export with `{ module_or_path: wasmBytes }`, so the WASM is never fetched again unverified;
 3. when `masterOverlayUrl` (`/api/v1/meta/deck-master?server=…&v=…`) is present on `init`, merges the live Next.js
-   columnar `master` tables (`moenotes.deck-master-overlay/1`) onto the verified `deck-data.json` snapshot, recomputes
-   the merged dataset SHA-256, and constructs `new DeckSolver(mergedBytes, { datasetId: mergedSha256 })` (falling back
-   cleanly to the baseline `deckDataBytes` if the overlay endpoint is unreachable or rejected);
+   columnar `master` tables (`moenotes.deck-master-overlay/1`) onto the verified `deck-data.json` snapshot, hydrates any
+   `MasterLiveMusic` chart difficulties not yet present in `deck-data.json` via `chartsOverlayUrl`
+   (`/api/v1/meta/deck-charts?server=…&v=…&ids=…`, `moenotes.deck-chart-overlay/1`, converted in binary32 precision from
+   `moenotes-assets` `Live/MusicScore/<chartKey>/<score>.json`), recomputes the merged dataset SHA-256, and constructs
+   `new DeckSolver(mergedBytes, { datasetId: mergedSha256 })` (falling back cleanly to master-only or the baseline
+   `deckDataBytes` if an overlay endpoint is unreachable or rejected);
 4. checks that `solver.datasetId`, the lowercase hex SHA-256 of the active deck data bytes, matches the expected hash;
 5. reads `solver.capabilities()` as JSON text, when the solver has it, and returns the dataset catalogue in `ready`.
 

public/deck/deck-worker-core.mjs

@@ -88,6 +88,50 @@ export async function loadDeckSolverModule({ glue, wasm }) {
 
 const FILES = [['deckData', 'deck data'], ['wasm', 'engine WASM'], ['glue', 'engine glue']];
 const DECK_MASTER_OVERLAY_FORMAT = 'moenotes.deck-master-overlay/1';
+const DECK_CHART_OVERLAY_FORMAT = 'moenotes.deck-chart-overlay/1';
+const CHART_OVERLAY_BATCH_SIZE = 80;
+
+function findMissingChartScoreIds(master, existingCharts) {
+  if (!record(master)) return [];
+  const scoreTable = master.MasterLiveMusicScore;
+  const musicTable = master.MasterLiveMusic;
+  if (!record(scoreTable) || !Array.isArray(scoreTable.columns) || !Array.isArray(scoreTable.rows)) return [];
+  if (!record(musicTable) || !Array.isArray(musicTable.columns) || !Array.isArray(musicTable.rows)) return [];
+  const scoreIdCol = scoreTable.columns.indexOf('_id');
+  const fileNameCol = scoreTable.columns.indexOf('_musicScoreTextFileName');
+  if (scoreIdCol < 0 || fileNameCol < 0) return [];
+
+  const exportableScoreIds = new Set();
+  for (const row of scoreTable.rows) {
+    if (!Array.isArray(row) || row.length !== scoreTable.columns.length) continue;
+    const id = row[scoreIdCol];
+    const fileName = row[fileNameCol];
+    if (positiveId(id) && typeof fileName === 'string' && fileName.trim().length > 0) {
+      exportableScoreIds.add(id);
+    }
+  }
+  if (exportableScoreIds.size === 0) return [];
+
+  const existingScoreIds = new Set(
+    Array.isArray(existingCharts)
+      ? existingCharts.filter(chart => record(chart) && positiveId(chart.scoreId)).map(chart => chart.scoreId)
+      : [],
+  );
+  const diffCols = DIFFICULTIES.map(([, col]) => musicTable.columns.indexOf(col)).filter(idx => idx >= 0);
+  if (diffCols.length === 0) return [];
+
+  const missing = new Set();
+  for (const row of musicTable.rows) {
+    if (!Array.isArray(row) || row.length !== musicTable.columns.length) continue;
+    for (const colIdx of diffCols) {
+      const scoreId = row[colIdx];
+      if (positiveId(scoreId) && exportableScoreIds.has(scoreId) && !existingScoreIds.has(scoreId)) {
+        missing.add(scoreId);
+      }
+    }
+  }
+  return [...missing].sort((a, b) => a - b);
+}
 
 function invalidInit(message) {
   if (message.protocol !== DECK_WORKER_PROTOCOL) return 'Unsupported protocol ' + String(message.protocol);
@@ -97,6 +141,7 @@ function invalidInit(message) {
   }
   if (message.modelCommit !== null && (typeof message.modelCommit !== 'string' || !message.modelCommit)) return 'Invalid modelCommit';
   if (message.masterOverlayUrl !== undefined && (typeof message.masterOverlayUrl !== 'string' || !message.masterOverlayUrl)) return 'Invalid masterOverlayUrl';
+  if (message.chartsOverlayUrl !== undefined && (typeof message.chartsOverlayUrl !== 'string' || !message.chartsOverlayUrl)) return 'Invalid chartsOverlayUrl';
   return null;
 }
 
@@ -167,11 +212,22 @@ export function createDeckWorkerCore({
     return value === undefined ? null : JSON.stringify(value);
   }
 
-  async function downloadOverlay(rawUrl) {
+  function resolveEndpointUrl(rawUrl, anchorUrl) {
     if (typeof rawUrl !== 'string' || !rawUrl) return null;
-    const url = rawUrl.startsWith('/') && typeof globalThis.location?.origin === 'string' && globalThis.location.origin !== 'null'
-      ? new URL(rawUrl, globalThis.location.origin).href
-      : rawUrl;
+    if (rawUrl.startsWith('/')) {
+      if (typeof anchorUrl === 'string' && /^https?:\/\//i.test(anchorUrl)) {
+        return new URL(rawUrl, anchorUrl).href;
+      }
+      if (typeof globalThis.location?.origin === 'string' && globalThis.location.origin !== 'null') {
+        return new URL(rawUrl, globalThis.location.origin).href;
+      }
+    }
+    return rawUrl;
+  }
+
+  async function downloadOverlay(rawUrl) {
+    const url = resolveEndpointUrl(rawUrl, null);
+    if (!url) return null;
     try {
       const response = await fetch(url);
       if (!response.ok) return null;
@@ -183,11 +239,60 @@ export function createDeckWorkerCore({
     }
   }
 
+  async function downloadMissingCharts(rawBaseUrl, anchorUrl, missingScoreIds) {
+    const baseUrl = resolveEndpointUrl(rawBaseUrl, anchorUrl);
+    if (!baseUrl || !Array.isArray(missingScoreIds) || missingScoreIds.length === 0) {
+      return { charts: [], musicLengthsMs: {} };
+    }
+    const charts = [];
+    const musicLengthsMs = {};
+    const seen = new Set();
+    for (let offset = 0; offset < missingScoreIds.length; offset += CHART_OVERLAY_BATCH_SIZE) {
+      const chunk = missingScoreIds.slice(offset, offset + CHART_OVERLAY_BATCH_SIZE);
+      const separator = baseUrl.includes('?') ? '&' : '?';
+      const url = baseUrl + separator + 'ids=' + chunk.join(',');
+      try {
+        const response = await fetch(url);
+        if (!response.ok) continue;
+        const parsed = await response.json();
+        if (!record(parsed) || parsed.format !== DECK_CHART_OVERLAY_FORMAT || !Array.isArray(parsed.charts)) continue;
+        const lengths = record(parsed.musicLengthsMs) ? parsed.musicLengthsMs : {};
+        for (const chart of parsed.charts) {
+          if (!record(chart) || !positiveId(chart.scoreId) || seen.has(chart.scoreId)) continue;
+          seen.add(chart.scoreId);
+          charts.push(chart);
+          const lengthVal = lengths[String(chart.scoreId)] ?? chart.durationMs;
+          if (Number.isSafeInteger(lengthVal) && lengthVal > 0) {
+            musicLengthsMs[String(chart.scoreId)] = lengthVal;
+          }
+        }
+      } catch {
+        // Ignore chart overlay batch transport failures; unavailable songs stay filtered out of catalog.musics.
+      }
+    }
+    return { charts, musicLengthsMs };
+  }
+
+  async function tryCreateCandidateSolver(DeckSolver, candidateData) {
+    try {
+      const mergedBytes = new TextEncoder().encode(JSON.stringify(candidateData));
+      const mergedSha256 = hex(await digest(mergedBytes.buffer));
+      const candidate = new DeckSolver(mergedBytes, { datasetId: mergedSha256 });
+      if (candidate && typeof candidate.datasetId === 'string' && candidate.datasetId === mergedSha256) {
+        return { solver: candidate, datasetId: mergedSha256, data: candidateData };
+      }
+      candidate?.free?.();
+    } catch {
+      // Fall back cleanly if the engine rejects overlaid tables or hydrated charts.
+    }
+    return null;
+  }
+
   async function initialize(message) {
     const problem = invalidInit(message);
     if (problem) { post({ type: 'failed', code: 'protocol', message: problem }); return; }
     const key = JSON.stringify([message.deckDataUrl, message.deckDataSha256, message.deckDataBytes, message.wasmUrl, message.wasmSha256,
-      message.wasmBytes, message.glueUrl, message.glueSha256, message.glueBytes, message.modelCommit, message.masterOverlayUrl ?? null]);
+      message.wasmBytes, message.glueUrl, message.glueSha256, message.glueBytes, message.modelCommit, message.masterOverlayUrl ?? null, message.chartsOverlayUrl ?? null]);
     if (initKey !== null) {
       if (key !== initKey) post({ type: 'failed', code: 'protocol', message: 'This Worker is bound to another runtime' });
       else post(ready ?? failure);
@@ -206,24 +311,51 @@ export function createDeckWorkerCore({
       try { DeckSolver = await loadSolver({ glue, wasm }); }
       catch (error) { throw new Failure('init', 'Engine: ' + describe(error)); }
 
+      const baseMaster = record(data.master) ? data.master : {};
+      const mergedMaster = overlay ? { ...baseMaster, ...overlay.master } : baseMaster;
+      const baseCharts = Array.isArray(data.charts) ? data.charts : [];
+      const chartsOverlayUrl = message.chartsOverlayUrl ?? (record(overlay) && typeof overlay.chartsOverlayUrl === 'string' ? overlay.chartsOverlayUrl : undefined);
+      const missingScoreIds = chartsOverlayUrl ? findMissingChartScoreIds(mergedMaster, baseCharts) : [];
+      const hydrated = missingScoreIds.length > 0
+        ? await downloadMissingCharts(chartsOverlayUrl, message.masterOverlayUrl, missingScoreIds)
+        : { charts: [], musicLengthsMs: {} };
+
       let created = null;
       let expectedDatasetId = message.deckDataSha256;
       let activeData = data;
-      if (overlay) {
-        try {
-          const mergedData = { ...data, master: { ...(record(data.master) ? data.master : {}), ...overlay.master } };
-          const mergedBytes = new TextEncoder().encode(JSON.stringify(mergedData));
-          const mergedSha256 = hex(await digest(mergedBytes.buffer));
-          const candidate = new DeckSolver(mergedBytes, { datasetId: mergedSha256 });
-          if (candidate && typeof candidate.datasetId === 'string' && candidate.datasetId === mergedSha256) {
-            created = candidate;
-            expectedDatasetId = mergedSha256;
-            activeData = mergedData;
-          } else {
-            candidate?.free?.();
-          }
-        } catch {
-          // Fall back cleanly to the verified baseline deck-data.json if the engine rejects overlaid tables.
+
+      if (hydrated.charts.length > 0) {
+        const baseProvenance = record(data.provenance) ? data.provenance : {};
+        const baseReplay = record(baseProvenance.replay) ? baseProvenance.replay : {};
+        const baseLengths = record(baseReplay.musicLengthsMs) ? baseReplay.musicLengthsMs : {};
+        const withCharts = await tryCreateCandidateSolver(DeckSolver, {
+          ...data,
+          provenance: {
+            ...baseProvenance,
+            replay: {
+              ...baseReplay,
+              musicLengthsMs: { ...baseLengths, ...hydrated.musicLengthsMs },
+            },
+          },
+          master: mergedMaster,
+          charts: [...baseCharts, ...hydrated.charts],
+        });
+        if (withCharts) {
+          created = withCharts.solver;
+          expectedDatasetId = withCharts.datasetId;
+          activeData = withCharts.data;
+        }
+      }
+
+      if (!created && overlay) {
+        const withMasterOnly = await tryCreateCandidateSolver(DeckSolver, {
+          ...data,
+          master: mergedMaster,
+        });
+        if (withMasterOnly) {
+          created = withMasterOnly.solver;
+          expectedDatasetId = withMasterOnly.datasetId;
+          activeData = withMasterOnly.data;
         }
       }
 

src/app/api/v1/meta/deck-charts/route.ts

@@ -0,0 +1,57 @@
+import type { NextRequest } from "next/server";
+import { buildEnv } from "@/config/build-env";
+import { isGameServer, PRIMARY_SERVER } from "@/config/servers";
+import { forceRefreshMasterData, shouldBypassMasterDataCache } from "@/lib/masterdata/build-snapshot";
+import { getDeckChartOverlay, parseScoreIdList } from "@/lib/masterdata/deck-charts";
+
+export const dynamic = "force-dynamic";
+
+function isPurgeAuthorized(request: NextRequest): boolean {
+  const expected = buildEnv("MOENOTES_CACHE_PURGE_TOKEN")?.trim();
+  if (!expected) return true;
+  const auth = request.headers.get("authorization") ?? "";
+  const bearer = auth.toLowerCase().startsWith("bearer ") ? auth.slice(7).trim() : "";
+  const headerToken = request.headers.get("x-moenotes-purge-token")?.trim() ?? "";
+  return bearer === expected || headerToken === expected;
+}
+
+export async function GET(request: NextRequest) {
+  const rawServer = request.nextUrl.searchParams.get("server")?.trim();
+  if (rawServer && !isGameServer(rawServer)) {
+    return Response.json(
+      { error: "Invalid game server" },
+      { status: 400, headers: { "Cache-Control": "no-store" } },
+    );
+  }
+  const server = rawServer && isGameServer(rawServer) ? rawServer : PRIMARY_SERVER;
+  const scoreIds = parseScoreIdList(request.nextUrl.searchParams.get("ids"));
+  const retryParam = request.nextUrl.searchParams.get("retry")?.trim();
+  const isRetry = Boolean(retryParam && retryParam !== "0" && retryParam.toLowerCase() !== "false");
+
+  const requestedBypass = shouldBypassMasterDataCache({
+    searchParams: request.nextUrl.searchParams,
+    headers: request.headers,
+  });
+  const forceBypass = requestedBypass && isPurgeAuthorized(request);
+  if (forceBypass) {
+    await forceRefreshMasterData({ purgeTables: true });
+  }
+
+  const payload = await getDeckChartOverlay(server, scoreIds, {
+    ...(forceBypass ? { forceBypass: true } : {}),
+    ...(isRetry ? { retry: true } : {}),
+  });
+
+  const versionParam = request.nextUrl.searchParams.get("v")?.trim();
+  const allResolved = scoreIds.length > 0 && payload.charts.length === scoreIds.length;
+  const cacheControl = !forceBypass && !isRetry && versionParam && allResolved
+    ? "private, max-age=60, stale-while-revalidate=300"
+    : "no-store, no-cache, max-age=0, must-revalidate";
+
+  return Response.json(payload, {
+    headers: {
+      "Cache-Control": cacheControl,
+      ...(forceBypass ? { "X-Moenotes-Cache-Bypass": "1" } : {}),
+    },
+  });
+}

src/components/deck/use-deck-solver.ts

@@ -1,6 +1,14 @@
 import { useCallback, useEffect, useRef, useState } from "react";
 import type { GameServer } from "@/config/servers";
-import { buildDeckMasterOverlayUrl, resolveDeckRuntime, type DeckRuntimeFailureCode, type DeckRuntimeUnavailableReason, type DeckSolverRuntime } from "@/lib/deck/runtime-source";
+import {
+  buildDeckChartsOverlayUrl,
+  buildDeckMasterOverlayUrl,
+  resolveDeckRuntime,
+  resolveLiveDeckRuntimeFallback,
+  type DeckRuntimeFailureCode,
+  type DeckRuntimeUnavailableReason,
+  type DeckSolverRuntime,
+} from "@/lib/deck/runtime-source";
 import { DeckWorkerClient } from "@/lib/deck/worker-client";
 import type { DeckDataCatalog, DeckWorkerFailureCode } from "@/lib/deck/worker-protocol";
 import { parseCapabilities, type DeckSolverCapabilities } from "@/lib/deck/goals";
@@ -60,16 +68,23 @@ export function useDeckSolver(server: GameServer, active: boolean) {
     const abort = new AbortController();
     setEngine({ status: "loading" }); setJob({ status: "idle" }); setGrowthByTeam({});
     void (async () => {
-      const resolved = await resolveDeckRuntime(server, { signal: abort.signal }).catch(error => {
+      let resolved = await resolveDeckRuntime(server, { signal: abort.signal }).catch(error => {
         if (abort.signal.aborted) return null;
         return { status: "failed" as const, code: "network" as const, message: error instanceof Error ? error.message : String(error) };
       });
       if (!resolved || abort.signal.aborted) return;
+      if (resolved.status !== "available") {
+        const fallback = await resolveLiveDeckRuntimeFallback(server, { signal: abort.signal }).catch(() => null);
+        if (abort.signal.aborted) return;
+        if (fallback) resolved = { status: "available", runtime: fallback };
+      }
       if (resolved.status === "unavailable") { setEngine({ status: "unavailable", reason: resolved.reason }); return; }
       if (resolved.status === "failed") { setEngine({ status: "failed", code: resolved.code, message: resolved.message }); return; }
+      const localVersion = getLocalServerMasterDataVersion(server);
       const runtime: DeckSolverRuntime = {
         ...resolved.runtime,
-        masterOverlayUrl: buildDeckMasterOverlayUrl(server, getLocalServerMasterDataVersion(server)),
+        masterOverlayUrl: buildDeckMasterOverlayUrl(server, localVersion, undefined, attempt),
+        chartsOverlayUrl: buildDeckChartsOverlayUrl(server, localVersion, undefined, attempt),
       };
       const ready = await worker.prepare(runtime);
       if (abort.signal.aborted) return;

src/config/assets.ts

@@ -55,6 +55,7 @@ export const assetConfig = {
   deck: {
     workerUrl: "/deck/deck-worker.js",
     masterOverlayUrl: "/api/v1/meta/deck-master",
+    chartsOverlayUrl: "/api/v1/meta/deck-charts",
     recommendEngine: process.env.NODE_ENV === "development" && process.env.PUBLIC_DECK_RECOMMEND_ENGINE !== undefined
       ? (String(process.env.PUBLIC_DECK_RECOMMEND_ENGINE) || null)
       : "/haneoka-team-engine/runtime.json",