media-library : optimizations

tools/media-library/core/admin-rate-limit.js

@@ -0,0 +1,100 @@
+/**
+ * Shared rate limiter for admin.hlx.page (10 req/s per project).
+ * Same pattern as backfill tool: promise-chain acquire, 429 retry with backoff.
+ */
+
+import isPerfEnabled from './params.js';
+
+const ADMIN_API_RATE = 8; /* keep some headroom under the 10 RPS project limit */
+
+function waitForDelay(ms, signal = null) {
+  const duration = Math.max(0, ms);
+  if (signal?.aborted) return Promise.resolve();
+  return new Promise((resolve) => {
+    const timeoutId = setTimeout(resolve, duration);
+    if (signal) {
+      signal.addEventListener('abort', () => {
+        clearTimeout(timeoutId);
+        resolve();
+      }, { once: true });
+    }
+  });
+}
+
+function createRateLimiter(initialRate = ADMIN_API_RATE, getSignal = () => null) {
+  let intervalMs = Math.ceil(1000 / initialRate);
+  let queue = Promise.resolve();
+
+  return {
+    acquire() {
+      const gate = queue;
+      queue = queue.then(() => waitForDelay(intervalMs, getSignal()));
+      return gate;
+    },
+    handleResponse(res) {
+      const rate = parseFloat(res?.headers?.get?.('x-ratelimit-rate'), 10);
+      if (Number.isFinite(rate) && rate > 0) {
+        intervalMs = Math.ceil(1000 / rate);
+      }
+    },
+    backoff(seconds) {
+      const ms = Math.max(0, (typeof seconds === 'number' ? seconds : 1) * 1000);
+      queue = queue.then(() => waitForDelay(ms, getSignal()));
+    },
+    reset() {
+      queue = Promise.resolve();
+    },
+  };
+}
+
+let adminLimiter = null;
+
+export function getAdminRateLimiter() {
+  if (!adminLimiter) {
+    adminLimiter = createRateLimiter(ADMIN_API_RATE);
+  }
+  return adminLimiter;
+}
+
+/**
+ * Fetch admin.hlx.page URL with rate limit and 429/503 retry.
+ * Call only for URLs under https://admin.hlx.page.
+ */
+export async function fetchAdminWithRateLimit(
+  url,
+  options = {},
+  { maxRetries = 3, signal } = {},
+) {
+  const limiter = getAdminRateLimiter();
+  const fetchOptions = { ...options, credentials: 'include', signal };
+
+  async function attempt(attemptNumber) {
+    await limiter.acquire();
+    const res = await fetch(url, fetchOptions);
+    limiter.handleResponse(res);
+
+    if (res.status === 429 && attemptNumber < maxRetries) {
+      const headerVal = parseInt(
+        res.headers.get('x-retry-after') || res.headers.get('retry-after'),
+        10,
+      );
+      const retryAfter = Math.max(headerVal || 2 ** attemptNumber, 30);
+      if (isPerfEnabled()) {
+        // eslint-disable-next-line no-console
+        console.log(`[admin-api] 429 rate limit hit, backing off ${retryAfter}s (attempt ${attemptNumber + 1}/${maxRetries + 1})`);
+      }
+      limiter.backoff(retryAfter);
+      return attempt(attemptNumber + 1);
+    }
+
+    if (res.status === 503 && attemptNumber < maxRetries) {
+      const delaySec = 2 ** attemptNumber;
+      limiter.backoff(delaySec);
+      return attempt(attemptNumber + 1);
+    }
+
+    return res;
+  }
+
+  return attempt(0);
+}

tools/media-library/core/constants.js

@@ -4,8 +4,14 @@ export const IndexConfig = Object.freeze({
   INCREMENTAL_WINDOW_MS: 10000,
   API_PAGE_SIZE: 1000,
   MAX_CONCURRENT_FETCHES: 10,
+  MAX_CONCURRENT_PAGE_FETCHES: 4,
+  USAGE_MAP_PROGRESSIVE_BATCH_SIZE: 1000,
   STATUS_POLL_INTERVAL_MS: 1000,
   STATUS_POLL_MAX_DURATION_MS: 30 * 60 * 1000,
+  STATUS_POLL_CONCURRENCY: 3,
+  DISCOVERY_SMALL_SITE_THRESHOLD: 20_000,
+  DISCOVERY_TARGET_PATHS_PER_JOB: 20_000,
+  DISCOVERY_MAX_PATHS_PER_JOB: 250,
 });
 
 export const Operation = Object.freeze({
@@ -38,6 +44,13 @@ export const Paths = Object.freeze({
 export const CORS_PROXY_URL = 'https://media-library-cors-proxy.aem-poc-lab.workers.dev/';
 export const MEDIA_UNDERSCORE_PREFIX = 'media_';
 
+// External video detection regexes
+export const YOUTUBE_VIDEO_RE = /(?:youtube\.com\/(?:watch\?v=|embed\/|shorts\/)([^&\n?#/]+)|youtu\.be\/([^&\n?#/]+))/;
+export const VIMEO_VIDEO_RE = /(?:player\.)?vimeo\.com\/(?:video\/)?(\d+)(?:$|[/?#])/;
+export const DAILYMOTION_VIDEO_RE = /(?:dailymotion\.com\/video\/|dai\.ly\/)([^&\n?#]+)/;
+export const SCENE7_VIDEO_RE = /scene7\.com\/is\/content\//;
+export const DYNAMIC_MEDIA_VIDEO_RE = /\/is\/content\//;
+
 const mediaExtensions = {
   pdf: ['pdf'],
   svg: ['svg'],
@@ -63,11 +76,6 @@ export const ExternalMedia = Object.freeze({
   EXTENSION_REGEX: mediaExtensionRegex,
   HOST_PATTERNS: [
     { host: /adobeaemcloud\.com$/i, pathContains: 'urn:aaid:aem', typeFromPath: true },
-    { host: /youtube\.com$/i, type: MediaType.VIDEO },
-    { host: /youtu\.be$/i, type: MediaType.VIDEO },
-    { host: /vimeo\.com$/i, type: MediaType.VIDEO },
-    { host: /player\.vimeo\.com$/i, type: MediaType.VIDEO },
-    { host: /unsplash\.com$/i, type: categoryImg },
     { host: /images\.unsplash\.com$/i, type: categoryImg },
   ],
 });

tools/media-library/core/media.js

@@ -1,4 +1,13 @@
-import { MediaType } from './constants.js';
+import {
+  MediaType,
+  ExternalMedia,
+  Domains,
+  YOUTUBE_VIDEO_RE,
+  VIMEO_VIDEO_RE,
+  DAILYMOTION_VIDEO_RE,
+  SCENE7_VIDEO_RE,
+  DYNAMIC_MEDIA_VIDEO_RE,
+} from './constants.js';
 
 const IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg', 'avif'];
 const VIDEO_EXTENSIONS = ['mp4', 'webm', 'mov', 'avi'];
@@ -55,12 +64,11 @@ export function isExternalVideoUrl(url) {
   if (!url || typeof url !== 'string') return false;
 
   const supportedPatterns = [
-    /(?:youtube\.com\/(?:watch\?v=|embed\/|shorts\/|)[^&\n?#/]+|youtu\.be\/[^&\n?#/]+)/,
-    /(?:^https?:\/\/)?(?:www\.)?(?:youtube\.com|youtu\.be)(?:\/|$)/,
-    /vimeo\.com\/(\d+)/,
-    /(?:dailymotion\.com\/video\/|dai\.ly\/)/,
-    /scene7\.com\/is\/content\//,
-    /marketing\.adobe\.com\/is\/content\//,
+    YOUTUBE_VIDEO_RE,
+    VIMEO_VIDEO_RE,
+    DAILYMOTION_VIDEO_RE,
+    SCENE7_VIDEO_RE,
+    DYNAMIC_MEDIA_VIDEO_RE,
   ];
 
   return supportedPatterns.some((pattern) => pattern.test(url));
@@ -102,29 +110,27 @@ export function isFragmentMedia(media) {
 export function getVideoThumbnail(videoUrl) {
   if (!videoUrl) return null;
 
-  const youtubeMatch = videoUrl.match(/(?:youtube\.com\/(?:watch\?v=|embed\/|shorts\/|)([^&\n?#/]+)|youtu\.be\/([^&\n?#/]+))/);
+  const youtubeMatch = videoUrl.match(YOUTUBE_VIDEO_RE);
   if (youtubeMatch) {
     const id = youtubeMatch[1] || youtubeMatch[2];
     return id ? `https://img.youtube.com/vi/${id}/hqdefault.jpg` : null;
   }
 
-  // Vimeo thumbnails require oEmbed API, not supported client-side
-  // Falls through to null, shows placeholder
+  const vimeoMatch = videoUrl.match(/vimeo\.com\/(\d+)/);
+  if (vimeoMatch) {
+    const videoId = vimeoMatch[1];
+    return `https://i.vimeocdn.com/video/${videoId}_640.jpg`;
+  }
 
-  const dailymotionMatch = videoUrl.match(/(?:dailymotion\.com\/video\/|dai\.ly\/)([^&\n?#/]+)/);
+  const dailymotionMatch = videoUrl.match(DAILYMOTION_VIDEO_RE);
   if (dailymotionMatch) {
     const videoId = dailymotionMatch[1];
     return `https://www.dailymotion.com/thumbnail/video/${videoId}`;
   }
 
-  const dynamicMediaMatch = videoUrl.match(/(scene7\.com\/is\/content\/[^?]+)/);
-  if (dynamicMediaMatch) {
-    return `${dynamicMediaMatch[1]}?fmt=jpeg&wid=300&hei=200`;
-  }
-
-  const marketingMatch = videoUrl.match(/(marketing\.adobe\.com\/is\/content\/[^?]+)/);
-  if (marketingMatch) {
-    return `${marketingMatch[1]}?fmt=jpeg&wid=300&hei=200`;
+  if (DYNAMIC_MEDIA_VIDEO_RE.test(videoUrl)) {
+    const dynamicMediaBase = videoUrl.split('?')[0];
+    return `${dynamicMediaBase}?fmt=jpeg&wid=300&hei=200`;
   }
 
   return null;
@@ -133,25 +139,91 @@ export function getVideoThumbnail(videoUrl) {
 export function getVideoEmbedUrl(videoUrl) {
   if (!videoUrl) return null;
 
-  const youtubeMatch = videoUrl.match(/(?:youtube\.com\/(?:watch\?v=|embed\/|shorts\/|)([^&\n?#/]+)|youtu\.be\/([^&\n?#/]+))/);
+  const youtubeMatch = videoUrl.match(YOUTUBE_VIDEO_RE);
   if (youtubeMatch) {
     const id = youtubeMatch[1] || youtubeMatch[2];
     return id ? `https://www.youtube.com/embed/${id}` : null;
   }
 
-  const vimeoMatch = videoUrl.match(/vimeo\.com\/(\d+)/);
+  const vimeoMatch = videoUrl.match(VIMEO_VIDEO_RE);
   if (vimeoMatch) {
     return `https://player.vimeo.com/video/${vimeoMatch[1]}`;
   }
 
-  const dailymotionMatch = videoUrl.match(/(?:dailymotion\.com\/video\/|dai\.ly\/)([^&\n?#/]+)/);
+  const dailymotionMatch = videoUrl.match(DAILYMOTION_VIDEO_RE);
   if (dailymotionMatch) {
     return `https://www.dailymotion.com/embed/video/${dailymotionMatch[1]}`;
   }
 
   return null;
 }
 
+function isExternalUrl(url) {
+  if (!url || !url.startsWith('http')) return false;
+  return !Domains.SAME_ORIGIN.some((domain) => url.includes(domain));
+}
+
+export function getExternalMediaTypeInfo(url) {
+  if (!url || !url.startsWith('http') || !isExternalUrl(url)) return null;
+
+  try {
+    const parsed = new URL(url);
+    const pathPart = parsed.pathname.split('?')[0].split('#')[0];
+    const pathLower = pathPart.toLowerCase();
+
+    const extMatch = pathLower.match(ExternalMedia.EXTENSION_REGEX);
+    if (extMatch) {
+      const ext = extMatch[1].toLowerCase();
+      let type = MediaType.LINK;
+      if (ExternalMedia.EXTENSIONS.pdf.includes(ext)) type = MediaType.DOCUMENT;
+      else if (ExternalMedia.EXTENSIONS.svg.includes(ext)) type = MediaType.IMAGE;
+      else if (ExternalMedia.EXTENSIONS.image.includes(ext)) type = MediaType.IMAGE;
+      else if (ExternalMedia.EXTENSIONS.video.includes(ext)) type = MediaType.VIDEO;
+      const name = pathPart.split('/').pop() || parsed.hostname;
+      return { type, name };
+    }
+
+    if (isExternalVideoUrl(url)) {
+      const lastSegment = pathPart.split('/').pop() || parsed.hostname;
+      return { type: MediaType.VIDEO, name: lastSegment };
+    }
+
+    const host = parsed.hostname;
+    const matched = ExternalMedia.HOST_PATTERNS.find(
+      (pattern) => pattern.host.test(host)
+        && (!pattern.pathContains || parsed.pathname.includes(pattern.pathContains)),
+    );
+    if (matched) {
+      const { type: patternType } = matched;
+
+      if (matched.typeFromPath) {
+        const lastSegment = pathPart.split('/').pop() || '';
+        const segExt = lastSegment.split('.').pop()?.toLowerCase();
+        const imageExts = [...ExternalMedia.EXTENSIONS.image, ...ExternalMedia.EXTENSIONS.svg];
+        if (segExt && ExternalMedia.EXTENSIONS.video.includes(segExt)) {
+          return { type: MediaType.VIDEO, name: lastSegment };
+        }
+        if (segExt && ExternalMedia.EXTENSIONS.pdf.includes(segExt)) {
+          return { type: MediaType.DOCUMENT, name: lastSegment };
+        }
+        if (segExt && imageExts.includes(segExt)) {
+          return { type: MediaType.IMAGE, name: lastSegment };
+        }
+      }
+
+      if (patternType === ExternalMedia.CATEGORY_IMG) {
+        return { type: MediaType.IMAGE, name: pathPart.split('/').pop() || host };
+      }
+
+      return { type: MediaType.LINK, name: host };
+    }
+  } catch {
+    /* parse error */
+  }
+
+  return null;
+}
+
 export function getImageOrientation(width, height) {
   if (Math.abs(width - height) < 5) {
     return 'Square';

tools/media-library/core/params.js

@@ -0,0 +1,32 @@
+/**
+ * URL parameter utilities
+ */
+
+let perfEnabledCached = null;
+let perfLoggedOnce = false;
+
+/**
+ * Check if performance logging is enabled via ?debug=perf URL parameter
+ */
+export default function isPerfEnabled() {
+  if (typeof window === 'undefined') return false;
+
+  // Cache the result since URL params don't change during page lifetime
+  if (perfEnabledCached !== null) {
+    return perfEnabledCached;
+  }
+
+  const params = new URLSearchParams(window.location.search);
+  const debug = params.get('debug');
+  const enabled = debug === 'perf' || debug === 'true' || debug === '1';
+  perfEnabledCached = enabled;
+
+  // Log once when perf debugging is enabled
+  if (enabled && !perfLoggedOnce) {
+    // eslint-disable-next-line no-console -- perf mode: indicate debug=perf is active
+    console.log(`[Media Library] Performance logging enabled via ?debug=${debug}`);
+    perfLoggedOnce = true;
+  }
+
+  return enabled;
+}

tools/media-library/core/urls.js

@@ -1,5 +1,12 @@
 import { Domains, MEDIA_UNDERSCORE_PREFIX } from './constants.js';
 
+function toAemRuntimeHostname(hostname) {
+  if (!hostname) return hostname;
+  return hostname
+    .replace('.hlx.page', Domains.AEM_PAGE)
+    .replace('.hlx.live', Domains.AEM_LIVE);
+}
+
 export function isExternalUrl(url) {
   if (!url) return false;
   return !url.includes(Domains.AEM_LIVE) && !url.includes(Domains.AEM_PAGE);
@@ -10,6 +17,7 @@ export function resolveMediaUrl(mediaUrl, org, repo) {
 
   try {
     const url = new URL(mediaUrl);
+    url.hostname = toAemRuntimeHostname(url.hostname);
     return url.href;
   } catch {
     if (org && repo) {
@@ -133,7 +141,9 @@ export function getDedupeKey(url) {
   if (!url) return '';
 
   try {
-    const urlObj = new URL(url);
+    // Normalize hostname (.hlx → .aem) before extracting pathname for deduplication
+    const normalizedUrl = resolveMediaUrl(url);
+    const urlObj = new URL(normalizedUrl);
     const { pathname } = urlObj;
     const filename = pathname.split('/').pop();