From c6fbd04ee7620df47eb3a847e64640749c662dd3 Mon Sep 17 00:00:00 2001
From: Kiran Murugulla <murugull@adobe.com>
Date: Thu, 12 Mar 2026 21:45:22 -0400
Subject: [PATCH 01/10] adding bulk status splitting fy folder logic

---
 tools/media-library/core/admin-rate-limit.js |  93 +++++
 tools/media-library/core/constants.js        |   4 +
 tools/media-library/indexing/build.js        |  28 +-
 tools/media-library/indexing/bulk-status.js  | 414 +++++++++++++++++++
 tools/media-library/indexing/parse.js        |  19 +-
 tools/media-library/indexing/status-api.js   | 105 ++++-
 6 files changed, 628 insertions(+), 35 deletions(-)
 create mode 100644 tools/media-library/core/admin-rate-limit.js
 create mode 100644 tools/media-library/indexing/bulk-status.js

diff --git a/tools/media-library/core/admin-rate-limit.js b/tools/media-library/core/admin-rate-limit.js
new file mode 100644
index 00000000..02addd04
--- /dev/null
+++ b/tools/media-library/core/admin-rate-limit.js
@@ -0,0 +1,93 @@
+/**
+ * Shared rate limiter for admin.hlx.page (10 req/s per project).
+ * Same pattern as backfill tool: promise-chain acquire, 429 retry with backoff.
+ */
+
+const ADMIN_API_RATE = 10;
+
+function waitForDelay(ms, signal = null) {
+  const duration = Math.max(0, ms);
+  if (signal?.aborted) return Promise.resolve();
+  return new Promise((resolve) => {
+    const timeoutId = setTimeout(resolve, duration);
+    if (signal) {
+      signal.addEventListener('abort', () => {
+        clearTimeout(timeoutId);
+        resolve();
+      }, { once: true });
+    }
+  });
+}
+
+function createRateLimiter(initialRate = ADMIN_API_RATE, getSignal = () => null) {
+  let intervalMs = Math.ceil(1000 / initialRate);
+  let queue = Promise.resolve();
+
+  return {
+    acquire() {
+      const gate = queue;
+      queue = queue.then(() => waitForDelay(intervalMs, getSignal()));
+      return gate;
+    },
+    handleResponse(res) {
+      const rate = parseFloat(res?.headers?.get?.('x-ratelimit-rate'), 10);
+      if (Number.isFinite(rate) && rate > 0) {
+        intervalMs = Math.ceil(1000 / rate);
+      }
+    },
+    backoff(seconds) {
+      const ms = Math.max(0, (typeof seconds === 'number' ? seconds : 1) * 1000);
+      queue = queue.then(() => waitForDelay(ms, getSignal()));
+    },
+    reset() {
+      queue = Promise.resolve();
+    },
+  };
+}
+
+let adminLimiter = null;
+
+export function getAdminRateLimiter() {
+  if (!adminLimiter) {
+    adminLimiter = createRateLimiter(ADMIN_API_RATE);
+  }
+  return adminLimiter;
+}
+
+/**
+ * Fetch admin.hlx.page URL with rate limit and 429/503 retry.
+ * Call only for URLs under https://admin.hlx.page.
+ */
+export async function fetchAdminWithRateLimit(
+  url,
+  options = {},
+  { maxRetries = 3, signal } = {},
+) {
+  const limiter = getAdminRateLimiter();
+  const fetchOptions = { ...options, credentials: 'include', signal };
+
+  async function attempt(attemptNumber) {
+    await limiter.acquire();
+    const res = await fetch(url, fetchOptions);
+    limiter.handleResponse(res);
+
+    if (res.status === 429 && attemptNumber < maxRetries) {
+      const retryAfter = parseInt(
+        res.headers.get('x-retry-after') || res.headers.get('retry-after'),
+        10,
+      ) || (2 ** attemptNumber);
+      limiter.backoff(retryAfter);
+      return attempt(attemptNumber + 1);
+    }
+
+    if (res.status === 503 && attemptNumber < maxRetries) {
+      const delaySec = 2 ** attemptNumber;
+      limiter.backoff(delaySec);
+      return attempt(attemptNumber + 1);
+    }
+
+    return res;
+  }
+
+  return attempt(0);
+}
diff --git a/tools/media-library/core/constants.js b/tools/media-library/core/constants.js
index 25c0fc7d..ed9cb4ce 100644
--- a/tools/media-library/core/constants.js
+++ b/tools/media-library/core/constants.js
@@ -6,6 +6,10 @@ export const IndexConfig = Object.freeze({
   MAX_CONCURRENT_FETCHES: 10,
   STATUS_POLL_INTERVAL_MS: 1000,
   STATUS_POLL_MAX_DURATION_MS: 30 * 60 * 1000,
+  STATUS_POLL_CONCURRENCY: 3,
+  DISCOVERY_SMALL_SITE_THRESHOLD: 20_000,
+  DISCOVERY_TARGET_PATHS_PER_JOB: 20_000,
+  DISCOVERY_MAX_PATHS_PER_JOB: 250,
 });
 
 export const Operation = Object.freeze({
diff --git a/tools/media-library/indexing/build.js b/tools/media-library/indexing/build.js
index 6b741026..5d4e64ad 100644
--- a/tools/media-library/indexing/build.js
+++ b/tools/media-library/indexing/build.js
@@ -13,6 +13,7 @@ import {
   pollStatusJob,
   getStatusJobDetails,
 } from './status-api.js';
+import runBulkStatus from './bulk-status.js';
 import {
   processLinkedContent,
   processStandaloneUploads,
@@ -173,21 +174,8 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
       if (fromMedialog.length > 0) onProgressiveData(fromMedialog);
     };
 
-    const runStatusJob = async () => {
-      onProgress({ stage: 'fetching', message: 'Creating status job...' });
-      const { jobUrl } = await createBulkStatusJob(org, site, 'main');
-      onProgress({ stage: 'fetching', message: 'Polling status job for site discovery...' });
-      await pollStatusJob(
-        jobUrl,
-        IndexConfig.STATUS_POLL_INTERVAL_MS,
-        (progress) => {
-          const msg = `Status: ${progress?.processed ?? 0}/${progress?.total ?? 0}...`;
-          onProgress({ stage: 'fetching', message: msg });
-        },
-        IndexConfig.STATUS_POLL_MAX_DURATION_MS,
-      );
-      return getStatusJobDetails(jobUrl);
-    };
+    const pathNorm = path ? (path.trim().replace(/\/+$/, '') || '').replace(/^(?!\/)/, '/') : '';
+    const contentPathForStatus = pathNorm || null;
 
     let statusPromise = null;
     if (preValidatedStatusResources != null) {
@@ -204,8 +192,16 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
         fetchAllMediaLog(org, site, timeParams, onMedialogChunk),
       ]);
     } else {
+      const progressCallback = (p) => {
+        const msg = p.message || `Status: ${p.progress?.processed ?? 0}/${p.progress?.total ?? 0}...`;
+        onProgress({ stage: p.stage || 'fetching', message: msg });
+      };
       [statusResources, medialogResult] = await Promise.all([
-        runStatusJob(),
+        runBulkStatus(org, site, 'main', contentPathForStatus, {
+          onProgress: progressCallback,
+          pollInterval: IndexConfig.STATUS_POLL_INTERVAL_MS,
+          maxDurationMs: IndexConfig.STATUS_POLL_MAX_DURATION_MS,
+        }).then(({ resources: r }) => r),
         fetchAllMediaLog(org, site, timeParams, onMedialogChunk),
       ]);
     }
diff --git a/tools/media-library/indexing/bulk-status.js b/tools/media-library/indexing/bulk-status.js
new file mode 100644
index 00000000..9fe0c882
--- /dev/null
+++ b/tools/media-library/indexing/bulk-status.js
@@ -0,0 +1,414 @@
+import {
+  createBulkStatusJob,
+  pollStatusJob,
+  getStatusJobDetails,
+  getStatusJobDetailsRaw,
+  extractJobPaths,
+  extractJobIsComplete,
+  parseResourcesFromDetailsRaw,
+} from './status-api.js';
+import { IndexConfig } from '../core/constants.js';
+
+const REQ_PER_SEC = 10;
+const THROTTLE_MS = 1000 / REQ_PER_SEC;
+
+const LARGE_SITE_PATH_THRESHOLD = IndexConfig.DISCOVERY_SMALL_SITE_THRESHOLD ?? 20_000;
+const TARGET_PARTITION_RESOURCE_COUNT = IndexConfig.DISCOVERY_TARGET_PATHS_PER_JOB ?? 20_000;
+const MAX_PARTITION_PATHS = IndexConfig.DISCOVERY_MAX_PATHS_PER_JOB ?? 250;
+
+function normalizePath(p) {
+  if (!p || typeof p !== 'string') return '';
+  return p.startsWith('/') ? p : `/${p}`;
+}
+
+function createSlotQueue(intervalMs = THROTTLE_MS) {
+  const queue = [];
+  const interval = setInterval(() => {
+    const item = queue.shift();
+    if (item) {
+      Promise.resolve(item.fn()).then(item.resolve).catch(item.reject);
+    }
+  }, intervalMs);
+  return {
+    run(fn) {
+      return new Promise((resolve, reject) => {
+        queue.push({ fn, resolve, reject });
+      });
+    },
+    stop() { clearInterval(interval); },
+  };
+}
+
+async function runWithConcurrency(items, limit, fn) {
+  const results = [];
+  let next = 0;
+  async function runOne() {
+    const i = next;
+    next += 1;
+    if (i >= items.length) return;
+    results[i] = await fn(items[i], i);
+    if (next < items.length) await runOne();
+  }
+  const workers = Array(Math.min(limit, items.length))
+    .fill(null)
+    .map(() => runOne());
+  await Promise.all(workers);
+  return results;
+}
+
+function mergeResourceRecords(existing, incoming) {
+  return {
+    ...existing,
+    ...incoming,
+    previewLastModified: existing.previewLastModified || incoming.previewLastModified || '',
+    previewLastModifiedBy: existing.previewLastModifiedBy || incoming.previewLastModifiedBy || '',
+  };
+}
+
+function mergeResourcesByPath(...resourceLists) {
+  const byPath = new Map();
+  resourceLists.forEach((resources) => {
+    resources.forEach((resource) => {
+      if (!resource?.path) return;
+      const existing = byPath.get(resource.path);
+      if (!existing) {
+        byPath.set(resource.path, resource);
+      } else {
+        byPath.set(resource.path, mergeResourceRecords(existing, resource));
+      }
+    });
+  });
+  return Array.from(byPath.values());
+}
+
+function packPathBuckets(buckets) {
+  const partitions = [];
+  const sortedBuckets = [...buckets].sort((a, b) => (
+    b.estimatedCount - a.estimatedCount
+    || a.paths[0].localeCompare(b.paths[0])
+  ));
+
+  sortedBuckets.forEach((bucket) => {
+    const targetPartition = partitions.find((partition) => (
+      partition.estimatedCount + bucket.estimatedCount <= TARGET_PARTITION_RESOURCE_COUNT
+      && partition.paths.length + bucket.paths.length <= MAX_PARTITION_PATHS
+    ));
+
+    if (targetPartition) {
+      targetPartition.paths.push(...bucket.paths);
+      targetPartition.estimatedCount += bucket.estimatedCount;
+      return;
+    }
+
+    partitions.push({
+      paths: [...bucket.paths],
+      estimatedCount: bucket.estimatedCount,
+    });
+  });
+
+  return partitions.map((partition) => ({
+    ...partition,
+    paths: partition.paths.slice().sort((a, b) => a.localeCompare(b)),
+  }));
+}
+
+function buildPathPartitions(paths, base = null) {
+  const topLevelBuckets = new Map();
+  const rootPaths = new Set();
+
+  paths.forEach((path) => {
+    if (typeof path !== 'string' || !path.startsWith('/')) return;
+    if (base && !path.startsWith(base) && path !== base) return;
+    const pathNorm = path.replace(/\/$/, '');
+    let relPath;
+    if (!base) {
+      relPath = pathNorm;
+    } else if (pathNorm === base) {
+      relPath = '';
+    } else {
+      relPath = pathNorm.slice(base.length + 1);
+    }
+    const segments = relPath.split('/').filter(Boolean);
+
+    if (!segments.length) {
+      rootPaths.add(base || path);
+      return;
+    }
+
+    const exactPath = base ? `${base}/${segments[0]}` : `/${segments[0]}`;
+    const bucket = topLevelBuckets.get(exactPath) || {
+      exactPath,
+      wildcardPath: `${exactPath}/*`,
+      hasExactRoot: false,
+      hasWildcardContent: false,
+      estimatedCount: 0,
+    };
+
+    if (segments.length === 1) {
+      if (path.endsWith('/')) {
+        bucket.hasWildcardContent = true;
+        bucket.estimatedCount += 1;
+      } else {
+        bucket.hasExactRoot = true;
+      }
+    } else {
+      bucket.hasWildcardContent = true;
+      bucket.estimatedCount += 1;
+    }
+
+    topLevelBuckets.set(exactPath, bucket);
+  });
+
+  const folderBuckets = [];
+  topLevelBuckets.forEach((bucket) => {
+    if (bucket.hasWildcardContent) {
+      folderBuckets.push({
+        paths: bucket.hasExactRoot
+          ? [bucket.exactPath, bucket.wildcardPath]
+          : [bucket.wildcardPath],
+        estimatedCount: bucket.estimatedCount + (bucket.hasExactRoot ? 1 : 0),
+      });
+      return;
+    }
+
+    if (bucket.hasExactRoot) {
+      rootPaths.add(bucket.exactPath);
+    }
+  });
+
+  const rootBuckets = Array.from(rootPaths)
+    .sort((a, b) => a.localeCompare(b))
+    .map((path) => ({
+      paths: [path],
+      estimatedCount: 1,
+    }));
+  const partitions = packPathBuckets([...folderBuckets, ...rootBuckets]).sort((a, b) => (
+    b.estimatedCount - a.estimatedCount
+    || a.paths[0].localeCompare(b.paths[0])
+  ));
+
+  return {
+    folderBucketCount: folderBuckets.length,
+    rootPathCount: rootBuckets.length,
+    partitions,
+  };
+}
+
+async function runPartitionedStatusJobs(org, repo, ref, partitions, opts) {
+  const {
+    onProgress, slotQueue, pollConcurrency, pollInterval, maxDurationMs, perf,
+  } = opts;
+  if (!partitions.length) return [];
+
+  let resources = [];
+  const interval = partitions.length > 5 ? Math.min(pollInterval * 2, 3000) : pollInterval;
+  const jobDurationsMs = [];
+
+  const jobCreateStart = Date.now();
+  const jobs = await partitions.reduce(async (accPromise, partition) => {
+    const acc = await accPromise;
+    const paths = Array.isArray(partition) ? partition : (partition?.paths || []);
+    const { jobUrl } = await createBulkStatusJob(org, repo, ref, null, { paths });
+    acc.push({ jobUrl, paths });
+    return acc;
+  }, Promise.resolve([]));
+  const jobCreationMs = Date.now() - jobCreateStart;
+
+  await runWithConcurrency(jobs, pollConcurrency, async ({ jobUrl }, i) => {
+    const jobStart = Date.now();
+    await pollStatusJob(jobUrl, interval, (progress) => {
+      if (onProgress && progress) {
+        const pct = progress.processed && progress.total
+          ? Math.round((progress.processed / progress.total) * 100)
+          : 0;
+        onProgress({
+          stage: 'fetching',
+          message: `Status job ${i + 1}/${jobs.length}: ${pct}%`,
+          percent: 15,
+        });
+      }
+    }, maxDurationMs);
+    jobDurationsMs[i] = Date.now() - jobStart;
+  });
+
+  if (perf) {
+    perf.partitionJobMs = jobDurationsMs;
+    perf.partitionJobMaxMs = Math.max(...jobDurationsMs, 0);
+    perf.partitionCount = partitions.length;
+    perf.jobCount = partitions.length;
+    perf.jobCreationMs = jobCreationMs;
+    perf.pollingMs = Math.max(...jobDurationsMs, 0);
+  }
+
+  const detailsStart = Date.now();
+  await jobs.reduce(async (prev, { jobUrl }) => {
+    await prev;
+    const partResources = await slotQueue.run(() => getStatusJobDetails(jobUrl));
+    resources = mergeResourcesByPath(resources, partResources);
+  }, Promise.resolve());
+  if (perf) {
+    perf.partitionDetailsMs = Date.now() - detailsStart;
+  }
+
+  return resources;
+}
+
+async function runStatusJob(org, repo, ref, paths, opts = {}) {
+  const {
+    pathsOnly = false,
+    onProgress,
+    slotQueue,
+    pollInterval = IndexConfig.STATUS_POLL_INTERVAL_MS,
+    maxDurationMs = IndexConfig.STATUS_POLL_MAX_DURATION_MS,
+  } = opts;
+  const normalizedPaths = Array.isArray(paths) ? paths : [paths];
+  const { jobUrl } = await createBulkStatusJob(org, repo, ref, null, {
+    paths: normalizedPaths,
+    pathsOnly,
+  });
+
+  await pollStatusJob(jobUrl, pollInterval, (progress) => {
+    if (onProgress && progress) {
+      const pct = progress.processed && progress.total
+        ? Math.round((progress.processed / progress.total) * 100)
+        : 0;
+      onProgress({ progress: pct });
+    }
+  }, maxDurationMs);
+
+  const detailsRaw = await slotQueue.run(() => getStatusJobDetailsRaw(jobUrl));
+  const resources = pathsOnly ? [] : parseResourcesFromDetailsRaw(detailsRaw);
+  const isComplete = extractJobIsComplete(detailsRaw, pathsOnly);
+  const discoveredPaths = pathsOnly ? extractJobPaths(detailsRaw) : [];
+
+  return {
+    isComplete,
+    resources,
+    paths: discoveredPaths,
+  };
+}
+
+export default async function runBulkStatus(org, repo, ref, contentPath, options = {}) {
+  const {
+    onProgress,
+    pollInterval = IndexConfig.STATUS_POLL_INTERVAL_MS,
+    maxDurationMs = IndexConfig.STATUS_POLL_MAX_DURATION_MS,
+    pollConcurrency = IndexConfig.STATUS_POLL_CONCURRENCY ?? 3,
+  } = options;
+
+  const effectiveRef = ref || 'main';
+  const perf = {
+    jobCount: 0,
+    jobCreationMs: 0,
+    pollingMs: 0,
+    totalDurationMs: 0,
+  };
+  const startTime = Date.now();
+  const base = contentPath ? normalizePath(contentPath).replace(/\/$/, '') : null;
+
+  const discoveryPaths = base ? [base, `${base}/*`] : ['/*'];
+  const slotQueue = createSlotQueue();
+
+  try {
+    const discoveryCreateStart = Date.now();
+    const pathDiscoveryJob = await runStatusJob(org, repo, effectiveRef, discoveryPaths, {
+      pathsOnly: true,
+      onProgress: (p) => {
+        if (onProgress) {
+          onProgress({
+            stage: 'discovery',
+            message: `Discovery: ${p.progress ?? 0}%`,
+            percent: 5,
+          });
+        }
+      },
+      slotQueue,
+      pollInterval,
+      maxDurationMs,
+    });
+    perf.discoveryMs = Date.now() - discoveryCreateStart;
+
+    const discoveredPaths = pathDiscoveryJob.paths;
+    const pathCount = discoveredPaths.length;
+    const partitionPlan = pathCount > 0 ? buildPathPartitions(discoveredPaths, base) : null;
+
+    if (pathDiscoveryJob.isComplete && pathCount === 0) {
+      perf.totalDurationMs = Date.now() - startTime;
+      return { resources: [], perf };
+    }
+
+    let resources = [];
+    perf.decision = 'single';
+
+    if (!pathDiscoveryJob.isComplete && partitionPlan) {
+      perf.decision = 'partitioned';
+      resources = await runPartitionedStatusJobs(
+        org,
+        repo,
+        effectiveRef,
+        partitionPlan.partitions,
+        {
+          onProgress, slotQueue, pollConcurrency, pollInterval, maxDurationMs, perf,
+        },
+      );
+    } else if (pathDiscoveryJob.isComplete && pathCount > LARGE_SITE_PATH_THRESHOLD) {
+      perf.decision = 'partitioned';
+      resources = await runPartitionedStatusJobs(
+        org,
+        repo,
+        effectiveRef,
+        partitionPlan.partitions,
+        {
+          onProgress, slotQueue, pollConcurrency, pollInterval, maxDurationMs, perf,
+        },
+      );
+    } else {
+      perf.decision = 'single';
+      const fullCreateStart = Date.now();
+      const primaryStatusJob = await runStatusJob(org, repo, effectiveRef, discoveryPaths, {
+        onProgress: (p) => {
+          if (onProgress) {
+            onProgress({
+              stage: 'fetching',
+              message: `Status job: ${p.progress ?? 0}%`,
+              percent: 15,
+            });
+          }
+        },
+        slotQueue,
+        pollInterval,
+        maxDurationMs,
+      });
+      perf.jobCreationMs = Date.now() - fullCreateStart;
+      perf.jobCount = 1;
+      resources = primaryStatusJob.resources;
+
+      if (!primaryStatusJob.isComplete && partitionPlan) {
+        perf.decision = 'partitioned-retry';
+        const partitioned = await runPartitionedStatusJobs(
+          org,
+          repo,
+          effectiveRef,
+          partitionPlan.partitions,
+          {
+            onProgress, slotQueue, pollConcurrency, pollInterval, maxDurationMs, perf,
+          },
+        );
+        resources = mergeResourcesByPath(resources, partitioned);
+      }
+    }
+
+    if (base) {
+      const prefix = base.endsWith('/') ? base : `${base}/`;
+      resources = resources.filter(
+        (r) => r.path === base || (r.path && r.path.startsWith(prefix)),
+      );
+    }
+
+    perf.totalDurationMs = Date.now() - startTime;
+
+    return { resources, perf };
+  } finally {
+    slotQueue.stop();
+  }
+}
diff --git a/tools/media-library/indexing/parse.js b/tools/media-library/indexing/parse.js
index e9e97081..1862b278 100644
--- a/tools/media-library/indexing/parse.js
+++ b/tools/media-library/indexing/parse.js
@@ -8,9 +8,9 @@ import {
   Paths,
   ICON_DOC_EXCLUDE,
   Domains,
-  CORS_PROXY_URL,
 } from '../core/constants.js';
 import { getDedupeKey } from '../core/urls.js';
+import { fetchAdminWithRateLimit } from '../core/admin-rate-limit.js';
 
 export { getDedupeKey };
 
@@ -205,19 +205,22 @@ function processConcurrently(items, fn, concurrency) {
   return Promise.all(workers).then(() => results);
 }
 
+const ADMIN_PREVIEW_BASE = 'https://admin.hlx.page/preview';
+
 /**
- * Fetch page markdown from preview URL.
+ * Fetch page markdown via admin.hlx.page (same as page-status diff).
+ * Uses shared admin rate limiter (10 req/s) and 429 retry with backoff.
  */
 export async function fetchPageMarkdown(pagePath, org, repo, ref = 'main') {
   try {
     const path = pagePath.startsWith('/') ? pagePath : `/${pagePath}`;
+    let fetchPath;
+    if (path.endsWith('/')) fetchPath = `${path}index.md`;
+    else if (path.endsWith('.md')) fetchPath = path;
+    else fetchPath = `${path}.md`;
+    const url = `${ADMIN_PREVIEW_BASE}/${org}/${repo}/${ref}${fetchPath}`;
 
-    // Use CORS proxy to fetch page markdown
-    // Proxy URL is a Cloudflare Worker that forwards requests with CORS headers
-    const pageUrl = `https://${ref}--${repo}--${org}.aem.page${path}`;
-    const url = `${CORS_PROXY_URL}?url=${encodeURIComponent(pageUrl)}`;
-
-    const resp = await fetch(url);
+    const resp = await fetchAdminWithRateLimit(url, {}, { maxRetries: 3 });
     if (!resp.ok) return null;
     return resp.text();
   } catch {
diff --git a/tools/media-library/indexing/status-api.js b/tools/media-library/indexing/status-api.js
index 907ce86f..aa99ec41 100644
--- a/tools/media-library/indexing/status-api.js
+++ b/tools/media-library/indexing/status-api.js
@@ -1,12 +1,12 @@
 import { ensureLogin } from '../../../blocks/profile/profile.js';
 import { IndexConfig } from '../core/constants.js';
+import { fetchAdminWithRateLimit } from '../core/admin-rate-limit.js';
 
 const STATUS_BASE = 'https://admin.hlx.page/status';
 
 function fetchWithAuth(url, options = {}) {
-  return fetch(url, {
+  return fetchAdminWithRateLimit(url, {
     ...options,
-    credentials: 'include',
     headers: {
       'Content-Type': 'application/json',
       ...options.headers,
@@ -14,16 +14,28 @@ function fetchWithAuth(url, options = {}) {
   });
 }
 
-export async function createBulkStatusJob(org, repo, ref = 'main') {
+export async function createBulkStatusJob(org, repo, ref = 'main', contentPath = null, options = {}) {
   await ensureLogin(org, repo);
 
+  let paths;
+  if (options.paths && options.paths.length > 0) {
+    paths = options.paths;
+  } else {
+    const normalized = contentPath && typeof contentPath === 'string' && contentPath.trim()
+      ? contentPath.trim().replace(/\/+$/, '').replace(/^(?!\/)/, '/')
+      : null;
+    paths = normalized ? [normalized, `${normalized}/*`] : ['/*'];
+  }
+
+  const payload = { paths, select: ['preview'] };
+  if (options.pathsOnly) {
+    payload.pathsOnly = true;
+  }
+
   const url = `${STATUS_BASE}/${org}/${repo}/${ref}/*`;
   const resp = await fetchWithAuth(url, {
     method: 'POST',
-    body: JSON.stringify({
-      paths: ['/*'],
-      select: ['preview'],
-    }),
+    body: JSON.stringify(payload),
   });
 
   if (!resp.ok) {
@@ -92,14 +104,85 @@ export async function pollStatusJob(
   return pollStatusJob(jobUrl, pollIntervalMs, onProgress, maxDurationMs, startedAt);
 }
 
-export async function getStatusJobDetails(jobUrl) {
+export async function getStatusJobDetailsRaw(jobUrl) {
   const detailsUrl = `${jobUrl}/details`;
   const resp = await fetchWithAuth(detailsUrl);
-
   if (!resp.ok) {
     throw new Error(`Failed to fetch job details: ${resp.status}`);
   }
+  return resp.json();
+}
 
-  const { data } = await resp.json();
-  return data?.resources || [];
+function asJobData(raw) {
+  const obj = raw && typeof raw === 'object' ? raw : {};
+  const nested = obj.job && typeof obj.job === 'object' ? obj.job : null;
+  const fromData = obj.data?.job && typeof obj.data.job === 'object' ? obj.data.job : null;
+  const job = nested && Object.keys(nested).length > 0 ? nested : fromData || obj;
+  return job;
+}
+
+function asObject(value) {
+  return value && typeof value === 'object' ? value : {};
+}
+
+export function parseResourcesFromDetailsRaw(raw) {
+  const jobData = asJobData(raw);
+  const dataRoot = jobData?.data != null ? asObject(jobData.data) : {};
+  const { resources } = dataRoot;
+  if (Array.isArray(resources)) return resources;
+  if (resources && typeof resources === 'object') {
+    const list = [];
+    Object.values(resources).forEach((part) => {
+      if (Array.isArray(part)) list.push(...part);
+    });
+    return list;
+  }
+  return [];
+}
+
+export function extractJobPhase(rawJobData) {
+  const jobData = asJobData(rawJobData);
+  const dataRoot = asObject(jobData?.data);
+  return typeof dataRoot.phase === 'string' ? dataRoot.phase : '';
+}
+
+export function extractJobState(rawJobData) {
+  if (typeof rawJobData?.state === 'string' && rawJobData.state) return rawJobData.state;
+  const jobData = asJobData(rawJobData);
+  return typeof jobData?.state === 'string' ? jobData.state : '';
+}
+
+export function extractJobIsComplete(rawJobData, pathsOnly) {
+  const state = extractJobState(rawJobData);
+  const phase = extractJobPhase(rawJobData);
+  const jobData = asJobData(rawJobData);
+  const error = jobData?.error;
+  const cancelled = jobData?.cancelled === true;
+
+  if (state !== 'stopped' || error || cancelled) return false;
+  if (phase === 'completed') return true;
+  if (!pathsOnly) {
+    const resources = parseResourcesFromDetailsRaw(rawJobData);
+    return resources.length > 0;
+  }
+  return false;
+}
+
+export function extractJobPaths(rawJobData) {
+  const jobData = asJobData(rawJobData);
+  const dataRoot = asObject(jobData?.data);
+  const resources = asObject(dataRoot.resources);
+  const paths = new Set();
+  Object.values(resources).forEach((partitionPaths) => {
+    if (!Array.isArray(partitionPaths)) return;
+    partitionPaths.forEach((p) => {
+      if (typeof p === 'string' && p.startsWith('/')) paths.add(p);
+    });
+  });
+  return Array.from(paths);
+}
+
+export async function getStatusJobDetails(jobUrl) {
+  const raw = await getStatusJobDetailsRaw(jobUrl);
+  return parseResourcesFromDetailsRaw(raw);
 }

From 0501aa277fe2f43e55af40c037af689bff3b6e20 Mon Sep 17 00:00:00 2001
From: Kiran Murugulla <murugull@adobe.com>
Date: Fri, 13 Mar 2026 11:02:04 -0400
Subject: [PATCH 02/10] adding bulk status and perf. optimizations for larger
 repositories

---
 tools/media-library/core/constants.js     |   2 +
 tools/media-library/core/params.js        |  31 ++++
 tools/media-library/core/utils.js         |  10 +-
 tools/media-library/indexing/build.js     | 140 ++++++++++++++++---
 tools/media-library/indexing/parse.js     | 163 +++++++++++++++++++---
 tools/media-library/indexing/reconcile.js |  47 ++++++-
 tools/media-library/media-library.js      |  47 ++++++-
 tools/media-library/views/grid/grid.js    |   8 +-
 8 files changed, 400 insertions(+), 48 deletions(-)
 create mode 100644 tools/media-library/core/params.js

diff --git a/tools/media-library/core/constants.js b/tools/media-library/core/constants.js
index ed9cb4ce..6517a9c9 100644
--- a/tools/media-library/core/constants.js
+++ b/tools/media-library/core/constants.js
@@ -4,6 +4,8 @@ export const IndexConfig = Object.freeze({
   INCREMENTAL_WINDOW_MS: 10000,
   API_PAGE_SIZE: 1000,
   MAX_CONCURRENT_FETCHES: 10,
+  MAX_CONCURRENT_PAGE_FETCHES: 4,
+  USAGE_MAP_PROGRESSIVE_BATCH_SIZE: 1000,
   STATUS_POLL_INTERVAL_MS: 1000,
   STATUS_POLL_MAX_DURATION_MS: 30 * 60 * 1000,
   STATUS_POLL_CONCURRENCY: 3,
diff --git a/tools/media-library/core/params.js b/tools/media-library/core/params.js
new file mode 100644
index 00000000..f691d506
--- /dev/null
+++ b/tools/media-library/core/params.js
@@ -0,0 +1,31 @@
+/**
+ * URL parameter utilities
+ */
+
+let perfEnabledCached = null;
+let perfLoggedOnce = false;
+
+/**
+ * Check if performance logging is enabled via ?debug=perf URL parameter
+ */
+export function isPerfEnabled() {
+  if (typeof window === 'undefined') return false;
+
+  // Cache the result since URL params don't change during page lifetime
+  if (perfEnabledCached !== null) {
+    return perfEnabledCached;
+  }
+
+  const params = new URLSearchParams(window.location.search);
+  const debug = params.get('debug');
+  const enabled = debug === 'perf' || debug === 'true' || debug === '1';
+  perfEnabledCached = enabled;
+
+  // Log once for debugging
+  if (enabled && !perfLoggedOnce) {
+    console.log('[Media Library] Performance logging enabled via ?debug=' + debug);
+    perfLoggedOnce = true;
+  }
+
+  return enabled;
+}
diff --git a/tools/media-library/core/utils.js b/tools/media-library/core/utils.js
index e460ecca..22bdba50 100644
--- a/tools/media-library/core/utils.js
+++ b/tools/media-library/core/utils.js
@@ -50,7 +50,7 @@ export function pluralize(singular, plural, count) {
 }
 
 /**
- * Sorts media items for display: newest first by timestamp, then by doc path depth, then by name.
+ * Sorts media items for display: newest first by timestamp, then by doc path depth, then by name, then by URL for stability.
  */
 export function sortMediaData(mediaData) {
   if (!mediaData || mediaData.length === 0) return mediaData ?? [];
@@ -69,6 +69,12 @@ export function sortMediaData(mediaData) {
 
     const nameA = (a.name || '').toLowerCase();
     const nameB = (b.name || '').toLowerCase();
-    return nameA.localeCompare(nameB);
+    const nameDiff = nameA.localeCompare(nameB);
+    if (nameDiff !== 0) return nameDiff;
+
+    // Final tiebreaker: URL/hash for absolute stability
+    const urlA = (a.url || a.hash || '').toLowerCase();
+    const urlB = (b.url || b.hash || '').toLowerCase();
+    return urlA.localeCompare(urlB);
   });
 }
diff --git a/tools/media-library/indexing/build.js b/tools/media-library/indexing/build.js
index 5d4e64ad..c7691e6e 100644
--- a/tools/media-library/indexing/build.js
+++ b/tools/media-library/indexing/build.js
@@ -23,6 +23,7 @@ import {
 import { toAbsoluteFilePath } from './parse.js';
 import { IndexConfig } from '../core/constants.js';
 import { incrementalTimeParams, initialTimeParams } from '../core/storage.js';
+import { isPerfEnabled } from '../core/params.js';
 
 const PROGRESSIVE_DISPLAY_CAP = 3000;
 
@@ -41,6 +42,7 @@ export async function buildMediaDataFromEntries(
   onProgress = null,
   onProgressiveData = null,
   path = '',
+  perf = null,
 ) {
   const pathNorm = normalizePathForFilter(path);
   let medialogScoped = medialogEntries;
@@ -50,7 +52,7 @@ export async function buildMediaDataFromEntries(
     );
   }
 
-  const { linkedEntries } = await processLinkedContent(
+  const { linkedEntries, parseStats } = await processLinkedContent(
     auditlogEntries,
     medialogScoped,
     org,
@@ -58,8 +60,18 @@ export async function buildMediaDataFromEntries(
     'main',
     onProgress,
     path,
+    onProgressiveData,
   );
 
+  // Track markdown parsing stats if perf object provided
+  if (perf && parseStats) {
+    perf.markdownParse.pages = parseStats.pages || 0;
+    perf.markdownParse.durationMs = parseStats.durationMs || 0;
+    perf.markdownParse.success = parseStats.success || 0;
+    perf.markdownParse.fail = parseStats.fail || 0;
+    perf.markdownParse.fetchTimes = parseStats.fetchTimes || { avg: 0, min: 0, max: 0 };
+  }
+
   const referencedHashes = new Set();
   [...medialogScoped, ...linkedEntries].forEach((e) => {
     if (e.resourcePath) {
@@ -137,6 +149,44 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
   } = options;
 
   const buildMode = incremental ? 'incremental' : 'full';
+  const buildStartTime = Date.now();
+
+  if (isPerfEnabled()) {
+    console.log(`[MediaLibrary:build] Starting ${buildMode} build for ${org}/${site}${path || '/'}`);
+  }
+
+  // Performance instrumentation
+  const perf = {
+    mode: buildMode,
+    org,
+    site,
+    path: path || '/',
+    medialog: {
+      streamed: 0,
+      chunks: 0,
+      durationMs: 0,
+    },
+    auditlog: {
+      streamed: 0,
+      chunks: 0,
+      durationMs: 0,
+    },
+    statusAPI: {
+      durationMs: 0,
+      resourcesDiscovered: 0,
+      pagesDiscovered: 0,
+      filesDiscovered: 0,
+    },
+    markdownParse: {
+      pages: 0,
+      durationMs: 0,
+      success: 0,
+      fail: 0,
+      fetchTimes: { avg: 0, min: 0, max: 0 },
+    },
+    totalDurationMs: 0,
+  };
+
   const mediaMap = new Map();
   const pagesByPath = new Map();
   const filesByPath = new Map();
@@ -148,7 +198,12 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
   if (incremental) {
     const timeParams = incrementalTimeParams(metadata?.lastFetchTime);
 
+    const medialogStart = Date.now();
+    const auditlogStart = Date.now();
+
     const onAuditChunk = (entries) => {
+      perf.auditlog.chunks += 1;
+      perf.auditlog.streamed += entries.length;
       applyAuditChunkToMaps(entries, pagesByPath, filesByPath, deletedPaths);
       const earlyLinked = buildEarlyLinkedPlaceholders(filesByPath, deletedPaths, org, site);
       const fromLinked = transformToMediaData([], earlyLinked);
@@ -156,6 +211,8 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
     };
 
     const onMedialogChunk = (entries) => {
+      perf.medialog.chunks += 1;
+      perf.medialog.streamed += entries.length;
       mergeEntriesIntoMediaMap(entries, mediaMap);
       const fromMedialog = getMediaItemsFromMap(mediaMap);
       if (fromMedialog.length > 0) onProgressiveData(fromMedialog);
@@ -165,10 +222,17 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
       fetchAllMediaLog(org, site, timeParams, onMedialogChunk),
       fetchAllAuditLog(org, site, timeParams, onAuditChunk),
     ]);
+
+    perf.medialog.durationMs = Date.now() - medialogStart;
+    perf.auditlog.durationMs = Date.now() - auditlogStart;
   } else {
     const timeParams = initialTimeParams();
 
+    const medialogStart = Date.now();
+
     const onMedialogChunk = (entries) => {
+      perf.medialog.chunks += 1;
+      perf.medialog.streamed += entries.length;
       mergeEntriesIntoMediaMap(entries, mediaMap);
       const fromMedialog = getMediaItemsFromMap(mediaMap);
       if (fromMedialog.length > 0) onProgressiveData(fromMedialog);
@@ -186,26 +250,48 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
 
     let statusResources;
     let medialogResult;
-    if (statusPromise) {
-      [statusResources, medialogResult] = await Promise.all([
-        statusPromise,
-        fetchAllMediaLog(org, site, timeParams, onMedialogChunk),
-      ]);
-    } else {
-      const progressCallback = (p) => {
-        const msg = p.message || `Status: ${p.progress?.processed ?? 0}/${p.progress?.total ?? 0}...`;
-        onProgress({ stage: p.stage || 'fetching', message: msg });
-      };
-      [statusResources, medialogResult] = await Promise.all([
-        runBulkStatus(org, site, 'main', contentPathForStatus, {
-          onProgress: progressCallback,
-          pollInterval: IndexConfig.STATUS_POLL_INTERVAL_MS,
-          maxDurationMs: IndexConfig.STATUS_POLL_MAX_DURATION_MS,
-        }).then(({ resources: r }) => r),
-        fetchAllMediaLog(org, site, timeParams, onMedialogChunk),
-      ]);
+    const statusStart = Date.now();
+
+    if (isPerfEnabled()) {
+      console.log('[MediaLibrary:build] Fetching status API and medialog in parallel...');
     }
 
+    try {
+      if (statusPromise) {
+        [statusResources, medialogResult] = await Promise.all([
+          statusPromise,
+          fetchAllMediaLog(org, site, timeParams, onMedialogChunk),
+        ]);
+      } else {
+        const progressCallback = (p) => {
+          const msg = p.message || `Status: ${p.progress?.processed ?? 0}/${p.progress?.total ?? 0}...`;
+          onProgress({ stage: p.stage || 'fetching', message: msg });
+        };
+        [statusResources, medialogResult] = await Promise.all([
+          runBulkStatus(org, site, 'main', contentPathForStatus, {
+            onProgress: progressCallback,
+            pollInterval: IndexConfig.STATUS_POLL_INTERVAL_MS,
+            maxDurationMs: IndexConfig.STATUS_POLL_MAX_DURATION_MS,
+          }).then(({ resources: r }) => r),
+          fetchAllMediaLog(org, site, timeParams, onMedialogChunk),
+        ]);
+      }
+
+      if (isPerfEnabled()) {
+        console.log(`[MediaLibrary:build] Status API returned ${statusResources.length} resources`);
+        console.log(`[MediaLibrary:build] Medialog returned ${medialogResult.length} entries`);
+      }
+    } catch (error) {
+      if (isPerfEnabled()) {
+        console.error('[MediaLibrary:build] Error during fetch:', error);
+      }
+      throw error;
+    }
+
+    perf.statusAPI.durationMs = Date.now() - statusStart;
+    perf.statusAPI.resourcesDiscovered = statusResources.length;
+    perf.medialog.durationMs = Date.now() - medialogStart;
+
     const filteredResources = path
       ? statusResources.filter((r) => pathUnder(r.path, path))
       : statusResources;
@@ -239,6 +325,10 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
     newAuditlog = syntheticAudit;
   }
 
+  // Track pages and files from status/audit
+  perf.statusAPI.pagesDiscovered = [...pagesByPath.keys()].length;
+  perf.statusAPI.filesDiscovered = [...filesByPath.keys()].length;
+
   const mediaData = await buildMediaDataFromEntries(
     newMedialog,
     newAuditlog,
@@ -247,7 +337,17 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
     onProgress,
     onProgressiveData,
     path,
+    perf, // Pass perf object for markdown parse tracking
   );
 
-  return { mediaData, buildMode };
+  perf.totalDurationMs = Date.now() - buildStartTime;
+  perf.collectedAt = new Date().toISOString();
+
+  // Log performance metrics only when debug=perf is enabled
+  if (isPerfEnabled()) {
+    console.log('[MediaLibrary:build] Build completed successfully');
+    console.log('[MediaLibrary:perf]', JSON.stringify(perf, null, 2));
+  }
+
+  return { mediaData, buildMode, perf };
 }
diff --git a/tools/media-library/indexing/parse.js b/tools/media-library/indexing/parse.js
index 1862b278..3fb1a45a 100644
--- a/tools/media-library/indexing/parse.js
+++ b/tools/media-library/indexing/parse.js
@@ -11,6 +11,7 @@ import {
 } from '../core/constants.js';
 import { getDedupeKey } from '../core/urls.js';
 import { fetchAdminWithRateLimit } from '../core/admin-rate-limit.js';
+import { isPerfEnabled } from '../core/params.js';
 
 export { getDedupeKey };
 
@@ -210,6 +211,7 @@ const ADMIN_PREVIEW_BASE = 'https://admin.hlx.page/preview';
 /**
  * Fetch page markdown via admin.hlx.page (same as page-status diff).
  * Uses shared admin rate limiter (10 req/s) and 429 retry with backoff.
+ * Returns structured result: { markdown, status, reason }
  */
 export async function fetchPageMarkdown(pagePath, org, repo, ref = 'main') {
   try {
@@ -221,17 +223,21 @@ export async function fetchPageMarkdown(pagePath, org, repo, ref = 'main') {
     const url = `${ADMIN_PREVIEW_BASE}/${org}/${repo}/${ref}${fetchPath}`;
 
     const resp = await fetchAdminWithRateLimit(url, {}, { maxRetries: 3 });
-    if (!resp.ok) return null;
-    return resp.text();
-  } catch {
-    return null;
+    if (!resp.ok) {
+      return { markdown: null, status: resp.status, reason: `HTTP ${resp.status}` };
+    }
+    const markdown = await resp.text();
+    return { markdown, status: 200, reason: null };
+  } catch (error) {
+    return { markdown: null, status: null, reason: error.message || 'Network error' };
   }
 }
 
 /**
  * Build usage map by fetching pages and parsing markdown for PDF/SVG/fragment/external links.
+ * Enhanced with batching, failure tracking, retry logic, and progressive callbacks.
  */
-export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProgress = null) {
+export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProgress = null, onBatch = null) {
   const usageMap = {
     fragments: new Map(),
     pdfs: new Map(),
@@ -251,18 +257,15 @@ export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProg
 
   const uniquePages = [...latestTimestampByPath.keys()].filter((p) => !isHiddenPath(p));
 
-  const results = await processConcurrently(
-    uniquePages,
-    async (normalizedPath, i) => {
-      onProgress?.({ message: `Parsing page ${i + 1}/${uniquePages.length}: ${normalizedPath}` });
-      const md = await fetchPageMarkdown(normalizedPath, org, repo, ref);
-      return { normalizedPath, md };
-    },
-    IndexConfig.MAX_CONCURRENT_FETCHES,
-  );
+  const usageMapStartTime = Date.now();
+  const counters = { success: 0, fail: 0, parsed: 0 };
+  const fetchTimes = [];
+  const batchSize = IndexConfig.USAGE_MAP_PROGRESSIVE_BATCH_SIZE ?? 1000;
+  const failureReasons = new Map(); // reason -> count
+  const failedPathsByReason = new Map(); // reason -> path[]
 
-  results.forEach(({ normalizedPath, md }) => {
-    if (!md) return;
+  const processResultIntoUsageMap = ({ normalizedPath, md }) => {
+    if (md == null) return; // allow empty string (200 with empty body)
 
     const addToMap = (map, path) => {
       if (!map.has(path)) map.set(path, new Set());
@@ -294,7 +297,87 @@ export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProg
     svgs.forEach((s) => addToMap(usageMap.svgs, s));
     icons.forEach((s) => addToMap(usageMap.svgs, s));
     externalUrls.forEach((u) => addToExternalMedia(u));
-  });
+  };
+
+  // Process pages in batches for progressive display
+  for (let batchStart = 0; batchStart < uniquePages.length; batchStart += batchSize) {
+    const batch = uniquePages.slice(batchStart, batchStart + batchSize);
+    const batchResults = await processConcurrently(
+      batch,
+      async (normalizedPath, i) => {
+        const globalIndex = batchStart + i;
+        onProgress?.({ message: `Parsing page ${globalIndex + 1}/${uniquePages.length}: ${normalizedPath}` });
+        const fetchStart = Date.now();
+        const result = await fetchPageMarkdown(normalizedPath, org, repo, ref);
+        const fetchTime = Date.now() - fetchStart;
+        fetchTimes.push(fetchTime);
+
+        const isSuccess = result?.status === 200;
+        const md = isSuccess ? (result.markdown ?? '') : (result?.markdown || null);
+        if (isSuccess) {
+          counters.success += 1;
+        } else {
+          counters.fail += 1;
+          const reason = result?.reason || `HTTP ${result?.status ?? 'unknown'}`;
+          const count = failureReasons.get(reason) || 0;
+          failureReasons.set(reason, count + 1);
+          if (!failedPathsByReason.has(reason)) failedPathsByReason.set(reason, []);
+          failedPathsByReason.get(reason).push(normalizedPath);
+        }
+        counters.parsed += 1;
+        return { normalizedPath, md };
+      },
+      IndexConfig.MAX_CONCURRENT_PAGE_FETCHES,
+    );
+
+    batchResults.forEach(processResultIntoUsageMap);
+
+    // Call onBatch after each batch for progressive UI updates
+    if (onBatch) {
+      onBatch(usageMap);
+    }
+  }
+
+  // Retry failed pages serially (concurrency=1 to avoid overload)
+  const allFailedPaths = [];
+  failedPathsByReason.forEach((paths) => allFailedPaths.push(...paths));
+  if (allFailedPaths.length > 0) {
+    onProgress?.({ message: `Retrying ${allFailedPaths.length} failed pages...` });
+    const retryResults = await processConcurrently(
+      allFailedPaths,
+      async (normalizedPath) => {
+        const result = await fetchPageMarkdown(normalizedPath, org, repo, ref);
+        const isSuccess = result?.status === 200;
+        const md = isSuccess ? (result.markdown ?? '') : (result?.markdown || null);
+        return { normalizedPath, md, isSuccess };
+      },
+      1, /* serial retries */
+    );
+    retryResults.forEach(processResultIntoUsageMap);
+    retryResults.forEach(({ isSuccess }) => {
+      if (isSuccess) {
+        counters.success += 1;
+        counters.fail -= 1;
+      }
+    });
+    // Remove recovered paths from failedPathsByReason and failureReasons
+    const recovered = new Set(retryResults.filter((r) => r.isSuccess).map((r) => r.normalizedPath));
+    if (recovered.size > 0) {
+      failedPathsByReason.forEach((paths, reason) => {
+        const remaining = paths.filter((p) => !recovered.has(p));
+        if (remaining.length === 0) {
+          failedPathsByReason.delete(reason);
+          failureReasons.delete(reason);
+        } else {
+          failedPathsByReason.set(reason, remaining);
+          failureReasons.set(reason, remaining.length);
+        }
+      });
+      if (isPerfEnabled()) {
+        console.log(`[buildUsageMap] Retry: ${recovered.size}/${allFailedPaths.length} recovered`);
+      }
+    }
+  }
 
   // Convert Sets to arrays for consumers
   ['fragments', 'pdfs', 'svgs'].forEach((key) => {
@@ -306,5 +389,49 @@ export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProg
     usageMap.externalMedia.set(url, { ...data, pages: [...data.pages] });
   });
 
-  return usageMap;
+  // Performance logging
+  const avgFetchTime = fetchTimes.length > 0
+    ? Math.round(fetchTimes.reduce((sum, t) => sum + t, 0) / fetchTimes.length)
+    : 0;
+  const maxFetchTime = fetchTimes.length > 0 ? Math.max(...fetchTimes) : 0;
+  const minFetchTime = fetchTimes.length > 0 ? Math.min(...fetchTimes) : 0;
+  const durationMs = Date.now() - usageMapStartTime;
+  const fragCount = usageMap.fragments?.size ?? 0;
+  const pdfCount = usageMap.pdfs?.size ?? 0;
+  const svgCount = usageMap.svgs?.size ?? 0;
+  const extCount = usageMap.externalMedia?.size ?? 0;
+  const itemsFound = fragCount + pdfCount + svgCount + extCount;
+
+  if (isPerfEnabled()) {
+    const endIso = new Date().toISOString();
+    const durationSec = Math.round(durationMs / 1000);
+    const lines = [
+      `[buildUsageMap] Ended at ${endIso} (duration: ${durationSec}s)`,
+      `  Pages: ${counters.success} success, ${counters.fail} failed of ${uniquePages.length} total`,
+      `  Fetch times: avg=${avgFetchTime}ms, min=${minFetchTime}ms, max=${maxFetchTime}ms`,
+      `  Items: frag=${fragCount}, pdf=${pdfCount}, svg=${svgCount}, ext=${extCount} (${itemsFound} total)`,
+    ];
+    const sortedReasons = [...failureReasons.entries()].sort((a, b) => b[1] - a[1]);
+    if (counters.fail > 0) {
+      if (sortedReasons.length > 0) {
+        lines.push('  Failure breakdown:');
+        sortedReasons.forEach(([reason, count]) => {
+          lines.push(`    ${reason}: ${count} (${Math.round((count / counters.fail) * 100)}%)`);
+          const paths = failedPathsByReason.get(reason) || [];
+          paths.slice(0, 10).forEach((p) => lines.push(`      - ${p}`));
+          if (paths.length > 10) {
+            lines.push(`      ... and ${paths.length - 10} more`);
+          }
+        });
+      }
+    }
+    console.log(lines.join('\n'));
+  }
+
+  return {
+    usageMap,
+    counters,
+    durationMs,
+    fetchTimes: { avg: avgFetchTime, min: minFetchTime, max: maxFetchTime },
+  };
 }
diff --git a/tools/media-library/indexing/reconcile.js b/tools/media-library/indexing/reconcile.js
index 97cb9d5d..79f6debf 100644
--- a/tools/media-library/indexing/reconcile.js
+++ b/tools/media-library/indexing/reconcile.js
@@ -187,6 +187,7 @@ export async function processLinkedContent(
   ref = 'main',
   onProgress = null,
   path = '',
+  onProgressiveData = null,
 ) {
   const aud = auditlogEntries || [];
   const med = medialogEntries || [];
@@ -210,11 +211,51 @@ export async function processLinkedContent(
   });
 
   if (pageEntries.length === 0) {
-    return { added: [], linkedEntries: [] };
+    return { added: [], linkedEntries: [], parseStats: null };
   }
 
   onProgress?.({ stage: 'parsing', message: 'Building usage map from page content...' });
-  const usageMap = await buildUsageMap(pageEntries, org, repo, ref, onProgress);
+
+  // Create onBatch callback for progressive updates during parsing
+  const onBatch = onProgressiveData
+    ? (usageMapPartial) => {
+      // Build linked entries from current usage map state
+      const linkedPartial = [];
+      const allLinkedPathsPartial = new Set(filesByPath.keys());
+      ['pdfs', 'svgs', 'fragments'].forEach((key) => {
+        usageMapPartial[key]?.forEach((_, fp) => allLinkedPathsPartial.add(fp));
+      });
+
+      allLinkedPathsPartial.forEach((filePath) => {
+        if (deletedPaths.has(filePath)) return;
+        let key = 'fragments';
+        if (isPdf(filePath)) key = 'pdfs';
+        else if (isSvg(filePath)) key = 'svgs';
+        const linkedPages = usageMapPartial[key]?.get(filePath) || [];
+        const status = linkedPages.length > 0 ? 'referenced' : 'discovering';
+        const fileEvent = filesByPath.get(filePath) || { timestamp: 0, user: '' };
+        linkedPages.forEach((doc) => {
+          linkedPartial.push(toLinkedContentEntry(filePath, doc, fileEvent, status, org, repo));
+        });
+      });
+
+      if (linkedPartial.length > 0) {
+        onProgressiveData(linkedPartial);
+      }
+    }
+    : null;
+
+  const result = await buildUsageMap(pageEntries, org, repo, ref, onProgress, onBatch);
+  const usageMap = result.usageMap || result; // Support both old and new return format
+  const parseStats = result.counters
+    ? {
+      pages: result.counters.parsed || 0,
+      durationMs: result.durationMs ?? 0,
+      success: result.counters.success || 0,
+      fail: result.counters.fail || 0,
+      fetchTimes: result.fetchTimes || { avg: 0, min: 0, max: 0 },
+    }
+    : null;
 
   const allLinkedPaths = new Set(filesByPath.keys());
   ['pdfs', 'svgs', 'fragments'].forEach((key) => {
@@ -254,5 +295,5 @@ export async function processLinkedContent(
     });
   });
 
-  return { added: linkedEntries, linkedEntries };
+  return { added: linkedEntries, linkedEntries, parseStats };
 }
diff --git a/tools/media-library/media-library.js b/tools/media-library/media-library.js
index 2611ef9d..39fac38e 100644
--- a/tools/media-library/media-library.js
+++ b/tools/media-library/media-library.js
@@ -311,7 +311,8 @@ async function init() {
 
     let cancelProgressiveThrottle = () => {};
     try {
-      if (!(await ensureLogin(orgKey, siteKey))) {
+      const isLoggedIn = await ensureLogin(orgKey, siteKey);
+      if (!isLoggedIn) {
         logMediaLibraryError(ErrorCodes.AUTH_REQUIRED, { context: 'build' });
         showNotification(t('NOTIFY_ERROR'), t('AUTH_REQUIRED'), 'danger');
         updateAppState({
@@ -357,17 +358,32 @@ async function init() {
 
       updateAppState({ isValidating: false, isIndexing: true, progressiveMediaData: [] });
 
+      const indexStartTime = Date.now();
+      // eslint-disable-next-line no-console -- index lifecycle: start (always logged)
+      console.log('[Media Library] Index started');
+
       const progressiveMap = new Map();
       let throttleTimer = null;
+      let progressiveDirty = false;
       cancelProgressiveThrottle = () => {
         if (throttleTimer) {
           clearTimeout(throttleTimer);
           throttleTimer = null;
         }
       };
+      let lastProgressiveCount = 0;
       const flushProgressive = () => {
         cancelProgressiveThrottle();
         const toEmit = Array.from(progressiveMap.values());
+
+        // Use insertion order during indexing to avoid cards jumping when new items arrive.
+        // Final build will apply proper sort.
+        if (!progressiveDirty && toEmit.length === lastProgressiveCount && toEmit.length > 0) {
+          return;
+        }
+        lastProgressiveCount = toEmit.length;
+        progressiveDirty = false;
+
         const capped = toEmit.length > PROGRESSIVE_DISPLAY_CAP
           ? toEmit.slice(0, PROGRESSIVE_DISPLAY_CAP)
           : toEmit;
@@ -376,10 +392,12 @@ async function init() {
       const onProgressiveData = (items) => {
         if (!items?.length) return;
         items.forEach((item) => {
-          const key = getDedupeKey(item.url);
+          const key = getDedupeKey(item?.url || item?.path || '');
+          if (!key || key.length < 2) return; // Skip invalid/empty keys (e.g. url='"')
           const existing = progressiveMap.get(key);
           if (!existing || (item.timestamp ?? 0) >= (existing.timestamp ?? 0)) {
             progressiveMap.set(key, item);
+            progressiveDirty = true;
           }
         });
         if (throttleTimer) clearTimeout(throttleTimer);
@@ -390,7 +408,7 @@ async function init() {
         onProgressiveData(cachedData);
       }
 
-      const { mediaData, buildMode } = await fetchAndBuildMediaData(orgKey, siteKey, {
+      const { mediaData, buildMode, perf } = await fetchAndBuildMediaData(orgKey, siteKey, {
         incremental,
         metadata,
         path: pathKey,
@@ -399,6 +417,11 @@ async function init() {
         statusResources: statusResourcesForBuild,
       });
 
+      const indexDurationSec = Math.round((Date.now() - indexStartTime) / 1000);
+      const pagesParsed = perf?.markdownParse?.pages ?? 0;
+      // eslint-disable-next-line no-console -- index lifecycle: end, duration, pages (always logged)
+      console.log(`[Media Library] Index completed in ${indexDurationSec}s, ${pagesParsed} pages`);
+
       const finalMediaData = incremental && hasCache
         ? (() => {
           const mergedMap = new Map();
@@ -594,6 +617,24 @@ async function init() {
   );
 
   registerToolReady(Promise.resolve(unsubscribe));
+
+  // Auto-load if org and site are in URL params
+  const orgParam = searchParams.get('org');
+  const siteParam = searchParams.get('site');
+  if (orgParam && siteParam) {
+    // Pre-populate form fields
+    orgInput.value = orgParam;
+    siteInput.value = siteParam;
+    if (pathParam && pathInput) {
+      pathInput.value = pathParam;
+    }
+
+    // Auto-load after a short delay to ensure DOM is ready
+    setTimeout(() => {
+      const path = getPathFromInput();
+      loadMediaData(orgParam, siteParam, path);
+    }, 100);
+  }
 }
 
 init();
diff --git a/tools/media-library/views/grid/grid.js b/tools/media-library/views/grid/grid.js
index 05ff4235..66820627 100644
--- a/tools/media-library/views/grid/grid.js
+++ b/tools/media-library/views/grid/grid.js
@@ -64,8 +64,12 @@ function getFilteredMedia(state) {
 
 function getDisplayData(state) {
   const hasProgressive = state.isIndexing && state.progressiveMediaData?.length > 0;
-  const data = hasProgressive ? state.progressiveMediaData : getFilteredMedia(state);
-  return sortMediaData(data);
+  if (hasProgressive) {
+    // Use order as-is during indexing (already sorted by URL in coordinator).
+    // Re-sorting by timestamp/doc/name would move cards as data is updated.
+    return state.progressiveMediaData;
+  }
+  return sortMediaData(getFilteredMedia(state));
 }
 
 function renderMediaPreview(media, org, site) {

From ae0f35dfe97379e6654bf08412fd4920daea5a65 Mon Sep 17 00:00:00 2001
From: Kiran Murugulla <murugull@adobe.com>
Date: Fri, 13 Mar 2026 11:07:43 -0400
Subject: [PATCH 03/10] lint fixes

---
 tools/media-library/core/params.js    | 7 ++++---
 tools/media-library/core/utils.js     | 6 ++++--
 tools/media-library/indexing/build.js | 9 ++++++++-
 tools/media-library/indexing/parse.js | 5 ++++-
 tools/media-library/media-library.js  | 4 ++--
 5 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/tools/media-library/core/params.js b/tools/media-library/core/params.js
index f691d506..0cbe87b0 100644
--- a/tools/media-library/core/params.js
+++ b/tools/media-library/core/params.js
@@ -8,7 +8,7 @@ let perfLoggedOnce = false;
 /**
  * Check if performance logging is enabled via ?debug=perf URL parameter
  */
-export function isPerfEnabled() {
+export default function isPerfEnabled() {
   if (typeof window === 'undefined') return false;
 
   // Cache the result since URL params don't change during page lifetime
@@ -21,9 +21,10 @@ export function isPerfEnabled() {
   const enabled = debug === 'perf' || debug === 'true' || debug === '1';
   perfEnabledCached = enabled;
 
-  // Log once for debugging
+  // Log once when perf debugging is enabled
   if (enabled && !perfLoggedOnce) {
-    console.log('[Media Library] Performance logging enabled via ?debug=' + debug);
+    // eslint-disable-next-line no-console -- perf mode: indicate debug=perf is active
+    console.log(`[Media Library] Performance logging enabled via ?debug=${debug}`);
     perfLoggedOnce = true;
   }
 
diff --git a/tools/media-library/core/utils.js b/tools/media-library/core/utils.js
index 22bdba50..882af163 100644
--- a/tools/media-library/core/utils.js
+++ b/tools/media-library/core/utils.js
@@ -21,7 +21,9 @@ export function escapeAttr(str) {
 export function safeUrlForAttr(url) {
   if (!url || typeof url !== 'string') return '';
   const t = url.trim();
-  if (t.startsWith('https://') || t.startsWith('http://') || (t.startsWith('/') && !t.startsWith('//'))) {
+  const hasValidScheme = t.startsWith('https://') || t.startsWith('http://');
+  const isRelPath = t.startsWith('/') && !t.startsWith('//');
+  if (hasValidScheme || isRelPath) {
     return escapeAttr(t);
   }
   return '';
@@ -50,7 +52,7 @@ export function pluralize(singular, plural, count) {
 }
 
 /**
- * Sorts media items for display: newest first by timestamp, then by doc path depth, then by name, then by URL for stability.
+ * Sorts media items for display: newest by timestamp, then doc depth, name, URL.
  */
 export function sortMediaData(mediaData) {
   if (!mediaData || mediaData.length === 0) return mediaData ?? [];
diff --git a/tools/media-library/indexing/build.js b/tools/media-library/indexing/build.js
index c7691e6e..d1fcc074 100644
--- a/tools/media-library/indexing/build.js
+++ b/tools/media-library/indexing/build.js
@@ -23,7 +23,7 @@ import {
 import { toAbsoluteFilePath } from './parse.js';
 import { IndexConfig } from '../core/constants.js';
 import { incrementalTimeParams, initialTimeParams } from '../core/storage.js';
-import { isPerfEnabled } from '../core/params.js';
+import isPerfEnabled from '../core/params.js';
 
 const PROGRESSIVE_DISPLAY_CAP = 3000;
 
@@ -152,6 +152,7 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
   const buildStartTime = Date.now();
 
   if (isPerfEnabled()) {
+    // eslint-disable-next-line no-console -- perf debug when ?debug=perf
     console.log(`[MediaLibrary:build] Starting ${buildMode} build for ${org}/${site}${path || '/'}`);
   }
 
@@ -253,6 +254,7 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
     const statusStart = Date.now();
 
     if (isPerfEnabled()) {
+      // eslint-disable-next-line no-console -- perf debug when ?debug=perf
       console.log('[MediaLibrary:build] Fetching status API and medialog in parallel...');
     }
 
@@ -278,11 +280,14 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
       }
 
       if (isPerfEnabled()) {
+        // eslint-disable-next-line no-console -- perf debug when ?debug=perf
         console.log(`[MediaLibrary:build] Status API returned ${statusResources.length} resources`);
+        // eslint-disable-next-line no-console -- perf debug when ?debug=perf
         console.log(`[MediaLibrary:build] Medialog returned ${medialogResult.length} entries`);
       }
     } catch (error) {
       if (isPerfEnabled()) {
+        // eslint-disable-next-line no-console -- perf debug: log fetch error when ?debug=perf
         console.error('[MediaLibrary:build] Error during fetch:', error);
       }
       throw error;
@@ -345,7 +350,9 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
 
   // Log performance metrics only when debug=perf is enabled
   if (isPerfEnabled()) {
+    // eslint-disable-next-line no-console -- perf debug when ?debug=perf
     console.log('[MediaLibrary:build] Build completed successfully');
+    // eslint-disable-next-line no-console -- perf debug when ?debug=perf
     console.log('[MediaLibrary:perf]', JSON.stringify(perf, null, 2));
   }
 
diff --git a/tools/media-library/indexing/parse.js b/tools/media-library/indexing/parse.js
index 3fb1a45a..bd4f279a 100644
--- a/tools/media-library/indexing/parse.js
+++ b/tools/media-library/indexing/parse.js
@@ -11,7 +11,7 @@ import {
 } from '../core/constants.js';
 import { getDedupeKey } from '../core/urls.js';
 import { fetchAdminWithRateLimit } from '../core/admin-rate-limit.js';
-import { isPerfEnabled } from '../core/params.js';
+import isPerfEnabled from '../core/params.js';
 
 export { getDedupeKey };
 
@@ -302,6 +302,7 @@ export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProg
   // Process pages in batches for progressive display
   for (let batchStart = 0; batchStart < uniquePages.length; batchStart += batchSize) {
     const batch = uniquePages.slice(batchStart, batchStart + batchSize);
+    // eslint-disable-next-line no-await-in-loop -- batch: complete each before next
     const batchResults = await processConcurrently(
       batch,
       async (normalizedPath, i) => {
@@ -374,6 +375,7 @@ export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProg
         }
       });
       if (isPerfEnabled()) {
+        // eslint-disable-next-line no-console -- perf debug when ?debug=perf
         console.log(`[buildUsageMap] Retry: ${recovered.size}/${allFailedPaths.length} recovered`);
       }
     }
@@ -425,6 +427,7 @@ export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProg
         });
       }
     }
+    // eslint-disable-next-line no-console -- perf debug when ?debug=perf
     console.log(lines.join('\n'));
   }
 
diff --git a/tools/media-library/media-library.js b/tools/media-library/media-library.js
index 39fac38e..ed517d9b 100644
--- a/tools/media-library/media-library.js
+++ b/tools/media-library/media-library.js
@@ -419,8 +419,8 @@ async function init() {
 
       const indexDurationSec = Math.round((Date.now() - indexStartTime) / 1000);
       const pagesParsed = perf?.markdownParse?.pages ?? 0;
-      // eslint-disable-next-line no-console -- index lifecycle: end, duration, pages (always logged)
-      console.log(`[Media Library] Index completed in ${indexDurationSec}s, ${pagesParsed} pages`);
+      // eslint-disable-next-line no-console -- index lifecycle (duration, pages)
+      console.log(`[Media Library] Index done: ${indexDurationSec}s, ${pagesParsed} pages`);
 
       const finalMediaData = incremental && hasCache
         ? (() => {

From 87f54d0ff4aaef7fd1e94e37e2d2314c2716045d Mon Sep 17 00:00:00 2001
From: Kiran Murugulla <murugull@adobe.com>
Date: Fri, 13 Mar 2026 14:43:30 -0400
Subject: [PATCH 04/10] fix : correcting the filter criteria for index

---
 tools/media-library/indexing/build.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tools/media-library/indexing/build.js b/tools/media-library/indexing/build.js
index d1fcc074..76eec870 100644
--- a/tools/media-library/indexing/build.js
+++ b/tools/media-library/indexing/build.js
@@ -50,6 +50,11 @@ export async function buildMediaDataFromEntries(
     medialogScoped = medialogEntries.filter(
       (m) => m.resourcePath && pathUnder(m.resourcePath, pathNorm),
     );
+  } else {
+    // Match DA-NX: keep referenced (resourcePath) or standalone; drop orphaned.
+    medialogScoped = medialogEntries.filter(
+      (m) => m.resourcePath || (m.originalFilename && !m.resourcePath),
+    );
   }
 
   const { linkedEntries, parseStats } = await processLinkedContent(

From dde3d8743ecb238c746005ab68432301de33ddc5 Mon Sep 17 00:00:00 2001
From: Kiran Murugulla <murugull@adobe.com>
Date: Fri, 13 Mar 2026 15:15:06 -0400
Subject: [PATCH 05/10] fix: use pathUnder in buildPathPartitions to enforce
 path boundary

---
 tools/media-library/indexing/bulk-status.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/media-library/indexing/bulk-status.js b/tools/media-library/indexing/bulk-status.js
index 9fe0c882..374191fd 100644
--- a/tools/media-library/indexing/bulk-status.js
+++ b/tools/media-library/indexing/bulk-status.js
@@ -8,6 +8,7 @@ import {
   parseResourcesFromDetailsRaw,
 } from './status-api.js';
 import { IndexConfig } from '../core/constants.js';
+import { pathUnder } from '../core/urls.js';
 
 const REQ_PER_SEC = 10;
 const THROTTLE_MS = 1000 / REQ_PER_SEC;
@@ -118,7 +119,7 @@ function buildPathPartitions(paths, base = null) {
 
   paths.forEach((path) => {
     if (typeof path !== 'string' || !path.startsWith('/')) return;
-    if (base && !path.startsWith(base) && path !== base) return;
+    if (base && !pathUnder(path, base)) return;
     const pathNorm = path.replace(/\/$/, '');
     let relPath;
     if (!base) {

From 4646444caf20824b68d2ded89b9c53359f569830 Mon Sep 17 00:00:00 2001
From: Kiran Murugulla <murugull@adobe.com>
Date: Fri, 13 Mar 2026 15:39:27 -0400
Subject: [PATCH 06/10] fix path boundary check, Set.size access, and Math.max 
  RangeError

---
 tools/media-library/indexing/bulk-status.js | 3 +--
 tools/media-library/indexing/parse.js       | 4 ++--
 tools/media-library/indexing/reconcile.js   | 2 +-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tools/media-library/indexing/bulk-status.js b/tools/media-library/indexing/bulk-status.js
index 374191fd..ae418648 100644
--- a/tools/media-library/indexing/bulk-status.js
+++ b/tools/media-library/indexing/bulk-status.js
@@ -8,7 +8,6 @@ import {
   parseResourcesFromDetailsRaw,
 } from './status-api.js';
 import { IndexConfig } from '../core/constants.js';
-import { pathUnder } from '../core/urls.js';
 
 const REQ_PER_SEC = 10;
 const THROTTLE_MS = 1000 / REQ_PER_SEC;
@@ -119,7 +118,7 @@ function buildPathPartitions(paths, base = null) {
 
   paths.forEach((path) => {
     if (typeof path !== 'string' || !path.startsWith('/')) return;
-    if (base && !pathUnder(path, base)) return;
+    if (base && path !== base && !path.startsWith(`${base}/`)) return;
     const pathNorm = path.replace(/\/$/, '');
     let relPath;
     if (!base) {
diff --git a/tools/media-library/indexing/parse.js b/tools/media-library/indexing/parse.js
index bd4f279a..413820cd 100644
--- a/tools/media-library/indexing/parse.js
+++ b/tools/media-library/indexing/parse.js
@@ -395,8 +395,8 @@ export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProg
   const avgFetchTime = fetchTimes.length > 0
     ? Math.round(fetchTimes.reduce((sum, t) => sum + t, 0) / fetchTimes.length)
     : 0;
-  const maxFetchTime = fetchTimes.length > 0 ? Math.max(...fetchTimes) : 0;
-  const minFetchTime = fetchTimes.length > 0 ? Math.min(...fetchTimes) : 0;
+  const maxFetchTime = fetchTimes.length > 0 ? fetchTimes.reduce((a, b) => Math.max(a, b)) : 0;
+  const minFetchTime = fetchTimes.length > 0 ? fetchTimes.reduce((a, b) => Math.min(a, b)) : 0;
   const durationMs = Date.now() - usageMapStartTime;
   const fragCount = usageMap.fragments?.size ?? 0;
   const pdfCount = usageMap.pdfs?.size ?? 0;
diff --git a/tools/media-library/indexing/reconcile.js b/tools/media-library/indexing/reconcile.js
index 79f6debf..9d01ecc9 100644
--- a/tools/media-library/indexing/reconcile.js
+++ b/tools/media-library/indexing/reconcile.js
@@ -232,7 +232,7 @@ export async function processLinkedContent(
         if (isPdf(filePath)) key = 'pdfs';
         else if (isSvg(filePath)) key = 'svgs';
         const linkedPages = usageMapPartial[key]?.get(filePath) || [];
-        const status = linkedPages.length > 0 ? 'referenced' : 'discovering';
+        const status = (linkedPages.size ?? linkedPages.length) > 0 ? 'referenced' : 'discovering';
         const fileEvent = filesByPath.get(filePath) || { timestamp: 0, user: '' };
         linkedPages.forEach((doc) => {
           linkedPartial.push(toLinkedContentEntry(filePath, doc, fileEvent, status, org, repo));

From 26b5d119b697272ac18896f639d55527648b3e53 Mon Sep 17 00:00:00 2001
From: Kiran Murugulla <murugull@adobe.com>
Date: Fri, 13 Mar 2026 20:20:05 -0400
Subject: [PATCH 07/10] fix : collecting only pages from status api

---
 tools/media-library/indexing/build.js | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/tools/media-library/indexing/build.js b/tools/media-library/indexing/build.js
index 76eec870..a9a21d26 100644
--- a/tools/media-library/indexing/build.js
+++ b/tools/media-library/indexing/build.js
@@ -20,7 +20,7 @@ import {
   applyAuditChunkToMaps,
   buildEarlyLinkedPlaceholders,
 } from './reconcile.js';
-import { toAbsoluteFilePath } from './parse.js';
+import { toAbsoluteFilePath, isPage } from './parse.js';
 import { IndexConfig } from '../core/constants.js';
 import { incrementalTimeParams, initialTimeParams } from '../core/storage.js';
 import isPerfEnabled from '../core/params.js';
@@ -316,15 +316,17 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
       );
     }
 
-    const syntheticAudit = filteredResources.map((r) => {
-      const entryPath = toAbsoluteFilePath(r.path);
-      return {
-        path: entryPath,
-        route: 'preview',
-        method: 'UPDATE',
-        timestamp: Date.now(),
-      };
-    });
+    const syntheticAudit = filteredResources
+      .filter((r) => isPage(r.path))
+      .map((r) => {
+        const entryPath = toAbsoluteFilePath(r.path);
+        return {
+          path: entryPath,
+          route: 'preview',
+          method: 'UPDATE',
+          timestamp: Date.now(),
+        };
+      });
 
     applyAuditChunkToMaps(syntheticAudit, pagesByPath, filesByPath, deletedPaths);
     const earlyLinked = buildEarlyLinkedPlaceholders(filesByPath, deletedPaths, org, site);

From 51ce8f02f2a0646d21220331a9c08fee74e8cfe6 Mon Sep 17 00:00:00 2001
From: Kiran Murugulla <murugull@adobe.com>
Date: Sat, 14 Mar 2026 23:23:06 -0400
Subject: [PATCH 08/10] fix: adding the optimization enhancements

---
 tools/media-library/core/admin-rate-limit.js  |  13 +-
 tools/media-library/core/constants.js         |  12 +-
 tools/media-library/core/media.js             | 116 +++++++++++---
 tools/media-library/core/urls.js              |  12 +-
 tools/media-library/indexing/build.js         |  53 +++++--
 tools/media-library/indexing/parse.js         | 142 +++++++++++-------
 tools/media-library/indexing/reconcile.js     |  18 ++-
 tools/media-library/media-library.js          |   9 +-
 .../views/mediainfo/mediainfo.js              |   5 +-
 9 files changed, 268 insertions(+), 112 deletions(-)

diff --git a/tools/media-library/core/admin-rate-limit.js b/tools/media-library/core/admin-rate-limit.js
index 02addd04..3a783812 100644
--- a/tools/media-library/core/admin-rate-limit.js
+++ b/tools/media-library/core/admin-rate-limit.js
@@ -3,7 +3,9 @@
  * Same pattern as backfill tool: promise-chain acquire, 429 retry with backoff.
  */
 
-const ADMIN_API_RATE = 10;
+import isPerfEnabled from './params.js';
+
+const ADMIN_API_RATE = 8; /* keep some headroom under the 10 RPS project limit */
 
 function waitForDelay(ms, signal = null) {
   const duration = Math.max(0, ms);
@@ -72,10 +74,15 @@ export async function fetchAdminWithRateLimit(
     limiter.handleResponse(res);
 
     if (res.status === 429 && attemptNumber < maxRetries) {
-      const retryAfter = parseInt(
+      const headerVal = parseInt(
         res.headers.get('x-retry-after') || res.headers.get('retry-after'),
         10,
-      ) || (2 ** attemptNumber);
+      );
+      const retryAfter = Math.max(headerVal || 2 ** attemptNumber, 30);
+      if (isPerfEnabled()) {
+        // eslint-disable-next-line no-console
+        console.log(`[admin-api] 429 rate limit hit, backing off ${retryAfter}s (attempt ${attemptNumber + 1}/${maxRetries + 1})`);
+      }
       limiter.backoff(retryAfter);
       return attempt(attemptNumber + 1);
     }
diff --git a/tools/media-library/core/constants.js b/tools/media-library/core/constants.js
index 6517a9c9..e9700e71 100644
--- a/tools/media-library/core/constants.js
+++ b/tools/media-library/core/constants.js
@@ -44,6 +44,13 @@ export const Paths = Object.freeze({
 export const CORS_PROXY_URL = 'https://media-library-cors-proxy.aem-poc-lab.workers.dev/';
 export const MEDIA_UNDERSCORE_PREFIX = 'media_';
 
+// External video detection regexes
+export const YOUTUBE_VIDEO_RE = /(?:youtube\.com\/(?:watch\?v=|embed\/|shorts\/)([^&\n?#/]+)|youtu\.be\/([^&\n?#/]+))/;
+export const VIMEO_VIDEO_RE = /(?:player\.)?vimeo\.com\/(?:video\/)?(\d+)(?:$|[/?#])/;
+export const DAILYMOTION_VIDEO_RE = /(?:dailymotion\.com\/video\/|dai\.ly\/)([^&\n?#]+)/;
+export const SCENE7_VIDEO_RE = /scene7\.com\/is\/content\//;
+export const DYNAMIC_MEDIA_VIDEO_RE = /\/is\/content\//;
+
 const mediaExtensions = {
   pdf: ['pdf'],
   svg: ['svg'],
@@ -69,11 +76,6 @@ export const ExternalMedia = Object.freeze({
   EXTENSION_REGEX: mediaExtensionRegex,
   HOST_PATTERNS: [
     { host: /adobeaemcloud\.com$/i, pathContains: 'urn:aaid:aem', typeFromPath: true },
-    { host: /youtube\.com$/i, type: MediaType.VIDEO },
-    { host: /youtu\.be$/i, type: MediaType.VIDEO },
-    { host: /vimeo\.com$/i, type: MediaType.VIDEO },
-    { host: /player\.vimeo\.com$/i, type: MediaType.VIDEO },
-    { host: /unsplash\.com$/i, type: categoryImg },
     { host: /images\.unsplash\.com$/i, type: categoryImg },
   ],
 });
diff --git a/tools/media-library/core/media.js b/tools/media-library/core/media.js
index 79229db2..9c2da6f3 100644
--- a/tools/media-library/core/media.js
+++ b/tools/media-library/core/media.js
@@ -1,4 +1,13 @@
-import { MediaType } from './constants.js';
+import {
+  MediaType,
+  ExternalMedia,
+  Domains,
+  YOUTUBE_VIDEO_RE,
+  VIMEO_VIDEO_RE,
+  DAILYMOTION_VIDEO_RE,
+  SCENE7_VIDEO_RE,
+  DYNAMIC_MEDIA_VIDEO_RE,
+} from './constants.js';
 
 const IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg', 'avif'];
 const VIDEO_EXTENSIONS = ['mp4', 'webm', 'mov', 'avi'];
@@ -55,12 +64,11 @@ export function isExternalVideoUrl(url) {
   if (!url || typeof url !== 'string') return false;
 
   const supportedPatterns = [
-    /(?:youtube\.com\/(?:watch\?v=|embed\/|shorts\/|)[^&\n?#/]+|youtu\.be\/[^&\n?#/]+)/,
-    /(?:^https?:\/\/)?(?:www\.)?(?:youtube\.com|youtu\.be)(?:\/|$)/,
-    /vimeo\.com\/(\d+)/,
-    /(?:dailymotion\.com\/video\/|dai\.ly\/)/,
-    /scene7\.com\/is\/content\//,
-    /marketing\.adobe\.com\/is\/content\//,
+    YOUTUBE_VIDEO_RE,
+    VIMEO_VIDEO_RE,
+    DAILYMOTION_VIDEO_RE,
+    SCENE7_VIDEO_RE,
+    DYNAMIC_MEDIA_VIDEO_RE,
   ];
 
   return supportedPatterns.some((pattern) => pattern.test(url));
@@ -102,29 +110,27 @@ export function isFragmentMedia(media) {
 export function getVideoThumbnail(videoUrl) {
   if (!videoUrl) return null;
 
-  const youtubeMatch = videoUrl.match(/(?:youtube\.com\/(?:watch\?v=|embed\/|shorts\/|)([^&\n?#/]+)|youtu\.be\/([^&\n?#/]+))/);
+  const youtubeMatch = videoUrl.match(YOUTUBE_VIDEO_RE);
   if (youtubeMatch) {
     const id = youtubeMatch[1] || youtubeMatch[2];
     return id ? `https://img.youtube.com/vi/${id}/hqdefault.jpg` : null;
   }
 
-  // Vimeo thumbnails require oEmbed API, not supported client-side
-  // Falls through to null, shows placeholder
+  const vimeoMatch = videoUrl.match(/vimeo\.com\/(\d+)/);
+  if (vimeoMatch) {
+    const videoId = vimeoMatch[1];
+    return `https://i.vimeocdn.com/video/${videoId}_640.jpg`;
+  }
 
-  const dailymotionMatch = videoUrl.match(/(?:dailymotion\.com\/video\/|dai\.ly\/)([^&\n?#/]+)/);
+  const dailymotionMatch = videoUrl.match(DAILYMOTION_VIDEO_RE);
   if (dailymotionMatch) {
     const videoId = dailymotionMatch[1];
     return `https://www.dailymotion.com/thumbnail/video/${videoId}`;
   }
 
-  const dynamicMediaMatch = videoUrl.match(/(scene7\.com\/is\/content\/[^?]+)/);
-  if (dynamicMediaMatch) {
-    return `${dynamicMediaMatch[1]}?fmt=jpeg&wid=300&hei=200`;
-  }
-
-  const marketingMatch = videoUrl.match(/(marketing\.adobe\.com\/is\/content\/[^?]+)/);
-  if (marketingMatch) {
-    return `${marketingMatch[1]}?fmt=jpeg&wid=300&hei=200`;
+  if (DYNAMIC_MEDIA_VIDEO_RE.test(videoUrl)) {
+    const dynamicMediaBase = videoUrl.split('?')[0];
+    return `${dynamicMediaBase}?fmt=jpeg&wid=300&hei=200`;
   }
 
   return null;
@@ -133,18 +139,18 @@ export function getVideoThumbnail(videoUrl) {
 export function getVideoEmbedUrl(videoUrl) {
   if (!videoUrl) return null;
 
-  const youtubeMatch = videoUrl.match(/(?:youtube\.com\/(?:watch\?v=|embed\/|shorts\/|)([^&\n?#/]+)|youtu\.be\/([^&\n?#/]+))/);
+  const youtubeMatch = videoUrl.match(YOUTUBE_VIDEO_RE);
   if (youtubeMatch) {
     const id = youtubeMatch[1] || youtubeMatch[2];
     return id ? `https://www.youtube.com/embed/${id}` : null;
   }
 
-  const vimeoMatch = videoUrl.match(/vimeo\.com\/(\d+)/);
+  const vimeoMatch = videoUrl.match(VIMEO_VIDEO_RE);
   if (vimeoMatch) {
     return `https://player.vimeo.com/video/${vimeoMatch[1]}`;
   }
 
-  const dailymotionMatch = videoUrl.match(/(?:dailymotion\.com\/video\/|dai\.ly\/)([^&\n?#/]+)/);
+  const dailymotionMatch = videoUrl.match(DAILYMOTION_VIDEO_RE);
   if (dailymotionMatch) {
     return `https://www.dailymotion.com/embed/video/${dailymotionMatch[1]}`;
   }
@@ -152,6 +158,72 @@ export function getVideoEmbedUrl(videoUrl) {
   return null;
 }
 
+function isExternalUrl(url) {
+  if (!url || !url.startsWith('http')) return false;
+  return !Domains.SAME_ORIGIN.some((domain) => url.includes(domain));
+}
+
+export function getExternalMediaTypeInfo(url) {
+  if (!url || !url.startsWith('http') || !isExternalUrl(url)) return null;
+
+  try {
+    const parsed = new URL(url);
+    const pathPart = parsed.pathname.split('?')[0].split('#')[0];
+    const pathLower = pathPart.toLowerCase();
+
+    const extMatch = pathLower.match(ExternalMedia.EXTENSION_REGEX);
+    if (extMatch) {
+      const ext = extMatch[1].toLowerCase();
+      let type = MediaType.LINK;
+      if (ExternalMedia.EXTENSIONS.pdf.includes(ext)) type = MediaType.DOCUMENT;
+      else if (ExternalMedia.EXTENSIONS.svg.includes(ext)) type = MediaType.IMAGE;
+      else if (ExternalMedia.EXTENSIONS.image.includes(ext)) type = MediaType.IMAGE;
+      else if (ExternalMedia.EXTENSIONS.video.includes(ext)) type = MediaType.VIDEO;
+      const name = pathPart.split('/').pop() || parsed.hostname;
+      return { type, name };
+    }
+
+    if (isExternalVideoUrl(url)) {
+      const lastSegment = pathPart.split('/').pop() || parsed.hostname;
+      return { type: MediaType.VIDEO, name: lastSegment };
+    }
+
+    const host = parsed.hostname;
+    const matched = ExternalMedia.HOST_PATTERNS.find(
+      (pattern) => pattern.host.test(host)
+        && (!pattern.pathContains || parsed.pathname.includes(pattern.pathContains)),
+    );
+    if (matched) {
+      const { type: patternType } = matched;
+
+      if (matched.typeFromPath) {
+        const lastSegment = pathPart.split('/').pop() || '';
+        const segExt = lastSegment.split('.').pop()?.toLowerCase();
+        const imageExts = [...ExternalMedia.EXTENSIONS.image, ...ExternalMedia.EXTENSIONS.svg];
+        if (segExt && ExternalMedia.EXTENSIONS.video.includes(segExt)) {
+          return { type: MediaType.VIDEO, name: lastSegment };
+        }
+        if (segExt && ExternalMedia.EXTENSIONS.pdf.includes(segExt)) {
+          return { type: MediaType.DOCUMENT, name: lastSegment };
+        }
+        if (segExt && imageExts.includes(segExt)) {
+          return { type: MediaType.IMAGE, name: lastSegment };
+        }
+      }
+
+      if (patternType === ExternalMedia.CATEGORY_IMG) {
+        return { type: MediaType.IMAGE, name: pathPart.split('/').pop() || host };
+      }
+
+      return { type: MediaType.LINK, name: host };
+    }
+  } catch {
+    /* parse error */
+  }
+
+  return null;
+}
+
 export function getImageOrientation(width, height) {
   if (Math.abs(width - height) < 5) {
     return 'Square';
diff --git a/tools/media-library/core/urls.js b/tools/media-library/core/urls.js
index 867c8d75..4496a9ec 100644
--- a/tools/media-library/core/urls.js
+++ b/tools/media-library/core/urls.js
@@ -1,5 +1,12 @@
 import { Domains, MEDIA_UNDERSCORE_PREFIX } from './constants.js';
 
+function toAemRuntimeHostname(hostname) {
+  if (!hostname) return hostname;
+  return hostname
+    .replace('.hlx.page', Domains.AEM_PAGE)
+    .replace('.hlx.live', Domains.AEM_LIVE);
+}
+
 export function isExternalUrl(url) {
   if (!url) return false;
   return !url.includes(Domains.AEM_LIVE) && !url.includes(Domains.AEM_PAGE);
@@ -10,6 +17,7 @@ export function resolveMediaUrl(mediaUrl, org, repo) {
 
   try {
     const url = new URL(mediaUrl);
+    url.hostname = toAemRuntimeHostname(url.hostname);
     return url.href;
   } catch {
     if (org && repo) {
@@ -133,7 +141,9 @@ export function getDedupeKey(url) {
   if (!url) return '';
 
   try {
-    const urlObj = new URL(url);
+    // Normalize hostname (.hlx → .aem) before extracting pathname for deduplication
+    const normalizedUrl = resolveMediaUrl(url);
+    const urlObj = new URL(normalizedUrl);
     const { pathname } = urlObj;
     const filename = pathname.split('/').pop();
 
diff --git a/tools/media-library/indexing/build.js b/tools/media-library/indexing/build.js
index a9a21d26..3a750819 100644
--- a/tools/media-library/indexing/build.js
+++ b/tools/media-library/indexing/build.js
@@ -1,4 +1,4 @@
-import { toCanonicalMediaKey, pathUnder } from '../core/urls.js';
+import { getDedupeKey, pathUnder } from '../core/urls.js';
 import { MediaLibraryError, ErrorCodes, logMediaLibraryError } from '../core/errors.js';
 import t from '../core/messages.js';
 import {
@@ -19,8 +19,9 @@ import {
   processStandaloneUploads,
   applyAuditChunkToMaps,
   buildEarlyLinkedPlaceholders,
+  toCanonicalPath,
 } from './reconcile.js';
-import { toAbsoluteFilePath, isPage } from './parse.js';
+import { toAbsoluteFilePath } from './parse.js';
 import { IndexConfig } from '../core/constants.js';
 import { incrementalTimeParams, initialTimeParams } from '../core/storage.js';
 import isPerfEnabled from '../core/params.js';
@@ -80,14 +81,30 @@ export async function buildMediaDataFromEntries(
   const referencedHashes = new Set();
   [...medialogScoped, ...linkedEntries].forEach((e) => {
     if (e.resourcePath) {
-      const k = toCanonicalMediaKey(e.path || e.mediaHash);
+      const k = getDedupeKey(e.path || e.mediaHash);
       if (k) referencedHashes.add(k);
     }
   });
 
   const standalone = processStandaloneUploads(medialogScoped, referencedHashes);
   const processedAuditlog = processAuditLog(auditlogEntries, org, site);
-  const allMedialog = [...medialogScoped, ...standalone];
+
+  // Filter out self-referencing entries from medialogScoped (they're now in standalone)
+  const medialogWithoutSelfReferencing = medialogScoped.filter((m) => {
+    if (!m.resourcePath) return true; // Keep unreferenced entries
+    const normResourcePath = toCanonicalPath(m.resourcePath);
+    const mediaFilePath = toCanonicalPath(m.path || m.mediaHash);
+    return mediaFilePath !== normResourcePath; // Filter out self-referencing
+  });
+
+  // Count matched vs unmatched (self-referencing)
+  const matchedCount = medialogWithoutSelfReferencing.filter((m) => m.resourcePath).length;
+  const unmatchedCount = medialogScoped.length - matchedCount - standalone.length;
+  perf.medialog.matched = matchedCount;
+  perf.medialog.standalone = standalone.length;
+  perf.medialog.unmatched = unmatchedCount;
+
+  const allMedialog = [...medialogWithoutSelfReferencing, ...standalone];
   const allAudit = [...processedAuditlog, ...linkedEntries];
   let mediaData = transformToMediaData(allMedialog, allAudit);
 
@@ -105,6 +122,11 @@ export async function buildMediaDataFromEntries(
     onProgressiveData(toEmit);
   }
 
+  if (isPerfEnabled()) {
+    // eslint-disable-next-line no-console
+    console.log(`[MediaLibrary:build] Final mediaData count: ${mediaData.length} (matched: ${matchedCount}, standalone: ${standalone.length}, unmatched: ${unmatchedCount})`);
+  }
+
   return mediaData;
 }
 
@@ -170,6 +192,9 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
     medialog: {
       streamed: 0,
       chunks: 0,
+      matched: 0,
+      standalone: 0,
+      unmatched: 0,
       durationMs: 0,
     },
     auditlog: {
@@ -316,17 +341,15 @@ export async function fetchAndBuildMediaData(org, site, options = {}) {
       );
     }
 
-    const syntheticAudit = filteredResources
-      .filter((r) => isPage(r.path))
-      .map((r) => {
-        const entryPath = toAbsoluteFilePath(r.path);
-        return {
-          path: entryPath,
-          route: 'preview',
-          method: 'UPDATE',
-          timestamp: Date.now(),
-        };
-      });
+    const syntheticAudit = filteredResources.map((r) => {
+      const entryPath = toAbsoluteFilePath(r.path);
+      return {
+        path: entryPath,
+        route: 'preview',
+        method: 'UPDATE',
+        timestamp: Date.now(),
+      };
+    });
 
     applyAuditChunkToMaps(syntheticAudit, pagesByPath, filesByPath, deletedPaths);
     const earlyLinked = buildEarlyLinkedPlaceholders(filesByPath, deletedPaths, org, site);
diff --git a/tools/media-library/indexing/parse.js b/tools/media-library/indexing/parse.js
index 413820cd..349e744f 100644
--- a/tools/media-library/indexing/parse.js
+++ b/tools/media-library/indexing/parse.js
@@ -3,7 +3,6 @@
  */
 import {
   IndexConfig,
-  ExternalMedia,
   MediaType,
   Paths,
   ICON_DOC_EXCLUDE,
@@ -11,6 +10,7 @@ import {
 } from '../core/constants.js';
 import { getDedupeKey } from '../core/urls.js';
 import { fetchAdminWithRateLimit } from '../core/admin-rate-limit.js';
+import { getExternalMediaTypeInfo } from '../core/media.js';
 import isPerfEnabled from '../core/params.js';
 
 export { getDedupeKey };
@@ -89,11 +89,83 @@ function toPath(href) {
   }
 }
 
+const HTML_MEDIA_ATTR_RE = /<(?:img|video|audio|source|iframe)\b[^>]*\b(src|srcset|poster)=["']([^"']+)["'][^>]*>/gi;
+
+function hasMalformedEncodedQuotes(target) {
+  return /^(?:%5c%22|%22)/i.test(target) || /(?:%5c%22|%22)$/i.test(target);
+}
+
+function stripMarkdownTitle(target) {
+  const trimmed = target.trim();
+  if (!trimmed) return '';
+
+  const angleMatch = trimmed.match(/^<([^>]+)>(?:\s+['"].*['"])?$/);
+  if (angleMatch) {
+    return angleMatch[1].trim();
+  }
+
+  const titleMatch = trimmed.match(/^(\S+)(?:\s+['"].*['"])?$/);
+  return titleMatch ? titleMatch[1] : trimmed;
+}
+
+function sanitizeExtractedUrl(rawUrl) {
+  if (!rawUrl || typeof rawUrl !== 'string') return null;
+
+  let value = stripMarkdownTitle(rawUrl);
+  if (!value || hasMalformedEncodedQuotes(value)) return null;
+
+  value = value.trim().replace(/&amp;/g, '&');
+
+  if (
+    (value.startsWith('"') && value.endsWith('"'))
+    || (value.startsWith('\'') && value.endsWith('\''))
+  ) {
+    value = value.slice(1, -1).trim();
+  }
+
+  if (!value || value.includes('\n') || value.includes('\r') || value.includes('\t')) {
+    return null;
+  }
+
+  if (
+    value.startsWith('\\"')
+    || value.endsWith('\\"')
+    || value.startsWith("\\'")
+    || value.endsWith("\\'")
+  ) {
+    return null;
+  }
+
+  if (!value.startsWith('http') && !value.startsWith('/')) {
+    return null;
+  }
+
+  return value;
+}
+
+function extractHtmlMediaUrls(md) {
+  return [...md.matchAll(HTML_MEDIA_ATTR_RE)].flatMap((match) => {
+    const [, attrName, rawValue] = match;
+    if (!rawValue) return [];
+    if (attrName.toLowerCase() === 'srcset') {
+      return rawValue
+        .split(',')
+        .map((candidate) => candidate.trim().split(/\s+/)[0])
+        .filter(Boolean);
+    }
+    return [rawValue];
+  });
+}
+
 function extractUrlsFromMarkdown(md) {
   if (!md || typeof md !== 'string') return [];
-  const fromLinks = [...md.matchAll(MD_LINK_RE)].map((m) => m[1].trim());
-  const fromAutolinks = [...md.matchAll(MD_AUTOLINK_RE)].map((m) => m[1].trim());
-  return [...fromLinks, ...fromAutolinks];
+  const candidates = [
+    ...[...md.matchAll(MD_LINK_RE)].map((m) => m[1]),
+    ...[...md.matchAll(MD_AUTOLINK_RE)].map((m) => m[1]),
+    ...extractHtmlMediaUrls(md),
+  ];
+
+  return [...new Set(candidates.map(sanitizeExtractedUrl).filter(Boolean))];
 }
 
 function isExternalUrl(url) {
@@ -102,54 +174,7 @@ function isExternalUrl(url) {
 }
 
 export function getExternalMediaType(url) {
-  if (!url || !url.startsWith('http') || !isExternalUrl(url)) return null;
-  try {
-    const parsed = new URL(url);
-    const pathPart = parsed.pathname.split('?')[0].split('#')[0];
-    const pathLower = pathPart.toLowerCase();
-    const extMatch = pathLower.match(ExternalMedia.EXTENSION_REGEX);
-    if (extMatch) {
-      const ext = extMatch[1].toLowerCase();
-      let type = MediaType.LINK;
-      if (ExternalMedia.EXTENSIONS.pdf.includes(ext)) type = MediaType.DOCUMENT;
-      else if (ExternalMedia.EXTENSIONS.svg.includes(ext)) type = MediaType.IMAGE;
-      else if (ExternalMedia.EXTENSIONS.image.includes(ext)) type = MediaType.IMAGE;
-      else if (ExternalMedia.EXTENSIONS.video.includes(ext)) type = MediaType.VIDEO;
-      const name = pathPart.split('/').pop() || parsed.hostname;
-      return { type, name };
-    }
-    const host = parsed.hostname;
-    const matched = ExternalMedia.HOST_PATTERNS.find(
-      (p) => p.host.test(host) && (!p.pathContains || parsed.pathname.includes(p.pathContains)),
-    );
-    if (matched) {
-      const { type: patternType } = matched;
-      if (matched.typeFromPath) {
-        const lastSegment = pathPart.split('/').pop() || '';
-        const segExt = lastSegment.split('.').pop()?.toLowerCase();
-        const imageExts = [...ExternalMedia.EXTENSIONS.image, ...ExternalMedia.EXTENSIONS.svg];
-        if (segExt && ExternalMedia.EXTENSIONS.video.includes(segExt)) {
-          return { type: MediaType.VIDEO, name: lastSegment };
-        }
-        if (segExt && ExternalMedia.EXTENSIONS.pdf.includes(segExt)) {
-          return { type: MediaType.DOCUMENT, name: lastSegment };
-        }
-        if (segExt && imageExts.includes(segExt)) {
-          return { type: MediaType.IMAGE, name: lastSegment };
-        }
-      }
-      if (patternType === ExternalMedia.CATEGORY_IMG) {
-        return { type: MediaType.IMAGE, name: pathPart.split('/').pop() || host };
-      }
-      if (patternType === MediaType.VIDEO) {
-        return { type: MediaType.VIDEO, name: host };
-      }
-      return { type: MediaType.LINK, name: host };
-    }
-  } catch {
-    /* parse error */
-  }
-  return null;
+  return getExternalMediaTypeInfo(url);
 }
 
 export function extractExternalMediaUrls(md) {
@@ -339,7 +364,11 @@ export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProg
     }
   }
 
-  // Retry failed pages serially (concurrency=1 to avoid overload)
+  // Retry failed pages with bounded concurrency pool (max 2) instead of serial
+  const retryConcurrency = Math.max(
+    1,
+    Math.min(2, IndexConfig.MAX_CONCURRENT_PAGE_FETCHES ?? 1),
+  );
   const allFailedPaths = [];
   failedPathsByReason.forEach((paths) => allFailedPaths.push(...paths));
   if (allFailedPaths.length > 0) {
@@ -348,11 +377,10 @@ export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProg
       allFailedPaths,
       async (normalizedPath) => {
         const result = await fetchPageMarkdown(normalizedPath, org, repo, ref);
-        const isSuccess = result?.status === 200;
-        const md = isSuccess ? (result.markdown ?? '') : (result?.markdown || null);
-        return { normalizedPath, md, isSuccess };
+        const md = result?.markdown || null;
+        return { normalizedPath, md };
       },
-      1, /* serial retries */
+      retryConcurrency,
     );
     retryResults.forEach(processResultIntoUsageMap);
     retryResults.forEach(({ isSuccess }) => {
diff --git a/tools/media-library/indexing/reconcile.js b/tools/media-library/indexing/reconcile.js
index 9d01ecc9..db3e30d2 100644
--- a/tools/media-library/indexing/reconcile.js
+++ b/tools/media-library/indexing/reconcile.js
@@ -16,7 +16,7 @@ import {
 import { Operation, Paths } from '../core/constants.js';
 import { getDedupeKey, pathUnder } from '../core/urls.js';
 
-function toCanonicalPath(path) {
+export function toCanonicalPath(path) {
   if (!path) return '';
   try {
     if (path.startsWith('http')) return new URL(path).pathname;
@@ -152,9 +152,19 @@ export function processStandaloneUploads(medialogEntries, referencedHashes) {
     ? referencedHashes
     : buildReferencedHashes(referencedHashes || []);
   const added = [];
-  const standaloneUploads = medialogEntries.filter(
-    (m) => !m.resourcePath && m.originalFilename,
-  );
+  const standaloneUploads = medialogEntries.filter((m) => {
+    // Case 1: No resourcePath + has originalFilename (existing logic)
+    if (!m.resourcePath && m.originalFilename) return true;
+
+    // Case 2: resourcePath === media path (self-referencing - DA-NX logic)
+    if (m.resourcePath) {
+      const normResourcePath = toCanonicalPath(m.resourcePath);
+      const mediaFilePath = toCanonicalPath(m.path || m.mediaHash);
+      if (mediaFilePath === normResourcePath) return true;
+    }
+
+    return false;
+  });
 
   standaloneUploads.forEach((media) => {
     const key = normalizeMediaKey(media.mediaHash || media.path);
diff --git a/tools/media-library/media-library.js b/tools/media-library/media-library.js
index ed517d9b..4853afb6 100644
--- a/tools/media-library/media-library.js
+++ b/tools/media-library/media-library.js
@@ -360,7 +360,7 @@ async function init() {
 
       const indexStartTime = Date.now();
       // eslint-disable-next-line no-console -- index lifecycle: start (always logged)
-      console.log('[Media Library] Index started');
+      console.log(`[Media Library] Index started at ${new Date(indexStartTime).toISOString()}`);
 
       const progressiveMap = new Map();
       let throttleTimer = null;
@@ -417,10 +417,11 @@ async function init() {
         statusResources: statusResourcesForBuild,
       });
 
-      const indexDurationSec = Math.round((Date.now() - indexStartTime) / 1000);
+      const indexEndTime = Date.now();
+      const indexDurationSec = Math.round((indexEndTime - indexStartTime) / 1000);
       const pagesParsed = perf?.markdownParse?.pages ?? 0;
-      // eslint-disable-next-line no-console -- index lifecycle (duration, pages)
-      console.log(`[Media Library] Index done: ${indexDurationSec}s, ${pagesParsed} pages`);
+      // eslint-disable-next-line no-console -- index lifecycle (start/end times, duration, pages)
+      console.log(`[Media Library] Index done: ${indexDurationSec}s, ${pagesParsed} pages (started: ${new Date(indexStartTime).toISOString()}, ended: ${new Date(indexEndTime).toISOString()})`);
 
       const finalMediaData = incremental && hasCache
         ? (() => {
diff --git a/tools/media-library/views/mediainfo/mediainfo.js b/tools/media-library/views/mediainfo/mediainfo.js
index f10f14e6..4edc0226 100644
--- a/tools/media-library/views/mediainfo/mediainfo.js
+++ b/tools/media-library/views/mediainfo/mediainfo.js
@@ -436,7 +436,10 @@ export default function createMediaInfoModal() {
   function doRender() {
     if (!media) return;
 
-    const displayName = escapeHtml(media.name || getFileName(media.url) || 'Media Details');
+    // Strip hash/query params from display name (e.g., #width=3000&height=3000)
+    const rawName = media.name || getFileName(media.url) || 'Media Details';
+    const cleanName = rawName.split('#')[0].split('?')[0];
+    const displayName = escapeHtml(cleanName);
     const refCount = usageData?.length ?? 0;
     const refLabel = refCount !== 1 ? 'References' : 'Reference';
 

From aaff0e18274a2ab9753c8f710b3d8042906f4f82 Mon Sep 17 00:00:00 2001
From: Kiran Murugulla <murugull@adobe.com>
Date: Sat, 14 Mar 2026 23:31:03 -0400
Subject: [PATCH 09/10] fix: autoloading when dom is ready, removing settimeout

---
 tools/media-library/media-library.js | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tools/media-library/media-library.js b/tools/media-library/media-library.js
index 4853afb6..f7c274b3 100644
--- a/tools/media-library/media-library.js
+++ b/tools/media-library/media-library.js
@@ -630,11 +630,9 @@ async function init() {
       pathInput.value = pathParam;
     }
 
-    // Auto-load after a short delay to ensure DOM is ready
-    setTimeout(() => {
-      const path = getPathFromInput();
-      loadMediaData(orgParam, siteParam, path);
-    }, 100);
+    // Auto-load immediately (DOM is ready since module scripts are deferred)
+    const path = getPathFromInput();
+    loadMediaData(orgParam, siteParam, path);
   }
 }
 

From 78733cc33826d1e15316479e0ea11fd437fa2dc5 Mon Sep 17 00:00:00 2001
From: Kiran Murugulla <murugull@adobe.com>
Date: Mon, 16 Mar 2026 09:15:46 -0400
Subject: [PATCH 10/10] fix : add null guards, fix retry isSuccess tracking,
 and gate   console.log with isPerfEnable

---
 tools/media-library/indexing/build.js |  8 +++++---
 tools/media-library/indexing/parse.js |  5 +++--
 tools/media-library/media-library.js  | 10 ++++++----
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/tools/media-library/indexing/build.js b/tools/media-library/indexing/build.js
index 3a750819..3b231fb7 100644
--- a/tools/media-library/indexing/build.js
+++ b/tools/media-library/indexing/build.js
@@ -100,9 +100,11 @@ export async function buildMediaDataFromEntries(
   // Count matched vs unmatched (self-referencing)
   const matchedCount = medialogWithoutSelfReferencing.filter((m) => m.resourcePath).length;
   const unmatchedCount = medialogScoped.length - matchedCount - standalone.length;
-  perf.medialog.matched = matchedCount;
-  perf.medialog.standalone = standalone.length;
-  perf.medialog.unmatched = unmatchedCount;
+  if (perf) {
+    perf.medialog.matched = matchedCount;
+    perf.medialog.standalone = standalone.length;
+    perf.medialog.unmatched = unmatchedCount;
+  }
 
   const allMedialog = [...medialogWithoutSelfReferencing, ...standalone];
   const allAudit = [...processedAuditlog, ...linkedEntries];
diff --git a/tools/media-library/indexing/parse.js b/tools/media-library/indexing/parse.js
index 349e744f..85d55ca4 100644
--- a/tools/media-library/indexing/parse.js
+++ b/tools/media-library/indexing/parse.js
@@ -377,8 +377,9 @@ export async function buildUsageMap(pageEntries, org, repo, ref = 'main', onProg
       allFailedPaths,
       async (normalizedPath) => {
         const result = await fetchPageMarkdown(normalizedPath, org, repo, ref);
-        const md = result?.markdown || null;
-        return { normalizedPath, md };
+        const isSuccess = result?.status === 200;
+        const md = isSuccess ? (result.markdown ?? '') : (result?.markdown || null);
+        return { normalizedPath, md, isSuccess };
       },
       retryConcurrency,
     );
diff --git a/tools/media-library/media-library.js b/tools/media-library/media-library.js
index f7c274b3..c3d73ad9 100644
--- a/tools/media-library/media-library.js
+++ b/tools/media-library/media-library.js
@@ -36,6 +36,7 @@ import {
   fetchAndBuildMediaData,
   validatePathWithStatus,
 } from './indexing/build.js';
+import isPerfEnabled from './core/params.js';
 
 import createMediaInfoModal from './views/mediainfo/mediainfo.js';
 import { setMediaLibraryContext } from './core/context.js';
@@ -359,8 +360,6 @@ async function init() {
       updateAppState({ isValidating: false, isIndexing: true, progressiveMediaData: [] });
 
       const indexStartTime = Date.now();
-      // eslint-disable-next-line no-console -- index lifecycle: start (always logged)
-      console.log(`[Media Library] Index started at ${new Date(indexStartTime).toISOString()}`);
 
       const progressiveMap = new Map();
       let throttleTimer = null;
@@ -420,8 +419,11 @@ async function init() {
       const indexEndTime = Date.now();
       const indexDurationSec = Math.round((indexEndTime - indexStartTime) / 1000);
       const pagesParsed = perf?.markdownParse?.pages ?? 0;
-      // eslint-disable-next-line no-console -- index lifecycle (start/end times, duration, pages)
-      console.log(`[Media Library] Index done: ${indexDurationSec}s, ${pagesParsed} pages (started: ${new Date(indexStartTime).toISOString()}, ended: ${new Date(indexEndTime).toISOString()})`);
+
+      if (isPerfEnabled()) {
+        // eslint-disable-next-line no-console -- perf debug when ?debug=perf
+        console.log(`[Media Library] Index done: ${indexDurationSec}s, ${pagesParsed} pages (started: ${new Date(indexStartTime).toISOString()}, ended: ${new Date(indexEndTime).toISOString()})`);
+      }
 
       const finalMediaData = incremental && hasCache
         ? (() => {