Skip to content

chore: better DOM for large bulk #73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
davidnuescheler wants to merge 3 commits into
base: main
Choose a base branch
from
Open

chore: better DOM for large bulk #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
70927be
chore: better DOM for large bulk
davidnuescheler Mar 13, 2025
dbe0a87
chore: LCP fix?
davidnuescheler Mar 13, 2025
45ddae1
chore: remove logs
davidnuescheler Mar 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tools/bulk/index.html
View file
Open in desktop

@fkakatie fkakatie Mar 13, 2025

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🙄

Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<link rel="stylesheet" href="/tools/bulk/styles.css" />
</head>

<body class="bulk-ops">
<body class="bulk-ops appear">
<header></header>
<main>
<!-- TITLE -->
Expand Down
130 changes: 88 additions & 42 deletions tools/bulk/scripts.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
const log = document.getElementById('logger');
const adminVersion = new URLSearchParams(window.location.search).get('hlx-admin-version');
const adminVersionSuffix = adminVersion ? `?hlx-admin-version=${adminVersion}` : '';
const simpleJobStatus = { data: { resources: [] }, links: { job: '' } };
let startTime = null;
let total = 0;
let operation = '';
let slow = false;
let forceUpdate = false;

const append = (string, status = 'unknown') => {
const logAppend = (string, status = 'unknown') => {
const p = document.createElement('p');
p.textContent = string;
if (status !== 'unknown') {
Expand All @@ -13,23 +19,78 @@ const append = (string, status = 'unknown') => {
return p;
};

function updateBulkStatus(element, jobStatus) {
const getProgressSummary = () => {
const progress = {
processed: 0,
success: 0,
warnings: 0,
errors: 0,
};
if (jobStatus.data && jobStatus.data.resources) {
jobStatus.data.resources.forEach((res) => {
if (res.status) progress.processed += 1;
if (res.status === 200 || res.status === 204) {
progress.success += 1;
} else if (res.status === 404) {
progress.warnings += 1;
} else if (res.status >= 500) {
progress.errors += 1;
}
});
}
return progress;
};

const {
processed,
success,
warnings,
errors,
} = getProgressSummary(jobStatus);
const VERB = {
preview: 'previewed',
live: 'published',
unpreview: 'unpreviewed',
unpublish: 'unpublished',
index: 'indexed',
cache: 'purged',
};
const [, , owner, repo] = new URL(jobStatus.links.job).pathname.split('/');
element.innerHTML = `<span class="status-pill status-success">${success}</span> <span class="status-pill status-warning">${warnings}</span> <span class="status-pill status-error">${errors}</span> ${processed}/${total} URL(s) bulk ${VERB[operation]} [${((new Date().valueOf() - startTime.valueOf()) / 1000).toFixed(2)}s]
<div class="status-details" aria-label="Details" aria-expanded="false"><textarea></textarea></div>`;
element.querySelector('.status-success').addEventListener('click', () => {
element.querySelector('.status-details').setAttribute('aria-expanded', 'true');
element.querySelector('.status-details textarea').textContent = jobStatus.data.resources.filter((r) => r.status === 200 || r.status === 204).map((r) => `https://main--${repo}--${owner}.aem.page${r.path}`).join('\n');
});
element.querySelector('.status-warning').addEventListener('click', () => {
element.querySelector('.status-details').setAttribute('aria-expanded', 'true');
element.querySelector('.status-details textarea').textContent = jobStatus.data.resources.filter((r) => r.status === 404).map((r) => `https://main--${repo}--${owner}.aem.page${r.path}`).join('\n');
});
element.querySelector('.status-error').addEventListener('click', () => {
element.querySelector('.status-details').setAttribute('aria-expanded', 'true');
element.querySelector('.status-details textarea').textContent = jobStatus.data.resources.filter((r) => r.status >= 500).map((r) => `https://main--${repo}--${owner}.aem.page${r.path}`).join('\n');
});
}

function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}

document.getElementById('urls-form').addEventListener('submit', async (e) => {
document.getElementById('urls-form').disabled = true;
startTime = new Date();
e.preventDefault();
let counter = 0;
const urls = document.getElementById('urls').value
.split('\n')
.map((u) => u.trim())
.filter((u) => u.length > 0);
const total = urls.length;
const operation = document.getElementById('operation').dataset.value;
const slow = document.getElementById('slow').checked;
const forceUpdate = document.getElementById('force').checked;
total = urls.length;
operation = document.getElementById('operation').dataset.value;
slow = document.getElementById('slow').checked;
forceUpdate = document.getElementById('force').checked;

const executeOperation = async (url) => {
const { hostname, pathname } = new URL(url);
Expand All @@ -48,34 +109,33 @@ document.getElementById('urls-form').addEventListener('submit', async (e) => {
const resp = await fetch(adminURL, {
method,
});
resp.text().then(() => {
counter += 1;
append(`${counter}/${total}: ${adminURL}`, resp.status);
document.getElementById('total').textContent = `${counter}/${total}`;
simpleJobStatus.data.resources.push({
status: resp.status,
path: pathname,
});
};

const dequeue = async () => {
const dequeue = async (bulkLog) => {
while (urls.length) {
const url = urls.shift();
// eslint-disable-next-line no-await-in-loop
await executeOperation(url, total);
updateBulkStatus(bulkLog, simpleJobStatus);
// eslint-disable-next-line no-await-in-loop
if (slow) await sleep(1500);
}
updateBulkStatus(bulkLog, simpleJobStatus);
document.getElementById('urls-form').disabled = false;
};

const doBulkOperation = async () => {
if (total > 0) {
const VERB = {
preview: 'preview',
live: 'publish',
};
const { hostname } = new URL(urls[0]); // use first URL to determine project details
const [branch, repo, owner] = hostname.split('.')[0].split('--');
const bulkText = `$1/${total} URL(s) bulk ${VERB[operation]}ed on ${owner}/${repo} ${forceUpdate ? '(force update)' : ''}`;
const bulkLog = append(bulkText.replace('$1', 0));
const bulkLog = logAppend('');
updateBulkStatus(bulkLog, { data: { resources: [] }, links: { job: `https://admin.hlx.page/job/${owner}/${repo}/main/simplejob` } });
const paths = urls.map((url) => new URL(url).pathname);

const bulkResp = await fetch(`https://admin.hlx.page/${operation}/${owner}/${repo}/${branch}/*${adminVersionSuffix}`, {
method: 'POST',
body: JSON.stringify({
Expand All @@ -87,41 +147,24 @@ document.getElementById('urls-form').addEventListener('submit', async (e) => {
},
});
if (!bulkResp.ok) {
append(`Failed to bulk ${VERB[operation]} ${paths.length} URLs on ${origin}: ${await bulkResp.text()}`);
logAppend(`Failed on ${origin}: ${await bulkResp.text()}`);
} else {
const { job } = await bulkResp.json();
const { name } = job;
const jobStatusPoll = window.setInterval(async () => {
try {
const jobResp = await fetch(`https://admin.hlx.page/job/${owner}/${repo}/${branch}/${VERB[operation]}/${name}/details`);
const verb = operation === 'live' ? 'publish' : operation;
const jobResp = await fetch(`https://admin.hlx.page/job/${owner}/${repo}/${branch}/${verb}/${name}/details`);
const jobStatus = await jobResp.json();
const {
state,
progress: {
processed = 0,
} = {},
startTime,
stopTime,
data: {
resources = [],
} = {},
} = jobStatus;
if (state === 'stopped') {
if (jobStatus.state === 'stopped') {
// job done, stop polling
window.clearInterval(jobStatusPoll);
// show job summary
resources.forEach((res) => append(`${res.path} (${res.status})`, res.status));
bulkLog.textContent = bulkText.replace('$1', processed);
const duration = (new Date(stopTime).valueOf()
- new Date(startTime).valueOf()) / 1000;
append(`bulk ${operation} completed in ${duration}s`);
} else {
// show job progress
bulkLog.textContent = bulkText.replace('$1', processed);
}
updateBulkStatus(bulkLog, jobStatus);
} catch (error) {
// eslint-disable-next-line no-console
console.error(`failed to get status for job ${name}: ${error}`);
document.getElementById('urls-form').disabled = false;
window.clearInterval(jobStatusPoll);
}
}, 1000);
Expand All @@ -133,13 +176,16 @@ document.getElementById('urls-form').addEventListener('submit', async (e) => {
// use bulk preview/publish API
doBulkOperation(urls);
} else {
append(`URLs: ${urls.length}`);
let concurrency = ['live', 'unpublish', 'unpreview'].includes(operation) ? 40 : 3;
if (slow) {
concurrency = 1;
}
const bulkLog = logAppend('');
const [, repo, owner] = new URL(urls[0]).hostname.split('--');
simpleJobStatus.data = { resources: [], links: { job: `https://admin.hlx.page/job/${owner}/${repo}/main/simplejob` } };
updateBulkStatus(bulkLog, simpleJobStatus);
for (let i = 0; i < concurrency; i += 1) {
dequeue(urls);
dequeue(bulkLog);
}
}
});
Expand Down
40 changes: 39 additions & 1 deletion tools/bulk/styles.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,42 @@

.bulk-ops .status-light.http3::before {
color: var(--yellow-900);
}
}

.status-pill {
display: inline-block;
padding: 0.5em 1em;
border-radius: 1.5em;
min-width: 5em;
color: #fff;
text-align: center;
}

.status-success {
background-color: var(--green-900);
}

.status-warning {
background-color: var(--yellow-900);
}

.status-error {
background-color: var(--red-900);
}

.status-details[aria-expanded="false"] {
display: none;
}

.status-details[aria-expanded="true"] {
margin-top: 1em;
display: block;
}

.status-details textarea {
width: 100%;
height: 400px;
font-family: var(--code-font-family);
font-size: var(--body-size-s);
}