diff --git a/packages/cli/readme.md b/packages/cli/readme.md index 8834921..657ee32 100644 --- a/packages/cli/readme.md +++ b/packages/cli/readme.md @@ -443,3 +443,15 @@ Remove a channel-engine instance in dev env for tenant `asdasd` as an OSC super ``` PAT_SECRET= osc --env dev admin remove-instance asdasd channel-engine mychannel ``` + +Sync the fork of an OSC service with its upstream repository (looks up the repository URL in the catalog, which needs the platform API key): + +``` +osc admin sync-fork eyevinn-web-runner --wait +``` + +Sync a fork by repository URL. This skips the catalog lookup and works with the dedicated sync-fork API key (`OSC_API_KEY=`). Give either `` or `--repository-url`, not both. The URL must be `https://github.com//[.git]`. With `--wait` the remake order is followed by polling the order list for the repository (`GET /maker?repositoryUrl=`) instead of fetching the order by id, and any failure exits non-zero. The sync status has no "changed" flag, so this is a heuristic: if no new or changed order appears within 60 seconds after the sync job has succeeded, the command exits 0 with "no remake order appeared within 60s (fork was likely already up to date)". A remake that only starts after those 60 seconds is not waited for. + +``` +osc --env prod admin sync-fork --repository-url https://github.com/Eyevinn/open-live --wait +``` diff --git a/packages/cli/src/admin/cmd.syncfork.test.ts b/packages/cli/src/admin/cmd.syncfork.test.ts new file mode 100644 index 0000000..e946e23 --- /dev/null +++ b/packages/cli/src/admin/cmd.syncfork.test.ts @@ -0,0 +1,248 @@ +import { FetchError, createFetch } from '@osaas/client-core'; +import { Command } from 'commander'; +import cmdAdmin from './cmd'; + +jest.mock('@osaas/client-core', () => ({ + ...jest.requireActual('@osaas/client-core'), + createFetch: jest.fn() +})); + +const mockFetch = createFetch as jest.Mock; +const REPO = 'https://github.com/Eyevinn/open-live'; + +type Phase = string; + +function setup(opts: { + makerLists: { orderId: string; phase: Phase }[][]; // after the sync, consumed in order (last repeats) + syncStatus?: object; +}) { + const lists = [...opts.makerLists]; + let first = true; + mockFetch.mockImplementation(async (url: URL, init?: { method?: string }) => { + const u = url.toString(); + if (u.startsWith('https://catalog.svc')) { + return { serviceMetadata: { repoUrl: REPO } }; + } + if (u.includes('/sync-fork/by-repo')) { + return ( + opts.syncStatus ?? { jobId: 'j', repositoryUrl: REPO, status: 'done' } + ); + } + if (u.endsWith('/sync-fork') && init?.method === 'POST') { + return { jobId: 'j1' }; + } + if (u.includes('/maker?repositoryUrl=')) { + if (first) { + first = false; + return (opts.makerLists.length ? opts.makerLists[0] : []).map((o) => ({ + order: { orderId: o.orderId, status: { phase: o.phase } } + })); + } + const next = lists.length > 1 ? lists.shift()! : lists[0]; + return next.map((o) => ({ + order: { orderId: o.orderId, status: { phase: o.phase } } + })); + } + throw new Error(`unexpected request ${u}`); + }); +} + +async function run(args: string[]) { + const cli = new Command(); + cli.option('--env '); + cli.addCommand(cmdAdmin()); + const p = cli.parseAsync( + ['node', 'cli', '--env', 'dev', 'admin', ...args].filter( + Boolean + ) as string[] + ); + // advance fake timers while the command polls + for (let i = 0; i < 60; i++) { + await jest.advanceTimersByTimeAsync(5000); + } + await p; +} + +const urls = () => mockFetch.mock.calls.map((c) => c[0].toString() as string); + +describe('admin sync-fork --repository-url', () => { + let logSpy: jest.SpyInstance; + let errSpy: jest.SpyInstance; + + beforeEach(() => { + jest.useFakeTimers(); + mockFetch.mockReset(); + process.exitCode = undefined; + process.env.OSC_API_KEY = 'narrow'; + logSpy = jest.spyOn(console, 'log').mockImplementation(() => undefined); + errSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); + }); + + afterEach(() => { + jest.useRealTimers(); + logSpy.mockRestore(); + errSpy.mockRestore(); + process.exitCode = undefined; + }); + + it('skips the catalog lookup and triggers sync without --wait', async () => { + setup({ makerLists: [[]] }); + await run(['sync-fork', '--repository-url', REPO]); + expect(urls().some((u) => u.includes('catalog.svc'))).toBe(false); + expect(urls().some((u) => u.endsWith('/sync-fork'))).toBe(true); + expect(process.exitCode).toBeUndefined(); + }); + + it('serviceId path still performs the catalog lookup', async () => { + setup({ makerLists: [[]] }); + await run(['sync-fork', 'eyevinn-open-live']); + expect(urls()[0]).toBe( + 'https://catalog.svc.dev.osaas.io/service/eyevinn-open-live' + ); + expect(process.exitCode).toBeUndefined(); + }); + + it('errors when neither serviceId nor --repository-url is given', async () => { + await run(['sync-fork']); + expect(mockFetch).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }); + + it('errors when both are given', async () => { + await run(['sync-fork', 'eyevinn-open-live', '--repository-url', REPO]); + expect(mockFetch).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }); + + it.each([ + 'https://gitlab.com/Eyevinn/open-live', + 'http://github.com/Eyevinn/open-live', + 'https://github.com/Eyevinn', + 'https://github.com/Eyevinn/open-live/tree/main', + 'https://github.com.evil.io/Eyevinn/open-live', + 'https://github.com/Eyevinn/open-live?x=1', + 'not a url' + ])('rejects invalid URL %s before any network call', async (bad) => { + await run(['sync-fork', '--repository-url', bad]); + expect(mockFetch).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }); + + it.each([REPO, `${REPO}.git`])('accepts valid URL %s', async (good) => { + setup({ makerLists: [[]] }); + await run(['sync-fork', '--repository-url', good]); + expect(mockFetch).toHaveBeenCalled(); + expect(process.exitCode).toBeUndefined(); + }); + + it('--wait: new order goes through active phases to DONE, exit 0, no GET /maker/', async () => { + setup({ + makerLists: [ + [{ orderId: 'old', phase: 'DONE' }], // snapshot + [{ orderId: 'old', phase: 'DONE' }], // not yet created + [ + { orderId: 'old', phase: 'DONE' }, + { orderId: 'new', phase: 'PENDING' } + ], + [ + { orderId: 'old', phase: 'DONE' }, + { orderId: 'new', phase: 'RUNNING' } + ], + [ + { orderId: 'old', phase: 'DONE' }, + { orderId: 'new', phase: 'DONE' } + ] + ] + }); + await run(['sync-fork', '--repository-url', REPO, '--wait']); + expect(process.exitCode).toBeUndefined(); + expect(urls().some((u) => /\/maker\/[^?]/.test(u))).toBe(false); + expect(urls().some((u) => u.includes('catalog.svc'))).toBe(false); + }); + + it('--wait: rebuild reusing the orderId (DONE -> PENDING -> DONE) is followed', async () => { + setup({ + makerLists: [ + [{ orderId: 'o1', phase: 'DONE' }], + [{ orderId: 'o1', phase: 'PENDING' }], + [{ orderId: 'o1', phase: 'DONE' }] + ] + }); + await run(['sync-fork', '--repository-url', REPO, '--wait']); + expect(process.exitCode).toBeUndefined(); + }); + + it('--wait: failed order phase exits non-zero', async () => { + setup({ + makerLists: [ + [], + [{ orderId: 'new', phase: 'RUNNING' }], + [{ orderId: 'new', phase: 'ERROR' }] + ] + }); + await run(['sync-fork', '--repository-url', REPO, '--wait']); + expect(process.exitCode).toBe(1); + expect(errSpy.mock.calls.join('\n')).toContain('ERROR'); + }); + + it('--wait: no-change run (only stale DONE order) exits 0 after the 60s grace and stops polling', async () => { + setup({ makerLists: [[{ orderId: 'old', phase: 'DONE' }]] }); + await run(['sync-fork', '--repository-url', REPO, '--wait']); + expect(process.exitCode).toBeUndefined(); + expect(logSpy.mock.calls.join('\n')).toContain( + 'Sync fork succeeded; no remake order appeared within 60s (fork was likely already up to date)' + ); + // 1 snapshot + polls at 0s, 5s, ..., 60s = 13, and nothing after the grace + const makerCalls = urls().filter((u) => + u.includes('/maker?repositoryUrl=') + ); + expect(makerCalls).toHaveLength(14); + expect(urls().some((u) => /\/maker\/[^?]/.test(u))).toBe(false); + }); + + it('--wait: order appearing late inside the grace is still waited to DONE', async () => { + const stale = [{ orderId: 'old', phase: 'DONE' }]; + setup({ + makerLists: [ + stale, + ...Array(10).fill(stale), + [...stale, { orderId: 'new', phase: 'RUNNING' }], + [...stale, { orderId: 'new', phase: 'DONE' }] + ] + }); + await run(['sync-fork', '--repository-url', REPO, '--wait']); + expect(process.exitCode).toBeUndefined(); + expect(logSpy.mock.calls.join('\n')).not.toContain('no remake order'); + }); + + it('--wait: order appearing inside the grace and failing exits 1', async () => { + const stale = [{ orderId: 'old', phase: 'DONE' }]; + setup({ + makerLists: [stale, stale, [...stale, { orderId: 'new', phase: 'ERROR' }]] + }); + await run(['sync-fork', '--repository-url', REPO, '--wait']); + expect(process.exitCode).toBe(1); + }); + + it('--wait: sync-fork failure exits non-zero', async () => { + setup({ + makerLists: [[]], + syncStatus: { + jobId: 'j', + repositoryUrl: REPO, + status: 'failed', + error: { type: 'conflict', message: 'merge conflict' } + } + }); + await run(['sync-fork', '--repository-url', REPO, '--wait']); + expect(process.exitCode).toBe(1); + }); + + it('request errors on the url path exit non-zero', async () => { + mockFetch.mockRejectedValue( + new FetchError({ message: 'forbidden', httpCode: 403 }) + ); + await run(['sync-fork', '--repository-url', REPO]); + expect(process.exitCode).toBe(1); + }); +}); diff --git a/packages/cli/src/admin/cmd.ts b/packages/cli/src/admin/cmd.ts index 0e097be..3d64a6b 100644 --- a/packages/cli/src/admin/cmd.ts +++ b/packages/cli/src/admin/cmd.ts @@ -10,9 +10,13 @@ import { waitForOrder } from '@osaas/client-core'; import { + assertGithubRepositoryUrl, getServiceRepoUrl, getSyncForkStatus, + listOrdersByRepositoryUrl, + REMAKE_GRACE_MS, triggerSyncFork, + waitForRemakeByRepositoryUrl, waitForSyncFork } from './syncfork'; import { @@ -323,23 +327,66 @@ export default function cmdAdmin() { .command('sync-fork') .description('Trigger a fork sync for an OSC service') .option('-w, --wait', 'Wait for sync fork and remake to complete') - .argument('', 'The OSC service ID (e.g. eyevinn-web-runner)') + .option( + '--repository-url ', + 'GitHub repository URL (https://github.com//) to sync instead of a service ID. Skips the catalog lookup and works with the dedicated sync-fork API key. With --wait the remake is followed by polling the order list for the repository' + ) + .argument('[serviceId]', 'The OSC service ID (e.g. eyevinn-web-runner)') .action(async (serviceId, options, command) => { + const byUrl = options.repositoryUrl !== undefined; + // On the --repository-url path every failure exits non-zero + const fail = () => { + if (byUrl) { + process.exitCode = 1; + } + }; + if (byUrl && serviceId) { + console.error( + 'Specify either or --repository-url, not both' + ); + process.exitCode = 1; + return; + } + if (!byUrl && !serviceId) { + console.error('Either or --repository-url is required'); + process.exitCode = 1; + return; + } + if (byUrl) { + try { + assertGithubRepositoryUrl(options.repositoryUrl); + } catch (err) { + console.error((err as Error).message); + process.exitCode = 1; + return; + } + } try { const globalOpts = command.optsWithGlobals(); const environment = globalOpts?.env || 'prod'; const platform = new Platform({ environment }); - Log().info( - `Looking up repository URL for service ${serviceId} in ${environment}` - ); - const repoUrl = await getServiceRepoUrl( - serviceId, - environment, - platform.getApiKey() - ); + let repoUrl: string; + if (byUrl) { + repoUrl = options.repositoryUrl; + } else { + Log().info( + `Looking up repository URL for service ${serviceId} in ${environment}` + ); + repoUrl = await getServiceRepoUrl( + serviceId, + environment, + platform.getApiKey() + ); + } Log().info(`Repository URL: ${repoUrl}`); + // Snapshot existing orders so the wait does not accept a stale one + const ordersBefore = + byUrl && options.wait + ? await listOrdersByRepositoryUrl(repoUrl, platform) + : []; + Log().info(`Triggering sync fork for ${repoUrl}`); const jobId = await triggerSyncFork(repoUrl, platform); Log().info(`Sync fork job started: ${jobId}`); @@ -357,6 +404,7 @@ export default function cmdAdmin() { `Conflict files: ${status.error.conflictFiles.join(', ')}` ); } + fail(); return; } } @@ -378,6 +426,32 @@ export default function cmdAdmin() { ); } } + fail(); + return; + } + + if (byUrl) { + const result = await waitForRemakeByRepositoryUrl( + repoUrl, + platform, + ordersBefore + ); + if (result.outcome === 'done') { + Log().info( + `Sync fork and remake for ${repoUrl} completed successfully` + ); + } else if (result.outcome === 'failed') { + console.error( + `Remake order ${result.orderId} for ${repoUrl} ended in phase ${result.phase}` + ); + process.exitCode = 1; + } else { + console.log( + `Sync fork succeeded; no remake order appeared within ${ + REMAKE_GRACE_MS / 1000 + }s (fork was likely already up to date)` + ); + } return; } @@ -409,6 +483,7 @@ export default function cmdAdmin() { } } catch (err) { console.log((err as Error).message); + fail(); } }); return admin; diff --git a/packages/cli/src/admin/syncfork.ts b/packages/cli/src/admin/syncfork.ts index 61d1d38..feb530c 100644 --- a/packages/cli/src/admin/syncfork.ts +++ b/packages/cli/src/admin/syncfork.ts @@ -111,3 +111,149 @@ export async function waitForSyncFork( } return undefined; } + +const GITHUB_SEGMENT = /^[A-Za-z0-9_.-]+$/; + +/** + * Validates that the value is https://github.com//[.git] and + * throws a descriptive error otherwise. The URL is passed on unmodified since + * maker-manager matches registered repositories on the exact string. + */ +export function assertGithubRepositoryUrl(value: string): void { + const invalid = () => + new Error( + `Invalid repository URL '${value}': expected https://github.com//[.git]` + ); + let url: URL; + try { + url = new URL(value); + } catch { + throw invalid(); + } + if ( + url.protocol !== 'https:' || + url.hostname !== 'github.com' || + url.port !== '' || + url.username !== '' || + url.password !== '' || + url.search !== '' || + url.hash !== '' || + value.endsWith('?') || + value.endsWith('#') + ) { + throw invalid(); + } + const segments = url.pathname.split('/'); + // ['', owner, repo] + if (segments.length !== 3 || segments[0] !== '') { + throw invalid(); + } + const owner = segments[1]; + const repo = segments[2].replace(/\.git$/, ''); + const ok = (s: string) => GITHUB_SEGMENT.test(s) && s !== '.' && s !== '..'; + if (!ok(owner) || !ok(repo)) { + throw invalid(); + } +} + +export interface RepoOrder { + orderId: string; + phase: string; +} + +/** + * Lists the orders for a repository with GET /maker?repositoryUrl=, the + * only query parameter, which is the one order lookup the dedicated sync-fork + * key is accepted on. Errors are propagated (404 means no orders). + */ +export async function listOrdersByRepositoryUrl( + repositoryUrl: string, + platform: Platform +): Promise { + const makerUrl = new URL( + `https://maker.svc.${platform.getEnvironment()}.osaas.io/maker` + ); + makerUrl.searchParams.append('repositoryUrl', repositoryUrl); + try { + const res = await createFetch< + { order: { orderId: string; status?: { phase?: string } } }[] + >(makerUrl, { + headers: { + Authorization: `Bearer ${platform.getApiKey()}`, + 'Content-Type': 'application/json' + } + }); + return res.map((o) => ({ + orderId: o.order.orderId, + phase: o.order.status?.phase || '' + })); + } catch (err) { + if (err instanceof FetchError && err.httpCode === 404) { + return []; + } + throw err; + } +} + +// Same non-terminal phases as waitForOrder / getOrderIdByRepositoryUrl in client-core +const ACTIVE_PHASES = new Set(['NEW', 'PENDING', 'RUNNING']); + +/** + * Heuristic: the by-repo status carries no "changed" flag, so after the sync + * job has succeeded we wait this long for a remake order to show up and then + * assume the fork was already up to date. + */ +export const REMAKE_GRACE_MS = 60000; +export const REMAKE_POLL_INTERVAL_MS = 5000; + +export type RemakeResult = + | { outcome: 'done'; orderId: string } + | { outcome: 'failed'; orderId: string; phase: string } + | { outcome: 'no_remake' }; + +/** + * Waits for the remake order started by a sync fork (call it once the sync + * job has succeeded; the grace period REMAKE_GRACE_MS runs from the call), using only + * GET /maker?repositoryUrl=. `before` is the order list captured before the + * sync was triggered. An order is only accepted when it is new (orderId not in + * `before`), is in an active phase, or its phase differs from the snapshot, so + * a stale earlier DONE order is never mistaken for the new remake. A rebuild + * can reuse the orderId, which is why the phase is compared as well. + */ +export async function waitForRemakeByRepositoryUrl( + repositoryUrl: string, + platform: Platform, + before: RepoOrder[] +): Promise { + const snapshot = new Map(before.map((o) => [o.orderId, o.phase])); + let trackedId: string | undefined; + let elapsedMs = 0; + for (;;) { + const orders = await listOrdersByRepositoryUrl(repositoryUrl, platform); + let order: RepoOrder | undefined; + if (trackedId) { + order = orders.find((o) => o.orderId === trackedId); + } else { + order = orders.find( + (o) => + !snapshot.has(o.orderId) || + ACTIVE_PHASES.has(o.phase) || + snapshot.get(o.orderId) !== o.phase + ); + if (order) { + trackedId = order.orderId; + } + } + if (order) { + if (!ACTIVE_PHASES.has(order.phase)) { + return order.phase === 'DONE' + ? { outcome: 'done', orderId: order.orderId } + : { outcome: 'failed', orderId: order.orderId, phase: order.phase }; + } + } else if (elapsedMs >= REMAKE_GRACE_MS) { + return { outcome: 'no_remake' }; + } + await delay(REMAKE_POLL_INTERVAL_MS); + elapsedMs += REMAKE_POLL_INTERVAL_MS; + } +}