diff --git a/.github/actions/update-sdk-schema/__test__/assets.ts b/.github/actions/update-sdk-schema/__test__/assets.ts index c6e6429..33e875b 100644 --- a/.github/actions/update-sdk-schema/__test__/assets.ts +++ b/.github/actions/update-sdk-schema/__test__/assets.ts @@ -14,14 +14,8 @@ const assetsMap = { path.resolve(__dirname, './v1.1.0/schema.yaml'), 'https://github.com/fingerprintjs/fingerprint-pro-server-api-openapi/releases/download/v1.2.0/fingerprint-server-api.yaml': path.resolve(__dirname, './v1.2.0/schema.yaml'), - 'https://github.com/fingerprintjs/fingerprint-pro-server-api-openapi/releases/download/v1.1.0/scopes': path.resolve( - __dirname, - './v1.1.0/scopes.yaml' - ), - 'https://github.com/fingerprintjs/fingerprint-pro-server-api-openapi/releases/download/v1.2.0/scopes': path.resolve( - __dirname, - './v1.2.0/scopes.yaml' - ), + 'https://raw.githubusercontent.com/fingerprintjs/fingerprint-pro-server-api-openapi/main/config/scopes.yaml': + path.resolve(__dirname, './v1.1.0/scopes.yaml'), } export function maybeMockAsset(url: string) { diff --git a/.github/actions/update-sdk-schema/action.yml b/.github/actions/update-sdk-schema/action.yml index 845805d..cb87c29 100644 --- a/.github/actions/update-sdk-schema/action.yml +++ b/.github/actions/update-sdk-schema/action.yml @@ -22,18 +22,27 @@ inputs: openApiRepository: description: 'OpenAPI repository in format of owner/repo' default: 'fingerprintjs/fingerprint-pro-server-api-openapi' + scopesRepository: + description: 'Repository to source scopes.yaml from, in format of owner/repo. Defaults to openApiRepository.' + default: '' + scopesConfigPath: + description: 'Path to scopes.yaml within scopesRepository' + default: 'config/scopes.yaml' + scopesRef: + description: 'Git ref (branch, tag, or SHA) to read scopesConfigPath from' + default: 'main' preRelease: description: 'Enable pre-release mode' - default: "false" + default: 'false' preReleaseTag: description: 'Tag suffix used for pre-releases' - default: "test" + default: 'test' allowedScopes: - description: "List of change scopes to allow, comma separated. If left empty, all scopes are allowed." - default: "" + description: 'List of change scopes to allow, comma separated. If left empty, all scopes are allowed.' + default: '' force: - description: "Whenever to force update to given tag, even if it was already updated before" - default: "false" + description: 'Whether to force update to given tag, even if it was already updated before' + default: 'false' runs: using: 'node24' main: 'dist/index.js' diff --git a/.github/actions/update-sdk-schema/config.ts b/.github/actions/update-sdk-schema/config.ts index 636ab72..dd2d4ee 100644 --- a/.github/actions/update-sdk-schema/config.ts +++ b/.github/actions/update-sdk-schema/config.ts @@ -9,12 +9,18 @@ export interface Config { preRelease: boolean owner: string repo: string + scopesOwner: string + scopesRepo: string + scopesConfigPath: string + scopesRef: string allowedScopes: string[] force: boolean } export function getConfig(): Config { const [owner, repo] = core.getInput('openApiRepository').split('/') + const scopesRepositoryInput = core.getInput('scopesRepository').trim() + const [scopesOwner, scopesRepo] = (scopesRepositoryInput || `${owner}/${repo}`).split('/') return { schemaSource: core.getInput('schemaSource'), @@ -26,6 +32,10 @@ export function getConfig(): Config { force: core.getInput('force') === 'true', owner, repo, + scopesOwner, + scopesRepo, + scopesConfigPath: core.getInput('scopesConfigPath') || 'config/scopes.yaml', + scopesRef: core.getInput('scopesRef') || 'main', allowedScopes: core .getInput('allowedScopes') .split(',') diff --git a/.github/actions/update-sdk-schema/const.ts b/.github/actions/update-sdk-schema/const.ts index e4d8bcd..4a5a03b 100644 --- a/.github/actions/update-sdk-schema/const.ts +++ b/.github/actions/update-sdk-schema/const.ts @@ -2,4 +2,3 @@ export const RELEASE_NOTES = 'changesets.zip' export const EXAMPLES_FILE = 'examples.zip' export const EXAMPLE_PATH_TO_REPLACE = 'examples/' export const CHANGESETS_PATH = '.changeset' -export const SCOPES_FILE = 'scopes.yaml' diff --git a/.github/actions/update-sdk-schema/dist/index.js b/.github/actions/update-sdk-schema/dist/index.js index d84c6e7..45b159d 100644 --- a/.github/actions/update-sdk-schema/dist/index.js +++ b/.github/actions/update-sdk-schema/dist/index.js @@ -49074,6 +49074,8 @@ function replacePackageName(changeset, name) { function getConfig() { const [owner, repo] = core.getInput('openApiRepository').split('/'); + const scopesRepositoryInput = core.getInput('scopesRepository').trim(); + const [scopesOwner, scopesRepo] = (scopesRepositoryInput || `${owner}/${repo}`).split('/'); return { schemaSource: core.getInput('schemaSource'), schemaPath: core.getInput('schemaPath'), @@ -49084,6 +49086,10 @@ function getConfig() { force: core.getInput('force') === 'true', owner, repo, + scopesOwner, + scopesRepo, + scopesConfigPath: core.getInput('scopesConfigPath') || 'config/scopes.yaml', + scopesRef: core.getInput('scopesRef') || 'main', allowedScopes: core.getInput('allowedScopes') .split(',') .map((v) => v.trim()) @@ -49144,7 +49150,13 @@ function findAsset(name, release) { async function downloadAsset(url) { try { console.info('Downloading asset:', url); - const response = await withRetry(() => fetch(url)); + const response = await withRetry(async () => { + const r = await fetch(url); + if (!r.ok) { + throw new Error(`Failed to download ${url}: ${r.status} ${r.statusText}`); + } + return r; + }); return Buffer.from(await response.arrayBuffer()); } catch (e) { @@ -49152,6 +49164,11 @@ async function downloadAsset(url) { throw e; } } +async function downloadRepoFile({ owner, repo, path, ref }) { + const normalizedPath = path.replace(/^\/+/, ''); + const url = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${normalizedPath}`; + return downloadAsset(url); +} async function getRelease({ config, tag, octokit }) { const { data } = await withRetry(() => octokit.rest.repos.getReleaseByTag({ owner: config.owner, @@ -49212,7 +49229,6 @@ const RELEASE_NOTES = 'changesets.zip'; const EXAMPLES_FILE = 'examples.zip'; const EXAMPLE_PATH_TO_REPLACE = 'examples/'; const CHANGESETS_PATH = '.changeset'; -const SCOPES_FILE = 'scopes.yaml'; ;// CONCATENATED MODULE: ./node_modules/.pnpm/js-yaml@4.1.0/node_modules/js-yaml/dist/js-yaml.mjs @@ -53129,12 +53145,6 @@ function walkJson(json, key, callback) { }); } -;// CONCATENATED MODULE: ./.github/actions/update-sdk-schema/scopes.ts - -function loadScopes(scopesYaml) { - return load(scopesYaml); -} - ;// CONCATENATED MODULE: ./.github/actions/update-sdk-schema/update-schema-for-tag.ts @@ -53144,8 +53154,7 @@ function loadScopes(scopesYaml) { - -async function updateSchemaForTag(tag, octokit, packageName, { schemaSource, schemaPath, examplesPath, repo, owner, allowedScopes, generateCommand }, cwd = process.cwd()) { +async function updateSchemaForTag(tag, octokit, packageName, scopes, { schemaSource, schemaPath, examplesPath, repo, owner, allowedScopes, generateCommand }, cwd = process.cwd()) { examplesPath = external_path_.join(cwd, examplesPath); schemaPath = external_path_.join(cwd, schemaPath); console.info('Updating schema for tag:', tag); @@ -53160,7 +53169,6 @@ async function updateSchemaForTag(tag, octokit, packageName, { schemaSource, sch const schemaAsset = findAsset(schemaSource, release.data); const releaseNotesAsset = findAsset(RELEASE_NOTES, release.data); const examplesAsset = findAsset(EXAMPLES_FILE, release.data); - const scopesAsset = findAsset(SCOPES_FILE, release.data); const changesets = await getReleaseNotes(releaseNotesAsset, allowedScopes, packageName).catch((e) => { console.error('Failed to get release notes', e); throw e; @@ -53170,8 +53178,7 @@ async function updateSchemaForTag(tag, octokit, packageName, { schemaSource, sch return; } const schema = await downloadAsset(schemaAsset.browser_download_url); - const scopes = await downloadAsset(scopesAsset.browser_download_url); - const filteredSchema = filterSchema(schema.toString(), loadScopes(scopes.toString()), allowedScopes); + const filteredSchema = filterSchema(schema.toString(), scopes, allowedScopes); console.info(`Writing schema (${tag}):\n`, filteredSchema); external_fs_.writeFileSync(schemaPath, filteredSchema); console.info('Schema written in', schemaPath); @@ -53235,6 +53242,12 @@ function writeSchemaVersion(version, cwd) { external_fs_.writeFileSync(external_path_.join(cwd, SCHEMA_VERSION_FILE), version); } +;// CONCATENATED MODULE: ./.github/actions/update-sdk-schema/scopes.ts + +function loadScopes(scopesYaml) { + return load(scopesYaml); +} + ;// CONCATENATED MODULE: ./.github/actions/update-sdk-schema/update-schema.ts @@ -53245,25 +53258,33 @@ function writeSchemaVersion(version, cwd) { + async function updateSchema({ config, tag, packageName, preReleaseTag = 'test', cwd }) { if (config.preRelease) { startPreRelease(preReleaseTag); } const octokit = (0,github.getOctokit)(config.githubToken); + const scopesBuffer = await downloadRepoFile({ + owner: config.scopesOwner, + repo: config.scopesRepo, + path: config.scopesConfigPath, + ref: config.scopesRef, + }); + const scopes = loadScopes(scopesBuffer.toString()); if (config.force) { const release = await getRelease({ config, tag, octokit, }); - await updateSchemaForTag(release.tag_name, octokit, packageName, config, cwd); + await updateSchemaForTag(release.tag_name, octokit, packageName, scopes, config, cwd); } else { // v1.0.0 is the first OpenAPI release that was created const schemaVersion = getLatestSchemaVersion() ?? 'v1.0.0'; const releases = await listReleasesBetween({ octokit, config, fromTag: schemaVersion, toTag: tag }); for (const release of releases) { - await updateSchemaForTag(release.tag_name, octokit, packageName, config, cwd); + await updateSchemaForTag(release.tag_name, octokit, packageName, scopes, config, cwd); } } writeSchemaVersion(tag, cwd); diff --git a/.github/actions/update-sdk-schema/github.ts b/.github/actions/update-sdk-schema/github.ts index 0df1cfb..c3748df 100644 --- a/.github/actions/update-sdk-schema/github.ts +++ b/.github/actions/update-sdk-schema/github.ts @@ -25,7 +25,13 @@ export function findAsset(name: string, release: Release) { export async function downloadAsset(url: string) { try { console.info('Downloading asset:', url) - const response = await withRetry(() => fetch(url)) + const response = await withRetry(async () => { + const r = await fetch(url) + if (!r.ok) { + throw new Error(`Failed to download ${url}: ${r.status} ${r.statusText}`) + } + return r + }) return Buffer.from(await response.arrayBuffer()) } catch (e) { @@ -35,6 +41,20 @@ export async function downloadAsset(url: string) { } } +interface DownloadRepoFileParams { + owner: string + repo: string + path: string + ref: string +} + +export async function downloadRepoFile({ owner, repo, path, ref }: DownloadRepoFileParams) { + const normalizedPath = path.replace(/^\/+/, '') + const url = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${normalizedPath}` + + return downloadAsset(url) +} + interface ListReleasesBetweenParams { octokit: GitHubClient config: Pick diff --git a/.github/actions/update-sdk-schema/update-schema-for-tag.ts b/.github/actions/update-sdk-schema/update-schema-for-tag.ts index ce28554..ee3b92b 100644 --- a/.github/actions/update-sdk-schema/update-schema-for-tag.ts +++ b/.github/actions/update-sdk-schema/update-schema-for-tag.ts @@ -1,24 +1,19 @@ import { Config } from './config' import { downloadAsset, findAsset, getReleaseNotes, GitHubClient } from './github' -import { - CHANGESETS_PATH, - EXAMPLE_PATH_TO_REPLACE, - EXAMPLES_FILE, - RELEASE_NOTES, - SCOPES_FILE, -} from './const' +import { CHANGESETS_PATH, EXAMPLE_PATH_TO_REPLACE, EXAMPLES_FILE, RELEASE_NOTES } from './const' import * as fs from 'fs' import * as unzipper from 'unzipper' import * as path from 'path' import * as cp from 'child_process' import { filterSchema } from './filter-schema' -import { loadScopes } from './scopes' +import { ScopesMap } from './scopes' import { withRetry } from './retry' export async function updateSchemaForTag( tag: string, octokit: GitHubClient, packageName: string, + scopes: ScopesMap, { schemaSource, schemaPath, examplesPath, repo, owner, allowedScopes, generateCommand }: Config, cwd = process.cwd() ) { @@ -40,7 +35,6 @@ export async function updateSchemaForTag( const schemaAsset = findAsset(schemaSource, release.data) const releaseNotesAsset = findAsset(RELEASE_NOTES, release.data) const examplesAsset = findAsset(EXAMPLES_FILE, release.data) - const scopesAsset = findAsset(SCOPES_FILE, release.data) const changesets = await getReleaseNotes(releaseNotesAsset, allowedScopes, packageName).catch((e) => { console.error('Failed to get release notes', e) @@ -54,8 +48,7 @@ export async function updateSchemaForTag( } const schema = await downloadAsset(schemaAsset.browser_download_url) - const scopes = await downloadAsset(scopesAsset.browser_download_url) - const filteredSchema = filterSchema(schema.toString(), loadScopes(scopes.toString()), allowedScopes) + const filteredSchema = filterSchema(schema.toString(), scopes, allowedScopes) console.info(`Writing schema (${tag}):\n`, filteredSchema) fs.writeFileSync(schemaPath, filteredSchema) console.info('Schema written in', schemaPath) diff --git a/.github/actions/update-sdk-schema/update-schema.test.ts b/.github/actions/update-sdk-schema/update-schema.test.ts index 1dc3e5c..69051e8 100644 --- a/.github/actions/update-sdk-schema/update-schema.test.ts +++ b/.github/actions/update-sdk-schema/update-schema.test.ts @@ -86,6 +86,10 @@ describe('Update schema', () => { config: { owner: 'test-owner', repo: 'test-repo', + scopesOwner: 'fingerprintjs', + scopesRepo: 'fingerprint-pro-server-api-openapi', + scopesConfigPath: 'config/scopes.yaml', + scopesRef: 'main', schemaSource: 'fingerprint-server-api-schema-for-sdks.yaml', allowedScopes: ['events', 'visitors', 'webhook'], githubToken: '', @@ -127,6 +131,10 @@ describe('Update schema', () => { config: { owner: 'test-owner', repo: 'test-repo', + scopesOwner: 'fingerprintjs', + scopesRepo: 'fingerprint-pro-server-api-openapi', + scopesConfigPath: 'config/scopes.yaml', + scopesRef: 'main', allowedScopes: ['events', 'visitors', 'webhook'], githubToken: '', schemaSource: 'fingerprint-server-api-schema-for-sdks.yaml', @@ -143,4 +151,63 @@ describe('Update schema', () => { expect(readTestPackageFile('.schema-version').toString()).toEqual('v1.2.0') expect(readTestPackageFile('res/fingerprint-server-api.yaml')).toMatchSnapshot('schema') }) + + it('fails fast when canonical scopes.yaml is not reachable', async () => { + listReleases.mockResolvedValue({ + data: [v120, v110], + }) + + getReleaseByTag.mockImplementation(({ tag }) => { + switch (tag) { + case 'v1.1.0': + return { data: v110 } + + case 'v1.2.0': + return { data: v120 } + + default: + throw new Error(`Unexpected tag: ${tag}`) + } + }) + + Object.assign(global, { + fetch: jest.fn().mockImplementation(async (url, opts) => { + if (url.startsWith('https://raw.githubusercontent.com/')) { + return new Response('Not Found', { status: 404, statusText: 'Not Found' }) + } + const response = maybeMockAsset(url) + if (response) { + return response + } + + return orgFetch(url, opts) + }), + }) + + await expect( + updateSchema({ + tag: 'v1.2.0', + cwd: TEST_PACKAGE_PATH, + config: { + owner: 'test-owner', + repo: 'test-repo', + scopesOwner: 'fingerprintjs', + scopesRepo: 'fingerprint-pro-server-api-openapi', + scopesConfigPath: 'config/does-not-exist.yaml', + scopesRef: 'main', + allowedScopes: ['events', 'visitors', 'webhook'], + githubToken: '', + schemaSource: 'fingerprint-server-api-schema-for-sdks.yaml', + examplesPath: 'examples', + generateCommand: 'touch ./.generated', + preRelease: false, + schemaPath: 'res/fingerprint-server-api.yaml', + force: false, + }, + packageName: 'test-package', + }) + ).rejects.toThrow(/404|circuit breaker/) + + expect(testPackageFileExists('.generated')).toBeFalsy() + }, 30000) }) diff --git a/.github/actions/update-sdk-schema/update-schema.ts b/.github/actions/update-sdk-schema/update-schema.ts index 5271a25..9e2830c 100644 --- a/.github/actions/update-sdk-schema/update-schema.ts +++ b/.github/actions/update-sdk-schema/update-schema.ts @@ -6,7 +6,8 @@ import { startPreRelease } from './changesets' import { Config, getConfig } from './config' import { updateSchemaForTag } from './update-schema-for-tag' import { getLatestSchemaVersion, writeSchemaVersion } from './schema-version' -import { getRelease, listReleasesBetween } from './github' +import { downloadRepoFile, getRelease, listReleasesBetween } from './github' +import { loadScopes } from './scopes' interface UpdateSchemaParams { config: Config @@ -23,6 +24,14 @@ export async function updateSchema({ config, tag, packageName, preReleaseTag = ' const octokit = getOctokit(config.githubToken) + const scopesBuffer = await downloadRepoFile({ + owner: config.scopesOwner, + repo: config.scopesRepo, + path: config.scopesConfigPath, + ref: config.scopesRef, + }) + const scopes = loadScopes(scopesBuffer.toString()) + if (config.force) { const release = await getRelease({ config, @@ -30,14 +39,14 @@ export async function updateSchema({ config, tag, packageName, preReleaseTag = ' octokit, }) - await updateSchemaForTag(release.tag_name, octokit, packageName, config, cwd) + await updateSchemaForTag(release.tag_name, octokit, packageName, scopes, config, cwd) } else { // v1.0.0 is the first OpenAPI release that was created const schemaVersion = getLatestSchemaVersion() ?? 'v1.0.0' const releases = await listReleasesBetween({ octokit, config, fromTag: schemaVersion, toTag: tag }) for (const release of releases) { - await updateSchemaForTag(release.tag_name, octokit, packageName, config, cwd) + await updateSchemaForTag(release.tag_name, octokit, packageName, scopes, config, cwd) } } diff --git a/.github/workflows/update-server-side-sdk-schema.yml b/.github/workflows/update-server-side-sdk-schema.yml index 6d1b4b4..719507f 100644 --- a/.github/workflows/update-server-side-sdk-schema.yml +++ b/.github/workflows/update-server-side-sdk-schema.yml @@ -64,12 +64,27 @@ on: description: 'Tag suffix used for pre-releases' default: 'test' allowed-scopes: - description: "List of change scopes to allow, comma separated. If left empty, all scopes are allowed." + description: 'List of change scopes to allow, comma separated. If left empty, all scopes are allowed.' required: false type: string - default: "" + default: '' + scopes-repository: + description: 'Repository to source scopes.yaml from, in format of owner/repo. Defaults to the OpenAPI repo.' + required: false + type: string + default: '' + scopes-config-path: + description: 'Path to scopes.yaml within scopes-repository' + required: false + type: string + default: 'config/scopes.yaml' + scopes-ref: + description: 'Git ref (branch, tag, or SHA) to read scopes-config-path from' + required: false + type: string + default: 'main' force: - description: "Whenever to force update to given tag, even if it was already updated before" + description: 'Whether to force update to given tag, even if it was already updated before' default: false type: boolean required: false @@ -132,6 +147,9 @@ jobs: githubToken: '${{ steps.app-token.outputs.token }}' preRelease: '${{ inputs.pre-release }}' allowedScopes: ${{ inputs.allowed-scopes }} + scopesRepository: ${{ inputs.scopes-repository }} + scopesConfigPath: ${{ inputs.scopes-config-path }} + scopesRef: ${{ inputs.scopes-ref }} preReleaseTag: '${{ inputs.pre-release-tag }}' force: '${{ inputs.force }}' @@ -143,7 +161,6 @@ jobs: message: 'feat: sync OpenAPI schema to ${{ inputs.tag }}' push: 'false' - - name: Create Pull Request uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c with: @@ -151,4 +168,3 @@ jobs: body: 'Schema sync for [${{ inputs.tag }}](https://github.com/fingerprintjs/fingerprint-pro-server-api-openapi/releases/tag/${{ inputs.tag }}) OpenAPI schema release.' token: ${{ steps.app-token.outputs.token }} branch: feat/open-api-${{ inputs.tag }} -