diff --git a/src/appdistribution/distribution.spec.ts b/src/appdistribution/distribution.spec.ts index 9b898737b9d..0c04855eb30 100644 --- a/src/appdistribution/distribution.spec.ts +++ b/src/appdistribution/distribution.spec.ts @@ -122,10 +122,11 @@ describe("appdistribution/distribution", () => { setTimeoutStub.restore(); }); - it("should fail immediately when a test execution fails", async () => { + it("should fail and report failed count when a test execution fails", async () => { const releaseTests: ReleaseTest[] = [{ name: "tests/1", deviceExecutions: [] }]; mockClient.getReleaseTest.resolves({ name: "tests/1", + displayName: "Login Case", deviceExecutions: [ { state: "FAILED", @@ -145,7 +146,38 @@ describe("appdistribution/distribution", () => { } expect(caughtError).to.exist; - expect(caughtError.message).to.match(/Automated test failed/); + expect(caughtError.message).to.match(/Automated test\(s\) failed/); + setTimeoutStub.restore(); + }); + + it("should write machine-readable JSON results file when specified", async () => { + const outputJsonStub = sinon.stub(fs, "outputJsonSync"); + const releaseTests: ReleaseTest[] = [{ name: "tests/1", deviceExecutions: [] }]; + mockClient.getReleaseTest.resolves({ + name: "tests/1", + displayName: "Login Case", + resultsBucket: "projects/123/buckets/my-bucket", + deviceExecutions: [ + { + state: "PASSED", + device: { model: "Pixel", version: "14", locale: "en_US", orientation: "PORTRAIT" }, + }, + ], + } as any); + + const setTimeoutStub = sinon.stub(global, "setTimeout").callsFake((fn) => fn() as any); + + await awaitTestResults(releaseTests, mockClient as any, "/tmp/results.json"); + + expect(outputJsonStub).to.have.been.calledWith( + "/tmp/results.json", + sinon.match({ + passed: true, + totalTestCases: 1, + passCount: 1, + failCount: 0, + }), + ); setTimeoutStub.restore(); }); }); diff --git a/src/appdistribution/distribution.ts b/src/appdistribution/distribution.ts index 6ea3ba6f34b..cf59da6aa61 100644 --- a/src/appdistribution/distribution.ts +++ b/src/appdistribution/distribution.ts @@ -120,56 +120,181 @@ export class Distribution { } } +export interface DeviceExecutionResult { + device: string; + state: string; + failedReason?: string; + inconclusiveReason?: string; + actionsJsonGcsUri?: string; +} + +export interface TestCaseResult { + displayName: string; + releaseTestName?: string; + passed: boolean; + deviceExecutions: DeviceExecutionResult[]; +} + +export interface TestResultSummary { + passed: boolean; + totalTestCases: number; + passCount: number; + failCount: number; + testCases: TestCaseResult[]; +} + /** Wait for release tests to complete */ export async function awaitTestResults( releaseTests: ReleaseTest[], requests: AppDistributionClient, -): Promise { - const releaseTestNames = new Set( + resultsFilePath?: string, +): Promise { + const pendingReleaseTestNames = new Set( releaseTests.map((rt) => rt.name).filter((n): n is string => !!n), ); + const completedReleaseTests = new Map(); + for (let i = 0; i < TEST_MAX_POLLING_RETRIES; i++) { - utils.logBullet(`${releaseTestNames.size} automated test results are pending...`); + if (pendingReleaseTestNames.size === 0) { + break; + } + utils.logBullet(`${pendingReleaseTestNames.size} automated test(s) pending completion...`); await delay(TEST_POLLING_INTERVAL_MILLIS); - for (const releaseTestName of releaseTestNames) { + + for (const releaseTestName of Array.from(pendingReleaseTestNames)) { const releaseTest = await requests.getReleaseTest(releaseTestName); - if (releaseTest.deviceExecutions.every((e) => e.state === "PASSED")) { - releaseTestNames.delete(releaseTestName); - if (releaseTestNames.size === 0) { - utils.logSuccess("Automated test(s) passed!"); - return; + const isFinished = releaseTest.deviceExecutions.every( + (e) => e.state && e.state !== "IN_PROGRESS", + ); + if (isFinished) { + pendingReleaseTestNames.delete(releaseTestName); + completedReleaseTests.set(releaseTestName, releaseTest); + + const displayName = releaseTest.displayName || releaseTestName; + const allPassed = releaseTest.deviceExecutions.every((e) => e.state === "PASSED"); + if (allPassed) { + const deviceList = releaseTest.deviceExecutions + .map((e) => deviceToString(e.device)) + .join(", "); + utils.logSuccess(`✔ Passed: "${displayName}" (${deviceList})`); } else { - continue; + const failedExecs = releaseTest.deviceExecutions.filter((e) => e.state !== "PASSED"); + for (const exec of failedExecs) { + const devStr = deviceToString(exec.device); + const reason = exec.failedReason || exec.inconclusiveReason || exec.state || "Failed"; + utils.logWarning(`✖ Failed: "${displayName}" on ${devStr}: ${reason}`); + const gcsUri = formatActionsGcsUri( + releaseTest.resultsBucket, + releaseTest.name, + exec.device.model, + ); + if (gcsUri) { + logger.info(` Log / Actions detail: ${gcsUri}`); + } + } } } - for (const execution of releaseTest.deviceExecutions) { - const device = deviceToString(execution.device); - switch (execution.state) { - case "PASSED": - case "IN_PROGRESS": - continue; - case "FAILED": - throw new FirebaseError( - `Automated test failed for ${device}: ${execution.failedReason}`, - { exit: 1 }, - ); - case "INCONCLUSIVE": - throw new FirebaseError( - `Automated test inconclusive for ${device}: ${execution.inconclusiveReason}`, - { exit: 1 }, - ); - default: - throw new FirebaseError( - `Unsupported automated test state for ${device}: ${execution.state}`, - { exit: 1 }, - ); + } + } + + if (pendingReleaseTestNames.size > 0) { + throw new FirebaseError("It took longer than expected to run your test(s), please try again.", { + exit: 1, + }); + } + + const summaryTestCases: TestCaseResult[] = []; + let totalFailures = 0; + let totalPasses = 0; + + for (const rt of releaseTests) { + const name = rt.name; + const finalRt = (name && completedReleaseTests.get(name)) || rt; + const displayName = finalRt.displayName || name || "Test Case"; + const casePassed = + finalRt.deviceExecutions.length > 0 && + finalRt.deviceExecutions.every((e) => e.state === "PASSED"); + + if (casePassed) { + totalPasses++; + } else { + totalFailures++; + } + + summaryTestCases.push({ + displayName, + releaseTestName: name, + passed: casePassed, + deviceExecutions: finalRt.deviceExecutions.map((e) => ({ + device: deviceToString(e.device), + state: e.state || "UNKNOWN", + failedReason: e.failedReason, + inconclusiveReason: e.inconclusiveReason, + actionsJsonGcsUri: formatActionsGcsUri(finalRt.resultsBucket, name, e.device.model), + })), + }); + } + + const summary: TestResultSummary = { + passed: totalFailures === 0, + totalTestCases: summaryTestCases.length, + passCount: totalPasses, + failCount: totalFailures, + testCases: summaryTestCases, + }; + + if (resultsFilePath) { + try { + fs.outputJsonSync(resultsFilePath, summary, { spaces: 2 }); + utils.logSuccess(`Wrote machine-readable test results to ${resultsFilePath}`); + } catch (err: unknown) { + logger.info(`Failed to write results file to ${resultsFilePath}: ${getErrMsg(err)}`); + } + } + + if (totalFailures > 0) { + utils.logWarning( + `\nAutomated test run finished with failures (${totalFailures} of ${summaryTestCases.length} test cases failed):`, + ); + for (const tc of summaryTestCases) { + if (!tc.passed) { + utils.logWarning(` - Case "${tc.displayName}":`); + for (const exec of tc.deviceExecutions) { + if (exec.state !== "PASSED") { + const reason = exec.failedReason || exec.inconclusiveReason || exec.state; + utils.logWarning(` • ${exec.device}: ${reason}`); + if (exec.actionsJsonGcsUri) { + utils.logWarning(` GCS actions log: ${exec.actionsJsonGcsUri}`); + } + } } } } + throw new FirebaseError( + `Automated test(s) failed: ${totalFailures} of ${summaryTestCases.length} test cases failed.`, + { exit: 1 }, + ); + } + + utils.logSuccess( + `Automated test(s) passed! (${totalPasses} of ${summaryTestCases.length} test cases passed)`, + ); + return summary; +} + +function formatActionsGcsUri( + resultsBucket: string | undefined, + releaseTestName: string | undefined, + model: string, +): string | undefined { + if (!resultsBucket || !releaseTestName) { + return undefined; } - throw new FirebaseError("It took longer than expected to run your test(s), please try again.", { - exit: 1, - }); + const bucketName = resultsBucket.includes("buckets/") + ? resultsBucket.split("buckets/")[1] + : resultsBucket.replace(/^gs:\/\//, ""); + const testId = releaseTestName.split("/").pop() || ""; + return `gs://${bucketName}/${testId}/${model}/actions.json`; } function delay(ms: number): Promise { diff --git a/src/commands/apptesting.ts b/src/commands/apptesting.ts index fd1089198e6..4d5477475f6 100644 --- a/src/commands/apptesting.ts +++ b/src/commands/apptesting.ts @@ -63,6 +63,10 @@ export const command = new Command("apptesting:execute [release-binary-file]") "--results-bucket ", "The name of a Google Cloud Storage bucket where raw test results will be stored. If this flag is not set, Firebase creates a default bucket for you. Note that the bucket must be owned by a billing-enabled project, and that using a non-default bucket will result in billing charges for the storage used.", ) + .option( + "--results-file ", + "Path to output a JSON file containing machine-readable test results.", + ) .option("--test-username ", "username for automatic login") .option( "--test-password ", @@ -141,7 +145,7 @@ export const command = new Command("apptesting:execute [release-binary-file]") `View progress and results in the Firebase Console:\n${release.firebaseConsoleUri}`, ); } else { - await awaitTestResults(releaseTests, client); + await awaitTestResults(releaseTests, client, options.resultsFile); utils.logBullet( `View detailed results in the Firebase Console:\n${release.firebaseConsoleUri}`, ); @@ -173,6 +177,10 @@ async function invokeTests( } return releaseTests; } catch (err: unknown) { - throw new FirebaseError("Test invocation failed", { original: getError(err) }); + const errObj = getError(err); + if (errObj instanceof FirebaseError) { + throw errObj; + } + throw new FirebaseError(`Test invocation failed: ${errObj.message}`, { original: errObj }); } }