diff --git a/server/src/Support/Installer/ArtifactDownloader.php b/server/src/Support/Installer/ArtifactDownloader.php index 13beeba..e7bf4ec 100644 --- a/server/src/Support/Installer/ArtifactDownloader.php +++ b/server/src/Support/Installer/ArtifactDownloader.php @@ -44,7 +44,7 @@ public function download(Artifact $artifact, string $directory): string } if (!Checksum::matches($target, $artifact->checksum)) { - throw new \RuntimeException(sprintf('Checksum mismatch for the %s artifact: expected %s, got %s.', $artifact->role, $artifact->checksum, Checksum::sha256($target))); + throw new ChecksumMismatchException(sprintf('Checksum mismatch for the %s artifact: expected %s, got %s.', $artifact->role, $artifact->checksum, Checksum::sha256($target))); } return $target; diff --git a/server/src/Support/Installer/ChecksumMismatchException.php b/server/src/Support/Installer/ChecksumMismatchException.php new file mode 100644 index 0000000..573d734 --- /dev/null +++ b/server/src/Support/Installer/ChecksumMismatchException.php @@ -0,0 +1,11 @@ +config->jobDirectory($ctx->id); foreach ($release->artifacts as $artifact) { $ctx->log->add(sprintf('Downloading %s artifact', $artifact->role)); - $ctx->downloads[$artifact->role] = $this->downloader->download($artifact, $directory . '/downloads'); + try { + $ctx->downloads[$artifact->role] = $this->downloader->download($artifact, $directory . '/downloads'); + } catch (ChecksumMismatchException $e) { + throw $e; + } catch (\RuntimeException $e) { + // The service always has the engine package too, so a dist it cannot serve right now (unreachable, + // 404, a licence refusal on a host the token is not sent to) costs a rebuild, not the install. + if ($artifact->role !== Artifact::DIST || $release->source !== ExtensionRef::REGISTRY_SERVICE) { + throw $e; + } + $this->skipPrebuilt($ctx, sprintf('The prebuilt dist could not be downloaded: %s', $e->getMessage())); + } } - if ($release->hasPrebuiltEngine()) { + if ($release->hasPrebuiltEngine() && isset($ctx->downloads[Artifact::DIST])) { $ctx->engineRelease = EngineRelease::extract($ctx->downloads[Artifact::DIST], $directory . '/dist', (string) $release->enginePackage); $ctx->log->add(sprintf('Verified %s@%s (%d files)', $ctx->engineRelease->name(), $ctx->engineRelease->version(), count($ctx->engineRelease->manifest['files'] ?? []))); } @@ -170,9 +181,10 @@ private function install(InstallContext $ctx): void if ($release->source !== ExtensionRef::REGISTRY_SERVICE) { throw new \RuntimeException($refusal); } - $ctx->log->add($refusal . ' Installing the engine package and rebuilding the console instead.'); - $ctx->result['prebuilt_skipped'] = $refusal; - $rebuild = true; + $this->skipPrebuilt($ctx, $refusal); + $rebuild = true; + } elseif ($release->source === ExtensionRef::REGISTRY_SERVICE && !isset($ctx->result['prebuilt_skipped'])) { + $this->skipPrebuilt($ctx, sprintf('%s %s has no prebuilt dist on the registry service.', $release->enginePackage, $release->version)); } $this->addEnginePackage($ctx, $release); }); @@ -264,6 +276,23 @@ private function prebuiltRefusal(InstallContext $ctx, Release $release): ?string return $compat->compatible ? null : 'The prebuilt engine is not compatible with this console: ' . $compat->summary(); } + /** + * Record why a release installs by rebuild instead of from its prebuilt dist: in the job log, in the result as + * `prebuilt_skipped`, and in the application log, so an operator can see what cost the rebuild. + */ + private function skipPrebuilt(InstallContext $ctx, string $reason): void + { + $ctx->log->add($reason . ' Installing the engine package and rebuilding the console instead.'); + $ctx->result['prebuilt_skipped'] = $reason; + if (function_exists('logger')) { + try { + logger()->warning('[registry-bridge] Prebuilt dist not used, rebuilding the console', ['job' => $ctx->id, 'reason' => $reason]); + } catch (\Throwable) { + // Outside a Laravel application (tests, scripts) the job log is the record. + } + } + } + /** * `pnpm add` the engine package for a console build. A runtime copy of the same engine is removed (the * snapshot restores it on failure), so the rebuilt console does not load it twice. @@ -325,7 +354,9 @@ private function buildConsole(InstallContext $ctx): void try { $ctx->snapshot?->flag('built'); - $this->exec($ctx, [$this->config->pnpmBinary, 'build', '--environment', $this->config->buildEnvironment], $this->config->consolePath, 'console.build', [OutputProgress::class, 'pnpmBuildOutputProgress']); + // FLEETBASE_HOST_MODULES=1 keeps dist/host-modules.json in every environment: without it a non-production + // rebuild drops the file and the console refuses every prebuilt engine afterwards. + $this->exec($ctx, [$this->config->pnpmBinary, 'build', '--environment', $this->config->buildEnvironment], $this->config->consolePath, 'console.build', [OutputProgress::class, 'pnpmBuildOutputProgress'], ['FLEETBASE_HOST_MODULES' => '1']); } finally { foreach ($preserved as $path => $copy) { Files::move($copy, $path); @@ -474,13 +505,14 @@ private function stage(InstallContext $ctx, string $stage, string $step, callabl /** * @param array $command * @param callable(string): int|null $progress maps an output line to a percentage + * @param array $env extra environment for this command, over the credentials' */ - private function exec(InstallContext $ctx, array $command, string $cwd, ?string $step = null, ?callable $progress = null): ProcessResult + private function exec(InstallContext $ctx, array $command, string $cwd, ?string $step = null, ?callable $progress = null, array $env = []): ProcessResult { $ctx->log->add('$ ' . implode(' ', $command)); $last = 0; - $env = $this->credentials->environment($command); + $env = array_merge($this->credentials->environment($command), $env); $result = $this->runner->run($command, $cwd, function (string $line) use ($ctx, $step, $progress, &$last) { $ctx->log->add($line); if ($step && $progress) { diff --git a/server/src/Support/Installer/ServiceReleaseSource.php b/server/src/Support/Installer/ServiceReleaseSource.php index d750fe6..10d97f7 100644 --- a/server/src/Support/Installer/ServiceReleaseSource.php +++ b/server/src/Support/Installer/ServiceReleaseSource.php @@ -18,11 +18,13 @@ * engine's npm scope. Credentials are not part of the release; they come from * ServicePackageCredentials per process. * - * When the release carries a prebuilt engine dist (`dist: {sha256, size, - * built_against}` on catalog releases), it becomes the release's DIST artifact - * at `/npm//-/dist/`, verified by that sha256 and - * downloaded with the instance token (PackageAuth::downloadHeaders), so the - * engine installs as a runtime engine without a console build. + * When the release carries a prebuilt engine dist (`dist: {url, sha256, size, + * built_against}` on catalog and update releases), it becomes the release's + * DIST artifact at `dist.url` (`/v1/instances/me/releases//dist`; a service + * that names no url serves it at `/npm//-/dist/`), + * verified by that sha256 and downloaded with the instance token + * (PackageAuth::downloadHeaders, only for URLs on the configured service), so + * the engine installs as a runtime engine without a console build. * * Licences are checked by the instance API when the install starts (see * ServiceInstallReporter), not here. @@ -113,7 +115,7 @@ public function release(ExtensionRef $extension, array $listing, ?string $versio composerConstraint: $packages['composer'] !== null ? $version : null, enginePackage: $packages['npm'], engineConstraint: $packages['npm'] !== null ? $version : null, - artifacts: $packages['npm'] !== null ? self::distArtifacts($auth, $packages['npm'], $version, $dist) : [], + artifacts: $packages['npm'] !== null ? $this->distArtifacts($auth, $packages['npm'], $version, $dist) : [], source: ExtensionRef::REGISTRY_SERVICE, engineTarball: $packages['npm'] !== null ? $auth->npmTarballUrl($packages['npm'], $version) : null, composerRepository: $repository !== null ? ['name' => PackageAuth::COMPOSER_REPOSITORY_NAME, ...$repository] : null, @@ -126,10 +128,10 @@ public function release(ExtensionRef $extension, array $listing, ?string $versio * * @return array */ - private static function distArtifacts(PackageAuth $auth, string $enginePackage, string $version, mixed $dist): array + private function distArtifacts(PackageAuth $auth, string $enginePackage, string $version, mixed $dist): array { $sha256 = is_array($dist) ? self::string($dist['sha256'] ?? null) : null; - $url = $auth->npmDistUrl($enginePackage, $version); + $url = (is_array($dist) ? $this->distUrl($dist['url'] ?? null) : null) ?? $auth->npmDistUrl($enginePackage, $version); if ($sha256 === null || !preg_match('/^[a-f0-9]{64}$/i', $sha256) || $url === null) { return []; } @@ -139,6 +141,23 @@ private static function distArtifacts(PackageAuth $auth, string $enginePackage, return [new Artifact(Artifact::DIST, $url, 'sha256-' . strtolower($sha256), $basename . '-' . $version . '-dist')]; } + /** + * `dist.url` as the service names it: an http(s) URL, or a path on the configured service. + */ + private function distUrl(mixed $url): ?string + { + $url = self::string($url); + if ($url === null) { + return null; + } + if (preg_match('#^https?://#i', $url)) { + return $url; + } + $serviceUrl = $this->config->serviceUrl(); + + return str_starts_with($url, '/') && $serviceUrl ? rtrim($serviceUrl, '/') . $url : null; + } + /** * @param array $listing * diff --git a/server/tests/Installer/RegistryServiceInstallTest.php b/server/tests/Installer/RegistryServiceInstallTest.php index c1a83df..aa0749a 100644 --- a/server/tests/Installer/RegistryServiceInstallTest.php +++ b/server/tests/Installer/RegistryServiceInstallTest.php @@ -155,7 +155,10 @@ function installReports(RegistryServiceFake $service): array expect(json_decode($env[1]['COMPOSER_AUTH'], true))->toBe(['bearer' => ['registry.example.test' => RegistryServiceFake::INSTANCE_TOKEN]]) ->and($env[2]['FLEETBASE_REGISTRY_TOKEN'])->toBe(RegistryServiceFake::INSTANCE_TOKEN) ->and($env[2]['npm_config_userconfig'])->toBe(PackageAuth::npmrcPath()) - ->and($env[4])->toBe([]); + ->and($env[3]['FLEETBASE_HOST_MODULES'])->toBe('1') + ->and($env[4])->toBe([]) + ->and($result['prebuilt_skipped'])->toBe(HELLO_NPM . ' 1.0.0 has no prebuilt dist on the registry service.') + ->and(implode("\n", $result['log']))->toContain('has no prebuilt dist on the registry service. Installing the engine package and rebuilding the console instead.'); foreach ($this->sandbox->runner->commands() as $command) { expect($command)->not->toContain(RegistryServiceFake::INSTANCE_TOKEN); } @@ -240,6 +243,36 @@ function installReports(RegistryServiceFake $service): array ->and($this->sandbox->manifest()->has(HELLO_NPM))->toBeFalse(); })->with(['built in', 'no host modules', 'incompatible']); +test('a dist the service cannot serve falls back to the engine package and a rebuild, with the reason logged', function () { + [$listing] = helloListingWithDist($this->sandbox); + $this->service->respondWith('GET', 'v1/catalog/extensions/' . HELLO_ID, 200, $listing); + $this->service->on('POST', 'v1/instances/me/installs', 'instances-installs-installing', 'instances-installs-installed'); + $fetcher = fn (string $url) => throw new \RuntimeException(sprintf('Could not download %s.', $url)); + + $result = serviceInstaller($this->sandbox, RegistryServiceFake::config(), $fetcher)->run(helloJob()); + + expect($result['status'])->toBe('completed') + ->and($result['mode'])->toBe('build') + ->and($result['stages'])->toBe(['resolve', 'download', 'snapshot', 'composer', 'engine', 'build', 'caches', 'record']) + ->and($result['prebuilt_skipped'])->toBe('The prebuilt dist could not be downloaded: Could not download ' . HELLO_DIST . '.') + ->and($this->sandbox->runner->commands())->toContain('pnpm add ' . HELLO_NPM . '@' . HELLO_TARBALL . ' --config.ignore-workspace-root-check=true', 'pnpm build --environment production') + ->and($this->sandbox->manifest()->has(HELLO_NPM))->toBeFalse(); +}); + +test('the dist downloads from dist.url when the service names one', function () { + $listing = helloListing(); + $listing['latest_release']['dist'] = ['url' => RegistryServiceFake::SERVICE_URL . '/v1/instances/me/releases/rel_fshrg663og/dist', 'size' => 10, 'sha256' => str_repeat('ab', 32), 'integrity' => null, 'built_against' => []]; + $source = new ServiceReleaseSource(RegistryClient::fromConfig(RegistryServiceFake::config()), RegistryServiceFake::config()); + + expect($source->release(helloRef(), $listing)->artifact('dist')?->uri)->toBe(RegistryServiceFake::SERVICE_URL . '/v1/instances/me/releases/rel_fshrg663og/dist'); + + $listing['latest_release']['dist']['url'] = '/v1/instances/me/releases/rel_fshrg663og/dist'; + expect($source->release(helloRef(), $listing)->artifact('dist')?->uri)->toBe(rtrim(RegistryServiceFake::SERVICE_URL, '/') . '/v1/instances/me/releases/rel_fshrg663og/dist'); + + $listing['latest_release']['dist']['url'] = 'ftp://elsewhere.test/dist'; + expect($source->release(helloRef(), $listing)->artifact('dist')?->uri)->toBe(HELLO_DIST); +}); + test('a release from an older service without dist metadata keeps the rebuild path', function () { $release = (new ServiceReleaseSource(RegistryClient::fromConfig(RegistryServiceFake::config()), RegistryServiceFake::config()))->release(helloRef(), helloListing());