Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion server/src/Support/Installer/ArtifactDownloader.php
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public function download(Artifact $artifact, string $directory): string
}

if (!Checksum::matches($target, $artifact->checksum)) {
throw new \RuntimeException(sprintf('Checksum mismatch for the %s artifact: expected %s, got %s.', $artifact->role, $artifact->checksum, Checksum::sha256($target)));
throw new ChecksumMismatchException(sprintf('Checksum mismatch for the %s artifact: expected %s, got %s.', $artifact->role, $artifact->checksum, Checksum::sha256($target)));
}

return $target;
Expand Down
11 changes: 11 additions & 0 deletions server/src/Support/Installer/ChecksumMismatchException.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<?php

namespace Fleetbase\RegistryBridge\Support\Installer;

/**
* A downloaded artifact does not match the checksum its release names. Never retried or worked around: the
* download is refused and the job fails.
*/
class ChecksumMismatchException extends \RuntimeException
{
}
48 changes: 40 additions & 8 deletions server/src/Support/Installer/ExtensionInstaller.php
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,20 @@ private function install(InstallContext $ctx): void
$directory = $this->config->jobDirectory($ctx->id);
foreach ($release->artifacts as $artifact) {
$ctx->log->add(sprintf('Downloading %s artifact', $artifact->role));
$ctx->downloads[$artifact->role] = $this->downloader->download($artifact, $directory . '/downloads');
try {
$ctx->downloads[$artifact->role] = $this->downloader->download($artifact, $directory . '/downloads');
} catch (ChecksumMismatchException $e) {
throw $e;
} catch (\RuntimeException $e) {
// The service always has the engine package too, so a dist it cannot serve right now (unreachable,
// 404, a licence refusal on a host the token is not sent to) costs a rebuild, not the install.
if ($artifact->role !== Artifact::DIST || $release->source !== ExtensionRef::REGISTRY_SERVICE) {
throw $e;
}
$this->skipPrebuilt($ctx, sprintf('The prebuilt dist could not be downloaded: %s', $e->getMessage()));
}
}
if ($release->hasPrebuiltEngine()) {
if ($release->hasPrebuiltEngine() && isset($ctx->downloads[Artifact::DIST])) {
$ctx->engineRelease = EngineRelease::extract($ctx->downloads[Artifact::DIST], $directory . '/dist', (string) $release->enginePackage);
$ctx->log->add(sprintf('Verified %s@%s (%d files)', $ctx->engineRelease->name(), $ctx->engineRelease->version(), count($ctx->engineRelease->manifest['files'] ?? [])));
}
Expand Down Expand Up @@ -170,9 +181,10 @@ private function install(InstallContext $ctx): void
if ($release->source !== ExtensionRef::REGISTRY_SERVICE) {
throw new \RuntimeException($refusal);
}
$ctx->log->add($refusal . ' Installing the engine package and rebuilding the console instead.');
$ctx->result['prebuilt_skipped'] = $refusal;
$rebuild = true;
$this->skipPrebuilt($ctx, $refusal);
$rebuild = true;
} elseif ($release->source === ExtensionRef::REGISTRY_SERVICE && !isset($ctx->result['prebuilt_skipped'])) {
$this->skipPrebuilt($ctx, sprintf('%s %s has no prebuilt dist on the registry service.', $release->enginePackage, $release->version));
}
$this->addEnginePackage($ctx, $release);
});
Expand Down Expand Up @@ -264,6 +276,23 @@ private function prebuiltRefusal(InstallContext $ctx, Release $release): ?string
return $compat->compatible ? null : 'The prebuilt engine is not compatible with this console: ' . $compat->summary();
}

/**
* Record why a release installs by rebuild instead of from its prebuilt dist: in the job log, in the result as
* `prebuilt_skipped`, and in the application log, so an operator can see what cost the rebuild.
*/
private function skipPrebuilt(InstallContext $ctx, string $reason): void
{
$ctx->log->add($reason . ' Installing the engine package and rebuilding the console instead.');
$ctx->result['prebuilt_skipped'] = $reason;
if (function_exists('logger')) {
try {
logger()->warning('[registry-bridge] Prebuilt dist not used, rebuilding the console', ['job' => $ctx->id, 'reason' => $reason]);
} catch (\Throwable) {
// Outside a Laravel application (tests, scripts) the job log is the record.
}
}
}

/**
* `pnpm add` the engine package for a console build. A runtime copy of the same engine is removed (the
* snapshot restores it on failure), so the rebuilt console does not load it twice.
Expand Down Expand Up @@ -325,7 +354,9 @@ private function buildConsole(InstallContext $ctx): void

try {
$ctx->snapshot?->flag('built');
$this->exec($ctx, [$this->config->pnpmBinary, 'build', '--environment', $this->config->buildEnvironment], $this->config->consolePath, 'console.build', [OutputProgress::class, 'pnpmBuildOutputProgress']);
// FLEETBASE_HOST_MODULES=1 keeps dist/host-modules.json in every environment: without it a non-production
// rebuild drops the file and the console refuses every prebuilt engine afterwards.
$this->exec($ctx, [$this->config->pnpmBinary, 'build', '--environment', $this->config->buildEnvironment], $this->config->consolePath, 'console.build', [OutputProgress::class, 'pnpmBuildOutputProgress'], ['FLEETBASE_HOST_MODULES' => '1']);
} finally {
foreach ($preserved as $path => $copy) {
Files::move($copy, $path);
Expand Down Expand Up @@ -474,13 +505,14 @@ private function stage(InstallContext $ctx, string $stage, string $step, callabl
/**
* @param array<int, string> $command
* @param callable(string): int|null $progress maps an output line to a percentage
* @param array<string, string> $env extra environment for this command, over the credentials'
*/
private function exec(InstallContext $ctx, array $command, string $cwd, ?string $step = null, ?callable $progress = null): ProcessResult
private function exec(InstallContext $ctx, array $command, string $cwd, ?string $step = null, ?callable $progress = null, array $env = []): ProcessResult
{
$ctx->log->add('$ ' . implode(' ', $command));
$last = 0;

$env = $this->credentials->environment($command);
$env = array_merge($this->credentials->environment($command), $env);
$result = $this->runner->run($command, $cwd, function (string $line) use ($ctx, $step, $progress, &$last) {
$ctx->log->add($line);
if ($step && $progress) {
Expand Down
35 changes: 27 additions & 8 deletions server/src/Support/Installer/ServiceReleaseSource.php
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@
* engine's npm scope. Credentials are not part of the release; they come from
* ServicePackageCredentials per process.
*
* When the release carries a prebuilt engine dist (`dist: {sha256, size,
* built_against}` on catalog releases), it becomes the release's DIST artifact
* at `<service>/npm/<engine>/-/dist/<version>`, verified by that sha256 and
* downloaded with the instance token (PackageAuth::downloadHeaders), so the
* engine installs as a runtime engine without a console build.
* When the release carries a prebuilt engine dist (`dist: {url, sha256, size,
* built_against}` on catalog and update releases), it becomes the release's
* DIST artifact at `dist.url` (`/v1/instances/me/releases/<id>/dist`; a service
* that names no url serves it at `<service>/npm/<engine>/-/dist/<version>`),
* verified by that sha256 and downloaded with the instance token
* (PackageAuth::downloadHeaders, only for URLs on the configured service), so
* the engine installs as a runtime engine without a console build.
*
* Licences are checked by the instance API when the install starts (see
* ServiceInstallReporter), not here.
Expand Down Expand Up @@ -113,7 +115,7 @@ public function release(ExtensionRef $extension, array $listing, ?string $versio
composerConstraint: $packages['composer'] !== null ? $version : null,
enginePackage: $packages['npm'],
engineConstraint: $packages['npm'] !== null ? $version : null,
artifacts: $packages['npm'] !== null ? self::distArtifacts($auth, $packages['npm'], $version, $dist) : [],
artifacts: $packages['npm'] !== null ? $this->distArtifacts($auth, $packages['npm'], $version, $dist) : [],
source: ExtensionRef::REGISTRY_SERVICE,
engineTarball: $packages['npm'] !== null ? $auth->npmTarballUrl($packages['npm'], $version) : null,
composerRepository: $repository !== null ? ['name' => PackageAuth::COMPOSER_REPOSITORY_NAME, ...$repository] : null,
Expand All @@ -126,10 +128,10 @@ public function release(ExtensionRef $extension, array $listing, ?string $versio
*
* @return array<int, Artifact>
*/
private static function distArtifacts(PackageAuth $auth, string $enginePackage, string $version, mixed $dist): array
private function distArtifacts(PackageAuth $auth, string $enginePackage, string $version, mixed $dist): array
{
$sha256 = is_array($dist) ? self::string($dist['sha256'] ?? null) : null;
$url = $auth->npmDistUrl($enginePackage, $version);
$url = (is_array($dist) ? $this->distUrl($dist['url'] ?? null) : null) ?? $auth->npmDistUrl($enginePackage, $version);
if ($sha256 === null || !preg_match('/^[a-f0-9]{64}$/i', $sha256) || $url === null) {
return [];
}
Expand All @@ -139,6 +141,23 @@ private static function distArtifacts(PackageAuth $auth, string $enginePackage,
return [new Artifact(Artifact::DIST, $url, 'sha256-' . strtolower($sha256), $basename . '-' . $version . '-dist')];
}

/**
* `dist.url` as the service names it: an http(s) URL, or a path on the configured service.
*/
private function distUrl(mixed $url): ?string
{
$url = self::string($url);
if ($url === null) {
return null;
}
if (preg_match('#^https?://#i', $url)) {
return $url;
}
$serviceUrl = $this->config->serviceUrl();

return str_starts_with($url, '/') && $serviceUrl ? rtrim($serviceUrl, '/') . $url : null;
}

/**
* @param array<string, mixed> $listing
*
Expand Down
35 changes: 34 additions & 1 deletion server/tests/Installer/RegistryServiceInstallTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,10 @@ function installReports(RegistryServiceFake $service): array
expect(json_decode($env[1]['COMPOSER_AUTH'], true))->toBe(['bearer' => ['registry.example.test' => RegistryServiceFake::INSTANCE_TOKEN]])
->and($env[2]['FLEETBASE_REGISTRY_TOKEN'])->toBe(RegistryServiceFake::INSTANCE_TOKEN)
->and($env[2]['npm_config_userconfig'])->toBe(PackageAuth::npmrcPath())
->and($env[4])->toBe([]);
->and($env[3]['FLEETBASE_HOST_MODULES'])->toBe('1')
->and($env[4])->toBe([])
->and($result['prebuilt_skipped'])->toBe(HELLO_NPM . ' 1.0.0 has no prebuilt dist on the registry service.')
->and(implode("\n", $result['log']))->toContain('has no prebuilt dist on the registry service. Installing the engine package and rebuilding the console instead.');
foreach ($this->sandbox->runner->commands() as $command) {
expect($command)->not->toContain(RegistryServiceFake::INSTANCE_TOKEN);
}
Expand Down Expand Up @@ -240,6 +243,36 @@ function installReports(RegistryServiceFake $service): array
->and($this->sandbox->manifest()->has(HELLO_NPM))->toBeFalse();
})->with(['built in', 'no host modules', 'incompatible']);

test('a dist the service cannot serve falls back to the engine package and a rebuild, with the reason logged', function () {
[$listing] = helloListingWithDist($this->sandbox);
$this->service->respondWith('GET', 'v1/catalog/extensions/' . HELLO_ID, 200, $listing);
$this->service->on('POST', 'v1/instances/me/installs', 'instances-installs-installing', 'instances-installs-installed');
$fetcher = fn (string $url) => throw new \RuntimeException(sprintf('Could not download %s.', $url));

$result = serviceInstaller($this->sandbox, RegistryServiceFake::config(), $fetcher)->run(helloJob());

expect($result['status'])->toBe('completed')
->and($result['mode'])->toBe('build')
->and($result['stages'])->toBe(['resolve', 'download', 'snapshot', 'composer', 'engine', 'build', 'caches', 'record'])
->and($result['prebuilt_skipped'])->toBe('The prebuilt dist could not be downloaded: Could not download ' . HELLO_DIST . '.')
->and($this->sandbox->runner->commands())->toContain('pnpm add ' . HELLO_NPM . '@' . HELLO_TARBALL . ' --config.ignore-workspace-root-check=true', 'pnpm build --environment production')
->and($this->sandbox->manifest()->has(HELLO_NPM))->toBeFalse();
});

test('the dist downloads from dist.url when the service names one', function () {
$listing = helloListing();
$listing['latest_release']['dist'] = ['url' => RegistryServiceFake::SERVICE_URL . '/v1/instances/me/releases/rel_fshrg663og/dist', 'size' => 10, 'sha256' => str_repeat('ab', 32), 'integrity' => null, 'built_against' => []];
$source = new ServiceReleaseSource(RegistryClient::fromConfig(RegistryServiceFake::config()), RegistryServiceFake::config());

expect($source->release(helloRef(), $listing)->artifact('dist')?->uri)->toBe(RegistryServiceFake::SERVICE_URL . '/v1/instances/me/releases/rel_fshrg663og/dist');

$listing['latest_release']['dist']['url'] = '/v1/instances/me/releases/rel_fshrg663og/dist';
expect($source->release(helloRef(), $listing)->artifact('dist')?->uri)->toBe(rtrim(RegistryServiceFake::SERVICE_URL, '/') . '/v1/instances/me/releases/rel_fshrg663og/dist');

$listing['latest_release']['dist']['url'] = 'ftp://elsewhere.test/dist';
expect($source->release(helloRef(), $listing)->artifact('dist')?->uri)->toBe(HELLO_DIST);
});

test('a release from an older service without dist metadata keeps the rebuild path', function () {
$release = (new ServiceReleaseSource(RegistryClient::fromConfig(RegistryServiceFake::config()), RegistryServiceFake::config()))->release(helloRef(), helloListing());

Expand Down
Loading