feat: improve popular package analysis

Author: savinmikhail

Makefile

@@ -1,3 +1,9 @@
+POPULAR_LIMIT ?= 20
+POPULAR_MIN_BYTES ?= 1024
+
+analyze-popular:
+	php check-top-packagist.php --limit=$(POPULAR_LIMIT) --min-bytes=$(POPULAR_MIN_BYTES)
+
 cs:
 	PHP_CS_FIXER_IGNORE_ENV=1 vendor/bin/php-cs-fixer fix --diff --verbose
 
@@ -20,4 +26,3 @@ composer-req:
 	vendor/bin/composer-require-checker check
 
 quality: cs rector composer-normalize composer-unused composer-req phpstan test
-

README.md

@@ -53,6 +53,28 @@ The workflow updates `dist-size-status.json`, and the badge reflects your reposi
 
 ## Usage
 
+### Analyze Popular Packages
+
+The repository includes a conservative discovery script for finding potential
+optimizations in popular Packagist packages:
+
+```bash
+make analyze-popular
+```
+
+The script analyzes a fresh package set by default, ranks observations by the
+uncompressed size of conservative development-only paths, and writes the full
+report to `var/results.json`. Observations below 1 KiB remain in the report but
+are not treated as actionable candidates. Use `--min-bytes=0` to change that
+threshold, `--resume` to skip packages already stored in `var/analyzed.json`,
+or `--config=/path/to/config.php` to select another pattern set.
+The Make target defaults to 20 packages; override it with
+`POPULAR_LIMIT=100 make analyze-popular`.
+
+Discovery results are leads, not ready-made pull requests. Before proposing a
+change, verify the exact source revision and compare real `git archive` output
+before and after applying the suggested `.gitattributes` rules.
+
 ### Check Current Project
 
 To check your current project (recommended during development):

check-top-packagist.php

@@ -3,130 +3,267 @@
 
 declare(strict_types=1);
 
-declare(ticks=1); // ensures signal checks happen between statements
+declare(ticks=1);
 
 require __DIR__ . '/vendor/autoload.php';
 
+use SavinMikhail\DistSizeOptimizer\Analysis\AnalysisReport;
 use SavinMikhail\DistSizeOptimizer\Command\CheckCommand;
+use SavinMikhail\DistSizeOptimizer\PackageManager\PackageManager;
 use Symfony\Component\Console\Input\ArrayInput;
 use Symfony\Component\Console\Output\BufferedOutput;
 
 const ANALYZED_FILE = __DIR__ . '/var/analyzed.json';
 const RESULT_FILE = __DIR__ . '/var/results.json';
+const DEFAULT_CONFIG_FILE = __DIR__ . '/export-ignore.safe.php';
 
 $interrupted = false;
 
 pcntl_signal(SIGINT, static function () use (&$interrupted): void {
-    echo "\n⛔️ Interrupted. Finishing up...\n";
+    echo "\nInterrupted. Saving the completed results...\n";
     $interrupted = true;
 });
 
+/** @return array{limit: int, minimumCandidateBytes: int, resume: bool, config: string} */
+function parseOptions(): array
+{
+    $options = getopt(short_options: '', long_options: ['limit:', 'min-bytes:', 'resume', 'config:']);
+    $limit = filter_var(value: $options['limit'] ?? 1_000, filter: FILTER_VALIDATE_INT);
+    $minimumCandidateBytes = filter_var(value: $options['min-bytes'] ?? 1_024, filter: FILTER_VALIDATE_INT);
+
+    if ($limit === false || $limit < 1) {
+        throw new InvalidArgumentException(message: '--limit must be a positive integer');
+    }
+
+    if ($minimumCandidateBytes === false || $minimumCandidateBytes < 0) {
+        throw new InvalidArgumentException(message: '--min-bytes must be a non-negative integer');
+    }
+
+    $configOption = $options['config'] ?? DEFAULT_CONFIG_FILE;
+    if (!is_string(value: $configOption)) {
+        throw new InvalidArgumentException(message: '--config must be a file path');
+    }
+
+    if (!is_file(filename: $configOption)) {
+        throw new InvalidArgumentException(message: "Config file not found: {$configOption}");
+    }
+
+    return [
+        'limit' => $limit,
+        'minimumCandidateBytes' => $minimumCandidateBytes,
+        'resume' => array_key_exists(key: 'resume', array: $options),
+        'config' => realpath(path: $configOption) ?: $configOption,
+    ];
+}
+
+/** @return array<string, array{status: string, packageMetadata: mixed, details: mixed, error?: string}> */
 function loadAnalyzed(): array
 {
-    if (!file_exists(ANALYZED_FILE)) {
+    if (!is_file(filename: ANALYZED_FILE)) {
         return [];
     }
 
-    return json_decode(file_get_contents(ANALYZED_FILE), true, 512, JSON_THROW_ON_ERROR);
+    $contents = file_get_contents(filename: ANALYZED_FILE);
+
+    return is_string(value: $contents)
+        ? json_decode(json: $contents, associative: true, flags: JSON_THROW_ON_ERROR)
+        : [];
 }
 
-function fetchTopPackages(int $limit = 1_000): array
+/** @return list<string> */
+function fetchTopPackages(int $limit): array
 {
     $packages = [];
-    $perPage = 100;
+    $perPage = min(100, $limit);
     $pages = (int) ceil($limit / $perPage);
 
     for ($page = 1; $page <= $pages; ++$page) {
         $url = "https://packagist.org/explore/popular.json?per_page={$perPage}&page={$page}";
-        $json = file_get_contents($url);
+        $json = file_get_contents(filename: $url);
 
-        if (!$json) {
-            throw new RuntimeException("Failed to fetch Packagist data for page {$page}");
+        if ($json === false) {
+            throw new RuntimeException(message: "Failed to fetch Packagist data for page {$page}");
         }
 
-        $data = json_decode($json, true, 512, JSON_THROW_ON_ERROR);
-        $pagePackages = array_map(static fn($pkg) => $pkg['name'], $data['packages']);
+        $data = json_decode(json: $json, associative: true, flags: JSON_THROW_ON_ERROR);
+        $pagePackages = array_map(
+            callback: static fn(array $package): string => $package['name'],
+            array: $data['packages'],
+        );
         $packages = array_merge($packages, $pagePackages);
 
-        // Stop early if we already have enough
-        if (count($packages) >= $limit) {
+        if (count(value: $packages) >= $limit) {
             break;
         }
     }
 
-    return array_slice($packages, 0, $limit);
+    return array_values(array: array_slice(array: $packages, offset: 0, length: $limit));
 }
 
-function runExportIgnoreCheck(string $package): array
+/** @return list<string> */
+function requireStringList(mixed $value, string $field): array
 {
-    $command = new CheckCommand();
-    $input = new ArrayInput([
+    if (!is_array(value: $value)) {
+        throw new RuntimeException(message: "The checker returned an invalid {$field} list");
+    }
+
+    $strings = [];
+    foreach ($value as $item) {
+        if (!is_string(value: $item)) {
+            throw new RuntimeException(message: "The checker returned a non-string {$field} entry");
+        }
+
+        $strings[] = $item;
+    }
+
+    return $strings;
+}
+
+/**
+ * @param array<mixed, mixed> $details
+ *
+ * @return array{files: string[], directories: string[], suggestions: string[], totalSizeBytes: int, humanReadableSize: string}
+ */
+function normalizeDetails(array $details): array
+{
+    $totalSizeBytes = $details['totalSizeBytes'] ?? null;
+    $humanReadableSize = $details['humanReadableSize'] ?? null;
+
+    if (!is_int(value: $totalSizeBytes) || !is_string(value: $humanReadableSize)) {
+        throw new RuntimeException(message: 'The checker returned invalid size information');
+    }
+
+    return [
+        'files' => requireStringList(value: $details['files'] ?? null, field: 'files'),
+        'directories' => requireStringList(value: $details['directories'] ?? null, field: 'directories'),
+        'suggestions' => requireStringList(value: $details['suggestions'] ?? null, field: 'suggestions'),
+        'totalSizeBytes' => $totalSizeBytes,
+        'humanReadableSize' => $humanReadableSize,
+    ];
+}
+
+/**
+ * @return array{status: 'ok'|'candidate'|'error', packageMetadata: null|array{name: string, version: string, sourceUrl: null|string, sourceReference: null|string, distUrl: null|string, distReference: null|string}, details: null|array{files: string[], directories: string[], suggestions: string[], totalSizeBytes: int, humanReadableSize: string}, error?: string}
+ */
+function analyzePackage(string $package, string $config): array
+{
+    $input = new ArrayInput(parameters: [
         'package' => $package,
         '--json' => true,
+        '--dry-run' => true,
+        '--config' => $config,
     ]);
     $output = new BufferedOutput();
+    $packageManager = new PackageManager();
 
     try {
-        $exitCode = $command->run($input, $output);
-        $result = json_decode($output->fetch(), true);
-
+        $exitCode = new CheckCommand(packageManager: $packageManager)->run(input: $input, output: $output);
+        $packageMetadata = $packageManager->getPackageMetadata();
         if ($exitCode === 0) {
-            return ['status' => '✅ OK', 'details' => null];
+            return ['status' => 'ok', 'packageMetadata' => $packageMetadata, 'details' => null];
         }
 
-        if (isset($result['files']) && count($result['files']) > 0 || isset($result['directories']) && count($result['directories']) > 0) {
-            return ['status' => '❌ Missing export-ignore', 'details' => $result];
+        $details = json_decode(json: $output->fetch(), associative: true, flags: JSON_THROW_ON_ERROR);
+        if (!is_array(value: $details)) {
+            throw new RuntimeException(message: 'The checker returned an invalid JSON report');
         }
 
-        return ['status' => '⚠️ Failed', 'details' => null];
-    } catch (Exception $e) {
-        return ['status' => '💥 Error: ' . $e->getMessage(), 'details' => null];
+        return [
+            'status' => 'candidate',
+            'packageMetadata' => $packageMetadata,
+            'details' => normalizeDetails(details: $details),
+        ];
+    } catch (Throwable $error) {
+        return [
+            'status' => 'error',
+            'packageMetadata' => null,
+            'details' => null,
+            'error' => $error->getMessage(),
+        ];
     }
 }
 
-function saveFailures(array $failures): void
+/** @param array<string, mixed> $data */
+function writeJson(string $path, array $data): void
 {
-    if (count($failures) === 0) {
-        echo "\n✅ No export-ignore issues found. Great job, open source!\n";
-
-        return;
+    $directory = dirname(path: $path);
+    if (!is_dir(filename: $directory)) {
+        mkdir(directory: $directory, permissions: 0o777, recursive: true);
     }
 
-    file_put_contents(RESULT_FILE, json_encode($failures, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
-    echo "\n📝 Saved detailed results to: " . RESULT_FILE . "\n";
+    file_put_contents(
+        filename: $path,
+        data: json_encode(value: $data, flags: JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n",
+    );
 }
 
-echo "🔍 Checking top Packagist packages...\n";
-
-$packages = fetchTopPackages(1_000);
-
-$results = [];
-$failures = [];
-
-$total = count($packages);
-$analyzed = loadAnalyzed();
-$newPackages = array_filter($packages, static fn($p) => !isset($analyzed[$p]));
-
-foreach (array_values($newPackages) as $i => $package) {
+$options = parseOptions();
+$packages = fetchTopPackages(limit: $options['limit']);
+$previousResults = $options['resume'] ? loadAnalyzed() : [];
+$results = $previousResults;
+$packagesToAnalyze = array_values(array_filter(
+    array: $packages,
+    callback: static fn(string $package): bool => !array_key_exists(key: $package, array: $previousResults),
+));
+
+echo sprintf(
+    "Checking %d popular Packagist packages with %s rules...\n",
+    count(value: $packagesToAnalyze),
+    basename(path: $options['config']),
+);
+
+foreach ($packagesToAnalyze as $index => $package) {
     if ($interrupted) {
         break;
     }
 
-    echo "📦 [{$i}/{$total}] Checking {$package}...\n";
-    $check = runExportIgnoreCheck($package);
-    $results[$package] = $check['status'];
+    echo sprintf("[%d/%d] %s\n", $index + 1, count(value: $packagesToAnalyze), $package);
+    $results[$package] = analyzePackage(package: $package, config: $options['config']);
+    writeJson(path: ANALYZED_FILE, data: $results);
+}
 
-    if ($check['status'] === '❌ Missing export-ignore' && isset($check['details'])) {
-        $failures[$package] = $check['details'];
+$selectedResults = array_intersect_key($results, array_flip($packages));
+$observations = new AnalysisReport()->rankCandidates(results: $selectedResults);
+$candidates = array_values(array_filter(
+    array: $observations,
+    callback: static fn(array $candidate): bool => $candidate['totalSizeBytes'] >= $options['minimumCandidateBytes'],
+));
+$report = [
+    'generatedAt' => gmdate(format: DATE_ATOM),
+    'requestedLimit' => $options['limit'],
+    'analyzedCount' => count(value: $selectedResults),
+    'minimumCandidateBytes' => $options['minimumCandidateBytes'],
+    'config' => basename(path: $options['config']),
+    'interrupted' => $interrupted,
+    'candidates' => $candidates,
+    'observations' => $observations,
+    'packages' => $selectedResults,
+];
+writeJson(path: RESULT_FILE, data: $report);
+
+echo sprintf("\nSaved the reproducible report to %s\n", RESULT_FILE);
+
+if ($candidates === []) {
+    echo sprintf(
+        "No conservative export-ignore candidates found above %d bytes.\n",
+        $options['minimumCandidateBytes'],
+    );
+
+    if ($observations !== []) {
+        echo sprintf(
+            "Largest observation below the threshold: %s (%s)\n",
+            $observations[0]['package'],
+            $observations[0]['humanReadableSize'],
+        );
     }
-}
 
-echo "\n📊 Results:\n";
-foreach ($results as $package => $status) {
-    echo "  {$package}: {$status}\n";
+    exit(0);
 }
 
-saveFailures($failures);
-
-$allAnalyzed = array_merge($analyzed, $results);
-file_put_contents(ANALYZED_FILE, json_encode($allAnalyzed, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
+$topCandidate = $candidates[0];
+echo sprintf(
+    "Top candidate: %s (%s across %d paths)\n",
+    $topCandidate['package'],
+    $topCandidate['humanReadableSize'],
+    count(value: $topCandidate['files']) + count(value: $topCandidate['directories']),
+);