feat(cli): add --threads auto to cap worker concurrency for large files (#77 P1)

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+### Features
+
+- **lint-md:** add `--threads auto` to cap worker concurrency for large Markdown files (issue #77, P1)
+
 ## [2.1.1](https://github.com/lint-md/cli/compare/v2.0.0...v2.1.1) (2026-06-30)
 
 ### Features

__tests__/batch-lint.spec.ts

@@ -1,9 +1,9 @@
 import { mkdtemp, rm, writeFile } from 'fs/promises';
-import { tmpdir } from 'os';
+import { availableParallelism, tmpdir } from 'os';
 import * as path from 'path';
 import { Piscina } from 'piscina';
 import type { LintMdRulesConfig } from '@lint-md/core';
-import { batchLint, runTasksWithLimit } from '../src/utils/batch-lint';
+import { batchLint, getMaxFileSize, resolveAdaptiveConcurrency, runTasksWithLimit } from '../src/utils/batch-lint';
 
 const RULES_NO_EMPTY_LIST: LintMdRulesConfig = {
   'no-empty-list': 2,
@@ -177,3 +177,142 @@ describe('batchLint', () => {
     });
   });
 });
+
+describe('getMaxFileSize', () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await mkdtemp(path.join(tmpdir(), 'batch-lint-max-'));
+  });
+
+  afterEach(async () => {
+    await rm(tmpDir, { recursive: true, force: true });
+  });
+
+  test('returns 0 for empty list', async () => {
+    expect(await getMaxFileSize([])).toBe(0);
+  });
+
+  test('returns size of the only file', async () => {
+    const file = path.join(tmpDir, 'only.md');
+    await writeFile(file, 'hello');
+    expect(await getMaxFileSize([file])).toBe(5);
+  });
+
+  test('returns size of the largest file among many', async () => {
+    const small = path.join(tmpDir, 'small.md');
+    const large = path.join(tmpDir, 'large.md');
+    const medium = path.join(tmpDir, 'medium.md');
+    await writeFile(small, 'a'.repeat(10));
+    await writeFile(large, 'b'.repeat(1000));
+    await writeFile(medium, 'c'.repeat(500));
+
+    expect(await getMaxFileSize([small, large, medium])).toBe(1000);
+  });
+
+  test('rejects when a file cannot be stat-ed', async () => {
+    const missing = path.join(tmpDir, 'missing.md');
+    await expect(getMaxFileSize([missing])).rejects.toThrow();
+  });
+});
+
+describe('resolveAdaptiveConcurrency', () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await mkdtemp(path.join(tmpdir(), 'batch-lint-adaptive-'));
+  });
+
+  afterEach(async () => {
+    await rm(tmpDir, { recursive: true, force: true });
+  });
+
+  const writeSizedFile = async (name: string, sizeBytes: number) => {
+    const file = path.join(tmpDir, name);
+    await writeFile(file, Buffer.alloc(sizeBytes));
+    return file;
+  };
+
+  describe('numeric threadCount (preserves existing behavior)', () => {
+    test('numeric 2 with 3 files → 2', async () => {
+      const files = await Promise.all([
+        writeSizedFile('a.md', 100),
+        writeSizedFile('b.md', 100),
+        writeSizedFile('c.md', 100),
+      ]);
+      expect(await resolveAdaptiveConcurrency(2, files)).toBe(2);
+    });
+
+    test('numeric threads > fileCount is clamped to fileCount', async () => {
+      const file = await writeSizedFile('only.md', 100);
+      expect(await resolveAdaptiveConcurrency(100, [file])).toBe(1);
+    });
+
+    test('numeric 0 is clamped to 1 (matches existing min clamp)', async () => {
+      const files = await Promise.all([
+        writeSizedFile('a.md', 100),
+        writeSizedFile('b.md', 100),
+      ]);
+      expect(await resolveAdaptiveConcurrency(0, files)).toBe(1);
+    });
+
+    test('numeric threads ignores file size', async () => {
+      const files = await Promise.all(
+        Array.from({ length: 8 }, (_, index) =>
+          writeSizedFile(`huge-${index}.md`, 10 * 1024 * 1024)
+        )
+      );
+
+      expect(await resolveAdaptiveConcurrency(8, files)).toBe(8);
+    });
+  });
+
+  describe('auto threadCount', () => {
+    test('empty file list → 0', async () => {
+      expect(await resolveAdaptiveConcurrency('auto', [])).toBe(0);
+    });
+
+    test('small files (< 1 MiB) use cpuLimit clamped to fileCount', async () => {
+      const files = await Promise.all([
+        writeSizedFile('a.md', 1024),
+        writeSizedFile('b.md', 2048),
+        writeSizedFile('c.md', 4096),
+      ]);
+      const cpuLimit = availableParallelism();
+      expect(await resolveAdaptiveConcurrency('auto', files)).toBe(Math.min(cpuLimit, files.length));
+    });
+
+    test('max file exactly 1 MiB caps at 2', async () => {
+      const files = await Promise.all([
+        writeSizedFile('small.md', 1024),
+        writeSizedFile('one-mib.md', 1024 * 1024),
+      ]);
+      expect(await resolveAdaptiveConcurrency('auto', files)).toBeLessThanOrEqual(2);
+    });
+
+    test('max file 1.5 MiB caps at 2', async () => {
+      const file = await writeSizedFile('medium.md', 1.5 * 1024 * 1024);
+      expect(await resolveAdaptiveConcurrency('auto', [file])).toBeLessThanOrEqual(2);
+    });
+
+    test('max file exactly 5 MiB forces 1', async () => {
+      const file = await writeSizedFile('five-mib.md', 5 * 1024 * 1024);
+      expect(await resolveAdaptiveConcurrency('auto', [file])).toBe(1);
+    });
+
+    test('max file 6 MiB forces 1', async () => {
+      const file = await writeSizedFile('six-mib.md', 6 * 1024 * 1024);
+      expect(await resolveAdaptiveConcurrency('auto', [file])).toBe(1);
+    });
+
+    test('single small file → 1', async () => {
+      const file = await writeSizedFile('only.md', 100);
+      expect(await resolveAdaptiveConcurrency('auto', [file])).toBe(1);
+    });
+
+    test('medium cap respects fileCount when files < 2', async () => {
+      const file = await writeSizedFile('one-mib.md', 1.2 * 1024 * 1024);
+      expect(await resolveAdaptiveConcurrency('auto', [file])).toBe(1);
+    });
+  });
+});

__tests__/configure.spec.ts

@@ -47,4 +47,8 @@ describe('getThreadCount', () => {
       expect.stringContaining('--threads must be a positive integer'),
     );
   });
+
+  test('"auto" → "auto"', () => {
+    expect(getThreadCount('auto')).toBe('auto');
+  });
 });

__tests__/threads-validation.spec.ts

@@ -1,4 +1,6 @@
 import { execFileSync } from 'child_process';
+import { mkdtempSync, rmSync, writeFileSync } from 'fs';
+import { tmpdir } from 'os';
 import * as path from 'path';
 
 const TSX = path.resolve(__dirname, '../node_modules/tsx/dist/cli.mjs');
@@ -41,4 +43,38 @@ describe('--threads validation across CLI paths', () => {
       expect(e.stderr).toContain('--threads must be a positive integer');
     }
   });
+
+  test('stdin + --threads auto → does not exit 1 (numeric validation skipped)', () => {
+    const result = execFileSync(
+      process.execPath,
+      [TSX, CLI, '--stdin', '--threads', 'auto'],
+      {
+        input: '# title\n',
+        encoding: 'utf8',
+        stdio: ['pipe', 'pipe', 'pipe'],
+      },
+    );
+    expect(result).toContain('Done in');
+  });
+
+  test('files + --threads auto → exit 0 on a small markdown file', () => {
+    const tmpDir = mkdtempSync(path.join(tmpdir(), 'threads-auto-test-'));
+    try {
+      const file = path.join(tmpDir, 'small.md');
+      writeFileSync(file, '# hello\n', 'utf8');
+
+      const result = execFileSync(
+        process.execPath,
+        [TSX, CLI, file, '--threads', 'auto'],
+        {
+          encoding: 'utf8',
+          stdio: ['pipe', 'pipe', 'pipe'],
+        },
+      );
+      expect(result).toContain('Done in');
+    }
+    finally {
+      rmSync(tmpDir, { recursive: true, force: true });
+    }
+  });
 });

scripts/benchmark-memory.mjs

@@ -21,7 +21,7 @@ Linux only: requires GNU /usr/bin/time -v.
 Options:
   --files <count>            Number of generated Markdown files (default: 8)
   --bytes-per-file <bytes>   Approximate bytes per file (default: 65536)
-  --threads <count>          lint-md worker count (default: 2)
+  --threads <count|auto>     lint-md worker count or "auto" (default: 2)
   --runs <count>             Number of benchmark repetitions (default: 1)
   --fix                      Benchmark fix mode
   -h, --help                 Show this help
@@ -35,6 +35,13 @@ const parsePositiveInteger = (value, option) => {
   return Number(value);
 };
 
+const parseThreads = (value, option) => {
+  if (value === 'auto') {
+    return 'auto';
+  }
+  return parsePositiveInteger(value, option);
+};
+
 const parseArgs = (args) => {
   const options = {
     files: 8,
@@ -69,7 +76,7 @@ const parseArgs = (args) => {
       options.bytesPerFile = parsePositiveInteger(value, arg);
     }
     else if (arg === '--threads') {
-      options.threads = parsePositiveInteger(value, arg);
+      options.threads = parseThreads(value, arg);
     }
     else if (arg === '--runs') {
       options.runs = parsePositiveInteger(value, arg);

src/lint-md.ts

@@ -2,13 +2,19 @@
 
 import * as process from 'process';
 import { readFileSync } from 'fs';
+import { availableParallelism } from 'os';
 import { program } from 'commander';
 import { lintMarkdown } from '@lint-md/core';
 import { version } from '../package.json';
 import { safeWriteFile } from './utils/safe-write-file';
-import { batchLint, runTasksWithLimit } from './utils/batch-lint';
+import {
+  batchLint,
+  getMaxFileSize,
+  resolveAdaptiveConcurrency,
+  runTasksWithLimit,
+} from './utils/batch-lint';
 import { getLintConfig, getThreadCount } from './utils/configure';
-import type { CLIOptions } from './types';
+import type { CLIOptions, ThreadCount } from './types';
 import { loadMdFiles } from './utils/load-md-files';
 import { getReportData } from './utils/get-report-data';
 
@@ -28,7 +34,7 @@ program
   .option('-d, --dev', 'open dev mode（开启开发者模式）')
   .option(
     '-t, --threads [thread-count]',
-    'The number of threads. The default is based on the number of available CPUs.（执行 Lint / Fix 的线程数，默认为 CPU 核心数）'
+    'Number of worker threads, or "auto" to cap concurrency for large files. Default: CPU count.（执行 Lint / Fix 的线程数，传 "auto" 时根据文件大小自适应）'
   )
   .option(
     '-s, --suppress-warnings',
@@ -53,7 +59,7 @@ program
     const { rules, excludeFiles, extensions } = getLintConfig(config);
 
     // --threads 参数校验，所有分支共用
-    const threadCount = getThreadCount(threads);
+    const threadCount: ThreadCount = getThreadCount(threads);
 
     // Handle stdin mode
     if (stdin) {
@@ -118,9 +124,23 @@ program
       return;
     }
 
+    const effectiveThreads = await resolveAdaptiveConcurrency(threadCount, mdFiles);
+
+    if (isDev && threadCount === 'auto') {
+      const maxFileSize = await getMaxFileSize(mdFiles);
+      const adaptiveApplied = maxFileSize >= 1024 * 1024;
+      const requested = availableParallelism();
+      if (adaptiveApplied && effectiveThreads < requested) {
+        const maxMiB = (maxFileSize / (1024 * 1024)).toFixed(2);
+        console.log(
+          `[lint-md] Adaptive concurrency: requested auto, effective ${effectiveThreads}, max file ${maxMiB} MiB`
+        );
+      }
+    }
+
     try {
       const lintResult = await batchLint(
-        threadCount,
+        effectiveThreads,
         mdFiles,
         isDev,
         isFixMode,
@@ -142,7 +162,7 @@ program
           lintResult
             .filter(({ fixedResult }) => fixedResult)
             .map(({ path, fixedResult }) => () => safeWriteFile(path, fixedResult!.result)),
-          threadCount
+          effectiveThreads
         );
       }
     }

src/types.ts

@@ -1,6 +1,8 @@
 /** CLI 配置 */
 import type { LintMdRulesConfig } from '@lint-md/core';
 
+export type ThreadCount = number | 'auto';
+
 export interface CLIConfig {
   excludeFiles?: string[]
   rules?: LintMdRulesConfig

src/utils/batch-lint.ts

@@ -1,8 +1,16 @@
 import path from 'path';
 import { existsSync } from 'fs';
+import { stat } from 'fs/promises';
+import { availableParallelism } from 'os';
 import { Piscina } from 'piscina';
 import type { LintMdRulesConfig } from '@lint-md/core';
-import type { BatchLintItem, LintWorkerOptions } from '../types';
+import type { BatchLintItem, LintWorkerOptions, ThreadCount } from '../types';
+
+const ONE_MIB = 1024 * 1024;
+const FIVE_MIB = 5 * ONE_MIB;
+const ADAPTIVE_MEDIUM_CAP = 2;
+const ADAPTIVE_LARGE_FILE_THRESHOLD = ONE_MIB;
+const ADAPTIVE_HUGE_FILE_THRESHOLD = FIVE_MIB;
 
 export async function runTasksWithLimit<T>(
   tasks: (() => Promise<T>)[],
@@ -31,6 +39,43 @@ const resolveWorkerFilename = (): string => {
   return path.resolve(__dirname, '../../lib/src/utils/lint-worker.js');
 };
 
+// TODO(perf, #78): getMaxFileSize() currently stats every file with one
+// Promise.all. For very large repositories, consider chunking stat calls
+// or short-circuiting once a file >= ADAPTIVE_HUGE_FILE_THRESHOLD is found.
+export const getMaxFileSize = async (filePaths: string[]): Promise<number> => {
+  if (filePaths.length === 0) {
+    return 0;
+  }
+  const stats = await Promise.all(filePaths.map(filePath => stat(filePath)));
+  return stats.reduce((max, current) => (current.size > max ? current.size : max), 0);
+};
+
+export const resolveAdaptiveConcurrency = async (
+  threadCount: ThreadCount,
+  mdFilePaths: string[]
+): Promise<number> => {
+  if (mdFilePaths.length === 0) {
+    return 0;
+  }
+
+  if (typeof threadCount === 'number') {
+    return Math.min(Math.max(threadCount, 1), mdFilePaths.length);
+  }
+
+  const maxFileSize = await getMaxFileSize(mdFilePaths);
+  const cpuLimit = availableParallelism();
+
+  let limit = cpuLimit;
+  if (maxFileSize >= ADAPTIVE_HUGE_FILE_THRESHOLD) {
+    limit = 1;
+  }
+  else if (maxFileSize >= ADAPTIVE_LARGE_FILE_THRESHOLD) {
+    limit = Math.min(limit, ADAPTIVE_MEDIUM_CAP);
+  }
+
+  return Math.min(Math.max(limit, 1), mdFilePaths.length);
+};
+
 export const batchLint = async (
   threadsCount: number,
   mdFilePaths: string[],