diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e579eb..bb2abc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### Features + +- **lint-md:** add `--threads auto` to cap worker concurrency for large Markdown files (issue #77, P1) + ## [2.1.1](https://github.com/lint-md/cli/compare/v2.0.0...v2.1.1) (2026-06-30) ### Features diff --git a/__tests__/batch-lint.spec.ts b/__tests__/batch-lint.spec.ts index 2e36f50..968394f 100644 --- a/__tests__/batch-lint.spec.ts +++ b/__tests__/batch-lint.spec.ts @@ -1,9 +1,9 @@ import { mkdtemp, rm, writeFile } from 'fs/promises'; -import { tmpdir } from 'os'; +import { availableParallelism, tmpdir } from 'os'; import * as path from 'path'; import { Piscina } from 'piscina'; import type { LintMdRulesConfig } from '@lint-md/core'; -import { batchLint, runTasksWithLimit } from '../src/utils/batch-lint'; +import { batchLint, getMaxFileSize, resolveAdaptiveConcurrency, runTasksWithLimit } from '../src/utils/batch-lint'; const RULES_NO_EMPTY_LIST: LintMdRulesConfig = { 'no-empty-list': 2, @@ -177,3 +177,142 @@ describe('batchLint', () => { }); }); }); + +describe('getMaxFileSize', () => { + let tmpDir: string; + + beforeEach(async () => { + tmpDir = await mkdtemp(path.join(tmpdir(), 'batch-lint-max-')); + }); + + afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }); + }); + + test('returns 0 for empty list', async () => { + expect(await getMaxFileSize([])).toBe(0); + }); + + test('returns size of the only file', async () => { + const file = path.join(tmpDir, 'only.md'); + await writeFile(file, 'hello'); + expect(await getMaxFileSize([file])).toBe(5); + }); + + test('returns size of the largest file among many', async () => { + const small = path.join(tmpDir, 'small.md'); + const large = path.join(tmpDir, 'large.md'); + const medium = path.join(tmpDir, 'medium.md'); + await writeFile(small, 'a'.repeat(10)); + await writeFile(large, 'b'.repeat(1000)); + await writeFile(medium, 'c'.repeat(500)); + + expect(await getMaxFileSize([small, large, medium])).toBe(1000); + }); + + test('rejects when a file cannot be stat-ed', async () => { + const missing = path.join(tmpDir, 'missing.md'); + await expect(getMaxFileSize([missing])).rejects.toThrow(); + }); +}); + +describe('resolveAdaptiveConcurrency', () => { + let tmpDir: string; + + beforeEach(async () => { + tmpDir = await mkdtemp(path.join(tmpdir(), 'batch-lint-adaptive-')); + }); + + afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }); + }); + + const writeSizedFile = async (name: string, sizeBytes: number) => { + const file = path.join(tmpDir, name); + await writeFile(file, Buffer.alloc(sizeBytes)); + return file; + }; + + describe('numeric threadCount (preserves existing behavior)', () => { + test('numeric 2 with 3 files → 2', async () => { + const files = await Promise.all([ + writeSizedFile('a.md', 100), + writeSizedFile('b.md', 100), + writeSizedFile('c.md', 100), + ]); + expect(await resolveAdaptiveConcurrency(2, files)).toBe(2); + }); + + test('numeric threads > fileCount is clamped to fileCount', async () => { + const file = await writeSizedFile('only.md', 100); + expect(await resolveAdaptiveConcurrency(100, [file])).toBe(1); + }); + + test('numeric 0 is clamped to 1 (matches existing min clamp)', async () => { + const files = await Promise.all([ + writeSizedFile('a.md', 100), + writeSizedFile('b.md', 100), + ]); + expect(await resolveAdaptiveConcurrency(0, files)).toBe(1); + }); + + test('numeric threads ignores file size', async () => { + const files = await Promise.all( + Array.from({ length: 8 }, (_, index) => + writeSizedFile(`huge-${index}.md`, 10 * 1024 * 1024) + ) + ); + + expect(await resolveAdaptiveConcurrency(8, files)).toBe(8); + }); + }); + + describe('auto threadCount', () => { + test('empty file list → 0', async () => { + expect(await resolveAdaptiveConcurrency('auto', [])).toBe(0); + }); + + test('small files (< 1 MiB) use cpuLimit clamped to fileCount', async () => { + const files = await Promise.all([ + writeSizedFile('a.md', 1024), + writeSizedFile('b.md', 2048), + writeSizedFile('c.md', 4096), + ]); + const cpuLimit = availableParallelism(); + expect(await resolveAdaptiveConcurrency('auto', files)).toBe(Math.min(cpuLimit, files.length)); + }); + + test('max file exactly 1 MiB caps at 2', async () => { + const files = await Promise.all([ + writeSizedFile('small.md', 1024), + writeSizedFile('one-mib.md', 1024 * 1024), + ]); + expect(await resolveAdaptiveConcurrency('auto', files)).toBeLessThanOrEqual(2); + }); + + test('max file 1.5 MiB caps at 2', async () => { + const file = await writeSizedFile('medium.md', 1.5 * 1024 * 1024); + expect(await resolveAdaptiveConcurrency('auto', [file])).toBeLessThanOrEqual(2); + }); + + test('max file exactly 5 MiB forces 1', async () => { + const file = await writeSizedFile('five-mib.md', 5 * 1024 * 1024); + expect(await resolveAdaptiveConcurrency('auto', [file])).toBe(1); + }); + + test('max file 6 MiB forces 1', async () => { + const file = await writeSizedFile('six-mib.md', 6 * 1024 * 1024); + expect(await resolveAdaptiveConcurrency('auto', [file])).toBe(1); + }); + + test('single small file → 1', async () => { + const file = await writeSizedFile('only.md', 100); + expect(await resolveAdaptiveConcurrency('auto', [file])).toBe(1); + }); + + test('medium cap respects fileCount when files < 2', async () => { + const file = await writeSizedFile('one-mib.md', 1.2 * 1024 * 1024); + expect(await resolveAdaptiveConcurrency('auto', [file])).toBe(1); + }); + }); +}); diff --git a/__tests__/configure.spec.ts b/__tests__/configure.spec.ts index ec815c2..1f114ba 100644 --- a/__tests__/configure.spec.ts +++ b/__tests__/configure.spec.ts @@ -47,4 +47,8 @@ describe('getThreadCount', () => { expect.stringContaining('--threads must be a positive integer'), ); }); + + test('"auto" → "auto"', () => { + expect(getThreadCount('auto')).toBe('auto'); + }); }); diff --git a/__tests__/threads-validation.spec.ts b/__tests__/threads-validation.spec.ts index ef94da3..d0a2db1 100644 --- a/__tests__/threads-validation.spec.ts +++ b/__tests__/threads-validation.spec.ts @@ -1,4 +1,6 @@ import { execFileSync } from 'child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; import * as path from 'path'; const TSX = path.resolve(__dirname, '../node_modules/tsx/dist/cli.mjs'); @@ -41,4 +43,38 @@ describe('--threads validation across CLI paths', () => { expect(e.stderr).toContain('--threads must be a positive integer'); } }); + + test('stdin + --threads auto → does not exit 1 (numeric validation skipped)', () => { + const result = execFileSync( + process.execPath, + [TSX, CLI, '--stdin', '--threads', 'auto'], + { + input: '# title\n', + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + }, + ); + expect(result).toContain('Done in'); + }); + + test('files + --threads auto → exit 0 on a small markdown file', () => { + const tmpDir = mkdtempSync(path.join(tmpdir(), 'threads-auto-test-')); + try { + const file = path.join(tmpDir, 'small.md'); + writeFileSync(file, '# hello\n', 'utf8'); + + const result = execFileSync( + process.execPath, + [TSX, CLI, file, '--threads', 'auto'], + { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + }, + ); + expect(result).toContain('Done in'); + } + finally { + rmSync(tmpDir, { recursive: true, force: true }); + } + }); }); diff --git a/scripts/benchmark-memory.mjs b/scripts/benchmark-memory.mjs index 5fec9cc..8e1112f 100644 --- a/scripts/benchmark-memory.mjs +++ b/scripts/benchmark-memory.mjs @@ -21,7 +21,7 @@ Linux only: requires GNU /usr/bin/time -v. Options: --files Number of generated Markdown files (default: 8) --bytes-per-file Approximate bytes per file (default: 65536) - --threads lint-md worker count (default: 2) + --threads lint-md worker count or "auto" (default: 2) --runs Number of benchmark repetitions (default: 1) --fix Benchmark fix mode -h, --help Show this help @@ -35,6 +35,13 @@ const parsePositiveInteger = (value, option) => { return Number(value); }; +const parseThreads = (value, option) => { + if (value === 'auto') { + return 'auto'; + } + return parsePositiveInteger(value, option); +}; + const parseArgs = (args) => { const options = { files: 8, @@ -69,7 +76,7 @@ const parseArgs = (args) => { options.bytesPerFile = parsePositiveInteger(value, arg); } else if (arg === '--threads') { - options.threads = parsePositiveInteger(value, arg); + options.threads = parseThreads(value, arg); } else if (arg === '--runs') { options.runs = parsePositiveInteger(value, arg); diff --git a/src/lint-md.ts b/src/lint-md.ts index 3869b56..bcf937f 100644 --- a/src/lint-md.ts +++ b/src/lint-md.ts @@ -2,13 +2,19 @@ import * as process from 'process'; import { readFileSync } from 'fs'; +import { availableParallelism } from 'os'; import { program } from 'commander'; import { lintMarkdown } from '@lint-md/core'; import { version } from '../package.json'; import { safeWriteFile } from './utils/safe-write-file'; -import { batchLint, runTasksWithLimit } from './utils/batch-lint'; +import { + batchLint, + getMaxFileSize, + resolveAdaptiveConcurrency, + runTasksWithLimit, +} from './utils/batch-lint'; import { getLintConfig, getThreadCount } from './utils/configure'; -import type { CLIOptions } from './types'; +import type { CLIOptions, ThreadCount } from './types'; import { loadMdFiles } from './utils/load-md-files'; import { getReportData } from './utils/get-report-data'; @@ -28,7 +34,7 @@ program .option('-d, --dev', 'open dev mode（开启开发者模式）') .option( '-t, --threads [thread-count]', - 'The number of threads. The default is based on the number of available CPUs.（执行 Lint / Fix 的线程数，默认为 CPU 核心数）' + 'Number of worker threads, or "auto" to cap concurrency for large files. Default: CPU count.（执行 Lint / Fix 的线程数，传 "auto" 时根据文件大小自适应）' ) .option( '-s, --suppress-warnings', @@ -53,7 +59,7 @@ program const { rules, excludeFiles, extensions } = getLintConfig(config); // --threads 参数校验，所有分支共用 - const threadCount = getThreadCount(threads); + const threadCount: ThreadCount = getThreadCount(threads); // Handle stdin mode if (stdin) { @@ -118,9 +124,23 @@ program return; } + const effectiveThreads = await resolveAdaptiveConcurrency(threadCount, mdFiles); + + if (isDev && threadCount === 'auto') { + const maxFileSize = await getMaxFileSize(mdFiles); + const adaptiveApplied = maxFileSize >= 1024 * 1024; + const requested = availableParallelism(); + if (adaptiveApplied && effectiveThreads < requested) { + const maxMiB = (maxFileSize / (1024 * 1024)).toFixed(2); + console.log( + `[lint-md] Adaptive concurrency: requested auto, effective ${effectiveThreads}, max file ${maxMiB} MiB` + ); + } + } + try { const lintResult = await batchLint( - threadCount, + effectiveThreads, mdFiles, isDev, isFixMode, @@ -142,7 +162,7 @@ program lintResult .filter(({ fixedResult }) => fixedResult) .map(({ path, fixedResult }) => () => safeWriteFile(path, fixedResult!.result)), - threadCount + effectiveThreads ); } } diff --git a/src/types.ts b/src/types.ts index f6437a8..bdf4139 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,8 @@ /** CLI 配置 */ import type { LintMdRulesConfig } from '@lint-md/core'; +export type ThreadCount = number | 'auto'; + export interface CLIConfig { excludeFiles?: string[] rules?: LintMdRulesConfig diff --git a/src/utils/batch-lint.ts b/src/utils/batch-lint.ts index a531482..e39137f 100644 --- a/src/utils/batch-lint.ts +++ b/src/utils/batch-lint.ts @@ -1,8 +1,16 @@ import path from 'path'; import { existsSync } from 'fs'; +import { stat } from 'fs/promises'; +import { availableParallelism } from 'os'; import { Piscina } from 'piscina'; import type { LintMdRulesConfig } from '@lint-md/core'; -import type { BatchLintItem, LintWorkerOptions } from '../types'; +import type { BatchLintItem, LintWorkerOptions, ThreadCount } from '../types'; + +const ONE_MIB = 1024 * 1024; +const FIVE_MIB = 5 * ONE_MIB; +const ADAPTIVE_MEDIUM_CAP = 2; +const ADAPTIVE_LARGE_FILE_THRESHOLD = ONE_MIB; +const ADAPTIVE_HUGE_FILE_THRESHOLD = FIVE_MIB; export async function runTasksWithLimit( tasks: (() => Promise)[], @@ -31,6 +39,43 @@ const resolveWorkerFilename = (): string => { return path.resolve(__dirname, '../../lib/src/utils/lint-worker.js'); }; +// TODO(perf, #78): getMaxFileSize() currently stats every file with one +// Promise.all. For very large repositories, consider chunking stat calls +// or short-circuiting once a file >= ADAPTIVE_HUGE_FILE_THRESHOLD is found. +export const getMaxFileSize = async (filePaths: string[]): Promise => { + if (filePaths.length === 0) { + return 0; + } + const stats = await Promise.all(filePaths.map(filePath => stat(filePath))); + return stats.reduce((max, current) => (current.size > max ? current.size : max), 0); +}; + +export const resolveAdaptiveConcurrency = async ( + threadCount: ThreadCount, + mdFilePaths: string[] +): Promise => { + if (mdFilePaths.length === 0) { + return 0; + } + + if (typeof threadCount === 'number') { + return Math.min(Math.max(threadCount, 1), mdFilePaths.length); + } + + const maxFileSize = await getMaxFileSize(mdFilePaths); + const cpuLimit = availableParallelism(); + + let limit = cpuLimit; + if (maxFileSize >= ADAPTIVE_HUGE_FILE_THRESHOLD) { + limit = 1; + } + else if (maxFileSize >= ADAPTIVE_LARGE_FILE_THRESHOLD) { + limit = Math.min(limit, ADAPTIVE_MEDIUM_CAP); + } + + return Math.min(Math.max(limit, 1), mdFilePaths.length); +}; + export const batchLint = async ( threadsCount: number, mdFilePaths: string[], diff --git a/src/utils/configure.ts b/src/utils/configure.ts index 0d83b46..fd7d625 100644 --- a/src/utils/configure.ts +++ b/src/utils/configure.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import { availableParallelism } from 'os'; import * as path from 'path'; import chalk from 'chalk'; -import type { CLIConfig } from '../types'; +import type { CLIConfig, ThreadCount } from '../types'; export const getLintConfig = (configFilePath?: string): Required => { if (configFilePath && !fs.existsSync(configFilePath)) { @@ -41,7 +41,13 @@ export const getLintConfig = (configFilePath?: string): Required => { }; }; -export const getThreadCount = (threadCount?: string | number | boolean): number => { +export const getThreadCount = ( + threadCount?: string | number | boolean +): ThreadCount => { + if (threadCount === 'auto') { + return 'auto'; + } + // 只接受 number 或 string，其他（undefined / boolean）视为未指定 if (typeof threadCount !== 'number' && typeof threadCount !== 'string') { return availableParallelism();