From 55df58d62bc6adc77c5ec54da8d8e5862a4c227a Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Tue, 13 Oct 2026 20:51:02 +0400 Subject: [PATCH] fix(tokenizer): stop reading keyword arguments and members named `function` as declarations The name of a function was read from the token before the next `(`, so any `function` token that declares nothing borrowed a name from later code: a named argument `f(function: $x)` followed by `if (` became a method `if`, and a constant, enum case or trait alias named `function`, or a grouped `use function` import, produced methods and functions that do not exist. A method named `function` was taken for a closure and left out. A declaration is now recognized only as `function`, an optional `&`, a name spelled as a PHP label and `(`, with comments allowed in between, and never right after `::`, where `Foo::function and ($x)` would otherwise declare `and`. Assisted-By: Claude Opus 5.5 --- core/Tokenizer/Reflection/TokenizedFile.php | 74 +++++++++++++------ .../Stub/KeywordNamedDeclarations.php | 62 +++++++++++++++- .../Unit/Reflection/TokenizedFileTest.php | 24 +++++- 3 files changed, 133 insertions(+), 27 deletions(-) diff --git a/core/Tokenizer/Reflection/TokenizedFile.php b/core/Tokenizer/Reflection/TokenizedFile.php index 4380106d..91becc36 100644 --- a/core/Tokenizer/Reflection/TokenizedFile.php +++ b/core/Tokenizer/Reflection/TokenizedFile.php @@ -30,6 +30,11 @@ final class TokenizedFile */ public const N_USES = 2; + /** + * Spelling of a PHP label: a function, method or class name. + */ + private const IDENTIFIER = '/^[a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*$/'; + public readonly Path $path; /** @@ -424,41 +429,34 @@ private function registerFunction(int $tokenID): void } } - // `use function Foo\bar;` carries a T_FUNCTION token too — it imports, it does not declare. - $prevID = $tokenID - 1; - while ($prevID >= 0 && $this->tokens[$prevID]->is(T_WHITESPACE)) { - --$prevID; - } - if ($prevID >= 0 && $this->tokens[$prevID]->is(T_USE)) { + // Only `function [&] name (` declares. Every other T_FUNCTION token declares nothing: + // a closure `function (`, an import `use function Foo\bar;`, or a member, argument, case or + // alias named `function` (`Foo::function()`, `f(function: $x)`, `run as function;`). + // A static member reference is rejected by what precedes it: `Foo::function and (...)` would + // otherwise pass for a declaration of `and`. After `->` and `?->` the lexer never yields + // T_FUNCTION in valid code. + $prevID = $this->previousMeaningfulToken($tokenID); + if ($prevID !== null && $this->tokens[$prevID]->is(T_DOUBLE_COLON)) { return; } - // The name (if any) is the token right before the parameter list "(". Anonymous functions - // have none there: `function (`, by-ref `function &(`, or an attributed closure - // `#[Attr] function (` all reach "(" with `function`/`&` immediately before it. A parameter - // type hint lives *inside* the parens, so it can never be misread as the name. Keyword-like - // method names (`list`, `print`, ...) are not T_STRING, so read by position, not by type. - $parenID = $tokenID + 1; - while (isset($this->tokens[$parenID]) && $this->tokens[$parenID]->text !== '(') { - ++$parenID; + $nameID = $this->nextMeaningfulToken($tokenID); + if ($nameID !== null && $this->tokens[$nameID]->text === '&') { + $nameID = $this->nextMeaningfulToken($nameID); } - if (!isset($this->tokens[$parenID])) { + // Keyword-like names (`list`, `print`, ...) are not T_STRING, so the name is checked by its + // spelling rather than by token type. + if ($nameID === null || \preg_match(self::IDENTIFIER, $this->tokens[$nameID]->text) !== 1) { return; } - $nameID = $parenID - 1; - while ($nameID > $tokenID && $this->tokens[$nameID]->is(T_WHITESPACE)) { - --$nameID; - } - - $nameToken = $this->tokens[$nameID]; - if ($nameToken->is(T_FUNCTION) || $nameToken->text === '&') { - // Anonymous function — nothing to declare. + $parenID = $this->nextMeaningfulToken($nameID); + if ($parenID === null || $this->tokens[$parenID]->text !== '(') { return; } - $name = $nameToken->text; + $name = $this->tokens[$nameID]->text; // Function if ($class === null) { @@ -796,6 +794,34 @@ private function endingToken(int $tokenID): int return $localID; } + /** + * Find the first non-whitespace, non-comment token after the given one. + */ + private function nextMeaningfulToken(int $tokenID): ?int + { + for ($localID = $tokenID + 1; $localID < $this->countTokens; ++$localID) { + if (!$this->tokens[$localID]->isIgnorable()) { + return $localID; + } + } + + return null; + } + + /** + * Find the last non-whitespace, non-comment token before the given one. + */ + private function previousMeaningfulToken(int $tokenID): ?int + { + for ($localID = $tokenID - 1; $localID >= 0; --$localID) { + if (!$this->tokens[$localID]->isIgnorable()) { + return $localID; + } + } + + return null; + } + /** * Get line number associated with token. */ diff --git a/tests/Tokenizer/Stub/KeywordNamedDeclarations.php b/tests/Tokenizer/Stub/KeywordNamedDeclarations.php index 25435864..7f4d67d2 100644 --- a/tests/Tokenizer/Stub/KeywordNamedDeclarations.php +++ b/tests/Tokenizer/Stub/KeywordNamedDeclarations.php @@ -4,10 +4,35 @@ namespace Tests\Tokenizer\Stub; -// Method names may be semi-reserved keywords. Their name token is not T_STRING, so the locator -// must read the name by position (the token before "(") rather than by token type. +// A grouped import carries `function` tokens that declare nothing. +use Tests\Tokenizer\Stub\Imported\{function first, function second}; + +// Keywords used as constant, case, and alias names declare nothing either. +enum KeywordCases +{ + case function; +} + +trait KeywordTrait +{ + public function run(): void {} +} + +final class KeywordAlias +{ + use KeywordTrait { + run as function; + } + + public const function = 1; +} + +// Method names may be semi-reserved keywords or contain non-ASCII letters. A keyword name token is +// not T_STRING, so the locator must check the name by its spelling rather than by token type. final class KeywordNames { + public function größe(): void {} + public function list(): array { return []; @@ -16,4 +41,37 @@ public function list(): array public function print(): void {} public function unset(): void {} + + public function function(): void {} + + public function & reference(): array + { + static $items = []; + return $items; + } + + public function /* name */ commented /* parameters */ (): void {} + + public function call(): void + { + // Keywords used as argument names and member names declare nothing. + $this->run(function: 1, class: 2); + if (true) { + KeywordCases::function; + } + if (true) { + self::function(); + $this->function(); + } + $this->run(function: (1)); + $same = KeywordAlias::function === (1); + $both = KeywordAlias::function and ($same); + $mask = KeywordAlias::function & intdiv(4, 2); + $spaced = KeywordAlias:: /* constant */ function and ($same); + } + + public function closure(): \Closure + { + return function /* anonymous */ () {}; + } } diff --git a/tests/Tokenizer/Unit/Reflection/TokenizedFileTest.php b/tests/Tokenizer/Unit/Reflection/TokenizedFileTest.php index 3c154fe8..3d15ff60 100644 --- a/tests/Tokenizer/Unit/Reflection/TokenizedFileTest.php +++ b/tests/Tokenizer/Unit/Reflection/TokenizedFileTest.php @@ -186,16 +186,23 @@ public function declarationsAreScopedToTheirBracedNamespace(): void ]); } - public function keywordNamedMethodsAreDetectedByPosition(): void + public function onlyFunctionTokensOfDeclarationsRegisterMethods(): void { $tokenized = self::tokenize('KeywordNamedDeclarations.php'); $methods = $tokenized->getMethodsFQN(); \sort($methods); Assert::same($methods, [ + 'Tests\Tokenizer\Stub\KeywordNames::call', + 'Tests\Tokenizer\Stub\KeywordNames::closure', + 'Tests\Tokenizer\Stub\KeywordNames::commented', + 'Tests\Tokenizer\Stub\KeywordNames::function', + 'Tests\Tokenizer\Stub\KeywordNames::größe', 'Tests\Tokenizer\Stub\KeywordNames::list', 'Tests\Tokenizer\Stub\KeywordNames::print', + 'Tests\Tokenizer\Stub\KeywordNames::reference', 'Tests\Tokenizer\Stub\KeywordNames::unset', + 'Tests\Tokenizer\Stub\KeywordTrait::run', ]); Assert::same($tokenized->getFunctions(), []); } @@ -447,6 +454,21 @@ public function globalBracedNamespaceIsRecognized(): void Assert::same($tokenized->getFunctions(), ['globalBracedFunction']); } + public function functionTokenBoundedByFileEdgesDeclaresNothing(): void + { + $path = \sys_get_temp_dir() . '/testo-tokenizer-' . \bin2hex(\random_bytes(4)) . '.php'; + \file_put_contents($path, 'getFunctions(), []); + Assert::same($tokenized->getMethodsFQN(), []); + } finally { + \unlink($path); + } + } + public function repeatedNamespaceBlockMergesDeclarationsFromBothBlocks(): void { $tokenized = self::tokenize('RepeatedNamespace.php');