fix(tokenizer): stop reading keyword arguments and members named function as declarations

core/Tokenizer/Reflection/TokenizedFile.php

@@ -30,6 +30,11 @@ final class TokenizedFile
      */
     public const N_USES = 2;
 
+    /**
+     * Spelling of a PHP label: a function, method or class name.
+     */
+    private const IDENTIFIER = '/^[a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*$/';
+
     public readonly Path $path;
 
     /**
@@ -424,41 +429,34 @@ private function registerFunction(int $tokenID): void
             }
         }
 
-        // `use function Foo\bar;` carries a T_FUNCTION token too — it imports, it does not declare.
-        $prevID = $tokenID - 1;
-        while ($prevID >= 0 && $this->tokens[$prevID]->is(T_WHITESPACE)) {
-            --$prevID;
-        }
-        if ($prevID >= 0 && $this->tokens[$prevID]->is(T_USE)) {
+        // Only `function [&] name (` declares. Every other T_FUNCTION token declares nothing:
+        // a closure `function (`, an import `use function Foo\bar;`, or a member, argument, case or
+        // alias named `function` (`Foo::function()`, `f(function: $x)`, `run as function;`).
+        // A static member reference is rejected by what precedes it: `Foo::function and (...)` would
+        // otherwise pass for a declaration of `and`. After `->` and `?->` the lexer never yields
+        // T_FUNCTION in valid code.
+        $prevID = $this->previousMeaningfulToken($tokenID);
+        if ($prevID !== null && $this->tokens[$prevID]->is(T_DOUBLE_COLON)) {
             return;
         }
 
-        // The name (if any) is the token right before the parameter list "(". Anonymous functions
-        // have none there: `function (`, by-ref `function &(`, or an attributed closure
-        // `#[Attr] function (` all reach "(" with `function`/`&` immediately before it. A parameter
-        // type hint lives *inside* the parens, so it can never be misread as the name. Keyword-like
-        // method names (`list`, `print`, ...) are not T_STRING, so read by position, not by type.
-        $parenID = $tokenID + 1;
-        while (isset($this->tokens[$parenID]) && $this->tokens[$parenID]->text !== '(') {
-            ++$parenID;
+        $nameID = $this->nextMeaningfulToken($tokenID);
+        if ($nameID !== null && $this->tokens[$nameID]->text === '&') {
+            $nameID = $this->nextMeaningfulToken($nameID);
         }
 
-        if (!isset($this->tokens[$parenID])) {
+        // Keyword-like names (`list`, `print`, ...) are not T_STRING, so the name is checked by its
+        // spelling rather than by token type.
+        if ($nameID === null || \preg_match(self::IDENTIFIER, $this->tokens[$nameID]->text) !== 1) {
             return;
         }
 
-        $nameID = $parenID - 1;
-        while ($nameID > $tokenID && $this->tokens[$nameID]->is(T_WHITESPACE)) {
-            --$nameID;
-        }
-
-        $nameToken = $this->tokens[$nameID];
-        if ($nameToken->is(T_FUNCTION) || $nameToken->text === '&') {
-            // Anonymous function — nothing to declare.
+        $parenID = $this->nextMeaningfulToken($nameID);
+        if ($parenID === null || $this->tokens[$parenID]->text !== '(') {
             return;
         }
 
-        $name = $nameToken->text;
+        $name = $this->tokens[$nameID]->text;
 
         // Function
         if ($class === null) {
@@ -796,6 +794,34 @@ private function endingToken(int $tokenID): int
         return $localID;
     }
 
+    /**
+     * Find the first non-whitespace, non-comment token after the given one.
+     */
+    private function nextMeaningfulToken(int $tokenID): ?int
+    {
+        for ($localID = $tokenID + 1; $localID < $this->countTokens; ++$localID) {
+            if (!$this->tokens[$localID]->isIgnorable()) {
+                return $localID;
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * Find the last non-whitespace, non-comment token before the given one.
+     */
+    private function previousMeaningfulToken(int $tokenID): ?int
+    {
+        for ($localID = $tokenID - 1; $localID >= 0; --$localID) {
+            if (!$this->tokens[$localID]->isIgnorable()) {
+                return $localID;
+            }
+        }
+
+        return null;
+    }
+
     /**
      * Get line number associated with token.
      */

tests/Tokenizer/Stub/KeywordNamedDeclarations.php

@@ -4,10 +4,35 @@
 
 namespace Tests\Tokenizer\Stub;
 
-// Method names may be semi-reserved keywords. Their name token is not T_STRING, so the locator
-// must read the name by position (the token before "(") rather than by token type.
+// A grouped import carries `function` tokens that declare nothing.
+use Tests\Tokenizer\Stub\Imported\{function first, function second};
+
+// Keywords used as constant, case, and alias names declare nothing either.
+enum KeywordCases
+{
+    case function;
+}
+
+trait KeywordTrait
+{
+    public function run(): void {}
+}
+
+final class KeywordAlias
+{
+    use KeywordTrait {
+        run as function;
+    }
+
+    public const function = 1;
+}
+
+// Method names may be semi-reserved keywords or contain non-ASCII letters. A keyword name token is
+// not T_STRING, so the locator must check the name by its spelling rather than by token type.
 final class KeywordNames
 {
+    public function größe(): void {}
+
     public function list(): array
     {
         return [];
@@ -16,4 +41,37 @@ public function list(): array
     public function print(): void {}
 
     public function unset(): void {}
+
+    public function function(): void {}
+
+    public function & reference(): array
+    {
+        static $items = [];
+        return $items;
+    }
+
+    public function /* name */ commented /* parameters */ (): void {}
+
+    public function call(): void
+    {
+        // Keywords used as argument names and member names declare nothing.
+        $this->run(function: 1, class: 2);
+        if (true) {
+            KeywordCases::function;
+        }
+        if (true) {
+            self::function();
+            $this->function();
+        }
+        $this->run(function: (1));
+        $same = KeywordAlias::function === (1);
+        $both = KeywordAlias::function and ($same);
+        $mask = KeywordAlias::function & intdiv(4, 2);
+        $spaced = KeywordAlias:: /* constant */ function and ($same);
+    }
+
+    public function closure(): \Closure
+    {
+        return function /* anonymous */ () {};
+    }
 }

tests/Tokenizer/Unit/Reflection/TokenizedFileTest.php

@@ -186,16 +186,23 @@ public function declarationsAreScopedToTheirBracedNamespace(): void
         ]);
     }
 
-    public function keywordNamedMethodsAreDetectedByPosition(): void
+    public function onlyFunctionTokensOfDeclarationsRegisterMethods(): void
     {
         $tokenized = self::tokenize('KeywordNamedDeclarations.php');
 
         $methods = $tokenized->getMethodsFQN();
         \sort($methods);
         Assert::same($methods, [
+            'Tests\Tokenizer\Stub\KeywordNames::call',
+            'Tests\Tokenizer\Stub\KeywordNames::closure',
+            'Tests\Tokenizer\Stub\KeywordNames::commented',
+            'Tests\Tokenizer\Stub\KeywordNames::function',
+            'Tests\Tokenizer\Stub\KeywordNames::größe',
             'Tests\Tokenizer\Stub\KeywordNames::list',
             'Tests\Tokenizer\Stub\KeywordNames::print',
+            'Tests\Tokenizer\Stub\KeywordNames::reference',
             'Tests\Tokenizer\Stub\KeywordNames::unset',
+            'Tests\Tokenizer\Stub\KeywordTrait::run',
         ]);
         Assert::same($tokenized->getFunctions(), []);
     }
@@ -447,6 +454,21 @@ public function globalBracedNamespaceIsRecognized(): void
         Assert::same($tokenized->getFunctions(), ['globalBracedFunction']);
     }
 
+    public function functionTokenBoundedByFileEdgesDeclaresNothing(): void
+    {
+        $path = \sys_get_temp_dir() . '/testo-tokenizer-' . \bin2hex(\random_bytes(4)) . '.php';
+        \file_put_contents($path, '<?php function');
+
+        try {
+            $tokenized = new TokenizedFile(new \SplFileInfo($path), $path);
+
+            Assert::same($tokenized->getFunctions(), []);
+            Assert::same($tokenized->getMethodsFQN(), []);
+        } finally {
+            \unlink($path);
+        }
+    }
+
     public function repeatedNamespaceBlockMergesDeclarationsFromBothBlocks(): void
     {
         $tokenized = self::tokenize('RepeatedNamespace.php');