cucumber · aurelien-reeves · Mar 30, 2022 · Mar 4, 2022 · Mar 5, 2022 · Mar 5, 2022
diff --git a/gherkin/php/src/AstNode.php b/gherkin/php/src/AstNode.php
@@ -76,21 +76,25 @@ public function getSingleUntyped(RuleType $ruleType, mixed $defaultValue = null)
     }
 
     /**
-     * @return list<Token>
+     * @return list<TokenMatch>
      */
-    public function getTokens(TokenType $tokenType): array
+    public function getTokenMatches(TokenType $tokenType): array
     {
-        $items = $this->getItems(Token::class, RuleType::cast($tokenType));
+        $items = $this->getItems(TokenMatch::class, RuleType::cast($tokenType));
 
         return $items;
     }
 
 
-    public function getToken(TokenType $tokenType): Token
+    public function getTokenMatch(TokenType $tokenType): TokenMatch
     {
         $ruleType = RuleType::cast($tokenType);
 
-        $item = $this->getSingle(Token::class, $ruleType) ?? new Token(null, new Location(-1, -1));
+        $item = $this->getSingle(TokenMatch::class, $ruleType);
+
+        if (!$item) {
+            throw new \LogicException('Requested token type was not in stack');
+        }
 
         return $item;
     }

diff --git a/gherkin/php/src/GherkinDocumentBuilder.php b/gherkin/php/src/GherkinDocumentBuilder.php
@@ -47,16 +47,16 @@ public function __construct(
 
     public function build(Token $token): void
     {
-        if (null === $token->matchedType) {
+        if (null === $token->match) {
             throw new LogicException('Token was not yet matched');
         }
 
-        $ruleType = RuleType::cast($token->matchedType);
+        $ruleType = RuleType::cast($token->match->tokenType);
 
-        if ($token->matchedType == TokenType::Comment) {
-            $this->comments[] = new Comment($this->getLocation($token, 0), $token->matchedText ?? '');
+        if ($token->match->tokenType == TokenType::Comment) {
+            $this->comments[] = new Comment($this->getLocation($token->match, 0), $token->match->text);
         } else {
-            $this->currentNode()->add($ruleType, $token);
+            $this->currentNode()->add($ruleType, $token->match);
         }
     }
 
@@ -116,7 +116,7 @@ private function getTransformedNode(AstNode $node): object|string|array|null
         };
     }
 
-    private function getLocation(Token $token, int $column): MessageLocation
+    private function getLocation(TokenMatch $token, int $column): MessageLocation
     {
         $column = ($column === 0) ? $token->location->column : $column;
 
@@ -139,7 +139,7 @@ private function getTableRows(AstNode $node): array
     {
         $rows = array_map(
             fn ($token) => new TableRow($this->getLocation($token, 0), $this->getCells($token), $this->idGenerator->newId()),
-            $node->getTokens(TokenType::TableRow),
+            $node->getTokenMatches(TokenType::TableRow),
         );
 
         $this->ensureCellCount($rows);
@@ -166,11 +166,11 @@ private function ensureCellCount(array $rows): void
     /**
      * @return list<TableCell>
      */
-    private function getCells(Token $token): array
+    private function getCells(TokenMatch $token): array
     {
         return array_map(
             fn ($cellItem) => new TableCell($this->getLocation($token, $cellItem->column), $cellItem->text),
-            $token->matchedItems ?? [],
+            $token->items,
         );
     }
 
@@ -181,10 +181,10 @@ private function getTags(AstNode $node): array
     {
         $tagsNode = $node->getSingle(AstNode::class, RuleType::Tags, new AstNode(RuleType::None));
 
-        $tokens = $tagsNode->getTokens(TokenType::TagLine);
+        $tokens = $tagsNode->getTokenMatches(TokenType::TagLine);
         $tags = [];
         foreach ($tokens as $token) {
-            foreach ($token->matchedItems ?? [] as $tagItem) {
+            foreach ($token->items as $tagItem) {
                 $tags[] = new Tag(
                     location: $this->getLocation($token, $tagItem->column),
                     name: $tagItem->text,
@@ -197,21 +197,21 @@ private function getTags(AstNode $node): array
     }
 
     /**
-     * @param array<Token> $lineTokens
+     * @param array<TokenMatch> $lineTokens
      */
     private function joinMatchedTextWithLinebreaks(array $lineTokens): string
     {
-        return join("\n", array_map(fn ($t) => $t->matchedText, $lineTokens));
+        return join("\n", array_map(fn ($t) => $t->text, $lineTokens));
     }
 
     private function transformStepNode(AstNode $node): Step
     {
-        $stepLine = $node->getToken(TokenType::StepLine);
+        $stepLine = $node->getTokenMatch(TokenType::StepLine);
 
         return new Step(
             location: $this->getLocation($stepLine, 0),
-            keyword: $stepLine->matchedKeyword ?? '',
-            text: $stepLine->matchedText ?? '',
+            keyword: $stepLine->keyword,
+            text: $stepLine->text,
             docString: $node->getSingle(DocString::class, RuleType::DocString),
             dataTable: $node->getSingle(DataTable::class, RuleType::DataTable),
             id: $this->idGenerator->newId(),
@@ -220,17 +220,17 @@ private function transformStepNode(AstNode $node): Step
 
     private function transformDocStringNode(AstNode $node): DocString
     {
-        $separatorToken = $node->getTokens(TokenType::DocStringSeparator)[0];
-        $mediaType = $separatorToken->matchedText ?: null;
-        $lineTokens = $node->getTokens(TokenType::Other);
+        $separatorToken = $node->getTokenMatches(TokenType::DocStringSeparator)[0];
+        $mediaType = $separatorToken->text;
+        $lineTokens = $node->getTokenMatches(TokenType::Other);
 
         $content = $this->joinMatchedTextWithLinebreaks($lineTokens);
 
         return new DocString(
             location: $this->getLocation($separatorToken, 0),
-            mediaType: $mediaType,
+            mediaType: $mediaType ?: null, // special case turns '' into null
             content: $content,
-            delimiter: $separatorToken->matchedKeyword ?? '',
+            delimiter: $separatorToken->keyword,
         );
     }
 
@@ -240,13 +240,13 @@ private function transformScenarioDefinitionNode(AstNode $node): ?Scenario
         if (null === $scenarioNode) {
             return null;
         }
-        $scenarioLine = $scenarioNode->getToken(TokenType::ScenarioLine);
+        $scenarioLine = $scenarioNode->getTokenMatch(TokenType::ScenarioLine);
 
         return new Scenario(
             location: $this->getLocation($scenarioLine, 0),
             tags: $this->getTags($node),
-            keyword: $scenarioLine->matchedKeyword ?? '',
-            name: $scenarioLine->matchedText ?? '',
+            keyword: $scenarioLine->keyword,
+            name: $scenarioLine->text,
             description: $this->getDescription($scenarioNode),
             steps: $this->getSteps($scenarioNode),
             examples: $scenarioNode->getItems(Examples::class, RuleType::ExamplesDefinition),
@@ -260,7 +260,7 @@ private function transformExamplesDefinitionNode(AstNode $node): ?Examples
         if (null === $examplesNode) {
             return null;
         }
-        $examplesLine = $examplesNode->getToken(TokenType::ExamplesLine);
+        $examplesLine = $examplesNode->getTokenMatch(TokenType::ExamplesLine);
         /** @var list<TableRow>|null $rows */
         $rows = $examplesNode->getSingleUntyped(RuleType::ExamplesTable);
         $tableHeader = is_array($rows) && count($rows) ? $rows[0] : null;
@@ -269,8 +269,8 @@ private function transformExamplesDefinitionNode(AstNode $node): ?Examples
         return new Examples(
             location: $this->getLocation($examplesLine, 0),
             tags: $this->getTags($node),
-            keyword: $examplesLine->matchedKeyword ?? '',
-            name: $examplesLine->matchedText ?? '',
+            keyword: $examplesLine->keyword,
+            name: $examplesLine->text,
             description: $this->getDescription($examplesNode),
             tableHeader: $tableHeader,
             tableBody: $tableBody,
@@ -293,12 +293,12 @@ private function transformExamplesTableNode(AstNode $node): array
 
     private function transformBackgroundNode(AstNode $node): Background
     {
-        $backgroundLine = $node->getToken(TokenType::BackgroundLine);
+        $backgroundLine = $node->getTokenMatch(TokenType::BackgroundLine);
 
         return new Background(
             location: $this->getLocation($backgroundLine, 0),
-            keyword: $backgroundLine->matchedKeyword ?? '',
-            name: $backgroundLine->matchedText ?? '',
+            keyword: $backgroundLine->keyword,
+            name: $backgroundLine->text,
             description: $this->getDescription($node),
             steps: $this->getSteps($node),
             id: $this->idGenerator->newId(),
@@ -307,7 +307,7 @@ private function transformBackgroundNode(AstNode $node): Background
 
     private function transformDescriptionNode(AstNode $node): string
     {
-        $lineTokens = $node->getTokens(TokenType::Other);
+        $lineTokens = $node->getTokenMatches(TokenType::Other);
 
         $lineText = preg_replace(
             '/(\\n\\s*)*$/u',
@@ -325,7 +325,7 @@ private function transformFeatureNode(AstNode $node): ?Feature
             return null;
         }
         $tags = $this->getTags($header);
-        $featureLine = $header->getToken(TokenType::FeatureLine);
+        $featureLine = $header->getTokenMatch(TokenType::FeatureLine);
 
         $children = [];
 
@@ -342,17 +342,14 @@ private function transformFeatureNode(AstNode $node): ?Feature
             $children[] = new FeatureChild($rule, null, null);
         }
 
-        $language = $featureLine->matchedGherkinDialect?->getLanguage();
-        if (null === $language) {
-            return null;
-        }
+        $language = $featureLine->gherkinDialect->getLanguage();
 
         return new Feature(
             location: $this->getLocation($featureLine, 0),
             tags: $tags,
             language: $language,
-            keyword: $featureLine->matchedKeyword ?? '',
-            name: $featureLine->matchedText ?? '',
+            keyword: $featureLine->keyword,
+            name: $featureLine->text,
             description: $this->getDescription($header),
             children: $children,
         );
@@ -362,7 +359,7 @@ private function transformRuleNode(AstNode $node): Rule
     {
         $header = $node->getSingle(AstNode::class, RuleType::RuleHeader, new AstNode(RuleType::RuleHeader));
 
-        $ruleLine = $header->getToken(TokenType::RuleLine);
+        $ruleLine = $header->getTokenMatch(TokenType::RuleLine);
 
         $children = [];
         $tags = $this->getTags($header);
@@ -379,8 +376,8 @@ private function transformRuleNode(AstNode $node): Rule
         return new Rule(
             location: $this->getLocation($ruleLine, 0),
             tags: $tags,
-            keyword: $ruleLine->matchedKeyword ?? '',
-            name: $ruleLine->matchedText ?? '',
+            keyword: $ruleLine->keyword,
+            name: $ruleLine->text,
             description: $this->getDescription($header),
             children: $children,
             id: $this->idGenerator->newId(),

diff --git a/gherkin/php/src/Token.php b/gherkin/php/src/Token.php
@@ -9,21 +9,11 @@
 final class Token
 {
     private const EOF_VALUE = 'EOF';
-    public ?TokenType $matchedType = null;
-    public ?string $matchedKeyword = null;
-    public ?string $matchedText = null;
-    public ?int $matchedIndent = null;
-    public ?GherkinDialect $matchedGherkinDialect = null;
-
-    /** @var list<GherkinLineSpan> */
-    public ?array $matchedItems = null;
+    public ?TokenMatch $match = null;
 
     public function __construct(
         public readonly ?GherkinLine $line,
-        /**
-         * Public because the token matcher adds indent to columns
-         */
-        public Location $location,
+        public readonly Location $location,
     ) {
     }
 
@@ -37,11 +27,33 @@ public function isEof(): bool
 
     public function getLocation(): Location
     {
-        return $this->location;
+        return $this->match ? $this->match->location : $this->location;
     }
 
     public function getTokenValue(): string
     {
         return $this->isEof() ? self::EOF_VALUE : $this->line->getLineText(-1);
     }
+
+    /**
+     * @param list<GherkinLineSpan> $items
+     */
+    public function match(
+        TokenType $matchedType,
+        GherkinDialect $gherkinDialect,
+        int $indent,
+        string $keyword,
+        string $text,
+        array $items,
+    ): void {
+        $this->match = new TokenMatch(
+            $matchedType,
+            $gherkinDialect,
+            $indent,
+            $keyword,
+            $text,
+            $items,
+            new Location($this->location->line, $indent + 1),
+        );
+    }
 }
diff --git a/gherkin/php/src/TokenFormatter.php b/gherkin/php/src/TokenFormatter.php
@@ -16,11 +16,11 @@ public function formatToken(Token $token): string
             "(%s:%s)%s:%s/%s/%s",
             $token->getLocation()->line,
             $token->getLocation()->column,
-            $token->matchedType?->name ??  '',
-            $token->matchedKeyword ?? '',
-            $token->matchedText ?? '',
-            $token->matchedItems === null ? ''
-                : join(',', array_map(fn ($linespan) => $linespan->column . ':' . $linespan->text, $token->matchedItems)),
+            $token->match?->tokenType->name ??  '',
+            $token->match?->keyword ?? '',
+            $token->match?->text ?? '',
+            $token->match === null ? ''
+                : join(',', array_map(fn ($linespan) => $linespan->column . ':' . $linespan->text, $token->match->items)),
         );
     }
 }
diff --git a/gherkin/php/src/TokenMatch.php b/gherkin/php/src/TokenMatch.php
@@ -0,0 +1,24 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Cucumber\Gherkin;
+
+use Cucumber\Gherkin\Parser\TokenType;
+
+final class TokenMatch
+{
+    /**
+     * @param list<GherkinLineSpan> $items
+     */
+    public function __construct(
+        public readonly TokenType $tokenType,
+        public readonly GherkinDialect $gherkinDialect,
+        public readonly int $indent,
+        public readonly string $keyword,
+        public readonly string $text,
+        public readonly array $items,
+        public readonly Location $location,
+    ) {
+    }
+}
diff --git a/gherkin/php/src/TokenMatcher.php b/gherkin/php/src/TokenMatcher.php
@@ -36,13 +36,15 @@ private function setTokenMatched(
         ?int $indent = null,
         ?array $items = null,
     ): void {
-        $token->matchedType = $matchedType;
-        $token->matchedKeyword = $keyword;
-        $token->matchedText = $text;
-        $token->matchedItems = $items;
-        $token->matchedGherkinDialect = $this->currentDialect;
-        $token->matchedIndent = $indent ?? $token->line?->indent() ?? 0;
-        $token->location = new Location($token->location->line, $token->matchedIndent + 1);
+        $matchedIndent = $indent ?? $token->line?->indent() ?? 0;
+        $token->match(
+            $matchedType,
+            $this->currentDialect,
+            $matchedIndent,
+            $keyword ?? '',
+            $text ?? '',
+            $items ?? [],
+        );
     }
 
     public function match_EOF(Token $token): bool
@@ -207,7 +209,7 @@ public function match_Language(Token $token): bool
 
             $this->setTokenMatched($token, TokenType::Language, $language);
 
-            $this->currentDialect = $this->dialectProvider->getDialect($language, $token->location);
+            $this->currentDialect = $this->dialectProvider->getDialect($language, $token->getLocation());
 
             return true;
         }