Tokenizer/PHP: change tokenization of long PHP open tags

jrfnl · jrfnl · commit b00f2e40cab2 · 2025-04-17T17:08:56.000+02:00
... to no longer include whitespace (single space/new line).

This ensures the tokenization of all PHP open tags - long open tags `&lt;?php`, short open tags `&lt;?` and short open echo tags `&lt;?=` - is consistent and should simplify handling of open tag tokens in sniffs and lower the cognitive load of sniff developers.

As per the detailed proposal in ticket 593.

Includes extensive unit tests, both ensuring the correct tokenization, as well as safeguarding consistency across the different open tags.

Includes the following additional updates to maintain the pre-existing behaviour:
* `AbstractPatternSniff`
    The `AbstractPatternSniff` tokenizes an arbitrary code pattern to determine the token pattern to sniff for. This tokenization now to takes the change in the PHP open tag tokenization into account.
* `Generic/EmptyPHPStatement` - replace now redundant special condition for PHP open tags + update "fixed" file for improved output + add extra test.
* `PSR12/OpenTag` - allow for the changed tokenization.
    The updated code also means that the fixer will no longer leave stray whitespace behind after a PHP open tag.
* `Squiz/CommentedOutCode` - allow for the changed tokenization.
* `Squiz/EmbeddedPhp` - code simplification.
* `FindStartOfStatementTest` - fixed token offset expectation as there is now an extra whitespace token.
* `ReplaceTabsInTokenMiscTest` - updated target finding

Fixes 593
diff --git a/src/Sniffs/AbstractPatternSniff.php b/src/Sniffs/AbstractPatternSniff.php
@@ -915,12 +915,12 @@ private function createTokenPattern($str)
 
         // Don't add a space after the closing php tag as it will add a new
         // whitespace token.
-        $tokenizer = new PHP('<?php '.$str.'?>', null);
+        $tokenizer = new PHP('<?php'."\n".$str.'?>', null);
         StatusWriter::resume();
 
         // Remove the <?php tag from the front and the end php tag from the back.
         $tokens = $tokenizer->getTokens();
-        $tokens = array_slice($tokens, 1, (count($tokens) - 2));
+        $tokens = array_slice($tokens, 2, (count($tokens) - 3));
 
         $patterns = [];
         foreach ($tokens as $patternInfo) {
diff --git a/src/Standards/Generic/Sniffs/CodeAnalysis/EmptyPHPStatementSniff.php b/src/Standards/Generic/Sniffs/CodeAnalysis/EmptyPHPStatementSniff.php
@@ -113,19 +113,15 @@ private function processSemicolon(File $phpcsFile, $stackPtr)
         if ($fix === true) {
             $phpcsFile->fixer->beginChangeset();
 
-            if ($tokens[$prevNonEmpty]['code'] === T_OPEN_TAG
-                || $tokens[$prevNonEmpty]['code'] === T_OPEN_TAG_WITH_ECHO
-            ) {
-                // Check for superfluous whitespace after the semicolon which should be
-                // removed as the `<?php ` open tag token already contains whitespace,
-                // either a space or a new line.
-                if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {
-                    $replacement = str_replace(' ', '', $tokens[($stackPtr + 1)]['content']);
-                    $phpcsFile->fixer->replaceToken(($stackPtr + 1), $replacement);
-                }
+            // Make sure there always remains one space between the open tag and the next content.
+            $replacement = ' ';
+            if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {
+                $replacement = '';
             }
 
-            for ($i = $stackPtr; $i > $prevNonEmpty; $i--) {
+            $phpcsFile->fixer->replaceToken($stackPtr, $replacement);
+
+            for ($i = ($stackPtr - 1); $i > $prevNonEmpty; $i--) {
                 if ($tokens[$i]['code'] !== T_SEMICOLON
                     && $tokens[$i]['code'] !== T_WHITESPACE
                 ) {
diff --git a/src/Standards/Generic/Tests/CodeAnalysis/EmptyPHPStatementUnitTest.2.inc b/src/Standards/Generic/Tests/CodeAnalysis/EmptyPHPStatementUnitTest.2.inc
@@ -1,5 +1,5 @@
 <!-- Tests with short open tag. -->
-
+<input name="<? ;something_else(); ?>" />
 <input name="<? ; something_else(); ?>" />
 <input name="<? something_else(); ; ?>" />
 
diff --git a/src/Standards/Generic/Tests/CodeAnalysis/EmptyPHPStatementUnitTest.2.inc.fixed b/src/Standards/Generic/Tests/CodeAnalysis/EmptyPHPStatementUnitTest.2.inc.fixed
@@ -1,6 +1,6 @@
 <!-- Tests with short open tag. -->
-
-<input name="<?something_else(); ?>" />
+<input name="<? something_else(); ?>" />
+<input name="<? something_else(); ?>" />
 <input name="<? something_else(); ?>" />
 
 /*
diff --git a/src/Standards/Generic/Tests/CodeAnalysis/EmptyPHPStatementUnitTest.php b/src/Standards/Generic/Tests/CodeAnalysis/EmptyPHPStatementUnitTest.php
@@ -93,6 +93,7 @@ public function getWarningList($testFile='')
             ];
         case 'EmptyPHPStatementUnitTest.2.inc':
             return [
+                2  => 1,
                 3  => 1,
                 4  => 1,
                 13 => 1,
diff --git a/src/Standards/PSR12/Sniffs/Files/OpenTagSniff.php b/src/Standards/PSR12/Sniffs/Files/OpenTagSniff.php
@@ -56,16 +56,24 @@ public function process(File $phpcsFile, $stackPtr)
             return $phpcsFile->numTokens;
         }
 
-        $next = $phpcsFile->findNext(T_INLINE_HTML, 0);
-        if ($next !== false) {
+        $hasInlineHTML = $phpcsFile->findNext(T_INLINE_HTML, 0);
+        if ($hasInlineHTML !== false) {
             // This rule only applies to PHP-only files.
             return $phpcsFile->numTokens;
         }
 
         $error = 'Opening PHP tag must be on a line by itself';
         $fix   = $phpcsFile->addFixableError($error, $stackPtr, 'NotAlone');
         if ($fix === true) {
+            $phpcsFile->fixer->beginChangeset();
+
+            // Remove whitespace between the open tag and the next non-empty token.
+            for ($i = ($stackPtr + 1); $i < $next; $i++) {
+                $phpcsFile->fixer->replaceToken($i, '');
+            }
+
             $phpcsFile->fixer->addNewline($stackPtr);
+            $phpcsFile->fixer->endChangeset();
         }
 
         return $phpcsFile->numTokens;
diff --git a/src/Standards/PSR12/Tests/Files/OpenTagUnitTest.2.inc.fixed b/src/Standards/PSR12/Tests/Files/OpenTagUnitTest.2.inc.fixed
@@ -1,2 +1,2 @@
-<?php 
+<?php
 echo 'hi';
diff --git a/src/Standards/Squiz/Sniffs/PHP/CommentedOutCodeSniff.php b/src/Standards/Squiz/Sniffs/PHP/CommentedOutCodeSniff.php
@@ -194,11 +194,13 @@ public function process(File $phpcsFile, $stackPtr)
         */
 
         // First token is always the opening tag.
-        if ($stringTokens[0]['code'] !== T_OPEN_TAG) {
+        if ($stringTokens[0]['code'] !== T_OPEN_TAG || $stringTokens[1]['code'] !== T_WHITESPACE) {
             return ($lastCommentBlockToken + 1);
         } else {
+            // Remove the PHP open tag + the whitespace token following it.
             array_shift($stringTokens);
-            --$numTokens;
+            array_shift($stringTokens);
+            $numTokens -= 2;
         }
 
         // Last token is always the closing tag, unless something went wrong.
diff --git a/src/Standards/Squiz/Sniffs/PHP/EmbeddedPhpSniff.php b/src/Standards/Squiz/Sniffs/PHP/EmbeddedPhpSniff.php
@@ -347,16 +347,7 @@ private function validateInlineEmbeddedPhp($phpcsFile, $stackPtr, $closeTag)
         }
 
         // Check that there is one, and only one space at the start of the statement.
-        $leadingSpace  = 0;
-        $isLongOpenTag = false;
-        if ($tokens[$stackPtr]['code'] === T_OPEN_TAG
-            && stripos($tokens[$stackPtr]['content'], '<?php') === 0
-        ) {
-            // The long open tag token in a single line tag set always contains a single space after it.
-            $leadingSpace  = 1;
-            $isLongOpenTag = true;
-        }
-
+        $leadingSpace = 0;
         if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {
             $leadingSpace += $tokens[($stackPtr + 1)]['length'];
         }
@@ -366,13 +357,11 @@ private function validateInlineEmbeddedPhp($phpcsFile, $stackPtr, $closeTag)
             $data  = [$leadingSpace];
             $fix   = $phpcsFile->addFixableError($error, $stackPtr, 'SpacingAfterOpen', $data);
             if ($fix === true) {
-                if ($isLongOpenTag === true) {
-                    $phpcsFile->fixer->replaceToken(($stackPtr + 1), '');
-                } else if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {
-                    // Short open tag with too much whitespace.
+                if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {
+                    // Open tag with too much whitespace.
                     $phpcsFile->fixer->replaceToken(($stackPtr + 1), ' ');
                 } else {
-                    // Short open tag without whitespace.
+                    // Open tag without whitespace.
                     $phpcsFile->fixer->addContent($stackPtr, ' ');
                 }
             }
diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
@@ -823,6 +823,59 @@ protected function tokenize($string)
                 continue;
             }//end if
 
+            /*
+                Split whitespace off from long PHP open tag tokens and potentially join the whitespace
+                with a subsequent whitespace token.
+            */
+
+            if ($tokenIsArray === true
+                && $token[0] === T_OPEN_TAG
+                && stripos($token[1], '<?php') === 0
+            ) {
+                $openTagAndWhiteSpace = str_split($token[1], 5);
+
+                $finalTokens[$newStackPtr] = [
+                    'code'    => T_OPEN_TAG,
+                    'type'    => 'T_OPEN_TAG',
+                    'content' => $openTagAndWhiteSpace[0],
+                ];
+                $newStackPtr++;
+
+                if (isset($openTagAndWhiteSpace[1]) === true) {
+                    // The original open tag token included whitespace.
+                    // Check whether a new whitespace token needs to be inserted or if the
+                    // whitespace needs to be joined with a pre-existing whitespace
+                    // token on the same line as the open tag.
+                    if (isset($tokens[($stackPtr + 1)]) === true
+                        && $openTagAndWhiteSpace[1] === ' '
+                        && is_array($tokens[($stackPtr + 1)]) === true
+                        && $tokens[($stackPtr + 1)][0] === T_WHITESPACE
+                    ) {
+                        // Adjusting the original token stack as the "new line may be split over two tokens"
+                        // check should still be run on this token.
+                        $tokens[($stackPtr + 1)][1] = $openTagAndWhiteSpace[1].$tokens[($stackPtr + 1)][1];
+
+                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                            StatusWriter::write("* removed whitespace from T_OPEN_TAG token $stackPtr and merged it with the next token T_WHITESPACE", 2);
+                        }
+                    } else {
+                        $finalTokens[$newStackPtr] = [
+                            'code'    => T_WHITESPACE,
+                            'type'    => 'T_WHITESPACE',
+                            'content' => $openTagAndWhiteSpace[1],
+                        ];
+
+                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                            StatusWriter::write("* T_OPEN_TAG token $stackPtr split into T_OPEN_TAG (without whitespace) and new T_WHITESPACE token", 2);
+                        }
+
+                        $newStackPtr++;
+                    }//end if
+                }//end if
+
+                continue;
+            }//end if
+
             /*
                 Parse doc blocks into something that can be easily iterated over.
             */
diff --git a/tests/Core/File/FindStartOfStatementTest.php b/tests/Core/File/FindStartOfStatementTest.php
@@ -520,7 +520,7 @@ public function testNestedMatch()
     public function testOpenTag()
     {
         $start  = $this->getTargetToken('/* testOpenTag */', T_OPEN_TAG);
-        $start += 2;
+        $start += 3;
         $found  = self::$phpcsFile->findStartOfStatement($start);
 
         $this->assertSame(($start - 1), $found);
diff --git a/tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.php b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.php
@@ -43,10 +43,18 @@ public function testLongOpenTagAtEndOfFile()
             $tokens[$stackPtr]['type'],
             'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
         );
-        $this->assertSame('<?php ', $tokens[$stackPtr]['content']);
+        $this->assertSame('<?php', $tokens[$stackPtr]['content']);
+
+        $this->assertArrayHasKey(($stackPtr + 1), $tokens, 'Missing whitespace token after open tag');
+        $this->assertSame(
+            T_WHITESPACE,
+            $tokens[($stackPtr + 1)]['code'],
+            'Missing whitespace token after open tag (code)'
+        );
+        $this->assertSame(' ', $tokens[($stackPtr + 1)]['content'], 'Missing whitespace token after open tag (content)');
 
         // Now make sure that this is the very last token in the file and there are no tokens after it.
-        $this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
+        $this->assertArrayNotHasKey(($stackPtr + 2), $tokens);
 
     }//end testLongOpenTagAtEndOfFile()
 
diff --git a/tests/Core/Tokenizers/PHP/PHPOpenTagTest.inc b/tests/Core/Tokenizers/PHP/PHPOpenTagTest.inc
@@ -0,0 +1,115 @@
+<?php /* testLongOpenTagWithNewLine */ ?>
+<?php
+echo 'with new line';
+?>
+
+<?php /* testLongOpenTagWithOneSpaceAndNewLine */ ?>
+<?php 
+echo 'with one space and new line';
+?>
+
+<?php /* testLongOpenTagWithTrailingWhiteSpaceAndNewLine */ ?>
+<?php    
+echo 'with trailing whitespace and new line';
+?>
+
+<?php /* testLongOpenTagOneSpace */ ?>
+<?php echo 'single line, one space'; ?>
+
+<?php /* testLongOpenTagMultiSpace */ ?>
+<?php      echo 'single line, multiple spaces'; ?>
+
+<?php /* testLongOpenTagWithDoubleNewLine */ ?>
+<?php
+
+echo 'with double new line';
+?>
+
+<?php /* testLongOpenTagWithTripleNewLine */ ?>
+<?php
+
+
+echo 'with triple new line';
+?>
+
+<?php /* testLongOpenTagWithNewLineAndIndentOnNextLine */ ?>
+    <?php
+    echo 'with new line and indent on next line';
+?>
+
+<!-- ====================================== -->
+
+<?php /* testCaseLongOpenTagWithNewLine */ ?>
+<?PHP
+echo 'with new line';
+?>
+
+<?php /* testCaseLongOpenTagWithOneSpaceAndNewLine */ ?>
+<?phP 
+echo 'with one space and new line';
+?>
+
+<?php /* testCaseLongOpenTagWithTrailingWhiteSpaceAndNewLine */ ?>
+<?Php    
+echo 'with trailing whitespace and new line';
+?>
+
+<?php /* testCaseLongOpenTagOneSpace */ ?>
+<?pHp echo 'single line, one space'; ?>
+
+<?php /* testCaseLongOpenTagMultiSpace */ ?>
+<?phP      echo 'single line, multiple spaces'; ?>
+
+<!-- ====================================== -->
+
+<?php /* testShortOpenEchoTagWithNewLine */ ?>
+<?=
+'with new line';
+?>
+
+<?php /* testShortOpenEchoTagWithOneSpaceAndNewLine */ ?>
+<?= 
+'with one space and new line';
+?>
+
+<?php /* testShortOpenEchoTagWithTrailingWhiteSpaceAndNewLine */ ?>
+<?=    
+'with trailing whitespace and new line';
+?>
+
+<?php /* testShortOpenEchoTagNoSpace */ ?>
+<?='single line, no space'; ?>
+
+<?php /* testShortOpenEchoTagOneSpace */ ?>
+<?= 'single line, one space'; ?>
+
+<?php /* testShortOpenEchoTagMultiSpace */ ?>
+<?=      'single line, multiple spaces'; ?>
+
+<!-- ====================================== -->
+
+<?php /* testShortOpenTagWithNewLine */ ?>
+<?
+echo 'with new line';
+?>
+
+<?php /* testShortOpenTagWithOneSpaceAndNewLine */ ?>
+<? 
+echo 'with one space and new line';
+?>
+
+<?php /* testShortOpenTagWithTrailingWhiteSpaceAndNewLine */ ?>
+<?    
+echo 'with trailing whitespace and new line';
+?>
+
+<?php /* testShortOpenTagNoSpace */ ?>
+<?echo 'single line, no space'; ?>
+
+<?php /* testShortOpenTagOneSpace */ ?>
+<? echo 'single line, one space'; ?>
+
+<?php /* testShortOpenTagMultiSpace */ ?>
+<?      echo 'single line, multiple spaces'; ?>
+
+<!-- ====================================== -->
diff --git a/tests/Core/Tokenizers/PHP/PHPOpenTagTest.php b/tests/Core/Tokenizers/PHP/PHPOpenTagTest.php
diff --git a/tests/Core/Tokenizers/Tokenizer/ReplaceTabsInTokenMiscTest.php b/tests/Core/Tokenizers/Tokenizer/ReplaceTabsInTokenMiscTest.php
