Skip to content

Commit 425803e

Browse files
gsherwoodjrfnl
gsherwood
and
jrfnl
committed
Tokenizer: remove backfill code for PHP < 7.2
Includes removing some related tests which have now become redundant. Co-authored-by: jrfnl <jrfnl@users.noreply.github.com>
1 parent 89f142f commit 425803e

File tree

7 files changed

+7
-354
lines changed

7 files changed

+7
-354
lines changed

‎src/Tokenizers/PHP.php‎

Lines changed: 5 additions & 194 deletions
Original file line numberDiff line numberDiff line change
@@ -607,27 +607,6 @@ protected function tokenize($string)
607607
echo PHP_EOL;
608608
}
609609

610-
/*
611-
Before PHP 5.5, the yield keyword was tokenized as
612-
T_STRING. So look for and change this token in
613-
earlier versions.
614-
*/
615-
616-
if (PHP_VERSION_ID < 50500
617-
&& $tokenIsArray === true
618-
&& $token[0] === T_STRING
619-
&& strtolower($token[1]) === 'yield'
620-
&& isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false
621-
) {
622-
// Could still be a context sensitive keyword or "yield from" and potentially multi-line,
623-
// so adjust the token stack in place.
624-
$token[0] = T_YIELD;
625-
626-
if (PHP_CODESNIFFER_VERBOSITY > 1) {
627-
echo "\t\t* token $stackPtr changed from T_STRING to T_YIELD".PHP_EOL;
628-
}
629-
}
630-
631610
/*
632611
Tokenize context sensitive keyword as string when it should be string.
633612
*/
@@ -1562,41 +1541,10 @@ protected function tokenize($string)
15621541
}//end if
15631542

15641543
/*
1565-
Before PHP 7.0, "yield from" was tokenized as
1566-
T_YIELD, T_WHITESPACE and T_STRING. So look for
1567-
and change this token in earlier versions.
1544+
Deal with "yield from" in various PHP versions.
15681545
*/
15691546

1570-
if (PHP_VERSION_ID < 70000
1571-
&& $tokenIsArray === true
1572-
&& $token[0] === T_YIELD
1573-
&& isset($tokens[($stackPtr + 1)]) === true
1574-
&& isset($tokens[($stackPtr + 2)]) === true
1575-
&& $tokens[($stackPtr + 1)][0] === T_WHITESPACE
1576-
&& strpos($tokens[($stackPtr + 1)][1], $this->eolChar) === false
1577-
&& $tokens[($stackPtr + 2)][0] === T_STRING
1578-
&& strtolower($tokens[($stackPtr + 2)][1]) === 'from'
1579-
) {
1580-
// Single-line "yield from" with only whitespace between.
1581-
$finalTokens[$newStackPtr] = [
1582-
'code' => T_YIELD_FROM,
1583-
'type' => 'T_YIELD_FROM',
1584-
'content' => $token[1].$tokens[($stackPtr + 1)][1].$tokens[($stackPtr + 2)][1],
1585-
];
1586-
1587-
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1588-
for ($i = ($stackPtr + 1); $i <= ($stackPtr + 2); $i++) {
1589-
$type = Tokens::tokenName($tokens[$i][0]);
1590-
$content = Common::prepareForOutput($tokens[$i][1]);
1591-
echo "\t\t* token $i merged into T_YIELD_FROM; was: $type => $content".PHP_EOL;
1592-
}
1593-
}
1594-
1595-
$newStackPtr++;
1596-
$stackPtr += 2;
1597-
1598-
continue;
1599-
} else if (PHP_VERSION_ID < 80300
1547+
if (PHP_VERSION_ID < 80300
16001548
&& $tokenIsArray === true
16011549
&& $token[0] === T_STRING
16021550
&& strtolower($token[1]) === 'from'
@@ -1625,8 +1573,7 @@ protected function tokenize($string)
16251573
}
16261574

16271575
continue;
1628-
} else if (PHP_VERSION_ID >= 70000
1629-
&& $tokenIsArray === true
1576+
} else if ($tokenIsArray === true
16301577
&& $token[0] === T_YIELD_FROM
16311578
&& strpos($token[1], $this->eolChar) !== false
16321579
&& preg_match('`^yield\s+from$`i', $token[1]) === 1
@@ -1719,92 +1666,15 @@ protected function tokenize($string)
17191666
}//end if
17201667

17211668
/*
1722-
Before PHP 5.6, the ... operator was tokenized as three
1723-
T_STRING_CONCAT tokens in a row. So look for and combine
1724-
these tokens in earlier versions.
1725-
*/
1726-
1727-
if ($tokenIsArray === false
1728-
&& $token[0] === '.'
1729-
&& isset($tokens[($stackPtr + 1)]) === true
1730-
&& isset($tokens[($stackPtr + 2)]) === true
1731-
&& $tokens[($stackPtr + 1)] === '.'
1732-
&& $tokens[($stackPtr + 2)] === '.'
1733-
) {
1734-
$newToken = [];
1735-
$newToken['code'] = T_ELLIPSIS;
1736-
$newToken['type'] = 'T_ELLIPSIS';
1737-
$newToken['content'] = '...';
1738-
$finalTokens[$newStackPtr] = $newToken;
1739-
1740-
$newStackPtr++;
1741-
$stackPtr += 2;
1742-
continue;
1743-
}
1744-
1745-
/*
1746-
Before PHP 5.6, the ** operator was tokenized as two
1747-
T_MULTIPLY tokens in a row. So look for and combine
1748-
these tokens in earlier versions.
1749-
*/
1750-
1751-
if ($tokenIsArray === false
1752-
&& $token[0] === '*'
1753-
&& isset($tokens[($stackPtr + 1)]) === true
1754-
&& $tokens[($stackPtr + 1)] === '*'
1755-
) {
1756-
$newToken = [];
1757-
$newToken['code'] = T_POW;
1758-
$newToken['type'] = 'T_POW';
1759-
$newToken['content'] = '**';
1760-
$finalTokens[$newStackPtr] = $newToken;
1761-
1762-
$newStackPtr++;
1763-
$stackPtr++;
1764-
continue;
1765-
}
1766-
1767-
/*
1768-
Before PHP 5.6, the **= operator was tokenized as
1769-
T_MULTIPLY followed by T_MUL_EQUAL. So look for and combine
1770-
these tokens in earlier versions.
1771-
*/
1772-
1773-
if ($tokenIsArray === false
1774-
&& $token[0] === '*'
1775-
&& isset($tokens[($stackPtr + 1)]) === true
1776-
&& is_array($tokens[($stackPtr + 1)]) === true
1777-
&& $tokens[($stackPtr + 1)][1] === '*='
1778-
) {
1779-
$newToken = [];
1780-
$newToken['code'] = T_POW_EQUAL;
1781-
$newToken['type'] = 'T_POW_EQUAL';
1782-
$newToken['content'] = '**=';
1783-
$finalTokens[$newStackPtr] = $newToken;
1784-
1785-
$newStackPtr++;
1786-
$stackPtr++;
1787-
continue;
1788-
}
1789-
1790-
/*
1791-
Before PHP 7, the ??= operator was tokenized as
1792-
T_INLINE_THEN, T_INLINE_THEN, T_EQUAL.
17931669
Between PHP 7.0 and 7.3, the ??= operator was tokenized as
17941670
T_COALESCE, T_EQUAL.
17951671
So look for and combine these tokens in earlier versions.
17961672
*/
17971673

1798-
if (($tokenIsArray === false
1799-
&& $token[0] === '?'
1800-
&& isset($tokens[($stackPtr + 1)]) === true
1801-
&& $tokens[($stackPtr + 1)][0] === '?'
1802-
&& isset($tokens[($stackPtr + 2)]) === true
1803-
&& $tokens[($stackPtr + 2)][0] === '=')
1804-
|| ($tokenIsArray === true
1674+
if ($tokenIsArray === true
18051675
&& $token[0] === T_COALESCE
18061676
&& isset($tokens[($stackPtr + 1)]) === true
1807-
&& $tokens[($stackPtr + 1)][0] === '=')
1677+
&& $tokens[($stackPtr + 1)][0] === '='
18081678
) {
18091679
$newToken = [];
18101680
$newToken['code'] = T_COALESCE_EQUAL;
@@ -1815,33 +1685,6 @@ protected function tokenize($string)
18151685
$newStackPtr++;
18161686
$stackPtr++;
18171687

1818-
if ($tokenIsArray === false) {
1819-
// Pre PHP 7.
1820-
$stackPtr++;
1821-
}
1822-
1823-
continue;
1824-
}
1825-
1826-
/*
1827-
Before PHP 7, the ?? operator was tokenized as
1828-
T_INLINE_THEN followed by T_INLINE_THEN.
1829-
So look for and combine these tokens in earlier versions.
1830-
*/
1831-
1832-
if ($tokenIsArray === false
1833-
&& $token[0] === '?'
1834-
&& isset($tokens[($stackPtr + 1)]) === true
1835-
&& $tokens[($stackPtr + 1)][0] === '?'
1836-
) {
1837-
$newToken = [];
1838-
$newToken['code'] = T_COALESCE;
1839-
$newToken['type'] = 'T_COALESCE';
1840-
$newToken['content'] = '??';
1841-
$finalTokens[$newStackPtr] = $newToken;
1842-
1843-
$newStackPtr++;
1844-
$stackPtr++;
18451688
continue;
18461689
}
18471690

@@ -2345,28 +2188,6 @@ function return types. We want to keep the parenthesis map clean,
23452188
}//end if
23462189
}//end if
23472190

2348-
/*
2349-
Before PHP 7, the <=> operator was tokenized as
2350-
T_IS_SMALLER_OR_EQUAL followed by T_GREATER_THAN.
2351-
So look for and combine these tokens in earlier versions.
2352-
*/
2353-
2354-
if ($tokenIsArray === true
2355-
&& $token[0] === T_IS_SMALLER_OR_EQUAL
2356-
&& isset($tokens[($stackPtr + 1)]) === true
2357-
&& $tokens[($stackPtr + 1)][0] === '>'
2358-
) {
2359-
$newToken = [];
2360-
$newToken['code'] = T_SPACESHIP;
2361-
$newToken['type'] = 'T_SPACESHIP';
2362-
$newToken['content'] = '<=>';
2363-
$finalTokens[$newStackPtr] = $newToken;
2364-
2365-
$newStackPtr++;
2366-
$stackPtr++;
2367-
continue;
2368-
}
2369-
23702191
/*
23712192
PHP doesn't assign a token to goto labels, so we have to.
23722193
These are just string tokens with a single colon after them. Double
@@ -2680,16 +2501,6 @@ function return types. We want to keep the parenthesis map clean,
26802501
}
26812502
}
26822503

2683-
// This is a special case when checking PHP 5.5+ code in PHP < 5.5
2684-
// where "finally" should be T_FINALLY instead of T_STRING.
2685-
if ($newToken['code'] === T_STRING
2686-
&& strtolower($newToken['content']) === 'finally'
2687-
&& $finalTokens[$lastNotEmptyToken]['code'] === T_CLOSE_CURLY_BRACKET
2688-
) {
2689-
$newToken['code'] = T_FINALLY;
2690-
$newToken['type'] = 'T_FINALLY';
2691-
}
2692-
26932504
// This is a special case for PHP 5.6 use function and use const
26942505
// where "function" and "const" should be T_STRING instead of T_FUNCTION
26952506
// and T_CONST.

‎src/Util/Tokens.php‎

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -90,46 +90,11 @@
9090
* `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!}
9191
*/
9292

93-
// Some PHP 5.5 tokens, replicated for lower versions.
94-
if (defined('T_FINALLY') === false) {
95-
define('T_FINALLY', 'PHPCS_T_FINALLY');
96-
}
97-
98-
if (defined('T_YIELD') === false) {
99-
define('T_YIELD', 'PHPCS_T_YIELD');
100-
}
101-
102-
// Some PHP 5.6 tokens, replicated for lower versions.
103-
if (defined('T_ELLIPSIS') === false) {
104-
define('T_ELLIPSIS', 'PHPCS_T_ELLIPSIS');
105-
}
106-
107-
if (defined('T_POW') === false) {
108-
define('T_POW', 'PHPCS_T_POW');
109-
}
110-
111-
if (defined('T_POW_EQUAL') === false) {
112-
define('T_POW_EQUAL', 'PHPCS_T_POW_EQUAL');
113-
}
114-
115-
// Some PHP 7 tokens, replicated for lower versions.
116-
if (defined('T_SPACESHIP') === false) {
117-
define('T_SPACESHIP', 'PHPCS_T_SPACESHIP');
118-
}
119-
120-
if (defined('T_COALESCE') === false) {
121-
define('T_COALESCE', 'PHPCS_T_COALESCE');
122-
}
123-
93+
// Some PHP 7.4 tokens, replicated for lower versions.
12494
if (defined('T_COALESCE_EQUAL') === false) {
12595
define('T_COALESCE_EQUAL', 'PHPCS_T_COALESCE_EQUAL');
12696
}
12797

128-
if (defined('T_YIELD_FROM') === false) {
129-
define('T_YIELD_FROM', 'PHPCS_T_YIELD_FROM');
130-
}
131-
132-
// Some PHP 7.4 tokens, replicated for lower versions.
13398
if (defined('T_BAD_CHARACTER') === false) {
13499
define('T_BAD_CHARACTER', 'PHPCS_T_BAD_CHARACTER');
135100
}

‎tests/Core/Tokenizers/PHP/FinallyTest.inc‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,5 @@
11
<?php
22

3-
try {
4-
// Do something.
5-
} catch(Exception $e) {
6-
// Do something.
7-
}
8-
/* testTryCatchFinally */
9-
finally {
10-
// Do something.
11-
}
12-
13-
/* testTryFinallyCatch */
14-
try {
15-
// Do something.
16-
} finally {
17-
// Do something.
18-
} catch(Exception $e) {
19-
// Do something.
20-
}
21-
22-
/* testTryFinally */
23-
try {
24-
// Do something.
25-
} FINALLY {
26-
// Do something.
27-
}
28-
293
class FinallyAsMethod {
304
/* testFinallyUsedAsClassConstantName */
315
const FINALLY = 'foo';

‎tests/Core/Tokenizers/PHP/FinallyTest.php‎

Lines changed: 0 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -15,46 +15,6 @@ final class FinallyTest extends AbstractTokenizerTestCase
1515
{
1616

1717

18-
/**
19-
* Test that the finally keyword is tokenized as such.
20-
*
21-
* @param string $testMarker The comment which prefaces the target token in the test file.
22-
*
23-
* @dataProvider dataFinallyKeyword
24-
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
25-
*
26-
* @return void
27-
*/
28-
public function testFinallyKeyword($testMarker)
29-
{
30-
$tokens = $this->phpcsFile->getTokens();
31-
$target = $this->getTargetToken($testMarker, [T_FINALLY, T_STRING]);
32-
$tokenArray = $tokens[$target];
33-
34-
$this->assertSame(T_FINALLY, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_FINALLY (code)');
35-
$this->assertSame('T_FINALLY', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_FINALLY (type)');
36-
37-
}//end testFinallyKeyword()
38-
39-
40-
/**
41-
* Data provider.
42-
*
43-
* @see testFinallyKeyword()
44-
*
45-
* @return array<string, array<string>>
46-
*/
47-
public static function dataFinallyKeyword()
48-
{
49-
return [
50-
'finally after try and catch' => ['/* testTryCatchFinally */'],
51-
'finally between try and catch' => ['/* testTryFinallyCatch */'],
52-
'finally after try, no catch' => ['/* testTryFinally */'],
53-
];
54-
55-
}//end dataFinallyKeyword()
56-
57-
5818
/**
5919
* Test that 'finally' when not used as the reserved keyword is tokenized as `T_STRING`.
6020
*

‎tests/Core/Tokenizers/PHP/YieldTest.inc‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,6 @@
22

33
function generator()
44
{
5-
/* testYield */
6-
yield 1;
7-
8-
/* testYieldFollowedByComment */
9-
YIELD/*comment*/ 2;
10-
115
/* testYieldFrom */
126
yield from gen2();
137

0 commit comments

Comments
 (0)