Skip to content

Commit 3b4c0be

Browse files
jrfnljrfnl
committed
Tokenizer/PHP: add tests for tokenization of yield and yield from
.. to document and safeguard the existing behaviour.
1 parent c16696c commit 3b4c0be

File tree

2 files changed

+285
-0
lines changed

2 files changed

+285
-0
lines changed

‎tests/Core/Tokenizers/PHP/YieldTest.inc‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
<?php
2+
3+
function generator()
4+
{
5+
/* testYield */
6+
yield 1;
7+
8+
/* testYieldFollowedByComment */
9+
YIELD/*comment*/ 2;
10+
11+
/* testYieldFrom */
12+
yield from gen2();
13+
14+
/* testYieldFromWithExtraSpacesBetween */
15+
Yield From gen2();
16+
17+
/* testYieldFromWithTabBetween */
18+
yield from gen2();
19+
20+
/* testYieldFromSplitByNewLines */
21+
yield
22+
23+
FROM
24+
gen2();
25+
}
26+
27+
/* testYieldUsedAsClassName */
28+
class Yield {
29+
/* testYieldUsedAsClassConstantName */
30+
const Type YIELD = 'foo';
31+
32+
/* testYieldUsedAsMethodName */
33+
public function yield() {
34+
/* testYieldUsedAsPropertyName1 */
35+
echo $obj->yield;
36+
37+
/* testYieldUsedAsPropertyName2 */
38+
echo $obj?->yield();
39+
40+
/* testYieldUsedForClassConstantAccess1 */
41+
echo MyClass::YIELD;
42+
/* testFromUsedForClassConstantAccess1 */
43+
echo MyClass::FROM;
44+
}
45+
46+
/* testYieldUsedAsMethodNameReturnByRef */
47+
public function &yield() {}
48+
}
49+
50+
function myGen() {
51+
/* testYieldLiveCoding */
52+
yield

‎tests/Core/Tokenizers/PHP/YieldTest.php‎

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of the `yield` and `yield from` keywords.
4+
*
5+
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
6+
* @copyright 2021 Squiz Pty Ltd (ABN 77 084 670 600)
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
11+
12+
use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
13+
use PHP_CodeSniffer\Util\Tokens;
14+
15+
/**
16+
* Tests the tokenization of the `yield` and `yield from` keywords.
17+
*
18+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
19+
*/
20+
final class YieldTest extends AbstractTokenizerTestCase
21+
{
22+
23+
24+
/**
25+
* Test that the yield keyword is tokenized as such.
26+
*
27+
* @param string $testMarker The comment which prefaces the target token in the test file.
28+
*
29+
* @dataProvider dataYieldKeyword
30+
*
31+
* @return void
32+
*/
33+
public function testYieldKeyword($testMarker)
34+
{
35+
$tokens = $this->phpcsFile->getTokens();
36+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING]);
37+
$tokenArray = $tokens[$target];
38+
39+
$this->assertSame(T_YIELD, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD (code)');
40+
$this->assertSame('T_YIELD', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD (type)');
41+
42+
}//end testYieldKeyword()
43+
44+
45+
/**
46+
* Data provider.
47+
*
48+
* @see testYieldKeyword()
49+
*
50+
* @return array<string, array<string>>
51+
*/
52+
public static function dataYieldKeyword()
53+
{
54+
return [
55+
'yield' => ['/* testYield */'],
56+
'yield followed by comment' => ['/* testYieldFollowedByComment */'],
57+
'yield at end of file, live coding' => ['/* testYieldLiveCoding */'],
58+
];
59+
60+
}//end dataYieldKeyword()
61+
62+
63+
/**
64+
* Test that the yield from keyword is tokenized as a single token when it in on a single line
65+
* and only has whitespace between the words.
66+
*
67+
* @param string $testMarker The comment which prefaces the target token in the test file.
68+
* @param string $content Optional. The test token content to search for.
69+
* Defaults to null.
70+
*
71+
* @dataProvider dataYieldFromKeywordSingleToken
72+
*
73+
* @return void
74+
*/
75+
public function testYieldFromKeywordSingleToken($testMarker, $content=null)
76+
{
77+
$tokens = $this->phpcsFile->getTokens();
78+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING], $content);
79+
$tokenArray = $tokens[$target];
80+
81+
$this->assertSame(T_YIELD_FROM, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (code)');
82+
$this->assertSame('T_YIELD_FROM', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (type)');
83+
84+
}//end testYieldFromKeywordSingleToken()
85+
86+
87+
/**
88+
* Data provider.
89+
*
90+
* @see testYieldFromKeywordSingleToken()
91+
*
92+
* @return array<string, array<string>>
93+
*/
94+
public static function dataYieldFromKeywordSingleToken()
95+
{
96+
return [
97+
'yield from' => [
98+
'testMarker' => '/* testYieldFrom */',
99+
],
100+
'yield from with extra space between' => [
101+
'testMarker' => '/* testYieldFromWithExtraSpacesBetween */',
102+
],
103+
'yield from with tab between' => [
104+
'testMarker' => '/* testYieldFromWithTabBetween */',
105+
],
106+
];
107+
108+
}//end dataYieldFromKeywordSingleToken()
109+
110+
111+
/**
112+
* Test that the yield from keyword is tokenized as a single token when it in on a single line
113+
* and only has whitespace between the words.
114+
*
115+
* @param string $testMarker The comment which prefaces the target token in the test file.
116+
* @param array<array<string, string>> $expectedTokens The tokenization expected.
117+
*
118+
* @dataProvider dataYieldFromKeywordMultiToken
119+
*
120+
* @return void
121+
*/
122+
public function testYieldFromKeywordMultiToken($testMarker, $expectedTokens)
123+
{
124+
$tokens = $this->phpcsFile->getTokens();
125+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING]);
126+
127+
foreach ($expectedTokens as $nr => $tokenInfo) {
128+
$this->assertSame(
129+
constant($tokenInfo['type']),
130+
$tokens[$target]['code'],
131+
'Token tokenized as '.Tokens::tokenName($tokens[$target]['code']).', not '.$tokenInfo['type'].' (code)'
132+
);
133+
$this->assertSame(
134+
$tokenInfo['type'],
135+
$tokens[$target]['type'],
136+
'Token tokenized as '.$tokens[$target]['type'].', not '.$tokenInfo['type'].' (type)'
137+
);
138+
$this->assertSame(
139+
$tokenInfo['content'],
140+
$tokens[$target]['content'],
141+
'Content of token '.($nr + 1).' ('.$tokens[$target]['type'].') does not match expectations'
142+
);
143+
144+
++$target;
145+
}
146+
147+
}//end testYieldFromKeywordMultiToken()
148+
149+
150+
/**
151+
* Data provider.
152+
*
153+
* @see testYieldFromKeywordMultiToken()
154+
*
155+
* @return array<string, array<string, string|array<array<string, string>>>>
156+
*/
157+
public static function dataYieldFromKeywordMultiToken()
158+
{
159+
return [
160+
'yield from with new line' => [
161+
'testMarker' => '/* testYieldFromSplitByNewLines */',
162+
'expectedTokens' => [
163+
[
164+
'type' => 'T_YIELD_FROM',
165+
'content' => 'yield
166+
',
167+
],
168+
[
169+
'type' => 'T_YIELD_FROM',
170+
'content' => '
171+
',
172+
],
173+
[
174+
'type' => 'T_YIELD_FROM',
175+
'content' => ' FROM',
176+
],
177+
[
178+
'type' => 'T_WHITESPACE',
179+
'content' => '
180+
',
181+
],
182+
],
183+
],
184+
];
185+
186+
}//end dataYieldFromKeywordMultiToken()
187+
188+
189+
/**
190+
* Test that 'yield' or 'from' when not used as the reserved keyword are tokenized as `T_STRING`.
191+
*
192+
* @param string $testMarker The comment which prefaces the target token in the test file.
193+
*
194+
* @dataProvider dataYieldNonKeyword
195+
*
196+
* @return void
197+
*/
198+
public function testYieldNonKeyword($testMarker)
199+
{
200+
$tokens = $this->phpcsFile->getTokens();
201+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING]);
202+
$tokenArray = $tokens[$target];
203+
204+
$this->assertSame(T_STRING, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (code)');
205+
$this->assertSame('T_STRING', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (type)');
206+
207+
}//end testYieldNonKeyword()
208+
209+
210+
/**
211+
* Data provider.
212+
*
213+
* @see testYieldNonKeyword()
214+
*
215+
* @return array<string, array<string>>
216+
*/
217+
public static function dataYieldNonKeyword()
218+
{
219+
return [
220+
'yield used as class name' => ['/* testYieldUsedAsClassName */'],
221+
'yield used as class constant name' => ['/* testYieldUsedAsClassConstantName */'],
222+
'yield used as method name' => ['/* testYieldUsedAsMethodName */'],
223+
'yield used as property access 1' => ['/* testYieldUsedAsPropertyName1 */'],
224+
'yield used as property access 2' => ['/* testYieldUsedAsPropertyName2 */'],
225+
'yield used as class constant access' => ['/* testYieldUsedForClassConstantAccess1 */'],
226+
'from used as class constant access' => ['/* testFromUsedForClassConstantAccess1 */'],
227+
'yield used as method name with ref' => ['/* testYieldUsedAsMethodNameReturnByRef */'],
228+
];
229+
230+
}//end dataYieldNonKeyword()
231+
232+
233+
}//end class

0 commit comments

Comments
 (0)