1<?php 2/** 3 * Tokenizes PHP code. 4 * 5 * @author Greg Sherwood <gsherwood@squiz.net> 6 * @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600) 7 * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence 8 */ 9 10namespace PHP_CodeSniffer\Tokenizers; 11 12use PHP_CodeSniffer\Util; 13 14class PHP extends Tokenizer 15{ 16 17 /** 18 * A list of tokens that are allowed to open a scope. 19 * 20 * This array also contains information about what kind of token the scope 21 * opener uses to open and close the scope, if the token strictly requires 22 * an opener, if the token can share a scope closer, and who it can be shared 23 * with. An example of a token that shares a scope closer is a CASE scope. 24 * 25 * @var array 26 */ 27 public $scopeOpeners = [ 28 T_IF => [ 29 'start' => [ 30 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET, 31 T_COLON => T_COLON, 32 ], 33 'end' => [ 34 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, 35 T_ENDIF => T_ENDIF, 36 T_ELSE => T_ELSE, 37 T_ELSEIF => T_ELSEIF, 38 ], 39 'strict' => false, 40 'shared' => false, 41 'with' => [ 42 T_ELSE => T_ELSE, 43 T_ELSEIF => T_ELSEIF, 44 ], 45 ], 46 T_TRY => [ 47 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 48 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 49 'strict' => true, 50 'shared' => false, 51 'with' => [], 52 ], 53 T_CATCH => [ 54 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 55 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 56 'strict' => true, 57 'shared' => false, 58 'with' => [], 59 ], 60 T_FINALLY => [ 61 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 62 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 63 'strict' => true, 64 'shared' => false, 65 'with' => [], 66 ], 67 T_ELSE => [ 68 'start' => [ 69 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET, 70 T_COLON => T_COLON, 71 ], 72 'end' => [ 73 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, 74 T_ENDIF => T_ENDIF, 75 ], 76 'strict' => false, 77 'shared' => false, 78 'with' => [ 79 T_IF => T_IF, 80 T_ELSEIF => T_ELSEIF, 81 ], 82 ], 83 T_ELSEIF => [ 84 'start' => [ 85 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET, 86 T_COLON => T_COLON, 87 ], 88 'end' => [ 89 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, 90 T_ENDIF => T_ENDIF, 91 T_ELSE => T_ELSE, 92 T_ELSEIF => T_ELSEIF, 93 ], 94 'strict' => false, 95 'shared' => false, 96 'with' => [ 97 T_IF => T_IF, 98 T_ELSE => T_ELSE, 99 ], 100 ], 101 T_FOR => [ 102 'start' => [ 103 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET, 104 T_COLON => T_COLON, 105 ], 106 'end' => [ 107 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, 108 T_ENDFOR => T_ENDFOR, 109 ], 110 'strict' => false, 111 'shared' => false, 112 'with' => [], 113 ], 114 T_FOREACH => [ 115 'start' => [ 116 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET, 117 T_COLON => T_COLON, 118 ], 119 'end' => [ 120 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, 121 T_ENDFOREACH => T_ENDFOREACH, 122 ], 123 'strict' => false, 124 'shared' => false, 125 'with' => [], 126 ], 127 T_INTERFACE => [ 128 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 129 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 130 'strict' => true, 131 'shared' => false, 132 'with' => [], 133 ], 134 T_FUNCTION => [ 135 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 136 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 137 'strict' => true, 138 'shared' => false, 139 'with' => [], 140 ], 141 T_CLASS => [ 142 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 143 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 144 'strict' => true, 145 'shared' => false, 146 'with' => [], 147 ], 148 T_TRAIT => [ 149 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 150 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 151 'strict' => true, 152 'shared' => false, 153 'with' => [], 154 ], 155 T_USE => [ 156 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 157 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 158 'strict' => false, 159 'shared' => false, 160 'with' => [], 161 ], 162 T_DECLARE => [ 163 'start' => [ 164 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET, 165 T_COLON => T_COLON, 166 ], 167 'end' => [ 168 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, 169 T_ENDDECLARE => T_ENDDECLARE, 170 ], 171 'strict' => false, 172 'shared' => false, 173 'with' => [], 174 ], 175 T_NAMESPACE => [ 176 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 177 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 178 'strict' => false, 179 'shared' => false, 180 'with' => [], 181 ], 182 T_WHILE => [ 183 'start' => [ 184 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET, 185 T_COLON => T_COLON, 186 ], 187 'end' => [ 188 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, 189 T_ENDWHILE => T_ENDWHILE, 190 ], 191 'strict' => false, 192 'shared' => false, 193 'with' => [], 194 ], 195 T_DO => [ 196 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 197 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 198 'strict' => true, 199 'shared' => false, 200 'with' => [], 201 ], 202 T_SWITCH => [ 203 'start' => [ 204 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET, 205 T_COLON => T_COLON, 206 ], 207 'end' => [ 208 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, 209 T_ENDSWITCH => T_ENDSWITCH, 210 ], 211 'strict' => true, 212 'shared' => false, 213 'with' => [], 214 ], 215 T_CASE => [ 216 'start' => [ 217 T_COLON => T_COLON, 218 T_SEMICOLON => T_SEMICOLON, 219 ], 220 'end' => [ 221 T_BREAK => T_BREAK, 222 T_RETURN => T_RETURN, 223 T_CONTINUE => T_CONTINUE, 224 T_THROW => T_THROW, 225 T_EXIT => T_EXIT, 226 ], 227 'strict' => true, 228 'shared' => true, 229 'with' => [ 230 T_DEFAULT => T_DEFAULT, 231 T_CASE => T_CASE, 232 T_SWITCH => T_SWITCH, 233 ], 234 ], 235 T_DEFAULT => [ 236 'start' => [ 237 T_COLON => T_COLON, 238 T_SEMICOLON => T_SEMICOLON, 239 ], 240 'end' => [ 241 T_BREAK => T_BREAK, 242 T_RETURN => T_RETURN, 243 T_CONTINUE => T_CONTINUE, 244 T_THROW => T_THROW, 245 T_EXIT => T_EXIT, 246 ], 247 'strict' => true, 248 'shared' => true, 249 'with' => [ 250 T_CASE => T_CASE, 251 T_SWITCH => T_SWITCH, 252 ], 253 ], 254 T_MATCH => [ 255 'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 256 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 257 'strict' => true, 258 'shared' => false, 259 'with' => [], 260 ], 261 T_START_HEREDOC => [ 262 'start' => [T_START_HEREDOC => T_START_HEREDOC], 263 'end' => [T_END_HEREDOC => T_END_HEREDOC], 264 'strict' => true, 265 'shared' => false, 266 'with' => [], 267 ], 268 T_START_NOWDOC => [ 269 'start' => [T_START_NOWDOC => T_START_NOWDOC], 270 'end' => [T_END_NOWDOC => T_END_NOWDOC], 271 'strict' => true, 272 'shared' => false, 273 'with' => [], 274 ], 275 ]; 276 277 /** 278 * A list of tokens that end the scope. 279 * 280 * This array is just a unique collection of the end tokens 281 * from the scopeOpeners array. The data is duplicated here to 282 * save time during parsing of the file. 283 * 284 * @var array 285 */ 286 public $endScopeTokens = [ 287 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, 288 T_ENDIF => T_ENDIF, 289 T_ENDFOR => T_ENDFOR, 290 T_ENDFOREACH => T_ENDFOREACH, 291 T_ENDWHILE => T_ENDWHILE, 292 T_ENDSWITCH => T_ENDSWITCH, 293 T_ENDDECLARE => T_ENDDECLARE, 294 T_BREAK => T_BREAK, 295 T_END_HEREDOC => T_END_HEREDOC, 296 T_END_NOWDOC => T_END_NOWDOC, 297 ]; 298 299 /** 300 * Known lengths of tokens. 301 * 302 * @var array<int, int> 303 */ 304 public $knownLengths = [ 305 T_ABSTRACT => 8, 306 T_AND_EQUAL => 2, 307 T_ARRAY => 5, 308 T_AS => 2, 309 T_BOOLEAN_AND => 2, 310 T_BOOLEAN_OR => 2, 311 T_BREAK => 5, 312 T_CALLABLE => 8, 313 T_CASE => 4, 314 T_CATCH => 5, 315 T_CLASS => 5, 316 T_CLASS_C => 9, 317 T_CLONE => 5, 318 T_CONCAT_EQUAL => 2, 319 T_CONST => 5, 320 T_CONTINUE => 8, 321 T_CURLY_OPEN => 2, 322 T_DEC => 2, 323 T_DECLARE => 7, 324 T_DEFAULT => 7, 325 T_DIR => 7, 326 T_DIV_EQUAL => 2, 327 T_DO => 2, 328 T_DOLLAR_OPEN_CURLY_BRACES => 2, 329 T_DOUBLE_ARROW => 2, 330 T_DOUBLE_COLON => 2, 331 T_ECHO => 4, 332 T_ELLIPSIS => 3, 333 T_ELSE => 4, 334 T_ELSEIF => 6, 335 T_EMPTY => 5, 336 T_ENDDECLARE => 10, 337 T_ENDFOR => 6, 338 T_ENDFOREACH => 10, 339 T_ENDIF => 5, 340 T_ENDSWITCH => 9, 341 T_ENDWHILE => 8, 342 T_EVAL => 4, 343 T_EXTENDS => 7, 344 T_FILE => 8, 345 T_FINAL => 5, 346 T_FINALLY => 7, 347 T_FN => 2, 348 T_FOR => 3, 349 T_FOREACH => 7, 350 T_FUNCTION => 8, 351 T_FUNC_C => 12, 352 T_GLOBAL => 6, 353 T_GOTO => 4, 354 T_HALT_COMPILER => 15, 355 T_IF => 2, 356 T_IMPLEMENTS => 10, 357 T_INC => 2, 358 T_INCLUDE => 7, 359 T_INCLUDE_ONCE => 12, 360 T_INSTANCEOF => 10, 361 T_INSTEADOF => 9, 362 T_INTERFACE => 9, 363 T_ISSET => 5, 364 T_IS_EQUAL => 2, 365 T_IS_GREATER_OR_EQUAL => 2, 366 T_IS_IDENTICAL => 3, 367 T_IS_NOT_EQUAL => 2, 368 T_IS_NOT_IDENTICAL => 3, 369 T_IS_SMALLER_OR_EQUAL => 2, 370 T_LINE => 8, 371 T_LIST => 4, 372 T_LOGICAL_AND => 3, 373 T_LOGICAL_OR => 2, 374 T_LOGICAL_XOR => 3, 375 T_MATCH => 5, 376 T_MATCH_ARROW => 2, 377 T_MATCH_DEFAULT => 7, 378 T_METHOD_C => 10, 379 T_MINUS_EQUAL => 2, 380 T_POW_EQUAL => 3, 381 T_MOD_EQUAL => 2, 382 T_MUL_EQUAL => 2, 383 T_NAMESPACE => 9, 384 T_NS_C => 13, 385 T_NS_SEPARATOR => 1, 386 T_NEW => 3, 387 T_NULLSAFE_OBJECT_OPERATOR => 3, 388 T_OBJECT_OPERATOR => 2, 389 T_OPEN_TAG_WITH_ECHO => 3, 390 T_OR_EQUAL => 2, 391 T_PLUS_EQUAL => 2, 392 T_PRINT => 5, 393 T_PRIVATE => 7, 394 T_PUBLIC => 6, 395 T_PROTECTED => 9, 396 T_REQUIRE => 7, 397 T_REQUIRE_ONCE => 12, 398 T_RETURN => 6, 399 T_STATIC => 6, 400 T_SWITCH => 6, 401 T_THROW => 5, 402 T_TRAIT => 5, 403 T_TRAIT_C => 9, 404 T_TRY => 3, 405 T_UNSET => 5, 406 T_USE => 3, 407 T_VAR => 3, 408 T_WHILE => 5, 409 T_XOR_EQUAL => 2, 410 T_YIELD => 5, 411 T_OPEN_CURLY_BRACKET => 1, 412 T_CLOSE_CURLY_BRACKET => 1, 413 T_OPEN_SQUARE_BRACKET => 1, 414 T_CLOSE_SQUARE_BRACKET => 1, 415 T_OPEN_PARENTHESIS => 1, 416 T_CLOSE_PARENTHESIS => 1, 417 T_COLON => 1, 418 T_STRING_CONCAT => 1, 419 T_INLINE_THEN => 1, 420 T_INLINE_ELSE => 1, 421 T_NULLABLE => 1, 422 T_NULL => 4, 423 T_FALSE => 5, 424 T_TRUE => 4, 425 T_SEMICOLON => 1, 426 T_EQUAL => 1, 427 T_MULTIPLY => 1, 428 T_DIVIDE => 1, 429 T_PLUS => 1, 430 T_MINUS => 1, 431 T_MODULUS => 1, 432 T_POW => 2, 433 T_SPACESHIP => 3, 434 T_COALESCE => 2, 435 T_COALESCE_EQUAL => 3, 436 T_BITWISE_AND => 1, 437 T_BITWISE_OR => 1, 438 T_BITWISE_XOR => 1, 439 T_SL => 2, 440 T_SR => 2, 441 T_SL_EQUAL => 3, 442 T_SR_EQUAL => 3, 443 T_GREATER_THAN => 1, 444 T_LESS_THAN => 1, 445 T_BOOLEAN_NOT => 1, 446 T_SELF => 4, 447 T_PARENT => 6, 448 T_COMMA => 1, 449 T_THIS => 4, 450 T_CLOSURE => 8, 451 T_BACKTICK => 1, 452 T_OPEN_SHORT_ARRAY => 1, 453 T_CLOSE_SHORT_ARRAY => 1, 454 T_TYPE_UNION => 1, 455 ]; 456 457 /** 458 * Contexts in which keywords should always be tokenized as T_STRING. 459 * 460 * @var array 461 */ 462 protected $tstringContexts = [ 463 T_OBJECT_OPERATOR => true, 464 T_NULLSAFE_OBJECT_OPERATOR => true, 465 T_FUNCTION => true, 466 T_CLASS => true, 467 T_INTERFACE => true, 468 T_TRAIT => true, 469 T_EXTENDS => true, 470 T_IMPLEMENTS => true, 471 T_ATTRIBUTE => true, 472 T_NEW => true, 473 T_CONST => true, 474 T_NS_SEPARATOR => true, 475 T_USE => true, 476 T_NAMESPACE => true, 477 T_PAAMAYIM_NEKUDOTAYIM => true, 478 ]; 479 480 /** 481 * A cache of different token types, resolved into arrays. 482 * 483 * @var array 484 * @see standardiseToken() 485 */ 486 private static $resolveTokenCache = []; 487 488 489 /** 490 * Creates an array of tokens when given some PHP code. 491 * 492 * Starts by using token_get_all() but does a lot of extra processing 493 * to insert information about the context of the token. 494 * 495 * @param string $string The string to tokenize. 496 * 497 * @return array 498 */ 499 protected function tokenize($string) 500 { 501 if (PHP_CODESNIFFER_VERBOSITY > 1) { 502 echo "\t*** START PHP TOKENIZING ***".PHP_EOL; 503 $isWin = false; 504 if (strtoupper(substr(PHP_OS, 0, 3)) === 'WIN') { 505 $isWin = true; 506 } 507 } 508 509 $tokens = @token_get_all($string); 510 $finalTokens = []; 511 512 $newStackPtr = 0; 513 $numTokens = count($tokens); 514 $lastNotEmptyToken = 0; 515 516 $insideInlineIf = []; 517 $insideUseGroup = false; 518 519 $commentTokenizer = new Comment(); 520 521 for ($stackPtr = 0; $stackPtr < $numTokens; $stackPtr++) { 522 // Special case for tokens we have needed to blank out. 523 if ($tokens[$stackPtr] === null) { 524 continue; 525 } 526 527 $token = (array) $tokens[$stackPtr]; 528 $tokenIsArray = isset($token[1]); 529 530 if (PHP_CODESNIFFER_VERBOSITY > 1) { 531 if ($tokenIsArray === true) { 532 $type = Util\Tokens::tokenName($token[0]); 533 $content = Util\Common::prepareForOutput($token[1]); 534 } else { 535 $newToken = self::resolveSimpleToken($token[0]); 536 $type = $newToken['type']; 537 $content = Util\Common::prepareForOutput($token[0]); 538 } 539 540 echo "\tProcess token "; 541 if ($tokenIsArray === true) { 542 echo "[$stackPtr]"; 543 } else { 544 echo " $stackPtr "; 545 } 546 547 echo ": $type => $content"; 548 }//end if 549 550 if ($newStackPtr > 0 551 && isset(Util\Tokens::$emptyTokens[$finalTokens[($newStackPtr - 1)]['code']]) === false 552 ) { 553 $lastNotEmptyToken = ($newStackPtr - 1); 554 } 555 556 /* 557 If we are using \r\n newline characters, the \r and \n are sometimes 558 split over two tokens. This normally occurs after comments. We need 559 to merge these two characters together so that our line endings are 560 consistent for all lines. 561 */ 562 563 if ($tokenIsArray === true && substr($token[1], -1) === "\r") { 564 if (isset($tokens[($stackPtr + 1)]) === true 565 && is_array($tokens[($stackPtr + 1)]) === true 566 && $tokens[($stackPtr + 1)][1][0] === "\n" 567 ) { 568 $token[1] .= "\n"; 569 if (PHP_CODESNIFFER_VERBOSITY > 1) { 570 if ($isWin === true) { 571 echo '\n'; 572 } else { 573 echo "\033[30;1m\\n\033[0m"; 574 } 575 } 576 577 if ($tokens[($stackPtr + 1)][1] === "\n") { 578 // This token's content has been merged into the previous, 579 // so we can skip it. 580 $tokens[($stackPtr + 1)] = ''; 581 } else { 582 $tokens[($stackPtr + 1)][1] = substr($tokens[($stackPtr + 1)][1], 1); 583 } 584 } 585 }//end if 586 587 if (PHP_CODESNIFFER_VERBOSITY > 1) { 588 echo PHP_EOL; 589 } 590 591 /* 592 Parse doc blocks into something that can be easily iterated over. 593 */ 594 595 if ($tokenIsArray === true 596 && ($token[0] === T_DOC_COMMENT 597 || ($token[0] === T_COMMENT && strpos($token[1], '/**') === 0)) 598 ) { 599 $commentTokens = $commentTokenizer->tokenizeString($token[1], $this->eolChar, $newStackPtr); 600 foreach ($commentTokens as $commentToken) { 601 $finalTokens[$newStackPtr] = $commentToken; 602 $newStackPtr++; 603 } 604 605 continue; 606 } 607 608 /* 609 PHP 8 tokenizes a new line after a slash and hash comment to the next whitespace token. 610 */ 611 612 if (PHP_VERSION_ID >= 80000 613 && $tokenIsArray === true 614 && ($token[0] === T_COMMENT && (strpos($token[1], '//') === 0 || strpos($token[1], '#') === 0)) 615 && isset($tokens[($stackPtr + 1)]) === true 616 && is_array($tokens[($stackPtr + 1)]) === true 617 && $tokens[($stackPtr + 1)][0] === T_WHITESPACE 618 ) { 619 $nextToken = $tokens[($stackPtr + 1)]; 620 621 // If the next token is a single new line, merge it into the comment token 622 // and set to it up to be skipped. 623 if ($nextToken[1] === "\n" || $nextToken[1] === "\r\n" || $nextToken[1] === "\n\r") { 624 $token[1] .= $nextToken[1]; 625 $tokens[($stackPtr + 1)] = null; 626 627 if (PHP_CODESNIFFER_VERBOSITY > 1) { 628 echo "\t\t* merged newline after comment into comment token $stackPtr".PHP_EOL; 629 } 630 } else { 631 // This may be a whitespace token consisting of multiple new lines. 632 if (strpos($nextToken[1], "\r\n") === 0) { 633 $token[1] .= "\r\n"; 634 $tokens[($stackPtr + 1)][1] = substr($nextToken[1], 2); 635 } else if (strpos($nextToken[1], "\n\r") === 0) { 636 $token[1] .= "\n\r"; 637 $tokens[($stackPtr + 1)][1] = substr($nextToken[1], 2); 638 } else if (strpos($nextToken[1], "\n") === 0) { 639 $token[1] .= "\n"; 640 $tokens[($stackPtr + 1)][1] = substr($nextToken[1], 1); 641 } 642 643 if (PHP_CODESNIFFER_VERBOSITY > 1) { 644 echo "\t\t* stripped first newline after comment and added it to comment token $stackPtr".PHP_EOL; 645 } 646 }//end if 647 }//end if 648 649 /* 650 PHP 8.1 introduced two dedicated tokens for the & character. 651 Retokenizing both of these to T_BITWISE_AND, which is the 652 token PHPCS already tokenized them as. 653 */ 654 655 if ($tokenIsArray === true 656 && ($token[0] === T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG 657 || $token[0] === T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG) 658 ) { 659 $finalTokens[$newStackPtr] = [ 660 'code' => T_BITWISE_AND, 661 'type' => 'T_BITWISE_AND', 662 'content' => $token[1], 663 ]; 664 $newStackPtr++; 665 continue; 666 } 667 668 /* 669 If this is a double quoted string, PHP will tokenize the whole 670 thing which causes problems with the scope map when braces are 671 within the string. So we need to merge the tokens together to 672 provide a single string. 673 */ 674 675 if ($tokenIsArray === false && ($token[0] === '"' || $token[0] === 'b"')) { 676 // Binary casts need a special token. 677 if ($token[0] === 'b"') { 678 $finalTokens[$newStackPtr] = [ 679 'code' => T_BINARY_CAST, 680 'type' => 'T_BINARY_CAST', 681 'content' => 'b', 682 ]; 683 $newStackPtr++; 684 } 685 686 $tokenContent = '"'; 687 $nestedVars = []; 688 for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { 689 $subToken = (array) $tokens[$i]; 690 $subTokenIsArray = isset($subToken[1]); 691 692 if ($subTokenIsArray === true) { 693 $tokenContent .= $subToken[1]; 694 if ($subToken[1] === '{' 695 && $subToken[0] !== T_ENCAPSED_AND_WHITESPACE 696 ) { 697 $nestedVars[] = $i; 698 } 699 } else { 700 $tokenContent .= $subToken[0]; 701 if ($subToken[0] === '}') { 702 array_pop($nestedVars); 703 } 704 } 705 706 if ($subTokenIsArray === false 707 && $subToken[0] === '"' 708 && empty($nestedVars) === true 709 ) { 710 // We found the other end of the double quoted string. 711 break; 712 } 713 }//end for 714 715 $stackPtr = $i; 716 717 // Convert each line within the double quoted string to a 718 // new token, so it conforms with other multiple line tokens. 719 $tokenLines = explode($this->eolChar, $tokenContent); 720 $numLines = count($tokenLines); 721 $newToken = []; 722 723 for ($j = 0; $j < $numLines; $j++) { 724 $newToken['content'] = $tokenLines[$j]; 725 if ($j === ($numLines - 1)) { 726 if ($tokenLines[$j] === '') { 727 break; 728 } 729 } else { 730 $newToken['content'] .= $this->eolChar; 731 } 732 733 $newToken['code'] = T_DOUBLE_QUOTED_STRING; 734 $newToken['type'] = 'T_DOUBLE_QUOTED_STRING'; 735 $finalTokens[$newStackPtr] = $newToken; 736 $newStackPtr++; 737 } 738 739 // Continue, as we're done with this token. 740 continue; 741 }//end if 742 743 /* 744 Detect binary casting and assign the casts their own token. 745 */ 746 747 if ($tokenIsArray === true 748 && $token[0] === T_CONSTANT_ENCAPSED_STRING 749 && (substr($token[1], 0, 2) === 'b"' 750 || substr($token[1], 0, 2) === "b'") 751 ) { 752 $finalTokens[$newStackPtr] = [ 753 'code' => T_BINARY_CAST, 754 'type' => 'T_BINARY_CAST', 755 'content' => 'b', 756 ]; 757 $newStackPtr++; 758 $token[1] = substr($token[1], 1); 759 } 760 761 if ($tokenIsArray === true 762 && $token[0] === T_STRING_CAST 763 && preg_match('`^\(\s*binary\s*\)$`i', $token[1]) === 1 764 ) { 765 $finalTokens[$newStackPtr] = [ 766 'code' => T_BINARY_CAST, 767 'type' => 'T_BINARY_CAST', 768 'content' => $token[1], 769 ]; 770 $newStackPtr++; 771 continue; 772 } 773 774 /* 775 If this is a heredoc, PHP will tokenize the whole 776 thing which causes problems when heredocs don't 777 contain real PHP code, which is almost never. 778 We want to leave the start and end heredoc tokens 779 alone though. 780 */ 781 782 if ($tokenIsArray === true && $token[0] === T_START_HEREDOC) { 783 // Add the start heredoc token to the final array. 784 $finalTokens[$newStackPtr] = self::standardiseToken($token); 785 786 // Check if this is actually a nowdoc and use a different token 787 // to help the sniffs. 788 $nowdoc = false; 789 if (strpos($token[1], "'") !== false) { 790 $finalTokens[$newStackPtr]['code'] = T_START_NOWDOC; 791 $finalTokens[$newStackPtr]['type'] = 'T_START_NOWDOC'; 792 $nowdoc = true; 793 } 794 795 $tokenContent = ''; 796 for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { 797 $subTokenIsArray = is_array($tokens[$i]); 798 if ($subTokenIsArray === true 799 && $tokens[$i][0] === T_END_HEREDOC 800 ) { 801 // We found the other end of the heredoc. 802 break; 803 } 804 805 if ($subTokenIsArray === true) { 806 $tokenContent .= $tokens[$i][1]; 807 } else { 808 $tokenContent .= $tokens[$i]; 809 } 810 } 811 812 if ($i === $numTokens) { 813 // We got to the end of the file and never 814 // found the closing token, so this probably wasn't 815 // a heredoc. 816 if (PHP_CODESNIFFER_VERBOSITY > 1) { 817 $type = $finalTokens[$newStackPtr]['type']; 818 echo "\t\t* failed to find the end of the here/nowdoc".PHP_EOL; 819 echo "\t\t* token $stackPtr changed from $type to T_STRING".PHP_EOL; 820 } 821 822 $finalTokens[$newStackPtr]['code'] = T_STRING; 823 $finalTokens[$newStackPtr]['type'] = 'T_STRING'; 824 $newStackPtr++; 825 continue; 826 } 827 828 $stackPtr = $i; 829 $newStackPtr++; 830 831 // Convert each line within the heredoc to a 832 // new token, so it conforms with other multiple line tokens. 833 $tokenLines = explode($this->eolChar, $tokenContent); 834 $numLines = count($tokenLines); 835 $newToken = []; 836 837 for ($j = 0; $j < $numLines; $j++) { 838 $newToken['content'] = $tokenLines[$j]; 839 if ($j === ($numLines - 1)) { 840 if ($tokenLines[$j] === '') { 841 break; 842 } 843 } else { 844 $newToken['content'] .= $this->eolChar; 845 } 846 847 if ($nowdoc === true) { 848 $newToken['code'] = T_NOWDOC; 849 $newToken['type'] = 'T_NOWDOC'; 850 } else { 851 $newToken['code'] = T_HEREDOC; 852 $newToken['type'] = 'T_HEREDOC'; 853 } 854 855 $finalTokens[$newStackPtr] = $newToken; 856 $newStackPtr++; 857 }//end for 858 859 // Add the end heredoc token to the final array. 860 $finalTokens[$newStackPtr] = self::standardiseToken($tokens[$stackPtr]); 861 862 if ($nowdoc === true) { 863 $finalTokens[$newStackPtr]['code'] = T_END_NOWDOC; 864 $finalTokens[$newStackPtr]['type'] = 'T_END_NOWDOC'; 865 } 866 867 $newStackPtr++; 868 869 // Continue, as we're done with this token. 870 continue; 871 }//end if 872 873 /* 874 As of PHP 8.0 fully qualified, partially qualified and namespace relative 875 identifier names are tokenized differently. 876 This "undoes" the new tokenization so the tokenization will be the same in 877 in PHP 5, 7 and 8. 878 */ 879 880 if (PHP_VERSION_ID >= 80000 881 && $tokenIsArray === true 882 && ($token[0] === T_NAME_QUALIFIED 883 || $token[0] === T_NAME_FULLY_QUALIFIED 884 || $token[0] === T_NAME_RELATIVE) 885 ) { 886 $name = $token[1]; 887 888 if ($token[0] === T_NAME_FULLY_QUALIFIED) { 889 $newToken = []; 890 $newToken['code'] = T_NS_SEPARATOR; 891 $newToken['type'] = 'T_NS_SEPARATOR'; 892 $newToken['content'] = '\\'; 893 $finalTokens[$newStackPtr] = $newToken; 894 ++$newStackPtr; 895 896 $name = ltrim($name, '\\'); 897 } 898 899 if ($token[0] === T_NAME_RELATIVE) { 900 $newToken = []; 901 $newToken['code'] = T_NAMESPACE; 902 $newToken['type'] = 'T_NAMESPACE'; 903 $newToken['content'] = substr($name, 0, 9); 904 $finalTokens[$newStackPtr] = $newToken; 905 ++$newStackPtr; 906 907 $newToken = []; 908 $newToken['code'] = T_NS_SEPARATOR; 909 $newToken['type'] = 'T_NS_SEPARATOR'; 910 $newToken['content'] = '\\'; 911 $finalTokens[$newStackPtr] = $newToken; 912 ++$newStackPtr; 913 914 $name = substr($name, 10); 915 } 916 917 $parts = explode('\\', $name); 918 $partCount = count($parts); 919 $lastPart = ($partCount - 1); 920 921 foreach ($parts as $i => $part) { 922 $newToken = []; 923 $newToken['code'] = T_STRING; 924 $newToken['type'] = 'T_STRING'; 925 $newToken['content'] = $part; 926 $finalTokens[$newStackPtr] = $newToken; 927 ++$newStackPtr; 928 929 if ($i !== $lastPart) { 930 $newToken = []; 931 $newToken['code'] = T_NS_SEPARATOR; 932 $newToken['type'] = 'T_NS_SEPARATOR'; 933 $newToken['content'] = '\\'; 934 $finalTokens[$newStackPtr] = $newToken; 935 ++$newStackPtr; 936 } 937 } 938 939 if (PHP_CODESNIFFER_VERBOSITY > 1) { 940 $type = Util\Tokens::tokenName($token[0]); 941 $content = Util\Common::prepareForOutput($token[1]); 942 echo "\t\t* token $stackPtr split into individual tokens; was: $type => $content".PHP_EOL; 943 } 944 945 continue; 946 }//end if 947 948 /* 949 PHP 8.0 Attributes 950 */ 951 952 if (PHP_VERSION_ID < 80000 953 && $token[0] === T_COMMENT 954 && strpos($token[1], '#[') === 0 955 ) { 956 $subTokens = $this->parsePhpAttribute($tokens, $stackPtr); 957 if ($subTokens !== null) { 958 array_splice($tokens, $stackPtr, 1, $subTokens); 959 $numTokens = count($tokens); 960 961 $tokenIsArray = true; 962 $token = $tokens[$stackPtr]; 963 } else { 964 $token[0] = T_ATTRIBUTE; 965 } 966 } 967 968 if ($tokenIsArray === true 969 && $token[0] === T_ATTRIBUTE 970 ) { 971 // Go looking for the close bracket. 972 $bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), ['[', '#['], ']'); 973 974 $newToken = []; 975 $newToken['code'] = T_ATTRIBUTE; 976 $newToken['type'] = 'T_ATTRIBUTE'; 977 $newToken['content'] = '#['; 978 $finalTokens[$newStackPtr] = $newToken; 979 980 $tokens[$bracketCloser] = []; 981 $tokens[$bracketCloser][0] = T_ATTRIBUTE_END; 982 $tokens[$bracketCloser][1] = ']'; 983 984 if (PHP_CODESNIFFER_VERBOSITY > 1) { 985 echo "\t\t* token $bracketCloser changed from T_CLOSE_SQUARE_BRACKET to T_ATTRIBUTE_END".PHP_EOL; 986 } 987 988 $newStackPtr++; 989 continue; 990 }//end if 991 992 /* 993 Tokenize the parameter labels for PHP 8.0 named parameters as a special T_PARAM_NAME 994 token and ensure that the colon after it is always T_COLON. 995 */ 996 997 if ($tokenIsArray === true 998 && ($token[0] === T_STRING 999 || preg_match('`^[a-zA-Z_\x80-\xff]`', $token[1]) === 1) 1000 ) { 1001 // Get the next non-empty token. 1002 for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { 1003 if (is_array($tokens[$i]) === false 1004 || isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false 1005 ) { 1006 break; 1007 } 1008 } 1009 1010 if (isset($tokens[$i]) === true 1011 && is_array($tokens[$i]) === false 1012 && $tokens[$i] === ':' 1013 ) { 1014 // Get the previous non-empty token. 1015 for ($j = ($stackPtr - 1); $j > 0; $j--) { 1016 if (is_array($tokens[$j]) === false 1017 || isset(Util\Tokens::$emptyTokens[$tokens[$j][0]]) === false 1018 ) { 1019 break; 1020 } 1021 } 1022 1023 if (is_array($tokens[$j]) === false 1024 && ($tokens[$j] === '(' 1025 || $tokens[$j] === ',') 1026 ) { 1027 $newToken = []; 1028 $newToken['code'] = T_PARAM_NAME; 1029 $newToken['type'] = 'T_PARAM_NAME'; 1030 $newToken['content'] = $token[1]; 1031 $finalTokens[$newStackPtr] = $newToken; 1032 1033 $newStackPtr++; 1034 1035 // Modify the original token stack so that future checks, like 1036 // determining T_COLON vs T_INLINE_ELSE can handle this correctly. 1037 $tokens[$stackPtr][0] = T_PARAM_NAME; 1038 1039 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1040 $type = Util\Tokens::tokenName($token[0]); 1041 echo "\t\t* token $stackPtr changed from $type to T_PARAM_NAME".PHP_EOL; 1042 } 1043 1044 continue; 1045 } 1046 }//end if 1047 }//end if 1048 1049 /* 1050 Before PHP 7.0, the "yield from" was tokenized as 1051 T_YIELD, T_WHITESPACE and T_STRING. So look for 1052 and change this token in earlier versions. 1053 */ 1054 1055 if (PHP_VERSION_ID < 70000 1056 && PHP_VERSION_ID >= 50500 1057 && $tokenIsArray === true 1058 && $token[0] === T_YIELD 1059 && isset($tokens[($stackPtr + 1)]) === true 1060 && isset($tokens[($stackPtr + 2)]) === true 1061 && $tokens[($stackPtr + 1)][0] === T_WHITESPACE 1062 && $tokens[($stackPtr + 2)][0] === T_STRING 1063 && strtolower($tokens[($stackPtr + 2)][1]) === 'from' 1064 ) { 1065 // Could be multi-line, so adjust the token stack. 1066 $token[0] = T_YIELD_FROM; 1067 $token[1] .= $tokens[($stackPtr + 1)][1].$tokens[($stackPtr + 2)][1]; 1068 1069 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1070 for ($i = ($stackPtr + 1); $i <= ($stackPtr + 2); $i++) { 1071 $type = Util\Tokens::tokenName($tokens[$i][0]); 1072 $content = Util\Common::prepareForOutput($tokens[$i][1]); 1073 echo "\t\t* token $i merged into T_YIELD_FROM; was: $type => $content".PHP_EOL; 1074 } 1075 } 1076 1077 $tokens[($stackPtr + 1)] = null; 1078 $tokens[($stackPtr + 2)] = null; 1079 } 1080 1081 /* 1082 Before PHP 5.5, the yield keyword was tokenized as 1083 T_STRING. So look for and change this token in 1084 earlier versions. 1085 Checks also if it is just "yield" or "yield from". 1086 */ 1087 1088 if (PHP_VERSION_ID < 50500 1089 && $tokenIsArray === true 1090 && $token[0] === T_STRING 1091 && strtolower($token[1]) === 'yield' 1092 ) { 1093 if (isset($tokens[($stackPtr + 1)]) === true 1094 && isset($tokens[($stackPtr + 2)]) === true 1095 && $tokens[($stackPtr + 1)][0] === T_WHITESPACE 1096 && $tokens[($stackPtr + 2)][0] === T_STRING 1097 && strtolower($tokens[($stackPtr + 2)][1]) === 'from' 1098 ) { 1099 // Could be multi-line, so just just the token stack. 1100 $token[0] = T_YIELD_FROM; 1101 $token[1] .= $tokens[($stackPtr + 1)][1].$tokens[($stackPtr + 2)][1]; 1102 1103 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1104 for ($i = ($stackPtr + 1); $i <= ($stackPtr + 2); $i++) { 1105 $type = Util\Tokens::tokenName($tokens[$i][0]); 1106 $content = Util\Common::prepareForOutput($tokens[$i][1]); 1107 echo "\t\t* token $i merged into T_YIELD_FROM; was: $type => $content".PHP_EOL; 1108 } 1109 } 1110 1111 $tokens[($stackPtr + 1)] = null; 1112 $tokens[($stackPtr + 2)] = null; 1113 } else { 1114 $newToken = []; 1115 $newToken['code'] = T_YIELD; 1116 $newToken['type'] = 'T_YIELD'; 1117 $newToken['content'] = $token[1]; 1118 $finalTokens[$newStackPtr] = $newToken; 1119 1120 $newStackPtr++; 1121 continue; 1122 }//end if 1123 }//end if 1124 1125 /* 1126 Before PHP 5.6, the ... operator was tokenized as three 1127 T_STRING_CONCAT tokens in a row. So look for and combine 1128 these tokens in earlier versions. 1129 */ 1130 1131 if ($tokenIsArray === false 1132 && $token[0] === '.' 1133 && isset($tokens[($stackPtr + 1)]) === true 1134 && isset($tokens[($stackPtr + 2)]) === true 1135 && $tokens[($stackPtr + 1)] === '.' 1136 && $tokens[($stackPtr + 2)] === '.' 1137 ) { 1138 $newToken = []; 1139 $newToken['code'] = T_ELLIPSIS; 1140 $newToken['type'] = 'T_ELLIPSIS'; 1141 $newToken['content'] = '...'; 1142 $finalTokens[$newStackPtr] = $newToken; 1143 1144 $newStackPtr++; 1145 $stackPtr += 2; 1146 continue; 1147 } 1148 1149 /* 1150 Before PHP 5.6, the ** operator was tokenized as two 1151 T_MULTIPLY tokens in a row. So look for and combine 1152 these tokens in earlier versions. 1153 */ 1154 1155 if ($tokenIsArray === false 1156 && $token[0] === '*' 1157 && isset($tokens[($stackPtr + 1)]) === true 1158 && $tokens[($stackPtr + 1)] === '*' 1159 ) { 1160 $newToken = []; 1161 $newToken['code'] = T_POW; 1162 $newToken['type'] = 'T_POW'; 1163 $newToken['content'] = '**'; 1164 $finalTokens[$newStackPtr] = $newToken; 1165 1166 $newStackPtr++; 1167 $stackPtr++; 1168 continue; 1169 } 1170 1171 /* 1172 Before PHP 5.6, the **= operator was tokenized as 1173 T_MULTIPLY followed by T_MUL_EQUAL. So look for and combine 1174 these tokens in earlier versions. 1175 */ 1176 1177 if ($tokenIsArray === false 1178 && $token[0] === '*' 1179 && isset($tokens[($stackPtr + 1)]) === true 1180 && is_array($tokens[($stackPtr + 1)]) === true 1181 && $tokens[($stackPtr + 1)][1] === '*=' 1182 ) { 1183 $newToken = []; 1184 $newToken['code'] = T_POW_EQUAL; 1185 $newToken['type'] = 'T_POW_EQUAL'; 1186 $newToken['content'] = '**='; 1187 $finalTokens[$newStackPtr] = $newToken; 1188 1189 $newStackPtr++; 1190 $stackPtr++; 1191 continue; 1192 } 1193 1194 /* 1195 Before PHP 7, the ??= operator was tokenized as 1196 T_INLINE_THEN, T_INLINE_THEN, T_EQUAL. 1197 Between PHP 7.0 and 7.3, the ??= operator was tokenized as 1198 T_COALESCE, T_EQUAL. 1199 So look for and combine these tokens in earlier versions. 1200 */ 1201 1202 if (($tokenIsArray === false 1203 && $token[0] === '?' 1204 && isset($tokens[($stackPtr + 1)]) === true 1205 && $tokens[($stackPtr + 1)][0] === '?' 1206 && isset($tokens[($stackPtr + 2)]) === true 1207 && $tokens[($stackPtr + 2)][0] === '=') 1208 || ($tokenIsArray === true 1209 && $token[0] === T_COALESCE 1210 && isset($tokens[($stackPtr + 1)]) === true 1211 && $tokens[($stackPtr + 1)][0] === '=') 1212 ) { 1213 $newToken = []; 1214 $newToken['code'] = T_COALESCE_EQUAL; 1215 $newToken['type'] = 'T_COALESCE_EQUAL'; 1216 $newToken['content'] = '??='; 1217 $finalTokens[$newStackPtr] = $newToken; 1218 1219 $newStackPtr++; 1220 $stackPtr++; 1221 1222 if ($tokenIsArray === false) { 1223 // Pre PHP 7. 1224 $stackPtr++; 1225 } 1226 1227 continue; 1228 } 1229 1230 /* 1231 Before PHP 7, the ?? operator was tokenized as 1232 T_INLINE_THEN followed by T_INLINE_THEN. 1233 So look for and combine these tokens in earlier versions. 1234 */ 1235 1236 if ($tokenIsArray === false 1237 && $token[0] === '?' 1238 && isset($tokens[($stackPtr + 1)]) === true 1239 && $tokens[($stackPtr + 1)][0] === '?' 1240 ) { 1241 $newToken = []; 1242 $newToken['code'] = T_COALESCE; 1243 $newToken['type'] = 'T_COALESCE'; 1244 $newToken['content'] = '??'; 1245 $finalTokens[$newStackPtr] = $newToken; 1246 1247 $newStackPtr++; 1248 $stackPtr++; 1249 continue; 1250 } 1251 1252 /* 1253 Before PHP 8, the ?-> operator was tokenized as 1254 T_INLINE_THEN followed by T_OBJECT_OPERATOR. 1255 So look for and combine these tokens in earlier versions. 1256 */ 1257 1258 if ($tokenIsArray === false 1259 && $token[0] === '?' 1260 && isset($tokens[($stackPtr + 1)]) === true 1261 && is_array($tokens[($stackPtr + 1)]) === true 1262 && $tokens[($stackPtr + 1)][0] === T_OBJECT_OPERATOR 1263 ) { 1264 $newToken = []; 1265 $newToken['code'] = T_NULLSAFE_OBJECT_OPERATOR; 1266 $newToken['type'] = 'T_NULLSAFE_OBJECT_OPERATOR'; 1267 $newToken['content'] = '?->'; 1268 $finalTokens[$newStackPtr] = $newToken; 1269 1270 $newStackPtr++; 1271 $stackPtr++; 1272 continue; 1273 } 1274 1275 /* 1276 Before PHP 7.4, underscores inside T_LNUMBER and T_DNUMBER 1277 tokens split the token with a T_STRING. So look for 1278 and change these tokens in earlier versions. 1279 */ 1280 1281 if (PHP_VERSION_ID < 70400 1282 && ($tokenIsArray === true 1283 && ($token[0] === T_LNUMBER 1284 || $token[0] === T_DNUMBER) 1285 && isset($tokens[($stackPtr + 1)]) === true 1286 && is_array($tokens[($stackPtr + 1)]) === true 1287 && $tokens[($stackPtr + 1)][0] === T_STRING 1288 && $tokens[($stackPtr + 1)][1][0] === '_') 1289 ) { 1290 $newContent = $token[1]; 1291 $newType = $token[0]; 1292 for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { 1293 if (is_array($tokens[$i]) === false) { 1294 break; 1295 } 1296 1297 if ($tokens[$i][0] === T_LNUMBER 1298 || $tokens[$i][0] === T_DNUMBER 1299 ) { 1300 $newContent .= $tokens[$i][1]; 1301 continue; 1302 } 1303 1304 if ($tokens[$i][0] === T_STRING 1305 && $tokens[$i][1][0] === '_' 1306 && ((strpos($newContent, '0x') === 0 1307 && preg_match('`^((?<!\.)_[0-9A-F][0-9A-F\.]*)+$`iD', $tokens[$i][1]) === 1) 1308 || (strpos($newContent, '0x') !== 0 1309 && substr($newContent, -1) !== '.' 1310 && substr(strtolower($newContent), -1) !== 'e' 1311 && preg_match('`^(?:(?<![\.e])_[0-9][0-9e\.]*)+$`iD', $tokens[$i][1]) === 1)) 1312 ) { 1313 $newContent .= $tokens[$i][1]; 1314 1315 // Support floats. 1316 if (substr(strtolower($tokens[$i][1]), -1) === 'e' 1317 && ($tokens[($i + 1)] === '-' 1318 || $tokens[($i + 1)] === '+') 1319 ) { 1320 $newContent .= $tokens[($i + 1)]; 1321 $i++; 1322 } 1323 1324 continue; 1325 }//end if 1326 1327 break; 1328 }//end for 1329 1330 if ($newType === T_LNUMBER 1331 && ((stripos($newContent, '0x') === 0 && hexdec(str_replace('_', '', $newContent)) > PHP_INT_MAX) 1332 || (stripos($newContent, '0b') === 0 && bindec(str_replace('_', '', $newContent)) > PHP_INT_MAX) 1333 || (stripos($newContent, '0x') !== 0 1334 && stripos($newContent, 'e') !== false || strpos($newContent, '.') !== false) 1335 || (strpos($newContent, '0') === 0 && stripos($newContent, '0x') !== 0 1336 && stripos($newContent, '0b') !== 0 && octdec(str_replace('_', '', $newContent)) > PHP_INT_MAX) 1337 || (strpos($newContent, '0') !== 0 && str_replace('_', '', $newContent) > PHP_INT_MAX)) 1338 ) { 1339 $newType = T_DNUMBER; 1340 } 1341 1342 $newToken = []; 1343 $newToken['code'] = $newType; 1344 $newToken['type'] = Util\Tokens::tokenName($newType); 1345 $newToken['content'] = $newContent; 1346 $finalTokens[$newStackPtr] = $newToken; 1347 1348 $newStackPtr++; 1349 $stackPtr = ($i - 1); 1350 continue; 1351 }//end if 1352 1353 /* 1354 Backfill the T_MATCH token for PHP versions < 8.0 and 1355 do initial correction for non-match expression T_MATCH tokens 1356 to T_STRING for PHP >= 8.0. 1357 A final check for non-match expression T_MATCH tokens is done 1358 in PHP::processAdditional(). 1359 */ 1360 1361 if ($tokenIsArray === true 1362 && (($token[0] === T_STRING 1363 && strtolower($token[1]) === 'match') 1364 || $token[0] === T_MATCH) 1365 ) { 1366 $isMatch = false; 1367 for ($x = ($stackPtr + 1); $x < $numTokens; $x++) { 1368 if (isset($tokens[$x][0], Util\Tokens::$emptyTokens[$tokens[$x][0]]) === true) { 1369 continue; 1370 } 1371 1372 if ($tokens[$x] !== '(') { 1373 // This is not a match expression. 1374 break; 1375 } 1376 1377 if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) { 1378 // Also not a match expression. 1379 break; 1380 } 1381 1382 $isMatch = true; 1383 break; 1384 }//end for 1385 1386 if ($isMatch === true && $token[0] === T_STRING) { 1387 $newToken = []; 1388 $newToken['code'] = T_MATCH; 1389 $newToken['type'] = 'T_MATCH'; 1390 $newToken['content'] = $token[1]; 1391 1392 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1393 echo "\t\t* token $stackPtr changed from T_STRING to T_MATCH".PHP_EOL; 1394 } 1395 1396 $finalTokens[$newStackPtr] = $newToken; 1397 $newStackPtr++; 1398 continue; 1399 } else if ($isMatch === false && $token[0] === T_MATCH) { 1400 // PHP 8.0, match keyword, but not a match expression. 1401 $newToken = []; 1402 $newToken['code'] = T_STRING; 1403 $newToken['type'] = 'T_STRING'; 1404 $newToken['content'] = $token[1]; 1405 1406 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1407 echo "\t\t* token $stackPtr changed from T_MATCH to T_STRING".PHP_EOL; 1408 } 1409 1410 $finalTokens[$newStackPtr] = $newToken; 1411 $newStackPtr++; 1412 continue; 1413 }//end if 1414 }//end if 1415 1416 /* 1417 Retokenize the T_DEFAULT in match control structures as T_MATCH_DEFAULT 1418 to prevent scope being set and the scope for switch default statements 1419 breaking. 1420 */ 1421 1422 if ($tokenIsArray === true 1423 && $token[0] === T_DEFAULT 1424 ) { 1425 if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false) { 1426 for ($x = ($stackPtr + 1); $x < $numTokens; $x++) { 1427 if ($tokens[$x] === ',') { 1428 // Skip over potential trailing comma (supported in PHP). 1429 continue; 1430 } 1431 1432 if (is_array($tokens[$x]) === false 1433 || isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === false 1434 ) { 1435 // Non-empty, non-comma content. 1436 break; 1437 } 1438 } 1439 1440 if (isset($tokens[$x]) === true 1441 && is_array($tokens[$x]) === true 1442 && $tokens[$x][0] === T_DOUBLE_ARROW 1443 ) { 1444 // Modify the original token stack for the double arrow so that 1445 // future checks can disregard the double arrow token more easily. 1446 // For match expression "case" statements, this is handled 1447 // in PHP::processAdditional(). 1448 $tokens[$x][0] = T_MATCH_ARROW; 1449 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1450 echo "\t\t* token $x changed from T_DOUBLE_ARROW to T_MATCH_ARROW".PHP_EOL; 1451 } 1452 1453 $newToken = []; 1454 $newToken['code'] = T_MATCH_DEFAULT; 1455 $newToken['type'] = 'T_MATCH_DEFAULT'; 1456 $newToken['content'] = $token[1]; 1457 1458 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1459 echo "\t\t* token $stackPtr changed from T_DEFAULT to T_MATCH_DEFAULT".PHP_EOL; 1460 } 1461 1462 $finalTokens[$newStackPtr] = $newToken; 1463 $newStackPtr++; 1464 continue; 1465 }//end if 1466 } else { 1467 // Definitely not the "default" keyword. 1468 $newToken = []; 1469 $newToken['code'] = T_STRING; 1470 $newToken['type'] = 'T_STRING'; 1471 $newToken['content'] = $token[1]; 1472 1473 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1474 echo "\t\t* token $stackPtr changed from T_DEFAULT to T_STRING".PHP_EOL; 1475 } 1476 1477 $finalTokens[$newStackPtr] = $newToken; 1478 $newStackPtr++; 1479 continue; 1480 }//end if 1481 }//end if 1482 1483 /* 1484 Convert ? to T_NULLABLE OR T_INLINE_THEN 1485 */ 1486 1487 if ($tokenIsArray === false && $token[0] === '?') { 1488 $newToken = []; 1489 $newToken['content'] = '?'; 1490 1491 /* 1492 * Check if the next non-empty token is one of the tokens which can be used 1493 * in type declarations. If not, it's definitely a ternary. 1494 * At this point, the only token types which need to be taken into consideration 1495 * as potential type declarations are identifier names, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR. 1496 */ 1497 1498 $lastRelevantNonEmpty = null; 1499 1500 for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { 1501 if (is_array($tokens[$i]) === true) { 1502 $tokenType = $tokens[$i][0]; 1503 } else { 1504 $tokenType = $tokens[$i]; 1505 } 1506 1507 if (isset(Util\Tokens::$emptyTokens[$tokenType]) === true) { 1508 continue; 1509 } 1510 1511 if ($tokenType === T_STRING 1512 || $tokenType === T_NAME_FULLY_QUALIFIED 1513 || $tokenType === T_NAME_RELATIVE 1514 || $tokenType === T_NAME_QUALIFIED 1515 || $tokenType === T_ARRAY 1516 || $tokenType === T_NAMESPACE 1517 || $tokenType === T_NS_SEPARATOR 1518 ) { 1519 $lastRelevantNonEmpty = $tokenType; 1520 continue; 1521 } 1522 1523 if (($tokenType !== T_CALLABLE 1524 && isset($lastRelevantNonEmpty) === false) 1525 || ($lastRelevantNonEmpty === T_ARRAY 1526 && $tokenType === '(') 1527 || (($lastRelevantNonEmpty === T_STRING 1528 || $lastRelevantNonEmpty === T_NAME_FULLY_QUALIFIED 1529 || $lastRelevantNonEmpty === T_NAME_RELATIVE 1530 || $lastRelevantNonEmpty === T_NAME_QUALIFIED) 1531 && ($tokenType === T_DOUBLE_COLON 1532 || $tokenType === '(' 1533 || $tokenType === ':')) 1534 ) { 1535 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1536 echo "\t\t* token $stackPtr changed from ? to T_INLINE_THEN".PHP_EOL; 1537 } 1538 1539 $newToken['code'] = T_INLINE_THEN; 1540 $newToken['type'] = 'T_INLINE_THEN'; 1541 1542 $insideInlineIf[] = $stackPtr; 1543 1544 $finalTokens[$newStackPtr] = $newToken; 1545 $newStackPtr++; 1546 continue 2; 1547 } 1548 1549 break; 1550 }//end for 1551 1552 /* 1553 * This can still be a nullable type or a ternary. 1554 * Do additional checking. 1555 */ 1556 1557 $prevNonEmpty = null; 1558 $lastSeenNonEmpty = null; 1559 1560 for ($i = ($stackPtr - 1); $i >= 0; $i--) { 1561 if (is_array($tokens[$i]) === true) { 1562 $tokenType = $tokens[$i][0]; 1563 } else { 1564 $tokenType = $tokens[$i]; 1565 } 1566 1567 if ($tokenType === T_STATIC 1568 && ($lastSeenNonEmpty === T_DOUBLE_COLON 1569 || $lastSeenNonEmpty === '(') 1570 ) { 1571 $lastSeenNonEmpty = $tokenType; 1572 continue; 1573 } 1574 1575 if ($prevNonEmpty === null 1576 && isset(Util\Tokens::$emptyTokens[$tokenType]) === false 1577 ) { 1578 // Found the previous non-empty token. 1579 if ($tokenType === ':' || $tokenType === ',' || $tokenType === T_ATTRIBUTE_END) { 1580 $newToken['code'] = T_NULLABLE; 1581 $newToken['type'] = 'T_NULLABLE'; 1582 1583 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1584 echo "\t\t* token $stackPtr changed from ? to T_NULLABLE".PHP_EOL; 1585 } 1586 1587 break; 1588 } 1589 1590 $prevNonEmpty = $tokenType; 1591 } 1592 1593 if ($tokenType === T_FUNCTION 1594 || $tokenType === T_FN 1595 || isset(Util\Tokens::$methodPrefixes[$tokenType]) === true 1596 || $tokenType === T_VAR 1597 ) { 1598 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1599 echo "\t\t* token $stackPtr changed from ? to T_NULLABLE".PHP_EOL; 1600 } 1601 1602 $newToken['code'] = T_NULLABLE; 1603 $newToken['type'] = 'T_NULLABLE'; 1604 break; 1605 } else if (in_array($tokenType, [T_DOUBLE_ARROW, T_OPEN_TAG, T_OPEN_TAG_WITH_ECHO, '=', '{', ';'], true) === true) { 1606 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1607 echo "\t\t* token $stackPtr changed from ? to T_INLINE_THEN".PHP_EOL; 1608 } 1609 1610 $newToken['code'] = T_INLINE_THEN; 1611 $newToken['type'] = 'T_INLINE_THEN'; 1612 1613 $insideInlineIf[] = $stackPtr; 1614 break; 1615 } 1616 1617 if (isset(Util\Tokens::$emptyTokens[$tokenType]) === false) { 1618 $lastSeenNonEmpty = $tokenType; 1619 } 1620 }//end for 1621 1622 $finalTokens[$newStackPtr] = $newToken; 1623 $newStackPtr++; 1624 continue; 1625 }//end if 1626 1627 /* 1628 Tokens after a double colon may look like scope openers, 1629 such as when writing code like Foo::NAMESPACE, but they are 1630 only ever variables or strings. 1631 */ 1632 1633 if ($stackPtr > 1 1634 && (is_array($tokens[($stackPtr - 1)]) === true 1635 && $tokens[($stackPtr - 1)][0] === T_PAAMAYIM_NEKUDOTAYIM) 1636 && $tokenIsArray === true 1637 && $token[0] !== T_STRING 1638 && $token[0] !== T_VARIABLE 1639 && $token[0] !== T_DOLLAR 1640 && isset(Util\Tokens::$emptyTokens[$token[0]]) === false 1641 ) { 1642 $newToken = []; 1643 $newToken['code'] = T_STRING; 1644 $newToken['type'] = 'T_STRING'; 1645 $newToken['content'] = $token[1]; 1646 $finalTokens[$newStackPtr] = $newToken; 1647 1648 $newStackPtr++; 1649 continue; 1650 } 1651 1652 /* 1653 Backfill the T_FN token for PHP versions < 7.4. 1654 */ 1655 1656 if ($tokenIsArray === true 1657 && $token[0] === T_STRING 1658 && strtolower($token[1]) === 'fn' 1659 ) { 1660 // Modify the original token stack so that 1661 // future checks (like looking for T_NULLABLE) can 1662 // detect the T_FN token more easily. 1663 $tokens[$stackPtr][0] = T_FN; 1664 $token[0] = T_FN; 1665 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1666 echo "\t\t* token $stackPtr changed from T_STRING to T_FN".PHP_EOL; 1667 } 1668 } 1669 1670 /* 1671 The string-like token after a function keyword should always be 1672 tokenized as T_STRING even if it appears to be a different token, 1673 such as when writing code like: function default(): foo 1674 so go forward and change the token type before it is processed. 1675 1676 Note: this should not be done for `function Level\Name` within a 1677 group use statement for the PHP 8 identifier name tokens as it 1678 would interfere with the re-tokenization of those. 1679 */ 1680 1681 if ($tokenIsArray === true 1682 && ($token[0] === T_FUNCTION 1683 || $token[0] === T_FN) 1684 && $finalTokens[$lastNotEmptyToken]['code'] !== T_USE 1685 ) { 1686 if ($token[0] === T_FUNCTION) { 1687 for ($x = ($stackPtr + 1); $x < $numTokens; $x++) { 1688 if (is_array($tokens[$x]) === false 1689 || (isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === false 1690 && $tokens[$x][1] !== '&') 1691 ) { 1692 // Non-empty content. 1693 break; 1694 } 1695 } 1696 1697 if ($x < $numTokens 1698 && is_array($tokens[$x]) === true 1699 && $tokens[$x][0] !== T_STRING 1700 && $tokens[$x][0] !== T_NAME_QUALIFIED 1701 ) { 1702 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1703 $oldType = Util\Tokens::tokenName($tokens[$x][0]); 1704 echo "\t\t* token $x changed from $oldType to T_STRING".PHP_EOL; 1705 } 1706 1707 $tokens[$x][0] = T_STRING; 1708 } 1709 }//end if 1710 1711 /* 1712 This is a special condition for T_ARRAY tokens used for 1713 function return types. We want to keep the parenthesis map clean, 1714 so let's tag these tokens as T_STRING. 1715 */ 1716 1717 // Go looking for the colon to start the return type hint. 1718 // Start by finding the closing parenthesis of the function. 1719 $parenthesisStack = []; 1720 $parenthesisCloser = false; 1721 for ($x = ($stackPtr + 1); $x < $numTokens; $x++) { 1722 if (is_array($tokens[$x]) === false && $tokens[$x] === '(') { 1723 $parenthesisStack[] = $x; 1724 } else if (is_array($tokens[$x]) === false && $tokens[$x] === ')') { 1725 array_pop($parenthesisStack); 1726 if (empty($parenthesisStack) === true) { 1727 $parenthesisCloser = $x; 1728 break; 1729 } 1730 } 1731 } 1732 1733 if ($parenthesisCloser !== false) { 1734 for ($x = ($parenthesisCloser + 1); $x < $numTokens; $x++) { 1735 if (is_array($tokens[$x]) === false 1736 || isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === false 1737 ) { 1738 // Non-empty content. 1739 if (is_array($tokens[$x]) === true && $tokens[$x][0] === T_USE) { 1740 // Found a use statements, so search ahead for the closing parenthesis. 1741 for ($x += 1; $x < $numTokens; $x++) { 1742 if (is_array($tokens[$x]) === false && $tokens[$x] === ')') { 1743 continue(2); 1744 } 1745 } 1746 } 1747 1748 break; 1749 } 1750 } 1751 1752 if (isset($tokens[$x]) === true 1753 && is_array($tokens[$x]) === false 1754 && $tokens[$x] === ':' 1755 ) { 1756 $allowed = [ 1757 T_STRING => T_STRING, 1758 T_NAME_FULLY_QUALIFIED => T_NAME_FULLY_QUALIFIED, 1759 T_NAME_RELATIVE => T_NAME_RELATIVE, 1760 T_NAME_QUALIFIED => T_NAME_QUALIFIED, 1761 T_ARRAY => T_ARRAY, 1762 T_CALLABLE => T_CALLABLE, 1763 T_SELF => T_SELF, 1764 T_PARENT => T_PARENT, 1765 T_NAMESPACE => T_NAMESPACE, 1766 T_STATIC => T_STATIC, 1767 T_NS_SEPARATOR => T_NS_SEPARATOR, 1768 ]; 1769 1770 $allowed += Util\Tokens::$emptyTokens; 1771 1772 // Find the start of the return type. 1773 for ($x += 1; $x < $numTokens; $x++) { 1774 if (is_array($tokens[$x]) === true 1775 && isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === true 1776 ) { 1777 // Whitespace or comments before the return type. 1778 continue; 1779 } 1780 1781 if (is_array($tokens[$x]) === false && $tokens[$x] === '?') { 1782 // Found a nullable operator, so skip it. 1783 // But also convert the token to save the tokenizer 1784 // a bit of time later on. 1785 $tokens[$x] = [ 1786 T_NULLABLE, 1787 '?', 1788 ]; 1789 1790 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1791 echo "\t\t* token $x changed from ? to T_NULLABLE".PHP_EOL; 1792 } 1793 1794 continue; 1795 } 1796 1797 break; 1798 }//end for 1799 }//end if 1800 }//end if 1801 }//end if 1802 1803 /* 1804 Before PHP 7, the <=> operator was tokenized as 1805 T_IS_SMALLER_OR_EQUAL followed by T_GREATER_THAN. 1806 So look for and combine these tokens in earlier versions. 1807 */ 1808 1809 if ($tokenIsArray === true 1810 && $token[0] === T_IS_SMALLER_OR_EQUAL 1811 && isset($tokens[($stackPtr + 1)]) === true 1812 && $tokens[($stackPtr + 1)][0] === '>' 1813 ) { 1814 $newToken = []; 1815 $newToken['code'] = T_SPACESHIP; 1816 $newToken['type'] = 'T_SPACESHIP'; 1817 $newToken['content'] = '<=>'; 1818 $finalTokens[$newStackPtr] = $newToken; 1819 1820 $newStackPtr++; 1821 $stackPtr++; 1822 continue; 1823 } 1824 1825 /* 1826 PHP doesn't assign a token to goto labels, so we have to. 1827 These are just string tokens with a single colon after them. Double 1828 colons are already tokenized and so don't interfere with this check. 1829 But we do have to account for CASE statements, that look just like 1830 goto labels. 1831 */ 1832 1833 if ($tokenIsArray === true 1834 && $token[0] === T_STRING 1835 && isset($tokens[($stackPtr + 1)]) === true 1836 && $tokens[($stackPtr + 1)] === ':' 1837 && (is_array($tokens[($stackPtr - 1)]) === false 1838 || $tokens[($stackPtr - 1)][0] !== T_PAAMAYIM_NEKUDOTAYIM) 1839 ) { 1840 $stopTokens = [ 1841 T_CASE => true, 1842 T_SEMICOLON => true, 1843 T_OPEN_TAG => true, 1844 T_OPEN_CURLY_BRACKET => true, 1845 T_INLINE_THEN => true, 1846 ]; 1847 1848 for ($x = ($newStackPtr - 1); $x > 0; $x--) { 1849 if (isset($stopTokens[$finalTokens[$x]['code']]) === true) { 1850 break; 1851 } 1852 } 1853 1854 if ($finalTokens[$x]['code'] !== T_CASE 1855 && $finalTokens[$x]['code'] !== T_INLINE_THEN 1856 ) { 1857 $finalTokens[$newStackPtr] = [ 1858 'content' => $token[1].':', 1859 'code' => T_GOTO_LABEL, 1860 'type' => 'T_GOTO_LABEL', 1861 ]; 1862 1863 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1864 echo "\t\t* token $stackPtr changed from T_STRING to T_GOTO_LABEL".PHP_EOL; 1865 echo "\t\t* skipping T_COLON token ".($stackPtr + 1).PHP_EOL; 1866 } 1867 1868 $newStackPtr++; 1869 $stackPtr++; 1870 continue; 1871 } 1872 }//end if 1873 1874 /* 1875 If this token has newlines in its content, split each line up 1876 and create a new token for each line. We do this so it's easier 1877 to ascertain where errors occur on a line. 1878 Note that $token[1] is the token's content. 1879 */ 1880 1881 if ($tokenIsArray === true && strpos($token[1], $this->eolChar) !== false) { 1882 $tokenLines = explode($this->eolChar, $token[1]); 1883 $numLines = count($tokenLines); 1884 $newToken = [ 1885 'type' => Util\Tokens::tokenName($token[0]), 1886 'code' => $token[0], 1887 'content' => '', 1888 ]; 1889 1890 for ($i = 0; $i < $numLines; $i++) { 1891 $newToken['content'] = $tokenLines[$i]; 1892 if ($i === ($numLines - 1)) { 1893 if ($tokenLines[$i] === '') { 1894 break; 1895 } 1896 } else { 1897 $newToken['content'] .= $this->eolChar; 1898 } 1899 1900 $finalTokens[$newStackPtr] = $newToken; 1901 $newStackPtr++; 1902 } 1903 } else { 1904 if ($tokenIsArray === true && $token[0] === T_STRING) { 1905 // Some T_STRING tokens should remain that way 1906 // due to their context. 1907 if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) { 1908 // Special case for syntax like: return new self 1909 // where self should not be a string. 1910 if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW 1911 && strtolower($token[1]) === 'self' 1912 ) { 1913 $finalTokens[$newStackPtr] = [ 1914 'content' => $token[1], 1915 'code' => T_SELF, 1916 'type' => 'T_SELF', 1917 ]; 1918 } else { 1919 $finalTokens[$newStackPtr] = [ 1920 'content' => $token[1], 1921 'code' => T_STRING, 1922 'type' => 'T_STRING', 1923 ]; 1924 } 1925 1926 $newStackPtr++; 1927 continue; 1928 }//end if 1929 }//end if 1930 1931 $newToken = null; 1932 if ($tokenIsArray === false) { 1933 if (isset(self::$resolveTokenCache[$token[0]]) === true) { 1934 $newToken = self::$resolveTokenCache[$token[0]]; 1935 } 1936 } else { 1937 $cacheKey = null; 1938 if ($token[0] === T_STRING) { 1939 $cacheKey = strtolower($token[1]); 1940 } else if ($token[0] !== T_CURLY_OPEN) { 1941 $cacheKey = $token[0]; 1942 } 1943 1944 if ($cacheKey !== null && isset(self::$resolveTokenCache[$cacheKey]) === true) { 1945 $newToken = self::$resolveTokenCache[$cacheKey]; 1946 $newToken['content'] = $token[1]; 1947 } 1948 } 1949 1950 if ($newToken === null) { 1951 $newToken = self::standardiseToken($token); 1952 } 1953 1954 // Convert colons that are actually the ELSE component of an 1955 // inline IF statement. 1956 if (empty($insideInlineIf) === false && $newToken['code'] === T_COLON) { 1957 $isInlineIf = true; 1958 1959 // Make sure this isn't a named parameter label. 1960 // Get the previous non-empty token. 1961 for ($i = ($stackPtr - 1); $i > 0; $i--) { 1962 if (is_array($tokens[$i]) === false 1963 || isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false 1964 ) { 1965 break; 1966 } 1967 } 1968 1969 if ($tokens[$i][0] === T_PARAM_NAME) { 1970 $isInlineIf = false; 1971 if (PHP_CODESNIFFER_VERBOSITY > 1) { 1972 echo "\t\t* token is parameter label, not T_INLINE_ELSE".PHP_EOL; 1973 } 1974 } 1975 1976 if ($isInlineIf === true) { 1977 // Make sure this isn't a return type separator. 1978 for ($i = ($stackPtr - 1); $i > 0; $i--) { 1979 if (is_array($tokens[$i]) === false 1980 || ($tokens[$i][0] !== T_DOC_COMMENT 1981 && $tokens[$i][0] !== T_COMMENT 1982 && $tokens[$i][0] !== T_WHITESPACE) 1983 ) { 1984 break; 1985 } 1986 } 1987 1988 if ($tokens[$i] === ')') { 1989 $parenCount = 1; 1990 for ($i--; $i > 0; $i--) { 1991 if ($tokens[$i] === '(') { 1992 $parenCount--; 1993 if ($parenCount === 0) { 1994 break; 1995 } 1996 } else if ($tokens[$i] === ')') { 1997 $parenCount++; 1998 } 1999 } 2000 2001 // We've found the open parenthesis, so if the previous 2002 // non-empty token is FUNCTION or USE, this is a return type. 2003 // Note that we need to skip T_STRING tokens here as these 2004 // can be function names. 2005 for ($i--; $i > 0; $i--) { 2006 if (is_array($tokens[$i]) === false 2007 || ($tokens[$i][0] !== T_DOC_COMMENT 2008 && $tokens[$i][0] !== T_COMMENT 2009 && $tokens[$i][0] !== T_WHITESPACE 2010 && $tokens[$i][0] !== T_STRING) 2011 ) { 2012 break; 2013 } 2014 } 2015 2016 if ($tokens[$i][0] === T_FUNCTION || $tokens[$i][0] === T_FN || $tokens[$i][0] === T_USE) { 2017 $isInlineIf = false; 2018 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2019 echo "\t\t* token is return type, not T_INLINE_ELSE".PHP_EOL; 2020 } 2021 } 2022 }//end if 2023 }//end if 2024 2025 // Check to see if this is a CASE or DEFAULT opener. 2026 if ($isInlineIf === true) { 2027 $inlineIfToken = $insideInlineIf[(count($insideInlineIf) - 1)]; 2028 for ($i = $stackPtr; $i > $inlineIfToken; $i--) { 2029 if (is_array($tokens[$i]) === true 2030 && ($tokens[$i][0] === T_CASE 2031 || $tokens[$i][0] === T_DEFAULT) 2032 ) { 2033 $isInlineIf = false; 2034 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2035 echo "\t\t* token is T_CASE or T_DEFAULT opener, not T_INLINE_ELSE".PHP_EOL; 2036 } 2037 2038 break; 2039 } 2040 2041 if (is_array($tokens[$i]) === false 2042 && ($tokens[$i] === ';' 2043 || $tokens[$i] === '{') 2044 ) { 2045 break; 2046 } 2047 } 2048 }//end if 2049 2050 if ($isInlineIf === true) { 2051 array_pop($insideInlineIf); 2052 $newToken['code'] = T_INLINE_ELSE; 2053 $newToken['type'] = 'T_INLINE_ELSE'; 2054 2055 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2056 echo "\t\t* token changed from T_COLON to T_INLINE_ELSE".PHP_EOL; 2057 } 2058 } 2059 }//end if 2060 2061 // This is a special condition for T_ARRAY tokens used for anything else 2062 // but array declarations, like type hinting function arguments as 2063 // being arrays. 2064 // We want to keep the parenthesis map clean, so let's tag these tokens as 2065 // T_STRING. 2066 if ($newToken['code'] === T_ARRAY) { 2067 for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { 2068 if (is_array($tokens[$i]) === false 2069 || isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false 2070 ) { 2071 // Non-empty content. 2072 break; 2073 } 2074 } 2075 2076 if ($tokens[$i] !== '(' && $i !== $numTokens) { 2077 $newToken['code'] = T_STRING; 2078 $newToken['type'] = 'T_STRING'; 2079 } 2080 } 2081 2082 // This is a special case when checking PHP 5.5+ code in PHP < 5.5 2083 // where "finally" should be T_FINALLY instead of T_STRING. 2084 if ($newToken['code'] === T_STRING 2085 && strtolower($newToken['content']) === 'finally' 2086 && $finalTokens[$lastNotEmptyToken]['code'] === T_CLOSE_CURLY_BRACKET 2087 ) { 2088 $newToken['code'] = T_FINALLY; 2089 $newToken['type'] = 'T_FINALLY'; 2090 } 2091 2092 // This is a special case for the PHP 5.5 classname::class syntax 2093 // where "class" should be T_STRING instead of T_CLASS. 2094 if (($newToken['code'] === T_CLASS 2095 || $newToken['code'] === T_FUNCTION) 2096 && $finalTokens[$lastNotEmptyToken]['code'] === T_DOUBLE_COLON 2097 ) { 2098 $newToken['code'] = T_STRING; 2099 $newToken['type'] = 'T_STRING'; 2100 } 2101 2102 // This is a special case for PHP 5.6 use function and use const 2103 // where "function" and "const" should be T_STRING instead of T_FUNCTION 2104 // and T_CONST. 2105 if (($newToken['code'] === T_FUNCTION 2106 || $newToken['code'] === T_CONST) 2107 && ($finalTokens[$lastNotEmptyToken]['code'] === T_USE || $insideUseGroup === true) 2108 ) { 2109 $newToken['code'] = T_STRING; 2110 $newToken['type'] = 'T_STRING'; 2111 } 2112 2113 // This is a special case for use groups in PHP 7+ where leaving 2114 // the curly braces as their normal tokens would confuse 2115 // the scope map and sniffs. 2116 if ($newToken['code'] === T_OPEN_CURLY_BRACKET 2117 && $finalTokens[$lastNotEmptyToken]['code'] === T_NS_SEPARATOR 2118 ) { 2119 $newToken['code'] = T_OPEN_USE_GROUP; 2120 $newToken['type'] = 'T_OPEN_USE_GROUP'; 2121 $insideUseGroup = true; 2122 } 2123 2124 if ($insideUseGroup === true && $newToken['code'] === T_CLOSE_CURLY_BRACKET) { 2125 $newToken['code'] = T_CLOSE_USE_GROUP; 2126 $newToken['type'] = 'T_CLOSE_USE_GROUP'; 2127 $insideUseGroup = false; 2128 } 2129 2130 $finalTokens[$newStackPtr] = $newToken; 2131 $newStackPtr++; 2132 }//end if 2133 }//end for 2134 2135 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2136 echo "\t*** END PHP TOKENIZING ***".PHP_EOL; 2137 } 2138 2139 return $finalTokens; 2140 2141 }//end tokenize() 2142 2143 2144 /** 2145 * Performs additional processing after main tokenizing. 2146 * 2147 * This additional processing checks for CASE statements that are using curly 2148 * braces for scope openers and closers. It also turns some T_FUNCTION tokens 2149 * into T_CLOSURE when they are not standard function definitions. It also 2150 * detects short array syntax and converts those square brackets into new tokens. 2151 * It also corrects some usage of the static and class keywords. It also 2152 * assigns tokens to function return types. 2153 * 2154 * @return void 2155 */ 2156 protected function processAdditional() 2157 { 2158 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2159 echo "\t*** START ADDITIONAL PHP PROCESSING ***".PHP_EOL; 2160 } 2161 2162 $this->createAttributesNestingMap(); 2163 2164 $numTokens = count($this->tokens); 2165 for ($i = ($numTokens - 1); $i >= 0; $i--) { 2166 // Check for any unset scope conditions due to alternate IF/ENDIF syntax. 2167 if (isset($this->tokens[$i]['scope_opener']) === true 2168 && isset($this->tokens[$i]['scope_condition']) === false 2169 ) { 2170 $this->tokens[$i]['scope_condition'] = $this->tokens[$this->tokens[$i]['scope_opener']]['scope_condition']; 2171 } 2172 2173 if ($this->tokens[$i]['code'] === T_FUNCTION) { 2174 /* 2175 Detect functions that are actually closures and 2176 assign them a different token. 2177 */ 2178 2179 if (isset($this->tokens[$i]['scope_opener']) === true) { 2180 for ($x = ($i + 1); $x < $numTokens; $x++) { 2181 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false 2182 && $this->tokens[$x]['code'] !== T_BITWISE_AND 2183 ) { 2184 break; 2185 } 2186 } 2187 2188 if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS) { 2189 $this->tokens[$i]['code'] = T_CLOSURE; 2190 $this->tokens[$i]['type'] = 'T_CLOSURE'; 2191 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2192 $line = $this->tokens[$i]['line']; 2193 echo "\t* token $i on line $line changed from T_FUNCTION to T_CLOSURE".PHP_EOL; 2194 } 2195 2196 for ($x = ($this->tokens[$i]['scope_opener'] + 1); $x < $this->tokens[$i]['scope_closer']; $x++) { 2197 if (isset($this->tokens[$x]['conditions'][$i]) === false) { 2198 continue; 2199 } 2200 2201 $this->tokens[$x]['conditions'][$i] = T_CLOSURE; 2202 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2203 $type = $this->tokens[$x]['type']; 2204 echo "\t\t* cleaned $x ($type) *".PHP_EOL; 2205 } 2206 } 2207 } 2208 }//end if 2209 2210 continue; 2211 } else if ($this->tokens[$i]['code'] === T_CLASS && isset($this->tokens[$i]['scope_opener']) === true) { 2212 /* 2213 Detect anonymous classes and assign them a different token. 2214 */ 2215 2216 for ($x = ($i + 1); $x < $numTokens; $x++) { 2217 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) { 2218 break; 2219 } 2220 } 2221 2222 if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS 2223 || $this->tokens[$x]['code'] === T_OPEN_CURLY_BRACKET 2224 || $this->tokens[$x]['code'] === T_EXTENDS 2225 || $this->tokens[$x]['code'] === T_IMPLEMENTS 2226 ) { 2227 $this->tokens[$i]['code'] = T_ANON_CLASS; 2228 $this->tokens[$i]['type'] = 'T_ANON_CLASS'; 2229 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2230 $line = $this->tokens[$i]['line']; 2231 echo "\t* token $i on line $line changed from T_CLASS to T_ANON_CLASS".PHP_EOL; 2232 } 2233 2234 if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS 2235 && isset($this->tokens[$x]['parenthesis_closer']) === true 2236 ) { 2237 $closer = $this->tokens[$x]['parenthesis_closer']; 2238 2239 $this->tokens[$i]['parenthesis_opener'] = $x; 2240 $this->tokens[$i]['parenthesis_closer'] = $closer; 2241 $this->tokens[$i]['parenthesis_owner'] = $i; 2242 $this->tokens[$x]['parenthesis_owner'] = $i; 2243 $this->tokens[$closer]['parenthesis_owner'] = $i; 2244 2245 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2246 $line = $this->tokens[$i]['line']; 2247 echo "\t\t* added parenthesis keys to T_ANON_CLASS token $i on line $line".PHP_EOL; 2248 } 2249 } 2250 2251 for ($x = ($this->tokens[$i]['scope_opener'] + 1); $x < $this->tokens[$i]['scope_closer']; $x++) { 2252 if (isset($this->tokens[$x]['conditions'][$i]) === false) { 2253 continue; 2254 } 2255 2256 $this->tokens[$x]['conditions'][$i] = T_ANON_CLASS; 2257 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2258 $type = $this->tokens[$x]['type']; 2259 echo "\t\t* cleaned $x ($type) *".PHP_EOL; 2260 } 2261 } 2262 }//end if 2263 2264 continue; 2265 } else if ($this->tokens[$i]['code'] === T_FN && isset($this->tokens[($i + 1)]) === true) { 2266 // Possible arrow function. 2267 for ($x = ($i + 1); $x < $numTokens; $x++) { 2268 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false 2269 && $this->tokens[$x]['code'] !== T_BITWISE_AND 2270 ) { 2271 // Non-whitespace content. 2272 break; 2273 } 2274 } 2275 2276 if (isset($this->tokens[$x]) === true && $this->tokens[$x]['code'] === T_OPEN_PARENTHESIS) { 2277 $ignore = Util\Tokens::$emptyTokens; 2278 $ignore += [ 2279 T_ARRAY => T_ARRAY, 2280 T_CALLABLE => T_CALLABLE, 2281 T_COLON => T_COLON, 2282 T_NAMESPACE => T_NAMESPACE, 2283 T_NS_SEPARATOR => T_NS_SEPARATOR, 2284 T_NULL => T_NULL, 2285 T_NULLABLE => T_NULLABLE, 2286 T_PARENT => T_PARENT, 2287 T_SELF => T_SELF, 2288 T_STATIC => T_STATIC, 2289 T_STRING => T_STRING, 2290 T_TYPE_UNION => T_TYPE_UNION, 2291 ]; 2292 2293 $closer = $this->tokens[$x]['parenthesis_closer']; 2294 for ($arrow = ($closer + 1); $arrow < $numTokens; $arrow++) { 2295 if (isset($ignore[$this->tokens[$arrow]['code']]) === false) { 2296 break; 2297 } 2298 } 2299 2300 if ($this->tokens[$arrow]['code'] === T_DOUBLE_ARROW) { 2301 $endTokens = [ 2302 T_COLON => true, 2303 T_COMMA => true, 2304 T_SEMICOLON => true, 2305 T_CLOSE_PARENTHESIS => true, 2306 T_CLOSE_SQUARE_BRACKET => true, 2307 T_CLOSE_CURLY_BRACKET => true, 2308 T_CLOSE_SHORT_ARRAY => true, 2309 T_OPEN_TAG => true, 2310 T_CLOSE_TAG => true, 2311 ]; 2312 2313 $inTernary = false; 2314 $lastEndToken = null; 2315 2316 for ($scopeCloser = ($arrow + 1); $scopeCloser < $numTokens; $scopeCloser++) { 2317 // Arrow function closer should never be shared with the closer of a match 2318 // control structure. 2319 if (isset($this->tokens[$scopeCloser]['scope_closer'], $this->tokens[$scopeCloser]['scope_condition']) === true 2320 && $scopeCloser === $this->tokens[$scopeCloser]['scope_closer'] 2321 && $this->tokens[$this->tokens[$scopeCloser]['scope_condition']]['code'] === T_MATCH 2322 ) { 2323 if ($arrow < $this->tokens[$scopeCloser]['scope_condition']) { 2324 // Match in return value of arrow function. Move on to the next token. 2325 continue; 2326 } 2327 2328 // Arrow function as return value for the last match case without trailing comma. 2329 if ($lastEndToken !== null) { 2330 $scopeCloser = $lastEndToken; 2331 break; 2332 } 2333 2334 for ($lastNonEmpty = ($scopeCloser - 1); $lastNonEmpty > $arrow; $lastNonEmpty--) { 2335 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$lastNonEmpty]['code']]) === false) { 2336 $scopeCloser = $lastNonEmpty; 2337 break 2; 2338 } 2339 } 2340 } 2341 2342 if (isset($endTokens[$this->tokens[$scopeCloser]['code']]) === true) { 2343 if ($lastEndToken !== null 2344 && ((isset($this->tokens[$scopeCloser]['parenthesis_opener']) === true 2345 && $this->tokens[$scopeCloser]['parenthesis_opener'] < $arrow) 2346 || (isset($this->tokens[$scopeCloser]['bracket_opener']) === true 2347 && $this->tokens[$scopeCloser]['bracket_opener'] < $arrow)) 2348 ) { 2349 for ($lastNonEmpty = ($scopeCloser - 1); $lastNonEmpty > $arrow; $lastNonEmpty--) { 2350 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$lastNonEmpty]['code']]) === false) { 2351 $scopeCloser = $lastNonEmpty; 2352 break; 2353 } 2354 } 2355 } 2356 2357 break; 2358 } 2359 2360 if ($inTernary === false 2361 && isset($this->tokens[$scopeCloser]['scope_closer'], $this->tokens[$scopeCloser]['scope_condition']) === true 2362 && $scopeCloser === $this->tokens[$scopeCloser]['scope_closer'] 2363 && $this->tokens[$this->tokens[$scopeCloser]['scope_condition']]['code'] === T_FN 2364 ) { 2365 // Found a nested arrow function that already has the closer set and is in 2366 // the same scope as us, so we can use its closer. 2367 break; 2368 } 2369 2370 if (isset($this->tokens[$scopeCloser]['scope_closer']) === true 2371 && $this->tokens[$scopeCloser]['code'] !== T_INLINE_ELSE 2372 && $this->tokens[$scopeCloser]['code'] !== T_END_HEREDOC 2373 && $this->tokens[$scopeCloser]['code'] !== T_END_NOWDOC 2374 ) { 2375 // We minus 1 here in case the closer can be shared with us. 2376 $scopeCloser = ($this->tokens[$scopeCloser]['scope_closer'] - 1); 2377 continue; 2378 } 2379 2380 if (isset($this->tokens[$scopeCloser]['parenthesis_closer']) === true) { 2381 $scopeCloser = $this->tokens[$scopeCloser]['parenthesis_closer']; 2382 $lastEndToken = $scopeCloser; 2383 continue; 2384 } 2385 2386 if (isset($this->tokens[$scopeCloser]['bracket_closer']) === true) { 2387 $scopeCloser = $this->tokens[$scopeCloser]['bracket_closer']; 2388 $lastEndToken = $scopeCloser; 2389 continue; 2390 } 2391 2392 if ($this->tokens[$scopeCloser]['code'] === T_INLINE_THEN) { 2393 $inTernary = true; 2394 continue; 2395 } 2396 2397 if ($this->tokens[$scopeCloser]['code'] === T_INLINE_ELSE) { 2398 if ($inTernary === false) { 2399 break; 2400 } 2401 2402 $inTernary = false; 2403 continue; 2404 } 2405 }//end for 2406 2407 if ($scopeCloser !== $numTokens) { 2408 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2409 $line = $this->tokens[$i]['line']; 2410 echo "\t=> token $i on line $line processed as arrow function".PHP_EOL; 2411 echo "\t\t* scope opener set to $arrow *".PHP_EOL; 2412 echo "\t\t* scope closer set to $scopeCloser *".PHP_EOL; 2413 echo "\t\t* parenthesis opener set to $x *".PHP_EOL; 2414 echo "\t\t* parenthesis closer set to $closer *".PHP_EOL; 2415 } 2416 2417 $this->tokens[$i]['code'] = T_FN; 2418 $this->tokens[$i]['type'] = 'T_FN'; 2419 $this->tokens[$i]['scope_condition'] = $i; 2420 $this->tokens[$i]['scope_opener'] = $arrow; 2421 $this->tokens[$i]['scope_closer'] = $scopeCloser; 2422 $this->tokens[$i]['parenthesis_owner'] = $i; 2423 $this->tokens[$i]['parenthesis_opener'] = $x; 2424 $this->tokens[$i]['parenthesis_closer'] = $closer; 2425 2426 $this->tokens[$arrow]['code'] = T_FN_ARROW; 2427 $this->tokens[$arrow]['type'] = 'T_FN_ARROW'; 2428 2429 $this->tokens[$arrow]['scope_condition'] = $i; 2430 $this->tokens[$arrow]['scope_opener'] = $arrow; 2431 $this->tokens[$arrow]['scope_closer'] = $scopeCloser; 2432 $this->tokens[$scopeCloser]['scope_condition'] = $i; 2433 $this->tokens[$scopeCloser]['scope_opener'] = $arrow; 2434 $this->tokens[$scopeCloser]['scope_closer'] = $scopeCloser; 2435 2436 $opener = $this->tokens[$i]['parenthesis_opener']; 2437 $closer = $this->tokens[$i]['parenthesis_closer']; 2438 $this->tokens[$opener]['parenthesis_owner'] = $i; 2439 $this->tokens[$closer]['parenthesis_owner'] = $i; 2440 2441 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2442 $line = $this->tokens[$arrow]['line']; 2443 echo "\t\t* token $arrow on line $line changed from T_DOUBLE_ARROW to T_FN_ARROW".PHP_EOL; 2444 } 2445 }//end if 2446 }//end if 2447 }//end if 2448 2449 // If after all that, the extra tokens are not set, this is not an arrow function. 2450 if (isset($this->tokens[$i]['scope_closer']) === false) { 2451 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2452 $line = $this->tokens[$i]['line']; 2453 echo "\t=> token $i on line $line is not an arrow function".PHP_EOL; 2454 echo "\t\t* token changed from T_FN to T_STRING".PHP_EOL; 2455 } 2456 2457 $this->tokens[$i]['code'] = T_STRING; 2458 $this->tokens[$i]['type'] = 'T_STRING'; 2459 } 2460 } else if ($this->tokens[$i]['code'] === T_OPEN_SQUARE_BRACKET) { 2461 if (isset($this->tokens[$i]['bracket_closer']) === false) { 2462 continue; 2463 } 2464 2465 // Unless there is a variable or a bracket before this token, 2466 // it is the start of an array being defined using the short syntax. 2467 $isShortArray = false; 2468 $allowed = [ 2469 T_CLOSE_SQUARE_BRACKET => T_CLOSE_SQUARE_BRACKET, 2470 T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, 2471 T_CLOSE_PARENTHESIS => T_CLOSE_PARENTHESIS, 2472 T_VARIABLE => T_VARIABLE, 2473 T_OBJECT_OPERATOR => T_OBJECT_OPERATOR, 2474 T_NULLSAFE_OBJECT_OPERATOR => T_NULLSAFE_OBJECT_OPERATOR, 2475 T_STRING => T_STRING, 2476 T_CONSTANT_ENCAPSED_STRING => T_CONSTANT_ENCAPSED_STRING, 2477 T_DOUBLE_QUOTED_STRING => T_DOUBLE_QUOTED_STRING, 2478 ]; 2479 $allowed += Util\Tokens::$magicConstants; 2480 2481 for ($x = ($i - 1); $x >= 0; $x--) { 2482 // If we hit a scope opener, the statement has ended 2483 // without finding anything, so it's probably an array 2484 // using PHP 7.1 short list syntax. 2485 if (isset($this->tokens[$x]['scope_opener']) === true) { 2486 $isShortArray = true; 2487 break; 2488 } 2489 2490 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) { 2491 if (isset($allowed[$this->tokens[$x]['code']]) === false) { 2492 $isShortArray = true; 2493 } 2494 2495 break; 2496 } 2497 } 2498 2499 if ($isShortArray === true) { 2500 $this->tokens[$i]['code'] = T_OPEN_SHORT_ARRAY; 2501 $this->tokens[$i]['type'] = 'T_OPEN_SHORT_ARRAY'; 2502 2503 $closer = $this->tokens[$i]['bracket_closer']; 2504 $this->tokens[$closer]['code'] = T_CLOSE_SHORT_ARRAY; 2505 $this->tokens[$closer]['type'] = 'T_CLOSE_SHORT_ARRAY'; 2506 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2507 $line = $this->tokens[$i]['line']; 2508 echo "\t* token $i on line $line changed from T_OPEN_SQUARE_BRACKET to T_OPEN_SHORT_ARRAY".PHP_EOL; 2509 $line = $this->tokens[$closer]['line']; 2510 echo "\t* token $closer on line $line changed from T_CLOSE_SQUARE_BRACKET to T_CLOSE_SHORT_ARRAY".PHP_EOL; 2511 } 2512 } 2513 2514 continue; 2515 } else if ($this->tokens[$i]['code'] === T_MATCH) { 2516 if (isset($this->tokens[$i]['scope_opener'], $this->tokens[$i]['scope_closer']) === false) { 2517 // Not a match expression after all. 2518 $this->tokens[$i]['code'] = T_STRING; 2519 $this->tokens[$i]['type'] = 'T_STRING'; 2520 2521 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2522 echo "\t\t* token $i changed from T_MATCH to T_STRING".PHP_EOL; 2523 } 2524 2525 if (isset($this->tokens[$i]['parenthesis_opener'], $this->tokens[$i]['parenthesis_closer']) === true) { 2526 $opener = $this->tokens[$i]['parenthesis_opener']; 2527 $closer = $this->tokens[$i]['parenthesis_closer']; 2528 unset( 2529 $this->tokens[$opener]['parenthesis_owner'], 2530 $this->tokens[$closer]['parenthesis_owner'] 2531 ); 2532 unset( 2533 $this->tokens[$i]['parenthesis_opener'], 2534 $this->tokens[$i]['parenthesis_closer'], 2535 $this->tokens[$i]['parenthesis_owner'] 2536 ); 2537 2538 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2539 echo "\t\t* cleaned parenthesis of token $i *".PHP_EOL; 2540 } 2541 } 2542 } else { 2543 // Retokenize the double arrows for match expression cases to `T_MATCH_ARROW`. 2544 $searchFor = [ 2545 T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET, 2546 T_OPEN_SQUARE_BRACKET => T_OPEN_SQUARE_BRACKET, 2547 T_OPEN_PARENTHESIS => T_OPEN_PARENTHESIS, 2548 T_OPEN_SHORT_ARRAY => T_OPEN_SHORT_ARRAY, 2549 T_DOUBLE_ARROW => T_DOUBLE_ARROW, 2550 ]; 2551 $searchFor += Util\Tokens::$scopeOpeners; 2552 2553 for ($x = ($this->tokens[$i]['scope_opener'] + 1); $x < $this->tokens[$i]['scope_closer']; $x++) { 2554 if (isset($searchFor[$this->tokens[$x]['code']]) === false) { 2555 continue; 2556 } 2557 2558 if (isset($this->tokens[$x]['scope_closer']) === true) { 2559 $x = $this->tokens[$x]['scope_closer']; 2560 continue; 2561 } 2562 2563 if (isset($this->tokens[$x]['parenthesis_closer']) === true) { 2564 $x = $this->tokens[$x]['parenthesis_closer']; 2565 continue; 2566 } 2567 2568 if (isset($this->tokens[$x]['bracket_closer']) === true) { 2569 $x = $this->tokens[$x]['bracket_closer']; 2570 continue; 2571 } 2572 2573 // This must be a double arrow, but make sure anyhow. 2574 if ($this->tokens[$x]['code'] === T_DOUBLE_ARROW) { 2575 $this->tokens[$x]['code'] = T_MATCH_ARROW; 2576 $this->tokens[$x]['type'] = 'T_MATCH_ARROW'; 2577 2578 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2579 echo "\t\t* token $x changed from T_DOUBLE_ARROW to T_MATCH_ARROW".PHP_EOL; 2580 } 2581 } 2582 }//end for 2583 }//end if 2584 2585 continue; 2586 } else if ($this->tokens[$i]['code'] === T_BITWISE_OR) { 2587 /* 2588 Convert "|" to T_TYPE_UNION or leave as T_BITWISE_OR. 2589 */ 2590 2591 $allowed = [ 2592 T_STRING => T_STRING, 2593 T_CALLABLE => T_CALLABLE, 2594 T_SELF => T_SELF, 2595 T_PARENT => T_PARENT, 2596 T_STATIC => T_STATIC, 2597 T_FALSE => T_FALSE, 2598 T_NULL => T_NULL, 2599 T_NAMESPACE => T_NAMESPACE, 2600 T_NS_SEPARATOR => T_NS_SEPARATOR, 2601 ]; 2602 2603 $suspectedType = null; 2604 $typeTokenCount = 0; 2605 2606 for ($x = ($i + 1); $x < $numTokens; $x++) { 2607 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true) { 2608 continue; 2609 } 2610 2611 if (isset($allowed[$this->tokens[$x]['code']]) === true) { 2612 ++$typeTokenCount; 2613 continue; 2614 } 2615 2616 if ($typeTokenCount > 0 2617 && ($this->tokens[$x]['code'] === T_BITWISE_AND 2618 || $this->tokens[$x]['code'] === T_ELLIPSIS) 2619 ) { 2620 // Skip past reference and variadic indicators for parameter types. 2621 continue; 2622 } 2623 2624 if ($this->tokens[$x]['code'] === T_VARIABLE) { 2625 // Parameter/Property defaults can not contain variables, so this could be a type. 2626 $suspectedType = 'property or parameter'; 2627 break; 2628 } 2629 2630 if ($this->tokens[$x]['code'] === T_DOUBLE_ARROW) { 2631 // Possible arrow function. 2632 $suspectedType = 'return'; 2633 break; 2634 } 2635 2636 if ($this->tokens[$x]['code'] === T_SEMICOLON) { 2637 // Possible abstract method or interface method. 2638 $suspectedType = 'return'; 2639 break; 2640 } 2641 2642 if ($this->tokens[$x]['code'] === T_OPEN_CURLY_BRACKET 2643 && isset($this->tokens[$x]['scope_condition']) === true 2644 && $this->tokens[$this->tokens[$x]['scope_condition']]['code'] === T_FUNCTION 2645 ) { 2646 $suspectedType = 'return'; 2647 } 2648 2649 break; 2650 }//end for 2651 2652 if ($typeTokenCount === 0 || isset($suspectedType) === false) { 2653 // Definitely not a union type, move on. 2654 continue; 2655 } 2656 2657 $typeTokenCount = 0; 2658 $unionOperators = [$i]; 2659 $confirmed = false; 2660 2661 for ($x = ($i - 1); $x >= 0; $x--) { 2662 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true) { 2663 continue; 2664 } 2665 2666 if (isset($allowed[$this->tokens[$x]['code']]) === true) { 2667 ++$typeTokenCount; 2668 continue; 2669 } 2670 2671 // Union types can't use the nullable operator, but be tolerant to parse errors. 2672 if ($typeTokenCount > 0 && $this->tokens[$x]['code'] === T_NULLABLE) { 2673 continue; 2674 } 2675 2676 if ($this->tokens[$x]['code'] === T_BITWISE_OR) { 2677 $unionOperators[] = $x; 2678 continue; 2679 } 2680 2681 if ($suspectedType === 'return' && $this->tokens[$x]['code'] === T_COLON) { 2682 $confirmed = true; 2683 break; 2684 } 2685 2686 if ($suspectedType === 'property or parameter' 2687 && (isset(Util\Tokens::$scopeModifiers[$this->tokens[$x]['code']]) === true 2688 || $this->tokens[$x]['code'] === T_VAR) 2689 ) { 2690 // This will also confirm constructor property promotion parameters, but that's fine. 2691 $confirmed = true; 2692 } 2693 2694 break; 2695 }//end for 2696 2697 if ($confirmed === false 2698 && $suspectedType === 'property or parameter' 2699 && isset($this->tokens[$i]['nested_parenthesis']) === true 2700 ) { 2701 $parens = $this->tokens[$i]['nested_parenthesis']; 2702 $last = end($parens); 2703 2704 if (isset($this->tokens[$last]['parenthesis_owner']) === true 2705 && $this->tokens[$this->tokens[$last]['parenthesis_owner']]['code'] === T_FUNCTION 2706 ) { 2707 $confirmed = true; 2708 } else { 2709 // No parenthesis owner set, this may be an arrow function which has not yet 2710 // had additional processing done. 2711 if (isset($this->tokens[$last]['parenthesis_opener']) === true) { 2712 for ($x = ($this->tokens[$last]['parenthesis_opener'] - 1); $x >= 0; $x--) { 2713 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true) { 2714 continue; 2715 } 2716 2717 break; 2718 } 2719 2720 if ($this->tokens[$x]['code'] === T_FN) { 2721 for (--$x; $x >= 0; $x--) { 2722 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true 2723 || $this->tokens[$x]['code'] === T_BITWISE_AND 2724 ) { 2725 continue; 2726 } 2727 2728 break; 2729 } 2730 2731 if ($this->tokens[$x]['code'] !== T_FUNCTION) { 2732 $confirmed = true; 2733 } 2734 } 2735 }//end if 2736 }//end if 2737 2738 unset($parens, $last); 2739 }//end if 2740 2741 if ($confirmed === false) { 2742 // Not a union type after all, move on. 2743 continue; 2744 } 2745 2746 foreach ($unionOperators as $x) { 2747 $this->tokens[$x]['code'] = T_TYPE_UNION; 2748 $this->tokens[$x]['type'] = 'T_TYPE_UNION'; 2749 2750 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2751 $line = $this->tokens[$x]['line']; 2752 echo "\t* token $x on line $line changed from T_BITWISE_OR to T_TYPE_UNION".PHP_EOL; 2753 } 2754 } 2755 2756 continue; 2757 } else if ($this->tokens[$i]['code'] === T_STATIC) { 2758 for ($x = ($i - 1); $x > 0; $x--) { 2759 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) { 2760 break; 2761 } 2762 } 2763 2764 if ($this->tokens[$x]['code'] === T_INSTANCEOF) { 2765 $this->tokens[$i]['code'] = T_STRING; 2766 $this->tokens[$i]['type'] = 'T_STRING'; 2767 2768 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2769 $line = $this->tokens[$i]['line']; 2770 echo "\t* token $i on line $line changed from T_STATIC to T_STRING".PHP_EOL; 2771 } 2772 } 2773 2774 continue; 2775 } else if ($this->tokens[$i]['code'] === T_TRUE 2776 || $this->tokens[$i]['code'] === T_FALSE 2777 || $this->tokens[$i]['code'] === T_NULL 2778 ) { 2779 for ($x = ($i + 1); $i < $numTokens; $x++) { 2780 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) { 2781 // Non-whitespace content. 2782 break; 2783 } 2784 } 2785 2786 if (isset($this->tstringContexts[$this->tokens[$x]['code']]) === true) { 2787 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2788 $line = $this->tokens[$i]['line']; 2789 $type = $this->tokens[$i]['type']; 2790 echo "\t* token $i on line $line changed from $type to T_STRING".PHP_EOL; 2791 } 2792 2793 $this->tokens[$i]['code'] = T_STRING; 2794 $this->tokens[$i]['type'] = 'T_STRING'; 2795 } 2796 } else if ($this->tokens[$i]['code'] === T_CONST) { 2797 // Context sensitive keywords support. 2798 for ($x = ($i + 1); $i < $numTokens; $x++) { 2799 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) { 2800 // Non-whitespace content. 2801 break; 2802 } 2803 } 2804 2805 if ($this->tokens[$x]['code'] !== T_STRING) { 2806 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2807 $line = $this->tokens[$x]['line']; 2808 $type = $this->tokens[$x]['type']; 2809 echo "\t* token $x on line $line changed from $type to T_STRING".PHP_EOL; 2810 } 2811 2812 $this->tokens[$x]['code'] = T_STRING; 2813 $this->tokens[$x]['type'] = 'T_STRING'; 2814 } 2815 }//end if 2816 2817 if (($this->tokens[$i]['code'] !== T_CASE 2818 && $this->tokens[$i]['code'] !== T_DEFAULT) 2819 || isset($this->tokens[$i]['scope_opener']) === false 2820 ) { 2821 // Only interested in CASE and DEFAULT statements from here on in. 2822 continue; 2823 } 2824 2825 $scopeOpener = $this->tokens[$i]['scope_opener']; 2826 $scopeCloser = $this->tokens[$i]['scope_closer']; 2827 2828 // If the first char after the opener is a curly brace 2829 // and that brace has been ignored, it is actually 2830 // opening this case statement and the opener and closer are 2831 // probably set incorrectly. 2832 for ($x = ($scopeOpener + 1); $x < $numTokens; $x++) { 2833 if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) { 2834 // Non-whitespace content. 2835 break; 2836 } 2837 } 2838 2839 if ($this->tokens[$x]['code'] === T_CASE || $this->tokens[$x]['code'] === T_DEFAULT) { 2840 // Special case for multiple CASE statements that share the same 2841 // closer. Because we are going backwards through the file, this next 2842 // CASE statement is already fixed, so just use its closer and don't 2843 // worry about fixing anything. 2844 $newCloser = $this->tokens[$x]['scope_closer']; 2845 $this->tokens[$i]['scope_closer'] = $newCloser; 2846 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2847 $oldType = $this->tokens[$scopeCloser]['type']; 2848 $newType = $this->tokens[$newCloser]['type']; 2849 $line = $this->tokens[$i]['line']; 2850 echo "\t* token $i (T_CASE) on line $line closer changed from $scopeCloser ($oldType) to $newCloser ($newType)".PHP_EOL; 2851 } 2852 2853 continue; 2854 } 2855 2856 if ($this->tokens[$x]['code'] !== T_OPEN_CURLY_BRACKET 2857 || isset($this->tokens[$x]['scope_condition']) === true 2858 ) { 2859 // Not a CASE/DEFAULT with a curly brace opener. 2860 continue; 2861 } 2862 2863 // The closer for this CASE/DEFAULT should be the closing curly brace and 2864 // not whatever it already is. The opener needs to be the opening curly 2865 // brace so everything matches up. 2866 $newCloser = $this->tokens[$x]['bracket_closer']; 2867 foreach ([$i, $x, $newCloser] as $index) { 2868 $this->tokens[$index]['scope_condition'] = $i; 2869 $this->tokens[$index]['scope_opener'] = $x; 2870 $this->tokens[$index]['scope_closer'] = $newCloser; 2871 } 2872 2873 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2874 $line = $this->tokens[$i]['line']; 2875 $tokenType = $this->tokens[$i]['type']; 2876 2877 $oldType = $this->tokens[$scopeOpener]['type']; 2878 $newType = $this->tokens[$x]['type']; 2879 echo "\t* token $i ($tokenType) on line $line opener changed from $scopeOpener ($oldType) to $x ($newType)".PHP_EOL; 2880 2881 $oldType = $this->tokens[$scopeCloser]['type']; 2882 $newType = $this->tokens[$newCloser]['type']; 2883 echo "\t* token $i ($tokenType) on line $line closer changed from $scopeCloser ($oldType) to $newCloser ($newType)".PHP_EOL; 2884 } 2885 2886 if ($this->tokens[$scopeOpener]['scope_condition'] === $i) { 2887 unset($this->tokens[$scopeOpener]['scope_condition']); 2888 unset($this->tokens[$scopeOpener]['scope_opener']); 2889 unset($this->tokens[$scopeOpener]['scope_closer']); 2890 } 2891 2892 if ($this->tokens[$scopeCloser]['scope_condition'] === $i) { 2893 unset($this->tokens[$scopeCloser]['scope_condition']); 2894 unset($this->tokens[$scopeCloser]['scope_opener']); 2895 unset($this->tokens[$scopeCloser]['scope_closer']); 2896 } else { 2897 // We were using a shared closer. All tokens that were 2898 // sharing this closer with us, except for the scope condition 2899 // and it's opener, need to now point to the new closer. 2900 $condition = $this->tokens[$scopeCloser]['scope_condition']; 2901 $start = ($this->tokens[$condition]['scope_opener'] + 1); 2902 for ($y = $start; $y < $scopeCloser; $y++) { 2903 if (isset($this->tokens[$y]['scope_closer']) === true 2904 && $this->tokens[$y]['scope_closer'] === $scopeCloser 2905 ) { 2906 $this->tokens[$y]['scope_closer'] = $newCloser; 2907 2908 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2909 $line = $this->tokens[$y]['line']; 2910 $tokenType = $this->tokens[$y]['type']; 2911 $oldType = $this->tokens[$scopeCloser]['type']; 2912 $newType = $this->tokens[$newCloser]['type']; 2913 echo "\t\t* token $y ($tokenType) on line $line closer changed from $scopeCloser ($oldType) to $newCloser ($newType)".PHP_EOL; 2914 } 2915 } 2916 } 2917 }//end if 2918 2919 unset($this->tokens[$x]['bracket_opener']); 2920 unset($this->tokens[$x]['bracket_closer']); 2921 unset($this->tokens[$newCloser]['bracket_opener']); 2922 unset($this->tokens[$newCloser]['bracket_closer']); 2923 $this->tokens[$scopeCloser]['conditions'][] = $i; 2924 2925 // Now fix up all the tokens that think they are 2926 // inside the CASE/DEFAULT statement when they are really outside. 2927 for ($x = $newCloser; $x < $scopeCloser; $x++) { 2928 foreach ($this->tokens[$x]['conditions'] as $num => $oldCond) { 2929 if ($oldCond === $this->tokens[$i]['code']) { 2930 $oldConditions = $this->tokens[$x]['conditions']; 2931 unset($this->tokens[$x]['conditions'][$num]); 2932 2933 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2934 $type = $this->tokens[$x]['type']; 2935 $oldConds = ''; 2936 foreach ($oldConditions as $condition) { 2937 $oldConds .= Util\Tokens::tokenName($condition).','; 2938 } 2939 2940 $oldConds = rtrim($oldConds, ','); 2941 2942 $newConds = ''; 2943 foreach ($this->tokens[$x]['conditions'] as $condition) { 2944 $newConds .= Util\Tokens::tokenName($condition).','; 2945 } 2946 2947 $newConds = rtrim($newConds, ','); 2948 2949 echo "\t\t* cleaned $x ($type) *".PHP_EOL; 2950 echo "\t\t\t=> conditions changed from $oldConds to $newConds".PHP_EOL; 2951 } 2952 2953 break; 2954 }//end if 2955 }//end foreach 2956 }//end for 2957 }//end for 2958 2959 if (PHP_CODESNIFFER_VERBOSITY > 1) { 2960 echo "\t*** END ADDITIONAL PHP PROCESSING ***".PHP_EOL; 2961 } 2962 2963 }//end processAdditional() 2964 2965 2966 /** 2967 * Takes a token produced from <code>token_get_all()</code> and produces a 2968 * more uniform token. 2969 * 2970 * @param string|array $token The token to convert. 2971 * 2972 * @return array The new token. 2973 */ 2974 public static function standardiseToken($token) 2975 { 2976 if (isset($token[1]) === false) { 2977 if (isset(self::$resolveTokenCache[$token[0]]) === true) { 2978 return self::$resolveTokenCache[$token[0]]; 2979 } 2980 } else { 2981 $cacheKey = null; 2982 if ($token[0] === T_STRING) { 2983 $cacheKey = strtolower($token[1]); 2984 } else if ($token[0] !== T_CURLY_OPEN) { 2985 $cacheKey = $token[0]; 2986 } 2987 2988 if ($cacheKey !== null && isset(self::$resolveTokenCache[$cacheKey]) === true) { 2989 $newToken = self::$resolveTokenCache[$cacheKey]; 2990 $newToken['content'] = $token[1]; 2991 return $newToken; 2992 } 2993 } 2994 2995 if (isset($token[1]) === false) { 2996 return self::resolveSimpleToken($token[0]); 2997 } 2998 2999 if ($token[0] === T_STRING) { 3000 switch ($cacheKey) { 3001 case 'false': 3002 $newToken['type'] = 'T_FALSE'; 3003 break; 3004 case 'true': 3005 $newToken['type'] = 'T_TRUE'; 3006 break; 3007 case 'null': 3008 $newToken['type'] = 'T_NULL'; 3009 break; 3010 case 'self': 3011 $newToken['type'] = 'T_SELF'; 3012 break; 3013 case 'parent': 3014 $newToken['type'] = 'T_PARENT'; 3015 break; 3016 default: 3017 $newToken['type'] = 'T_STRING'; 3018 break; 3019 } 3020 3021 $newToken['code'] = constant($newToken['type']); 3022 3023 self::$resolveTokenCache[$cacheKey] = $newToken; 3024 } else if ($token[0] === T_CURLY_OPEN) { 3025 $newToken = [ 3026 'code' => T_OPEN_CURLY_BRACKET, 3027 'type' => 'T_OPEN_CURLY_BRACKET', 3028 ]; 3029 } else { 3030 $newToken = [ 3031 'code' => $token[0], 3032 'type' => Util\Tokens::tokenName($token[0]), 3033 ]; 3034 3035 self::$resolveTokenCache[$token[0]] = $newToken; 3036 }//end if 3037 3038 $newToken['content'] = $token[1]; 3039 return $newToken; 3040 3041 }//end standardiseToken() 3042 3043 3044 /** 3045 * Converts simple tokens into a format that conforms to complex tokens 3046 * produced by token_get_all(). 3047 * 3048 * Simple tokens are tokens that are not in array form when produced from 3049 * token_get_all(). 3050 * 3051 * @param string $token The simple token to convert. 3052 * 3053 * @return array The new token in array format. 3054 */ 3055 public static function resolveSimpleToken($token) 3056 { 3057 $newToken = []; 3058 3059 switch ($token) { 3060 case '{': 3061 $newToken['type'] = 'T_OPEN_CURLY_BRACKET'; 3062 break; 3063 case '}': 3064 $newToken['type'] = 'T_CLOSE_CURLY_BRACKET'; 3065 break; 3066 case '[': 3067 $newToken['type'] = 'T_OPEN_SQUARE_BRACKET'; 3068 break; 3069 case ']': 3070 $newToken['type'] = 'T_CLOSE_SQUARE_BRACKET'; 3071 break; 3072 case '(': 3073 $newToken['type'] = 'T_OPEN_PARENTHESIS'; 3074 break; 3075 case ')': 3076 $newToken['type'] = 'T_CLOSE_PARENTHESIS'; 3077 break; 3078 case ':': 3079 $newToken['type'] = 'T_COLON'; 3080 break; 3081 case '.': 3082 $newToken['type'] = 'T_STRING_CONCAT'; 3083 break; 3084 case ';': 3085 $newToken['type'] = 'T_SEMICOLON'; 3086 break; 3087 case '=': 3088 $newToken['type'] = 'T_EQUAL'; 3089 break; 3090 case '*': 3091 $newToken['type'] = 'T_MULTIPLY'; 3092 break; 3093 case '/': 3094 $newToken['type'] = 'T_DIVIDE'; 3095 break; 3096 case '+': 3097 $newToken['type'] = 'T_PLUS'; 3098 break; 3099 case '-': 3100 $newToken['type'] = 'T_MINUS'; 3101 break; 3102 case '%': 3103 $newToken['type'] = 'T_MODULUS'; 3104 break; 3105 case '^': 3106 $newToken['type'] = 'T_BITWISE_XOR'; 3107 break; 3108 case '&': 3109 $newToken['type'] = 'T_BITWISE_AND'; 3110 break; 3111 case '|': 3112 $newToken['type'] = 'T_BITWISE_OR'; 3113 break; 3114 case '~': 3115 $newToken['type'] = 'T_BITWISE_NOT'; 3116 break; 3117 case '<': 3118 $newToken['type'] = 'T_LESS_THAN'; 3119 break; 3120 case '>': 3121 $newToken['type'] = 'T_GREATER_THAN'; 3122 break; 3123 case '!': 3124 $newToken['type'] = 'T_BOOLEAN_NOT'; 3125 break; 3126 case ',': 3127 $newToken['type'] = 'T_COMMA'; 3128 break; 3129 case '@': 3130 $newToken['type'] = 'T_ASPERAND'; 3131 break; 3132 case '$': 3133 $newToken['type'] = 'T_DOLLAR'; 3134 break; 3135 case '`': 3136 $newToken['type'] = 'T_BACKTICK'; 3137 break; 3138 default: 3139 $newToken['type'] = 'T_NONE'; 3140 break; 3141 }//end switch 3142 3143 $newToken['code'] = constant($newToken['type']); 3144 $newToken['content'] = $token; 3145 3146 self::$resolveTokenCache[$token] = $newToken; 3147 return $newToken; 3148 3149 }//end resolveSimpleToken() 3150 3151 3152 /** 3153 * Finds a "closer" token (closing parenthesis or square bracket for example) 3154 * Handle parenthesis balancing while searching for closing token 3155 * 3156 * @param array $tokens The list of tokens to iterate searching the closing token (as returned by token_get_all) 3157 * @param int $start The starting position 3158 * @param string|string[] $openerTokens The opening character 3159 * @param string $closerChar The closing character 3160 * 3161 * @return int|null The position of the closing token, if found. NULL otherwise. 3162 */ 3163 private function findCloser(array &$tokens, $start, $openerTokens, $closerChar) 3164 { 3165 $numTokens = count($tokens); 3166 $stack = [0]; 3167 $closer = null; 3168 $openerTokens = (array) $openerTokens; 3169 3170 for ($x = $start; $x < $numTokens; $x++) { 3171 if (in_array($tokens[$x], $openerTokens, true) === true 3172 || (is_array($tokens[$x]) === true && in_array($tokens[$x][1], $openerTokens, true) === true) 3173 ) { 3174 $stack[] = $x; 3175 } else if ($tokens[$x] === $closerChar) { 3176 array_pop($stack); 3177 if (empty($stack) === true) { 3178 $closer = $x; 3179 break; 3180 } 3181 } 3182 } 3183 3184 return $closer; 3185 3186 }//end findCloser() 3187 3188 3189 /** 3190 * PHP 8 attributes parser for PHP < 8 3191 * Handles single-line and multiline attributes. 3192 * 3193 * @param array $tokens The original array of tokens (as returned by token_get_all) 3194 * @param int $stackPtr The current position in token array 3195 * 3196 * @return array|null The array of parsed attribute tokens 3197 */ 3198 private function parsePhpAttribute(array &$tokens, $stackPtr) 3199 { 3200 3201 $token = $tokens[$stackPtr]; 3202 3203 $commentBody = substr($token[1], 2); 3204 $subTokens = @token_get_all('<?php '.$commentBody); 3205 3206 foreach ($subTokens as $i => $subToken) { 3207 if (is_array($subToken) === true 3208 && $subToken[0] === T_COMMENT 3209 && strpos($subToken[1], '#[') === 0 3210 ) { 3211 $reparsed = $this->parsePhpAttribute($subTokens, $i); 3212 if ($reparsed !== null) { 3213 array_splice($subTokens, $i, 1, $reparsed); 3214 } else { 3215 $subToken[0] = T_ATTRIBUTE; 3216 } 3217 } 3218 } 3219 3220 array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]); 3221 3222 // Go looking for the close bracket. 3223 $bracketCloser = $this->findCloser($subTokens, 1, '[', ']'); 3224 if (PHP_VERSION_ID < 80000 && $bracketCloser === null) { 3225 foreach (array_slice($tokens, ($stackPtr + 1)) as $token) { 3226 if (is_array($token) === true) { 3227 $commentBody .= $token[1]; 3228 } else { 3229 $commentBody .= $token; 3230 } 3231 } 3232 3233 $subTokens = @token_get_all('<?php '.$commentBody); 3234 array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]); 3235 3236 $bracketCloser = $this->findCloser($subTokens, 1, '[', ']'); 3237 if ($bracketCloser !== null) { 3238 array_splice($tokens, ($stackPtr + 1), count($tokens), array_slice($subTokens, ($bracketCloser + 1))); 3239 $subTokens = array_slice($subTokens, 0, ($bracketCloser + 1)); 3240 } 3241 } 3242 3243 if ($bracketCloser === null) { 3244 return null; 3245 } 3246 3247 return $subTokens; 3248 3249 }//end parsePhpAttribute() 3250 3251 3252 /** 3253 * Creates a map for the attributes tokens that surround other tokens. 3254 * 3255 * @return void 3256 */ 3257 private function createAttributesNestingMap() 3258 { 3259 $map = []; 3260 for ($i = 0; $i < $this->numTokens; $i++) { 3261 if (isset($this->tokens[$i]['attribute_opener']) === true 3262 && $i === $this->tokens[$i]['attribute_opener'] 3263 ) { 3264 if (empty($map) === false) { 3265 $this->tokens[$i]['nested_attributes'] = $map; 3266 } 3267 3268 if (isset($this->tokens[$i]['attribute_closer']) === true) { 3269 $map[$this->tokens[$i]['attribute_opener']] 3270 = $this->tokens[$i]['attribute_closer']; 3271 } 3272 } else if (isset($this->tokens[$i]['attribute_closer']) === true 3273 && $i === $this->tokens[$i]['attribute_closer'] 3274 ) { 3275 array_pop($map); 3276 if (empty($map) === false) { 3277 $this->tokens[$i]['nested_attributes'] = $map; 3278 } 3279 } else { 3280 if (empty($map) === false) { 3281 $this->tokens[$i]['nested_attributes'] = $map; 3282 } 3283 }//end if 3284 }//end for 3285 3286 }//end createAttributesNestingMap() 3287 3288 3289}//end class 3290