提交代码

vendor/nikic/php-parser/lib/PhpParser/Lexer/Emulative.php

@@ -0,0 +1,250 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer;
+
+use PhpParser\Error;
+use PhpParser\ErrorHandler;
+use PhpParser\Lexer;
+use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
+use PhpParser\Lexer\TokenEmulator\TokenEmulatorInterface;
+use PhpParser\Parser\Tokens;
+
+class Emulative extends Lexer
+{
+    const PHP_7_3 = '7.3.0dev';
+    const PHP_7_4 = '7.4.0dev';
+
+    const T_COALESCE_EQUAL = 1007;
+    const T_FN = 1008;
+
+    const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
+/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
+(?:.*\r?\n)*?
+(?<indentation>\h*)\2(?![a-zA-Z0-9_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
+REGEX;
+
+    /** @var mixed[] Patches used to reverse changes introduced in the code */
+    private $patches = [];
+
+    /** @var TokenEmulatorInterface[] */
+    private $tokenEmulators = [];
+
+    /**
+     * @param mixed[] $options
+     */
+    public function __construct(array $options = [])
+    {
+        parent::__construct($options);
+
+        $this->tokenEmulators[] = new FnTokenEmulator();
+        $this->tokenEmulators[] = new CoaleseEqualTokenEmulator();
+        $this->tokenEmulators[] = new NumericLiteralSeparatorEmulator();
+
+        $this->tokenMap[self::T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
+        $this->tokenMap[self::T_FN] = Tokens::T_FN;
+    }
+
+    public function startLexing(string $code, ErrorHandler $errorHandler = null) {
+        $this->patches = [];
+
+        if ($this->isEmulationNeeded($code) === false) {
+            // Nothing to emulate, yay
+            parent::startLexing($code, $errorHandler);
+            return;
+        }
+
+        $collector = new ErrorHandler\Collecting();
+
+        // 1. emulation of heredoc and nowdoc new syntax
+        $preparedCode = $this->processHeredocNowdoc($code);
+        parent::startLexing($preparedCode, $collector);
+        $this->fixupTokens();
+
+        $errors = $collector->getErrors();
+        if (!empty($errors)) {
+            $this->fixupErrors($errors);
+            foreach ($errors as $error) {
+                $errorHandler->handleError($error);
+            }
+        }
+
+        // add token emulation
+        foreach ($this->tokenEmulators as $emulativeToken) {
+            if ($emulativeToken->isEmulationNeeded($code)) {
+                $this->tokens = $emulativeToken->emulate($code, $this->tokens);
+            }
+        }
+    }
+
+    private function isHeredocNowdocEmulationNeeded(string $code): bool
+    {
+        // skip version where this works without emulation
+        if (version_compare(\PHP_VERSION, self::PHP_7_3, '>=')) {
+            return false;
+        }
+
+        return strpos($code, '<<<') !== false;
+    }
+
+    private function processHeredocNowdoc(string $code): string
+    {
+        if ($this->isHeredocNowdocEmulationNeeded($code) === false) {
+            return $code;
+        }
+
+        if (!preg_match_all(self::FLEXIBLE_DOC_STRING_REGEX, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
+            // No heredoc/nowdoc found
+            return $code;
+        }
+
+        // Keep track of how much we need to adjust string offsets due to the modifications we
+        // already made
+        $posDelta = 0;
+        foreach ($matches as $match) {
+            $indentation = $match['indentation'][0];
+            $indentationStart = $match['indentation'][1];
+
+            $separator = $match['separator'][0];
+            $separatorStart = $match['separator'][1];
+
+            if ($indentation === '' && $separator !== '') {
+                // Ordinary heredoc/nowdoc
+                continue;
+            }
+
+            if ($indentation !== '') {
+                // Remove indentation
+                $indentationLen = strlen($indentation);
+                $code = substr_replace($code, '', $indentationStart + $posDelta, $indentationLen);
+                $this->patches[] = [$indentationStart + $posDelta, 'add', $indentation];
+                $posDelta -= $indentationLen;
+            }
+
+            if ($separator === '') {
+                // Insert newline as separator
+                $code = substr_replace($code, "\n", $separatorStart + $posDelta, 0);
+                $this->patches[] = [$separatorStart + $posDelta, 'remove', "\n"];
+                $posDelta += 1;
+            }
+        }
+
+        return $code;
+    }
+
+    private function isEmulationNeeded(string $code): bool
+    {
+        foreach ($this->tokenEmulators as $emulativeToken) {
+            if ($emulativeToken->isEmulationNeeded($code)) {
+                return true;
+            }
+        }
+
+        return $this->isHeredocNowdocEmulationNeeded($code);
+    }
+
+    private function fixupTokens()
+    {
+        if (\count($this->patches) === 0) {
+            return;
+        }
+
+        // Load first patch
+        $patchIdx = 0;
+
+        list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
+
+        // We use a manual loop over the tokens, because we modify the array on the fly
+        $pos = 0;
+        for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
+            $token = $this->tokens[$i];
+            if (\is_string($token)) {
+                // We assume that patches don't apply to string tokens
+                $pos += \strlen($token);
+                continue;
+            }
+
+            $len = \strlen($token[1]);
+            $posDelta = 0;
+            while ($patchPos >= $pos && $patchPos < $pos + $len) {
+                $patchTextLen = \strlen($patchText);
+                if ($patchType === 'remove') {
+                    if ($patchPos === $pos && $patchTextLen === $len) {
+                        // Remove token entirely
+                        array_splice($this->tokens, $i, 1, []);
+                        $i--;
+                        $c--;
+                    } else {
+                        // Remove from token string
+                        $this->tokens[$i][1] = substr_replace(
+                            $token[1], '', $patchPos - $pos + $posDelta, $patchTextLen
+                        );
+                        $posDelta -= $patchTextLen;
+                    }
+                } elseif ($patchType === 'add') {
+                    // Insert into the token string
+                    $this->tokens[$i][1] = substr_replace(
+                        $token[1], $patchText, $patchPos - $pos + $posDelta, 0
+                    );
+                    $posDelta += $patchTextLen;
+                } else {
+                    assert(false);
+                }
+
+                // Fetch the next patch
+                $patchIdx++;
+                if ($patchIdx >= \count($this->patches)) {
+                    // No more patches, we're done
+                    return;
+                }
+
+                list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
+
+                // Multiple patches may apply to the same token. Reload the current one to check
+                // If the new patch applies
+                $token = $this->tokens[$i];
+            }
+
+            $pos += $len;
+        }
+
+        // A patch did not apply
+        assert(false);
+    }
+
+    /**
+     * Fixup line and position information in errors.
+     *
+     * @param Error[] $errors
+     */
+    private function fixupErrors(array $errors) {
+        foreach ($errors as $error) {
+            $attrs = $error->getAttributes();
+
+            $posDelta = 0;
+            $lineDelta = 0;
+            foreach ($this->patches as $patch) {
+                list($patchPos, $patchType, $patchText) = $patch;
+                if ($patchPos >= $attrs['startFilePos']) {
+                    // No longer relevant
+                    break;
+                }
+
+                if ($patchType === 'add') {
+                    $posDelta += strlen($patchText);
+                    $lineDelta += substr_count($patchText, "\n");
+                } else {
+                    $posDelta -= strlen($patchText);
+                    $lineDelta -= substr_count($patchText, "\n");
+                }
+            }
+
+            $attrs['startFilePos'] += $posDelta;
+            $attrs['endFilePos'] += $posDelta;
+            $attrs['startLine'] += $lineDelta;
+            $attrs['endLine'] += $lineDelta;
+            $error->setAttributes($attrs);
+        }
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php

@@ -0,0 +1,41 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class CoaleseEqualTokenEmulator implements TokenEmulatorInterface
+{
+    public function isEmulationNeeded(string $code) : bool
+    {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
+        return strpos($code, '??=') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        $line = 1;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            if (isset($tokens[$i + 1])) {
+                if ($tokens[$i][0] === T_COALESCE && $tokens[$i + 1] === '=') {
+                    array_splice($tokens, $i, 2, [
+                        [Emulative::T_COALESCE_EQUAL, '??=', $line]
+                    ]);
+                    $c--;
+                    continue;
+                }
+            }
+            if (\is_array($tokens[$i])) {
+                $line += substr_count($tokens[$i][1], "\n");
+            }
+        }
+
+        return $tokens;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php

@@ -0,0 +1,53 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class FnTokenEmulator implements TokenEmulatorInterface
+{
+    public function isEmulationNeeded(string $code) : bool
+    {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
+        return strpos($code, 'fn') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        foreach ($tokens as $i => $token) {
+            if ($token[0] === T_STRING && $token[1] === 'fn') {
+                $previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $i);
+                if ($previousNonSpaceToken !== null && $previousNonSpaceToken[0] === T_OBJECT_OPERATOR) {
+                    continue;
+                }
+
+                $tokens[$i][0] = Emulative::T_FN;
+            }
+        }
+
+        return $tokens;
+    }
+
+    /**
+     * @param mixed[] $tokens
+     * @return mixed[]|null
+     */
+    private function getPreviousNonSpaceToken(array $tokens, int $start)
+    {
+        for ($i = $start - 1; $i >= 0; --$i) {
+            if ($tokens[$i][0] === T_WHITESPACE) {
+                continue;
+            }
+
+            return $tokens[$i];
+        }
+
+        return null;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php

@@ -0,0 +1,98 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
+{
+    const BIN = '(?:0b[01]+(?:_[01]+)*)';
+    const HEX = '(?:0x[0-9a-f]+(?:_[0-9a-f]+)*)';
+    const DEC = '(?:[0-9]+(?:_[0-9]+)*)';
+    const SIMPLE_FLOAT = '(?:' . self::DEC . '\.' . self::DEC . '?|\.' . self::DEC . ')';
+    const EXP = '(?:e[+-]?' . self::DEC . ')';
+    const FLOAT = '(?:' . self::SIMPLE_FLOAT . self::EXP . '?|' . self::DEC . self::EXP . ')';
+    const NUMBER = '~' . self::FLOAT . '|' . self::BIN . '|' . self::HEX . '|' . self::DEC . '~iA';
+
+    public function isEmulationNeeded(string $code) : bool
+    {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
+        return preg_match('~[0-9a-f]_[0-9a-f]~i', $code) !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        $codeOffset = 0;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            $tokenLen = \strlen(\is_array($token) ? $token[1] : $token);
+
+            if ($token[0] !== T_LNUMBER && $token[0] !== T_DNUMBER) {
+                $codeOffset += $tokenLen;
+                continue;
+            }
+
+            $res = preg_match(self::NUMBER, $code, $matches, 0, $codeOffset);
+            assert($res, "No number at number token position");
+
+            $match = $matches[0];
+            $matchLen = \strlen($match);
+            if ($matchLen === $tokenLen) {
+                // Original token already holds the full number.
+                $codeOffset += $tokenLen;
+                continue;
+            }
+
+            $tokenKind = $this->resolveIntegerOrFloatToken($match);
+            $newTokens = [[$tokenKind, $match, $token[2]]];
+
+            $numTokens = 1;
+            $len = $tokenLen;
+            while ($matchLen > $len) {
+                $nextToken = $tokens[$i + $numTokens];
+                $nextTokenText = \is_array($nextToken) ? $nextToken[1] : $nextToken;
+                $nextTokenLen = \strlen($nextTokenText);
+
+                $numTokens++;
+                if ($matchLen < $len + $nextTokenLen) {
+                    // Split trailing characters into a partial token.
+                    assert(is_array($nextToken), "Partial token should be an array token");
+                    $partialText = substr($nextTokenText, $matchLen - $len);
+                    $newTokens[] = [$nextToken[0], $partialText, $nextToken[2]];
+                    break;
+                }
+
+                $len += $nextTokenLen;
+            }
+
+            array_splice($tokens, $i, $numTokens, $newTokens);
+            $c -= $numTokens - \count($newTokens);
+            $codeOffset += $matchLen;
+        }
+
+        return $tokens;
+    }
+
+    private function resolveIntegerOrFloatToken(string $str): int
+    {
+        $str = str_replace('_', '', $str);
+
+        if (stripos($str, '0b') === 0) {
+            $num = bindec($str);
+        } elseif (stripos($str, '0x') === 0) {
+            $num = hexdec($str);
+        } elseif (stripos($str, '0') === 0 && ctype_digit($str)) {
+            $num = octdec($str);
+        } else {
+            $num = +$str;
+        }
+
+        return is_float($num) ? T_DNUMBER : T_LNUMBER;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/TokenEmulatorInterface.php

@@ -0,0 +1,14 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+/** @internal */
+interface TokenEmulatorInterface
+{
+    public function isEmulationNeeded(string $code): bool;
+
+    /**
+     * @return array Modified Tokens
+     */
+    public function emulate(string $code, array $tokens): array;
+}