diff options
Diffstat (limited to 'vendor/theseer/tokenizer/src')
-rw-r--r-- | vendor/theseer/tokenizer/src/Exception.php | 5 | ||||
-rw-r--r-- | vendor/theseer/tokenizer/src/NamespaceUri.php | 25 | ||||
-rw-r--r-- | vendor/theseer/tokenizer/src/NamespaceUriException.php | 5 | ||||
-rw-r--r-- | vendor/theseer/tokenizer/src/Token.php | 35 | ||||
-rw-r--r-- | vendor/theseer/tokenizer/src/TokenCollection.php | 93 | ||||
-rw-r--r-- | vendor/theseer/tokenizer/src/TokenCollectionException.php | 5 | ||||
-rw-r--r-- | vendor/theseer/tokenizer/src/Tokenizer.php | 142 | ||||
-rw-r--r-- | vendor/theseer/tokenizer/src/XMLSerializer.php | 79 |
8 files changed, 389 insertions, 0 deletions
diff --git a/vendor/theseer/tokenizer/src/Exception.php b/vendor/theseer/tokenizer/src/Exception.php new file mode 100644 index 000000000..71fc117a5 --- /dev/null +++ b/vendor/theseer/tokenizer/src/Exception.php @@ -0,0 +1,5 @@ +<?php declare(strict_types = 1); +namespace TheSeer\Tokenizer; + +class Exception extends \Exception { +} diff --git a/vendor/theseer/tokenizer/src/NamespaceUri.php b/vendor/theseer/tokenizer/src/NamespaceUri.php new file mode 100644 index 000000000..14e9f0c8c --- /dev/null +++ b/vendor/theseer/tokenizer/src/NamespaceUri.php @@ -0,0 +1,25 @@ +<?php declare(strict_types = 1); +namespace TheSeer\Tokenizer; + +class NamespaceUri { + + /** @var string */ + private $value; + + public function __construct(string $value) { + $this->ensureValidUri($value); + $this->value = $value; + } + + public function asString(): string { + return $this->value; + } + + private function ensureValidUri($value): void { + if (\strpos($value, ':') === false) { + throw new NamespaceUriException( + \sprintf("Namespace URI '%s' must contain at least one colon", $value) + ); + } + } +} diff --git a/vendor/theseer/tokenizer/src/NamespaceUriException.php b/vendor/theseer/tokenizer/src/NamespaceUriException.php new file mode 100644 index 000000000..ab1c48d29 --- /dev/null +++ b/vendor/theseer/tokenizer/src/NamespaceUriException.php @@ -0,0 +1,5 @@ +<?php declare(strict_types = 1); +namespace TheSeer\Tokenizer; + +class NamespaceUriException extends Exception { +} diff --git a/vendor/theseer/tokenizer/src/Token.php b/vendor/theseer/tokenizer/src/Token.php new file mode 100644 index 000000000..ffcbe4009 --- /dev/null +++ b/vendor/theseer/tokenizer/src/Token.php @@ -0,0 +1,35 @@ +<?php declare(strict_types = 1); +namespace TheSeer\Tokenizer; + +class Token { + + /** @var int */ + private $line; + + /** @var string */ + private $name; + + /** @var string */ + private $value; + + /** + * Token constructor. + */ + public function __construct(int $line, string $name, string $value) { + $this->line = $line; + $this->name = $name; + $this->value = $value; + } + + public function getLine(): int { + return $this->line; + } + + public function getName(): string { + return $this->name; + } + + public function getValue(): string { + return $this->value; + } +} diff --git a/vendor/theseer/tokenizer/src/TokenCollection.php b/vendor/theseer/tokenizer/src/TokenCollection.php new file mode 100644 index 000000000..e5e6e401c --- /dev/null +++ b/vendor/theseer/tokenizer/src/TokenCollection.php @@ -0,0 +1,93 @@ +<?php declare(strict_types = 1); +namespace TheSeer\Tokenizer; + +class TokenCollection implements \ArrayAccess, \Iterator, \Countable { + + /** @var Token[] */ + private $tokens = []; + + /** @var int */ + private $pos; + + public function addToken(Token $token): void { + $this->tokens[] = $token; + } + + public function current(): Token { + return \current($this->tokens); + } + + public function key(): int { + return \key($this->tokens); + } + + public function next(): void { + \next($this->tokens); + $this->pos++; + } + + public function valid(): bool { + return $this->count() > $this->pos; + } + + public function rewind(): void { + \reset($this->tokens); + $this->pos = 0; + } + + public function count(): int { + return \count($this->tokens); + } + + public function offsetExists($offset): bool { + return isset($this->tokens[$offset]); + } + + /** + * @throws TokenCollectionException + */ + public function offsetGet($offset): Token { + if (!$this->offsetExists($offset)) { + throw new TokenCollectionException( + \sprintf('No Token at offest %s', $offset) + ); + } + + return $this->tokens[$offset]; + } + + /** + * @param Token $value + * + * @throws TokenCollectionException + */ + public function offsetSet($offset, $value): void { + if (!\is_int($offset)) { + $type = \gettype($offset); + + throw new TokenCollectionException( + \sprintf( + 'Offset must be of type integer, %s given', + $type === 'object' ? \get_class($value) : $type + ) + ); + } + + if (!$value instanceof Token) { + $type = \gettype($value); + + throw new TokenCollectionException( + \sprintf( + 'Value must be of type %s, %s given', + Token::class, + $type === 'object' ? \get_class($value) : $type + ) + ); + } + $this->tokens[$offset] = $value; + } + + public function offsetUnset($offset): void { + unset($this->tokens[$offset]); + } +} diff --git a/vendor/theseer/tokenizer/src/TokenCollectionException.php b/vendor/theseer/tokenizer/src/TokenCollectionException.php new file mode 100644 index 000000000..4291ce0c4 --- /dev/null +++ b/vendor/theseer/tokenizer/src/TokenCollectionException.php @@ -0,0 +1,5 @@ +<?php declare(strict_types = 1); +namespace TheSeer\Tokenizer; + +class TokenCollectionException extends Exception { +} diff --git a/vendor/theseer/tokenizer/src/Tokenizer.php b/vendor/theseer/tokenizer/src/Tokenizer.php new file mode 100644 index 000000000..f582d9587 --- /dev/null +++ b/vendor/theseer/tokenizer/src/Tokenizer.php @@ -0,0 +1,142 @@ +<?php declare(strict_types = 1); +namespace TheSeer\Tokenizer; + +class Tokenizer { + + /** + * Token Map for "non-tokens" + * + * @var array + */ + private $map = [ + '(' => 'T_OPEN_BRACKET', + ')' => 'T_CLOSE_BRACKET', + '[' => 'T_OPEN_SQUARE', + ']' => 'T_CLOSE_SQUARE', + '{' => 'T_OPEN_CURLY', + '}' => 'T_CLOSE_CURLY', + ';' => 'T_SEMICOLON', + '.' => 'T_DOT', + ',' => 'T_COMMA', + '=' => 'T_EQUAL', + '<' => 'T_LT', + '>' => 'T_GT', + '+' => 'T_PLUS', + '-' => 'T_MINUS', + '*' => 'T_MULT', + '/' => 'T_DIV', + '?' => 'T_QUESTION_MARK', + '!' => 'T_EXCLAMATION_MARK', + ':' => 'T_COLON', + '"' => 'T_DOUBLE_QUOTES', + '@' => 'T_AT', + '&' => 'T_AMPERSAND', + '%' => 'T_PERCENT', + '|' => 'T_PIPE', + '$' => 'T_DOLLAR', + '^' => 'T_CARET', + '~' => 'T_TILDE', + '`' => 'T_BACKTICK' + ]; + + public function parse(string $source): TokenCollection { + $result = new TokenCollection(); + + if ($source === '') { + return $result; + } + + $tokens = \token_get_all($source); + + $lastToken = new Token( + $tokens[0][2], + 'Placeholder', + '' + ); + + foreach ($tokens as $pos => $tok) { + if (\is_string($tok)) { + $token = new Token( + $lastToken->getLine(), + $this->map[$tok], + $tok + ); + $result->addToken($token); + $lastToken = $token; + + continue; + } + + $line = $tok[2]; + $values = \preg_split('/\R+/Uu', $tok[1]); + + foreach ($values as $v) { + $token = new Token( + $line, + \token_name($tok[0]), + $v + ); + $lastToken = $token; + $line++; + + if ($v === '') { + continue; + } + + $result->addToken($token); + } + } + + return $this->fillBlanks($result, $lastToken->getLine()); + } + + private function fillBlanks(TokenCollection $tokens, int $maxLine): TokenCollection { + $prev = new Token( + 0, + 'Placeholder', + '' + ); + + $final = new TokenCollection(); + + foreach ($tokens as $token) { + if ($prev === null) { + $final->addToken($token); + $prev = $token; + + continue; + } + + $gap = $token->getLine() - $prev->getLine(); + + while ($gap > 1) { + $linebreak = new Token( + $prev->getLine() + 1, + 'T_WHITESPACE', + '' + ); + $final->addToken($linebreak); + $prev = $linebreak; + $gap--; + } + + $final->addToken($token); + $prev = $token; + } + + $gap = $maxLine - $prev->getLine(); + + while ($gap > 0) { + $linebreak = new Token( + $prev->getLine() + 1, + 'T_WHITESPACE', + '' + ); + $final->addToken($linebreak); + $prev = $linebreak; + $gap--; + } + + return $final; + } +} diff --git a/vendor/theseer/tokenizer/src/XMLSerializer.php b/vendor/theseer/tokenizer/src/XMLSerializer.php new file mode 100644 index 000000000..e67a7fe8b --- /dev/null +++ b/vendor/theseer/tokenizer/src/XMLSerializer.php @@ -0,0 +1,79 @@ +<?php declare(strict_types = 1); +namespace TheSeer\Tokenizer; + +use DOMDocument; + +class XMLSerializer { + + /** @var \XMLWriter */ + private $writer; + + /** @var Token */ + private $previousToken; + + /** @var NamespaceUri */ + private $xmlns; + + /** + * XMLSerializer constructor. + * + * @param NamespaceUri $xmlns + */ + public function __construct(NamespaceUri $xmlns = null) { + if ($xmlns === null) { + $xmlns = new NamespaceUri('https://github.com/theseer/tokenizer'); + } + $this->xmlns = $xmlns; + } + + public function toDom(TokenCollection $tokens): DOMDocument { + $dom = new DOMDocument(); + $dom->preserveWhiteSpace = false; + $dom->loadXML($this->toXML($tokens)); + + return $dom; + } + + public function toXML(TokenCollection $tokens): string { + $this->writer = new \XMLWriter(); + $this->writer->openMemory(); + $this->writer->setIndent(true); + $this->writer->startDocument(); + $this->writer->startElement('source'); + $this->writer->writeAttribute('xmlns', $this->xmlns->asString()); + + if (\count($tokens) > 0) { + $this->writer->startElement('line'); + $this->writer->writeAttribute('no', '1'); + + $this->previousToken = $tokens[0]; + + foreach ($tokens as $token) { + $this->addToken($token); + } + } + + $this->writer->endElement(); + $this->writer->endElement(); + $this->writer->endDocument(); + + return $this->writer->outputMemory(); + } + + private function addToken(Token $token): void { + if ($this->previousToken->getLine() < $token->getLine()) { + $this->writer->endElement(); + + $this->writer->startElement('line'); + $this->writer->writeAttribute('no', (string)$token->getLine()); + $this->previousToken = $token; + } + + if ($token->getValue() !== '') { + $this->writer->startElement('token'); + $this->writer->writeAttribute('name', $token->getName()); + $this->writer->writeRaw(\htmlspecialchars($token->getValue(), \ENT_NOQUOTES | \ENT_DISALLOWED | \ENT_XML1)); + $this->writer->endElement(); + } + } +} |