summaryrefslogtreecommitdiff
path: root/vendor/theseer/tokenizer/src
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/theseer/tokenizer/src')
-rw-r--r--vendor/theseer/tokenizer/src/Exception.php5
-rw-r--r--vendor/theseer/tokenizer/src/NamespaceUri.php25
-rw-r--r--vendor/theseer/tokenizer/src/NamespaceUriException.php5
-rw-r--r--vendor/theseer/tokenizer/src/Token.php35
-rw-r--r--vendor/theseer/tokenizer/src/TokenCollection.php93
-rw-r--r--vendor/theseer/tokenizer/src/TokenCollectionException.php5
-rw-r--r--vendor/theseer/tokenizer/src/Tokenizer.php142
-rw-r--r--vendor/theseer/tokenizer/src/XMLSerializer.php79
8 files changed, 389 insertions, 0 deletions
diff --git a/vendor/theseer/tokenizer/src/Exception.php b/vendor/theseer/tokenizer/src/Exception.php
new file mode 100644
index 000000000..71fc117a5
--- /dev/null
+++ b/vendor/theseer/tokenizer/src/Exception.php
@@ -0,0 +1,5 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+class Exception extends \Exception {
+}
diff --git a/vendor/theseer/tokenizer/src/NamespaceUri.php b/vendor/theseer/tokenizer/src/NamespaceUri.php
new file mode 100644
index 000000000..14e9f0c8c
--- /dev/null
+++ b/vendor/theseer/tokenizer/src/NamespaceUri.php
@@ -0,0 +1,25 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+class NamespaceUri {
+
+ /** @var string */
+ private $value;
+
+ public function __construct(string $value) {
+ $this->ensureValidUri($value);
+ $this->value = $value;
+ }
+
+ public function asString(): string {
+ return $this->value;
+ }
+
+ private function ensureValidUri($value): void {
+ if (\strpos($value, ':') === false) {
+ throw new NamespaceUriException(
+ \sprintf("Namespace URI '%s' must contain at least one colon", $value)
+ );
+ }
+ }
+}
diff --git a/vendor/theseer/tokenizer/src/NamespaceUriException.php b/vendor/theseer/tokenizer/src/NamespaceUriException.php
new file mode 100644
index 000000000..ab1c48d29
--- /dev/null
+++ b/vendor/theseer/tokenizer/src/NamespaceUriException.php
@@ -0,0 +1,5 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+class NamespaceUriException extends Exception {
+}
diff --git a/vendor/theseer/tokenizer/src/Token.php b/vendor/theseer/tokenizer/src/Token.php
new file mode 100644
index 000000000..ffcbe4009
--- /dev/null
+++ b/vendor/theseer/tokenizer/src/Token.php
@@ -0,0 +1,35 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+class Token {
+
+ /** @var int */
+ private $line;
+
+ /** @var string */
+ private $name;
+
+ /** @var string */
+ private $value;
+
+ /**
+ * Token constructor.
+ */
+ public function __construct(int $line, string $name, string $value) {
+ $this->line = $line;
+ $this->name = $name;
+ $this->value = $value;
+ }
+
+ public function getLine(): int {
+ return $this->line;
+ }
+
+ public function getName(): string {
+ return $this->name;
+ }
+
+ public function getValue(): string {
+ return $this->value;
+ }
+}
diff --git a/vendor/theseer/tokenizer/src/TokenCollection.php b/vendor/theseer/tokenizer/src/TokenCollection.php
new file mode 100644
index 000000000..e5e6e401c
--- /dev/null
+++ b/vendor/theseer/tokenizer/src/TokenCollection.php
@@ -0,0 +1,93 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+class TokenCollection implements \ArrayAccess, \Iterator, \Countable {
+
+ /** @var Token[] */
+ private $tokens = [];
+
+ /** @var int */
+ private $pos;
+
+ public function addToken(Token $token): void {
+ $this->tokens[] = $token;
+ }
+
+ public function current(): Token {
+ return \current($this->tokens);
+ }
+
+ public function key(): int {
+ return \key($this->tokens);
+ }
+
+ public function next(): void {
+ \next($this->tokens);
+ $this->pos++;
+ }
+
+ public function valid(): bool {
+ return $this->count() > $this->pos;
+ }
+
+ public function rewind(): void {
+ \reset($this->tokens);
+ $this->pos = 0;
+ }
+
+ public function count(): int {
+ return \count($this->tokens);
+ }
+
+ public function offsetExists($offset): bool {
+ return isset($this->tokens[$offset]);
+ }
+
+ /**
+ * @throws TokenCollectionException
+ */
+ public function offsetGet($offset): Token {
+ if (!$this->offsetExists($offset)) {
+ throw new TokenCollectionException(
+ \sprintf('No Token at offest %s', $offset)
+ );
+ }
+
+ return $this->tokens[$offset];
+ }
+
+ /**
+ * @param Token $value
+ *
+ * @throws TokenCollectionException
+ */
+ public function offsetSet($offset, $value): void {
+ if (!\is_int($offset)) {
+ $type = \gettype($offset);
+
+ throw new TokenCollectionException(
+ \sprintf(
+ 'Offset must be of type integer, %s given',
+ $type === 'object' ? \get_class($value) : $type
+ )
+ );
+ }
+
+ if (!$value instanceof Token) {
+ $type = \gettype($value);
+
+ throw new TokenCollectionException(
+ \sprintf(
+ 'Value must be of type %s, %s given',
+ Token::class,
+ $type === 'object' ? \get_class($value) : $type
+ )
+ );
+ }
+ $this->tokens[$offset] = $value;
+ }
+
+ public function offsetUnset($offset): void {
+ unset($this->tokens[$offset]);
+ }
+}
diff --git a/vendor/theseer/tokenizer/src/TokenCollectionException.php b/vendor/theseer/tokenizer/src/TokenCollectionException.php
new file mode 100644
index 000000000..4291ce0c4
--- /dev/null
+++ b/vendor/theseer/tokenizer/src/TokenCollectionException.php
@@ -0,0 +1,5 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+class TokenCollectionException extends Exception {
+}
diff --git a/vendor/theseer/tokenizer/src/Tokenizer.php b/vendor/theseer/tokenizer/src/Tokenizer.php
new file mode 100644
index 000000000..f582d9587
--- /dev/null
+++ b/vendor/theseer/tokenizer/src/Tokenizer.php
@@ -0,0 +1,142 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+class Tokenizer {
+
+ /**
+ * Token Map for "non-tokens"
+ *
+ * @var array
+ */
+ private $map = [
+ '(' => 'T_OPEN_BRACKET',
+ ')' => 'T_CLOSE_BRACKET',
+ '[' => 'T_OPEN_SQUARE',
+ ']' => 'T_CLOSE_SQUARE',
+ '{' => 'T_OPEN_CURLY',
+ '}' => 'T_CLOSE_CURLY',
+ ';' => 'T_SEMICOLON',
+ '.' => 'T_DOT',
+ ',' => 'T_COMMA',
+ '=' => 'T_EQUAL',
+ '<' => 'T_LT',
+ '>' => 'T_GT',
+ '+' => 'T_PLUS',
+ '-' => 'T_MINUS',
+ '*' => 'T_MULT',
+ '/' => 'T_DIV',
+ '?' => 'T_QUESTION_MARK',
+ '!' => 'T_EXCLAMATION_MARK',
+ ':' => 'T_COLON',
+ '"' => 'T_DOUBLE_QUOTES',
+ '@' => 'T_AT',
+ '&' => 'T_AMPERSAND',
+ '%' => 'T_PERCENT',
+ '|' => 'T_PIPE',
+ '$' => 'T_DOLLAR',
+ '^' => 'T_CARET',
+ '~' => 'T_TILDE',
+ '`' => 'T_BACKTICK'
+ ];
+
+ public function parse(string $source): TokenCollection {
+ $result = new TokenCollection();
+
+ if ($source === '') {
+ return $result;
+ }
+
+ $tokens = \token_get_all($source);
+
+ $lastToken = new Token(
+ $tokens[0][2],
+ 'Placeholder',
+ ''
+ );
+
+ foreach ($tokens as $pos => $tok) {
+ if (\is_string($tok)) {
+ $token = new Token(
+ $lastToken->getLine(),
+ $this->map[$tok],
+ $tok
+ );
+ $result->addToken($token);
+ $lastToken = $token;
+
+ continue;
+ }
+
+ $line = $tok[2];
+ $values = \preg_split('/\R+/Uu', $tok[1]);
+
+ foreach ($values as $v) {
+ $token = new Token(
+ $line,
+ \token_name($tok[0]),
+ $v
+ );
+ $lastToken = $token;
+ $line++;
+
+ if ($v === '') {
+ continue;
+ }
+
+ $result->addToken($token);
+ }
+ }
+
+ return $this->fillBlanks($result, $lastToken->getLine());
+ }
+
+ private function fillBlanks(TokenCollection $tokens, int $maxLine): TokenCollection {
+ $prev = new Token(
+ 0,
+ 'Placeholder',
+ ''
+ );
+
+ $final = new TokenCollection();
+
+ foreach ($tokens as $token) {
+ if ($prev === null) {
+ $final->addToken($token);
+ $prev = $token;
+
+ continue;
+ }
+
+ $gap = $token->getLine() - $prev->getLine();
+
+ while ($gap > 1) {
+ $linebreak = new Token(
+ $prev->getLine() + 1,
+ 'T_WHITESPACE',
+ ''
+ );
+ $final->addToken($linebreak);
+ $prev = $linebreak;
+ $gap--;
+ }
+
+ $final->addToken($token);
+ $prev = $token;
+ }
+
+ $gap = $maxLine - $prev->getLine();
+
+ while ($gap > 0) {
+ $linebreak = new Token(
+ $prev->getLine() + 1,
+ 'T_WHITESPACE',
+ ''
+ );
+ $final->addToken($linebreak);
+ $prev = $linebreak;
+ $gap--;
+ }
+
+ return $final;
+ }
+}
diff --git a/vendor/theseer/tokenizer/src/XMLSerializer.php b/vendor/theseer/tokenizer/src/XMLSerializer.php
new file mode 100644
index 000000000..e67a7fe8b
--- /dev/null
+++ b/vendor/theseer/tokenizer/src/XMLSerializer.php
@@ -0,0 +1,79 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+use DOMDocument;
+
+class XMLSerializer {
+
+ /** @var \XMLWriter */
+ private $writer;
+
+ /** @var Token */
+ private $previousToken;
+
+ /** @var NamespaceUri */
+ private $xmlns;
+
+ /**
+ * XMLSerializer constructor.
+ *
+ * @param NamespaceUri $xmlns
+ */
+ public function __construct(NamespaceUri $xmlns = null) {
+ if ($xmlns === null) {
+ $xmlns = new NamespaceUri('https://github.com/theseer/tokenizer');
+ }
+ $this->xmlns = $xmlns;
+ }
+
+ public function toDom(TokenCollection $tokens): DOMDocument {
+ $dom = new DOMDocument();
+ $dom->preserveWhiteSpace = false;
+ $dom->loadXML($this->toXML($tokens));
+
+ return $dom;
+ }
+
+ public function toXML(TokenCollection $tokens): string {
+ $this->writer = new \XMLWriter();
+ $this->writer->openMemory();
+ $this->writer->setIndent(true);
+ $this->writer->startDocument();
+ $this->writer->startElement('source');
+ $this->writer->writeAttribute('xmlns', $this->xmlns->asString());
+
+ if (\count($tokens) > 0) {
+ $this->writer->startElement('line');
+ $this->writer->writeAttribute('no', '1');
+
+ $this->previousToken = $tokens[0];
+
+ foreach ($tokens as $token) {
+ $this->addToken($token);
+ }
+ }
+
+ $this->writer->endElement();
+ $this->writer->endElement();
+ $this->writer->endDocument();
+
+ return $this->writer->outputMemory();
+ }
+
+ private function addToken(Token $token): void {
+ if ($this->previousToken->getLine() < $token->getLine()) {
+ $this->writer->endElement();
+
+ $this->writer->startElement('line');
+ $this->writer->writeAttribute('no', (string)$token->getLine());
+ $this->previousToken = $token;
+ }
+
+ if ($token->getValue() !== '') {
+ $this->writer->startElement('token');
+ $this->writer->writeAttribute('name', $token->getName());
+ $this->writer->writeRaw(\htmlspecialchars($token->getValue(), \ENT_NOQUOTES | \ENT_DISALLOWED | \ENT_XML1));
+ $this->writer->endElement();
+ }
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-03 03:57:46 +0000