scanner = $scanner;
$this->events = $eventHandler;
$this->mode = $mode;
}
/**
* Begin parsing.
*
* This will begin scanning the document, tokenizing as it goes.
* Tokens are emitted into the event handler.
*
* Tokenizing will continue until the document is completely
* read. Errors are emitted into the event handler, but
* the parser will attempt to continue parsing until the
* entire input stream is read.
*/
public function parse()
{
do {
$this->consumeData();
// FIXME: Add infinite loop protection.
} while ($this->carryOn);
}
/**
* Set the text mode for the character data reader.
*
* HTML5 defines three different modes for reading text:
* - Normal: Read until a tag is encountered.
* - RCDATA: Read until a tag is encountered, but skip a few otherwise-
* special characters.
* - Raw: Read until a special closing tag is encountered (viz. pre, script)
*
* This allows those modes to be set.
*
* Normally, setting is done by the event handler via a special return code on
* startTag(), but it can also be set manually using this function.
*
* @param int $textmode One of Elements::TEXT_*.
* @param string $untilTag The tag that should stop RAW or RCDATA mode. Normal mode does not
* use this indicator.
*/
public function setTextMode($textmode, $untilTag = null)
{
$this->textMode = $textmode & (Elements::TEXT_RAW | Elements::TEXT_RCDATA);
$this->untilTag = $untilTag;
}
/**
* Consume a character and make a move.
* HTML5 8.2.4.1.
*/
protected function consumeData()
{
$tok = $this->scanner->current();
if ('&' === $tok) {
// Character reference
$ref = $this->decodeCharacterReference();
$this->buffer($ref);
$tok = $this->scanner->current();
}
// Parse tag
if ('<' === $tok) {
// Any buffered text data can go out now.
$this->flushBuffer();
$tok = $this->scanner->next();
if ('!' === $tok) {
$this->markupDeclaration();
} elseif ('/' === $tok) {
$this->endTag();
} elseif ('?' === $tok) {
$this->processingInstruction();
} elseif (ctype_alpha($tok)) {
$this->tagName();
} else {
$this->parseError('Illegal tag opening');
// TODO is this necessary ?
$this->characterData();
}
$tok = $this->scanner->current();
}
if (false === $tok) {
// Handle end of document
$this->eof();
} else {
// Parse character
switch ($this->textMode) {
case Elements::TEXT_RAW:
$this->rawText($tok);
break;
case Elements::TEXT_RCDATA:
$this->rcdata($tok);
break;
default:
if ('<' === $tok || '&' === $tok) {
break;
}
// NULL character
if ("\00" === $tok) {
$this->parseError('Received null character.');
$this->text .= $tok;
$this->scanner->consume();
break;
}
$this->text .= $this->scanner->charsUntil("<&\0");
}
}
return $this->carryOn;
}
/**
* Parse anything that looks like character data.
*
* Different rules apply based on the current text mode.
*
* @see Elements::TEXT_RAW Elements::TEXT_RCDATA.
*/
protected function characterData()
{
$tok = $this->scanner->current();
if (false === $tok) {
return false;
}
switch ($this->textMode) {
case Elements::TEXT_RAW:
return $this->rawText($tok);
case Elements::TEXT_RCDATA:
return $this->rcdata($tok);
default:
if ('<' === $tok || '&' === $tok) {
return false;
}
return $this->text($tok);
}
}
/**
* This buffers the current token as character data.
*
* @param string $tok The current token.
*
* @return bool
*/
protected function text($tok)
{
// This should never happen...
if (false === $tok) {
return false;
}
// NULL character
if ("\00" === $tok) {
$this->parseError('Received null character.');
}
$this->buffer($tok);
$this->scanner->consume();
return true;
}
/**
* Read text in RAW mode.
*
* @param string $tok The current token.
*
* @return bool
*/
protected function rawText($tok)
{
if (is_null($this->untilTag)) {
return $this->text($tok);
}
$sequence = '' . $this->untilTag . '>';
$txt = $this->readUntilSequence($sequence);
$this->events->text($txt);
$this->setTextMode(0);
return $this->endTag();
}
/**
* Read text in RCDATA mode.
*
* @param string $tok The current token.
*
* @return bool
*/
protected function rcdata($tok)
{
if (is_null($this->untilTag)) {
return $this->text($tok);
}
$sequence = '' . $this->untilTag;
$txt = '';
$caseSensitive = !Elements::isHtml5Element($this->untilTag);
while (false !== $tok && !('<' == $tok && ($this->scanner->sequenceMatches($sequence, $caseSensitive)))) {
if ('&' == $tok) {
$txt .= $this->decodeCharacterReference();
$tok = $this->scanner->current();
} else {
$txt .= $tok;
$tok = $this->scanner->next();
}
}
$len = strlen($sequence);
$this->scanner->consume($len);
$len += $this->scanner->whitespace();
if ('>' !== $this->scanner->current()) {
$this->parseError('Unclosed RCDATA end tag');
}
$this->scanner->unconsume($len);
$this->events->text($txt);
$this->setTextMode(0);
return $this->endTag();
}
/**
* If the document is read, emit an EOF event.
*/
protected function eof()
{
// fprintf(STDOUT, "EOF");
$this->flushBuffer();
$this->events->eof();
$this->carryOn = false;
}
/**
* Look for markup.
*/
protected function markupDeclaration()
{
$tok = $this->scanner->next();
// Comment:
if ('-' == $tok && '-' == $this->scanner->peek()) {
$this->scanner->consume(2);
return $this->comment();
} elseif ('D' == $tok || 'd' == $tok) { // Doctype
return $this->doctype();
} elseif ('[' == $tok) { // CDATA section
return $this->cdataSection();
}
// FINISH
$this->parseError('Expected . Emit an empty comment because 8.2.4.46 says to.
if ('>' == $tok) {
// Parse error. Emit the comment token.
$this->parseError("Expected comment data, got '>'");
$this->events->comment('');
$this->scanner->consume();
return true;
}
// Replace NULL with the replacement char.
if ("\0" == $tok) {
$tok = UTF8Utils::FFFD;
}
while (!$this->isCommentEnd()) {
$comment .= $tok;
$tok = $this->scanner->next();
}
$this->events->comment($comment);
$this->scanner->consume();
return true;
}
/**
* Check if the scanner has reached the end of a comment.
*
* @return bool
*/
protected function isCommentEnd()
{
$tok = $this->scanner->current();
// EOF
if (false === $tok) {
// Hit the end.
$this->parseError('Unexpected EOF in a comment.');
return true;
}
// If it doesn't start with -, not the end.
if ('-' != $tok) {
return false;
}
// Advance one, and test for '->'
if ('-' == $this->scanner->next() && '>' == $this->scanner->peek()) {
$this->scanner->consume(); // Consume the last '>'
return true;
}
// Unread '-';
$this->scanner->unconsume(1);
return false;
}
/**
* Parse a DOCTYPE.
*
* Parse a DOCTYPE declaration. This method has strong bearing on whether or
* not Quirksmode is enabled on the event handler.
*
* @todo This method is a little long. Should probably refactor.
*
* @return bool
*/
protected function doctype()
{
// Check that string is DOCTYPE.
if ($this->scanner->sequenceMatches('DOCTYPE', false)) {
$this->scanner->consume(7);
} else {
$chars = $this->scanner->charsWhile('DOCTYPEdoctype');
$this->parseError('Expected DOCTYPE, got %s', $chars);
return $this->bogusComment('scanner->whitespace();
$tok = $this->scanner->current();
// EOF: die.
if (false === $tok) {
$this->events->doctype('html5', EventHandler::DOCTYPE_NONE, '', true);
$this->eof();
return true;
}
// NULL char: convert.
if ("\0" === $tok) {
$this->parseError('Unexpected null character in DOCTYPE.');
}
$stop = " \n\f>";
$doctypeName = $this->scanner->charsUntil($stop);
// Lowercase ASCII, replace \0 with FFFD
$doctypeName = strtolower(strtr($doctypeName, "\0", UTF8Utils::FFFD));
$tok = $this->scanner->current();
// If false, emit a parse error, DOCTYPE, and return.
if (false === $tok) {
$this->parseError('Unexpected EOF in DOCTYPE declaration.');
$this->events->doctype($doctypeName, EventHandler::DOCTYPE_NONE, null, true);
return true;
}
// Short DOCTYPE, like
if ('>' == $tok) {
// DOCTYPE without a name.
if (0 == strlen($doctypeName)) {
$this->parseError('Expected a DOCTYPE name. Got nothing.');
$this->events->doctype($doctypeName, 0, null, true);
$this->scanner->consume();
return true;
}
$this->events->doctype($doctypeName);
$this->scanner->consume();
return true;
}
$this->scanner->whitespace();
$pub = strtoupper($this->scanner->getAsciiAlpha());
$white = $this->scanner->whitespace();
// Get ID, and flag it as pub or system.
if (('PUBLIC' == $pub || 'SYSTEM' == $pub) && $white > 0) {
// Get the sys ID.
$type = 'PUBLIC' == $pub ? EventHandler::DOCTYPE_PUBLIC : EventHandler::DOCTYPE_SYSTEM;
$id = $this->quotedString("\0>");
if (false === $id) {
$this->events->doctype($doctypeName, $type, $pub, false);
return true;
}
// Premature EOF.
if (false === $this->scanner->current()) {
$this->parseError('Unexpected EOF in DOCTYPE');
$this->events->doctype($doctypeName, $type, $id, true);
return true;
}
// Well-formed complete DOCTYPE.
$this->scanner->whitespace();
if ('>' == $this->scanner->current()) {
$this->events->doctype($doctypeName, $type, $id, false);
$this->scanner->consume();
return true;
}
// If we get here, we have scanner->charsUntil('>');
$this->parseError('Malformed DOCTYPE.');
$this->events->doctype($doctypeName, $type, $id, true);
$this->scanner->consume();
return true;
}
// Else it's a bogus DOCTYPE.
// Consume to > and trash.
$this->scanner->charsUntil('>');
$this->parseError('Expected PUBLIC or SYSTEM. Got %s.', $pub);
$this->events->doctype($doctypeName, 0, null, true);
$this->scanner->consume();
return true;
}
/**
* Utility for reading a quoted string.
*
* @param string $stopchars Characters (in addition to a close-quote) that should stop the string.
* E.g. sometimes '>' is higher precedence than '"' or "'".
*
* @return mixed String if one is found (quotations omitted).
*/
protected function quotedString($stopchars)
{
$tok = $this->scanner->current();
if ('"' == $tok || "'" == $tok) {
$this->scanner->consume();
$ret = $this->scanner->charsUntil($tok . $stopchars);
if ($this->scanner->current() == $tok) {
$this->scanner->consume();
} else {
// Parse error because no close quote.
$this->parseError('Expected %s, got %s', $tok, $this->scanner->current());
}
return $ret;
}
return false;
}
/**
* Handle a CDATA section.
*
* @return bool
*/
protected function cdataSection()
{
$cdata = '';
$this->scanner->consume();
$chars = $this->scanner->charsWhile('CDAT');
if ('CDATA' != $chars || '[' != $this->scanner->current()) {
$this->parseError('Expected [CDATA[, got %s', $chars);
return $this->bogusComment('scanner->next();
do {
if (false === $tok) {
$this->parseError('Unexpected EOF inside CDATA.');
$this->bogusComment('scanner->next();
} while (!$this->scanner->sequenceMatches(']]>'));
// Consume ]]>
$this->scanner->consume(3);
$this->events->cdata($cdata);
return true;
}
// ================================================================
// Non-HTML5
// ================================================================
/**
* Handle a processing instruction.
*
* XML processing instructions are supposed to be ignored in HTML5,
* treated as "bogus comments". However, since we're not a user
* agent, we allow them. We consume until ?> and then issue a
* EventListener::processingInstruction() event.
*
* @return bool
*/
protected function processingInstruction()
{
if ('?' != $this->scanner->current()) {
return false;
}
$tok = $this->scanner->next();
$procName = $this->scanner->getAsciiAlpha();
$white = $this->scanner->whitespace();
// If not a PI, send to bogusComment.
if (0 == strlen($procName) || 0 == $white || false == $this->scanner->current()) {
$this->parseError("Expected processing instruction name, got $tok");
$this->bogusComment('' . $tok . $procName);
return true;
}
$data = '';
// As long as it's not the case that the next two chars are ? and >.
while (!('?' == $this->scanner->current() && '>' == $this->scanner->peek())) {
$data .= $this->scanner->current();
$tok = $this->scanner->next();
if (false === $tok) {
$this->parseError('Unexpected EOF in processing instruction.');
$this->events->processingInstruction($procName, $data);
return true;
}
}
$this->scanner->consume(2); // Consume the closing tag
$this->events->processingInstruction($procName, $data);
return true;
}
// ================================================================
// UTILITY FUNCTIONS
// ================================================================
/**
* Read from the input stream until we get to the desired sequene
* or hit the end of the input stream.
*
* @param string $sequence
*
* @return string
*/
protected function readUntilSequence($sequence)
{
$buffer = '';
// Optimization for reading larger blocks faster.
$first = substr($sequence, 0, 1);
while (false !== $this->scanner->current()) {
$buffer .= $this->scanner->charsUntil($first);
// Stop as soon as we hit the stopping condition.
if ($this->scanner->sequenceMatches($sequence, false)) {
return $buffer;
}
$buffer .= $this->scanner->current();
$this->scanner->consume();
}
// If we get here, we hit the EOF.
$this->parseError('Unexpected EOF during text read.');
return $buffer;
}
/**
* Check if upcomming chars match the given sequence.
*
* This will read the stream for the $sequence. If it's
* found, this will return true. If not, return false.
* Since this unconsumes any chars it reads, the caller
* will still need to read the next sequence, even if
* this returns true.
*
* Example: $this->scanner->sequenceMatches('') will
* see if the input stream is at the start of a
* '' string.
*
* @param string $sequence
* @param bool $caseSensitive
*
* @return bool
*/
protected function sequenceMatches($sequence, $caseSensitive = true)
{
@trigger_error(__METHOD__ . ' method is deprecated since version 2.4 and will be removed in 3.0. Use Scanner::sequenceMatches() instead.', E_USER_DEPRECATED);
return $this->scanner->sequenceMatches($sequence, $caseSensitive);
}
/**
* Send a TEXT event with the contents of the text buffer.
*
* This emits an EventHandler::text() event with the current contents of the
* temporary text buffer. (The buffer is used to group as much PCDATA
* as we can instead of emitting lots and lots of TEXT events.)
*/
protected function flushBuffer()
{
if ('' === $this->text) {
return;
}
$this->events->text($this->text);
$this->text = '';
}
/**
* Add text to the temporary buffer.
*
* @see flushBuffer()
*
* @param string $str
*/
protected function buffer($str)
{
$this->text .= $str;
}
/**
* Emit a parse error.
*
* A parse error always returns false because it never consumes any
* characters.
*
* @param string $msg
*
* @return string
*/
protected function parseError($msg)
{
$args = func_get_args();
if (count($args) > 1) {
array_shift($args);
$msg = vsprintf($msg, $args);
}
$line = $this->scanner->currentLine();
$col = $this->scanner->columnOffset();
$this->events->parseError($msg, $line, $col);
return false;
}
/**
* Decode a character reference and return the string.
*
* If $inAttribute is set to true, a bare & will be returned as-is.
*
* @param bool $inAttribute Set to true if the text is inside of an attribute value.
* false otherwise.
*
* @return string
*/
protected function decodeCharacterReference($inAttribute = false)
{
// Next char after &.
$tok = $this->scanner->next();
$start = $this->scanner->position();
if (false === $tok) {
return '&';
}
// These indicate not an entity. We return just
// the &.
if ("\t" === $tok || "\n" === $tok || "\f" === $tok || ' ' === $tok || '&' === $tok || '<' === $tok) {
// $this->scanner->next();
return '&';
}
// Numeric entity
if ('#' === $tok) {
$tok = $this->scanner->next();
if (false === $tok) {
$this->parseError('Expected DEC; HEX;, got EOF');
$this->scanner->unconsume(1);
return '&';
}
// Hexidecimal encoding.
// X[0-9a-fA-F]+;
// x[0-9a-fA-F]+;
if ('x' === $tok || 'X' === $tok) {
$tok = $this->scanner->next(); // Consume x
// Convert from hex code to char.
$hex = $this->scanner->getHex();
if (empty($hex)) {
$this->parseError('Expected HEX;, got %s', $tok);
// We unconsume because we don't know what parser rules might
// be in effect for the remaining chars. For example. '>'
// might result in a specific parsing rule inside of tag
// contexts, while not inside of pcdata context.
$this->scanner->unconsume(2);
return '&';
}
$entity = CharacterReference::lookupHex($hex);
} // Decimal encoding.
// [0-9]+;
else {
// Convert from decimal to char.
$numeric = $this->scanner->getNumeric();
if (false === $numeric) {
$this->parseError('Expected DIGITS;, got %s', $tok);
$this->scanner->unconsume(2);
return '&';
}
$entity = CharacterReference::lookupDecimal($numeric);
}
} elseif ('=' === $tok && $inAttribute) {
return '&';
} else { // String entity.
// Attempt to consume a string up to a ';'.
// [a-zA-Z0-9]+;
$cname = $this->scanner->getAsciiAlphaNum();
$entity = CharacterReference::lookupName($cname);
// When no entity is found provide the name of the unmatched string
// and continue on as the & is not part of an entity. The & will
// be converted to & elsewhere.
if (null === $entity) {
if (!$inAttribute || '' === $cname) {
$this->parseError("No match in entity table for '%s'", $cname);
}
$this->scanner->unconsume($this->scanner->position() - $start);
return '&';
}
}
// The scanner has advanced the cursor for us.
$tok = $this->scanner->current();
// We have an entity. We're done here.
if (';' === $tok) {
$this->scanner->consume();
return $entity;
}
// Failing to match ; means unconsume the entire string.
$this->scanner->unconsume($this->scanner->position() - $start);
$this->parseError('Expected &ENTITY;, got &ENTITY%s (no trailing ;) ', $tok);
return '&';
}
}