scanner = $scanner; $this->events = $eventHandler; } /** * Begin parsing. * * This will begin scanning the document, tokenizing as it goes. * Tokens are emitted into the event handler. * * Tokenizing will continue until the document is completely * read. Errors are emitted into the event handler, but * the parser will attempt to continue parsing until the * entire input stream is read. */ public function parse() { $p = 0; do { $p = $this->scanner->position(); $this->consumeData(); // FIXME: Add infinite loop protection. } while ($this->carryOn); } /** * Set the text mode for the character data reader. * * HTML5 defines three different modes for reading text: * - Normal: Read until a tag is encountered. * - RCDATA: Read until a tag is encountered, but skip a few otherwise- * special characters. * - Raw: Read until a special closing tag is encountered (viz. pre, script) * * This allows those modes to be set. * * Normally, setting is done by the event handler via a special return code on * startTag(), but it can also be set manually using this function. * * @param integer $textmode * One of Elements::TEXT_* * @param string $untilTag * The tag that should stop RAW or RCDATA mode. Normal mode does not * use this indicator. */ public function setTextMode($textmode, $untilTag = NULL) { $this->textMode = $textmode & (Elements::TEXT_RAW | Elements::TEXT_RCDATA); $this->untilTag = $untilTag; } /** * Consume a character and make a move. * HTML5 8.2.4.1 */ protected function consumeData() { // Character Ref /* $this->characterReference() || $this->tagOpen() || $this->eof() || $this->characterData(); */ $this->characterReference(); $this->tagOpen(); $this->eof(); $this->characterData(); return $this->carryOn; } /** * Parse anything that looks like character data. * * Different rules apply based on the current text mode. * * @see Elements::TEXT_RAW Elements::TEXT_RCDATA. */ protected function characterData() { if ($this->scanner->current() === FALSE) { return FALSE; } switch ($this->textMode) { case Elements::TEXT_RAW: case Elements::TEXT_RCDATA: return $this->rawText(); default: $tok = $this->scanner->current(); if (strspn($tok, "<&")) { return FALSE; } return $this->text(); } } /** * This buffers the current token as character data. */ protected function text() { $tok = $this->scanner->current(); // This should never happen... if ($tok === FALSE) { return FALSE; } // Null if ($tok === "\00") { $this->parseError("Received NULL character."); } // fprintf(STDOUT, "Writing '%s'", $tok); $this->buffer($tok); $this->scanner->next(); return TRUE; } /** * Read text in RAW mode. */ protected function rawText() { if (is_null($this->untilTag)) { return $this->text(); } $sequence = 'untilTag . '>'; $txt = $this->readUntilSequence($sequence); $this->events->text($txt); $this->setTextMode(0); return $this->endTag(); } /** * If the document is read, emit an EOF event. */ protected function eof() { if ($this->scanner->current() === FALSE) { //fprintf(STDOUT, "EOF"); $this->flushBuffer(); $this->events->eof(); $this->carryOn = FALSE; return TRUE; } return FALSE; } /** * Handle character references (aka entities). * * This version is specific to PCDATA, as it buffers data into the * text buffer. For a generic version, see decodeCharacterReference(). * * HTML5 8.2.4.2 */ protected function characterReference() { $ref = $this->decodeCharacterReference(); if ($ref !== FALSE) { $this->buffer($ref); return TRUE; } return FALSE; } /** * Emit a tagStart event on encountering a tag. * * 8.2.4.8 */ protected function tagOpen() { if ($this->scanner->current() != '<') { return FALSE; } // Any buffered text data can go out now. $this->flushBuffer(); $this->scanner->next(); return $this->markupDeclaration() || $this->endTag() || $this->processingInstruction() || $this->tagName() || // This always returns false. $this->parseError("Illegal tag opening") || $this->characterData(); } /** * Look for markup. */ protected function markupDeclaration() { if ($this->scanner->current() != '!') { return FALSE; } $tok = $this->scanner->next(); // Comment: if ($tok == '-' && $this->scanner->peek() == '-') { $this->scanner->next(); // Consume the other '-' $this->scanner->next(); // Next char. return $this->comment(); } // Doctype elseif($tok == 'D' || $tok == 'd') { return $this->doctype(''); } // CDATA section elseif($tok == '[') { return $this->cdataSection(); } // FINISH $this->parseError("Expected . Emit an empty comment because 8.2.4.46 says to. if ($tok == '>') { // Parse error. Emit the comment token. $this->parseError("Expected comment data, got '>'"); $this->events->comment(''); $this->scanner->next(); return TRUE; } // Replace NULL with the replacement char. if ($tok == "\0") { $tok = UTF8Utils::FFFD; } while (!$this->isCommentEnd()) { $comment .= $tok; $tok = $this->scanner->next(); } $this->events->comment($comment); $this->scanner->next(); return TRUE; } /** * Check if the scanner has reached the end of a comment. */ protected function isCommentEnd() { // EOF if($this->scanner->current() === FALSE) { // Hit the end. $this->parseError("Unexpected EOF in a comment."); return TRUE; } // If it doesn't start with -, not the end. if($this->scanner->current() != '-') { return FALSE; } // Advance one, and test for '->' if ($this->scanner->next() == '-' && $this->scanner->peek() == '>') { $this->scanner->next(); // Consume the last '>' return TRUE; } // Unread '-'; $this->scanner->unconsume(1); return FALSE; } /** * Parse a DOCTYPE. * * Parse a DOCTYPE declaration. This method has strong bearing on whether or * not Quirksmode is enabled on the event handler. * * @todo This method is a little long. Should probably refactor. */ protected function doctype() { if (strcasecmp($this->scanner->current(), 'D')) { return FALSE; } // Check that string is DOCTYPE. $chars = $this->scanner->charsWhile("DOCTYPEdoctype"); if (strcasecmp($chars, 'DOCTYPE')) { $this->parseError('Expected DOCTYPE, got %s', $chars); return $this->bogusComment('scanner->whitespace(); $tok = $this->scanner->current(); // EOF: die. if ($tok === FALSE) { $this->events->doctype('html5',EventHandler::DOCTYPE_NONE,'', TRUE); return $this->eof(); } $doctypeName = ''; // NULL char: convert. if ($tok === "\0") { $this->parseError("Unexpected NULL character in DOCTYPE."); $doctypeName .= UTF8::FFFD; $tok = $this->scanner->next(); } $stop = " \n\f>"; $doctypeName = $this->scanner->charsUntil($stop); // Lowercase ASCII, replace \0 with FFFD $doctypeName = strtolower(strtr($doctypeName, "\0", UTF8Utils::FFFD)); $tok = $this->scanner->current(); // If FALSE, emit a parse error, DOCTYPE, and return. if ($tok === FALSE) { $this->parseError('Unexpected EOF in DOCTYPE declaration.'); $this->events->doctype($doctypeName, EventHandler::DOCTYPE_NONE, NULL, TRUE); return TRUE; } // Short DOCTYPE, like if ($tok == '>') { // DOCTYPE without a name. if (strlen($doctypeName) == 0) { $this->parseError("Expected a DOCTYPE name. Got nothing."); $this->events->doctype($doctypeName, 0, NULL, TRUE); $this->scanner->next(); return TRUE; } $this->events->doctype($doctypeName); $this->scanner->next(); return TRUE; } $this->scanner->whitespace(); $pub = strtoupper($this->scanner->getAsciiAlpha()); $white = strlen($this->scanner->whitespace()); $tok = $this->scanner->current(); // Get ID, and flag it as pub or system. if (($pub == 'PUBLIC' || $pub == 'SYSTEM') && $white > 0) { // Get the sys ID. $type = $pub == 'PUBLIC' ? EventHandler::DOCTYPE_PUBLIC : EventHandler::DOCTYPE_SYSTEM; $id = $this->quotedString("\0>"); if ($id === FALSE) { $this->events->doctype($doctypeName, $type, $pub, FALSE); return FALSE; } // Premature EOF. if ($this->scanner->current() === FALSE) { $this->parseError("Unexpected EOF in DOCTYPE"); $this->events->doctype($doctypeName, $type, $id, TRUE); return TRUE; } // Well-formed complete DOCTYPE. $this->scanner->whitespace(); if ($this->scanner->current() == '>') { $this->events->doctype($doctypeName, $type, $id, FALSE); $this->scanner->next(); return TRUE; } // If we get here, we have scanner->charsUntil(">"); $this->parseError("Malformed DOCTYPE."); $this->events->doctype($doctypeName, $type, $id, TRUE); $this->scanner->next(); return TRUE; } // Else it's a bogus DOCTYPE. // Consume to > and trash. $this->scanner->charsUntil('>'); $this->parseError("Expected PUBLIC or SYSTEM. Got %s.", $pub); $this->events->doctype($doctypeName, 0, NULL, TRUE); $this->scanner->next(); return TRUE; } /** * Utility for reading a quoted string. * * @param string $stopchars * Characters (in addition to a close-quote) that should stop the string. * E.g. sometimes '>' is higher precedence than '"' or "'". * @return mixed * String if one is found (quotations omitted) */ protected function quotedString($stopchars) { $tok = $this->scanner->current(); if ($tok == '"' || $tok == "'") { $this->scanner->next(); $ret = $this->scanner->charsUntil($tok . $stopchars); if ($this->scanner->current() == $tok) { $this->scanner->next(); } else { // Parse error because no close quote. $this->parseError("Expected %s, got %s", $tok, $this->scanner->current()); } return $ret; } return FALSE; } /** * Handle a CDATA section. */ protected function cdataSection() { if ($this->scanner->current() != '[') { return FALSE; } $cdata = ''; $this->scanner->next(); $chars = $this->scanner->charsWhile('CDAT'); if ($chars != 'CDATA' || $this->scanner->current() != '[') { $this->parseError('Expected [CDATA[, got %s', $chars); return $this->bogusComment('scanner->next(); do { if ($tok === FALSE) { $this->parseError('Unexpected EOF inside CDATA.'); $this->bogusComment('scanner->next(); } while (!$this->sequenceMatches(']]>')); // Consume ]]> $this->scanner->consume(3); $this->events->cdata($cdata); return TRUE; } // ================================================================ // Non-HTML5 // ================================================================ /** * Handle a processing instruction. * * XML processing instructions are supposed to be ignored in HTML5, * treated as "bogus comments". However, since we're not a user * agent, we allow them. We consume until ?> and then issue a * EventListener::processingInstruction() event. */ protected function processingInstruction() { if ($this->scanner->current() != '?') { return FALSE; } $tok = $this->scanner->next(); $procName = $this->scanner->getAsciiAlpha(); $white = strlen($this->scanner->whitespace()); // If not a PI, send to bogusComment. if (strlen($procName) == 0 || $white == 0 || $this->scanner->current() == FALSE) { $this->parseError("Expected processing instruction name, got $tok"); $this->bogusComment('. while (!($this->scanner->current() == '?' && $this->scanner->peek() == '>')) { $data .= $this->scanner->current(); $tok = $this->scanner->next(); if ($tok === FALSE) { $this->parseError("Unexpected EOF in processing instruction."); $this->events->processingInstruction($procName, $data); return TRUE; } } $this->scanner->next(); // > $this->scanner->next(); // Next token. $this->events->processingInstruction($procName, $data); return TRUE; } // ================================================================ // UTILITY FUNCTIONS // ================================================================ /** * Read from the input stream until we get to the desired sequene * or hit the end of the input stream. */ protected function readUntilSequence($sequence) { $buffer = ''; // Optimization for reading larger blocks faster. $first = substr($sequence, 0, 1); while ($this->scanner->current() !== FALSE) { $buffer .= $this->scanner->charsUntil($first); // Stop as soon as we hit the stopping condition. if ($this->sequenceMatches($sequence) || $this->sequenceMatches(strtoupper($sequence))) { return $buffer; } $buffer .= $this->scanner->current(); $this->scanner->next(); } // If we get here, we hit the EOF. $this->parseError("Unexpected EOF during text read."); return $buffer; } /** * Check if upcomming chars match the given sequence. * * This will read the stream for the $sequence. If it's * found, this will return TRUE. If not, return FALSE. * Since this unconsumes any chars it reads, the caller * will still need to read the next sequence, even if * this returns TRUE. * * Example: $this->sequenceMatches('') will * see if the input stream is at the start of a * '' string. */ protected function sequenceMatches($sequence) { $len = strlen($sequence); $buffer = ''; for ($i = 0; $i < $len; ++$i) { $buffer .= $this->scanner->current(); // EOF. Rewind and let the caller handle it. if ($this->scanner->current() === FALSE) { $this->scanner->unconsume($i); return FALSE; } $this->scanner->next(); } $this->scanner->unconsume($len); return $buffer == $sequence; } /** * Send a TEXT event with the contents of the text buffer. * * This emits an EventHandler::text() event with the current contents of the * temporary text buffer. (The buffer is used to group as much PCDATA * as we can instead of emitting lots and lots of TEXT events.) */ protected function flushBuffer() { if (empty($this->text)) { return; } $this->events->text($this->text); $this->text = ''; } /** * Add text to the temporary buffer. * * @see flushBuffer() */ protected function buffer($str) { $this->text .= $str; } /** * Emit a parse error. * * A parse error always returns FALSE because it never consumes any * characters. */ protected function parseError($msg) { $args = func_get_args(); if (count($args) > 1) { array_shift($args); $msg = vsprintf($msg, $args); } $line = $this->scanner->currentLine(); $col = $this->scanner->columnOffset(); $this->events->parseError($msg, $line, $col); return FALSE; } /** * Decode a character reference and return the string. * * Returns FALSE if the entity could not be found. If $inAttribute is set * to TRUE, a bare & will be returned as-is. * * @param boolean $inAttribute * Set to TRUE if the text is inside of an attribute value. * FALSE otherwise. */ protected function decodeCharacterReference($inAttribute = FALSE) { // If it fails this, it's definitely not an entity. if ($this->scanner->current() != '&') { return FALSE; } // Next char after &. $tok = $this->scanner->next(); $entity = ''; $start = $this->scanner->position(); if ($tok == FALSE) { return '&'; } // These indicate not an entity. We return just // the &. if (strspn($tok, static::WHITE . "&<") == 1) { //$this->scanner->next(); return '&'; } // Numeric entity if ($tok == '#') { $tok = $this->scanner->next(); // Hexidecimal encoding. // X[0-9a-fA-F]+; // x[0-9a-fA-F]+; if ($tok == 'x' || $tok == 'X') { $tok = $this->scanner->next(); // Consume x // Convert from hex code to char. $hex = $this->scanner->getHex(); if (empty($hex)) { $this->parseError("Expected &#xHEX;, got &#x%s", $tok); // We unconsume because we don't know what parser rules might // be in effect for the remaining chars. For example. '&#>' // might result in a specific parsing rule inside of tag // contexts, while not inside of pcdata context. $this->scanner->unconsume(2); return '&'; } $entity = CharacterReference::lookupHex($hex); } // Decimal encoding. // [0-9]+; else { // Convert from decimal to char. $numeric = $this->scanner->getNumeric(); if ($numeric === FALSE) { $this->parseError("Expected &#DIGITS;, got &#%s", $tok); $this->scanner->unconsume(2); return '&'; } $entity = CharacterReference::lookupDecimal($numeric); } } // String entity. else { // Attempt to consume a string up to a ';'. // [a-zA-Z0-9]+; $cname = $this->scanner->getAsciiAlpha(); $entity = CharacterReference::lookupName($cname); if ($entity == NULL) { $this->parseError("No match in entity table for '%s'", $entity); } } // The scanner has advanced the cursor for us. $tok = $this->scanner->current(); // We have an entity. We're done here. if ($tok == ';') { $this->scanner->next(); return $entity; } // If in an attribute, then failing to match ; means unconsume the // entire string. Otherwise, failure to match is an error. if ($inAttribute) { $this->scanner->unconsume($this->scanner->position() - $start); return '&'; } $this->parseError("Expected &ENTITY;, got &ENTITY%s (no trailing ;) ", $tok); return '&' . $entity; } }