diff options
author | Andres Rey <[email protected]> | 2018-03-19 13:37:02 +0000 |
---|---|---|
committer | GitHub <[email protected]> | 2018-03-19 13:37:02 +0000 |
commit | 45c58269371ecf72057bcd4b404aef87afb0de48 (patch) | |
tree | 4dc06a87abdb7f354f21498750df1f780764346e /src | |
parent | f0f69065301d640ccf7308cff7c1eb006d03ef45 (diff) | |
parent | a7b5fa260676c213d456b6a077b17dc52fcdbe0c (diff) |
Merge pull request #54 from andreskrey/development
Prepare for release
Diffstat (limited to 'src')
-rw-r--r-- | src/Configuration.php | 12 | ||||
-rw-r--r-- | src/Nodes/DOM/DOMDocument.php | 3 | ||||
-rw-r--r-- | src/Nodes/DOM/DOMEntity.php | 10 | ||||
-rw-r--r-- | src/Nodes/NodeTrait.php | 3 | ||||
-rw-r--r-- | src/Readability.php | 116 |
5 files changed, 120 insertions, 24 deletions
diff --git a/src/Configuration.php b/src/Configuration.php index a0f363f..951740e 100644 --- a/src/Configuration.php +++ b/src/Configuration.php @@ -115,6 +115,18 @@ class Configuration } /** + * @param LoggerInterface $logger + * + * @return Configuration + */ + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; + + return $this; + } + + /** * @return int */ public function getMaxTopCandidates() diff --git a/src/Nodes/DOM/DOMDocument.php b/src/Nodes/DOM/DOMDocument.php index a83f5b9..81e9c7d 100644 --- a/src/Nodes/DOM/DOMDocument.php +++ b/src/Nodes/DOM/DOMDocument.php @@ -20,10 +20,11 @@ class DOMDocument extends \DOMDocument $this->registerNodeClass('DOMDocumentFragment', DOMDocumentFragment::class); $this->registerNodeClass('DOMDocumentType', DOMDocumentType::class); $this->registerNodeClass('DOMElement', DOMElement::class); + $this->registerNodeClass('DOMEntity', DOMEntity::class); + $this->registerNodeClass('DOMEntityReference', DOMEntityReference::class); $this->registerNodeClass('DOMNode', DOMNode::class); $this->registerNodeClass('DOMNotation', DOMNotation::class); $this->registerNodeClass('DOMProcessingInstruction', DOMProcessingInstruction::class); $this->registerNodeClass('DOMText', DOMText::class); - $this->registerNodeClass('DOMEntityReference', DOMEntityReference::class); } } diff --git a/src/Nodes/DOM/DOMEntity.php b/src/Nodes/DOM/DOMEntity.php new file mode 100644 index 0000000..8493e73 --- /dev/null +++ b/src/Nodes/DOM/DOMEntity.php @@ -0,0 +1,10 @@ +<?php + +namespace andreskrey\Readability\Nodes\DOM; + +use andreskrey\Readability\Nodes\NodeTrait; + +class DOMEntity extends \DOMEntity +{ + use NodeTrait; +} diff --git a/src/Nodes/NodeTrait.php b/src/Nodes/NodeTrait.php index 5a3cd7f..13611c9 100644 --- a/src/Nodes/NodeTrait.php +++ b/src/Nodes/NodeTrait.php @@ -7,6 +7,9 @@ use andreskrey\Readability\Nodes\DOM\DOMElement; use andreskrey\Readability\Nodes\DOM\DOMNode; use andreskrey\Readability\Nodes\DOM\DOMText; +/** + * @method \DOMNode removeAttribute($name) + */ trait NodeTrait { /** diff --git a/src/Readability.php b/src/Readability.php index 91e703c..93fc810 100644 --- a/src/Readability.php +++ b/src/Readability.php @@ -78,6 +78,13 @@ class Readability private $logger; /** + * Collection of attempted text extractions. + * + * @var array + */ + private $attempts = []; + + /** * @var array */ private $defaultTagsToScore = [ @@ -155,54 +162,76 @@ class Readability * finding the -right- content. */ - $length = 0; - foreach ($result->getElementsByTagName('p') as $p) { - $length += mb_strlen($p->textContent); - } + $length = mb_strlen(preg_replace(NodeUtility::$regexps['onlyWhitespace'], '', $result->textContent)); $this->logger->info(sprintf('[Parsing] Article parsed. Amount of words: %s. Current threshold is: %s', $length, $this->configuration->getWordThreshold())); - if ($result && mb_strlen(preg_replace('/\s/', '', $result->textContent)) < $this->configuration->getWordThreshold()) { + $parseSuccessful = true; + + if ($result && $length < $this->configuration->getWordThreshold()) { $this->dom = $this->loadHTML($html); $root = $this->dom->getElementsByTagName('body')->item(0); + $parseSuccessful = false; if ($this->configuration->getStripUnlikelyCandidates()) { $this->logger->debug('[Parsing] Threshold not met, trying again setting StripUnlikelyCandidates as false'); $this->configuration->setStripUnlikelyCandidates(false); + $this->attempts[] = ['articleContent' => $result, 'textLength' => $length]; } elseif ($this->configuration->getWeightClasses()) { $this->logger->debug('[Parsing] Threshold not met, trying again setting WeightClasses as false'); $this->configuration->setWeightClasses(false); + $this->attempts[] = ['articleContent' => $result, 'textLength' => $length]; } elseif ($this->configuration->getCleanConditionally()) { $this->logger->debug('[Parsing] Threshold not met, trying again setting CleanConditionally as false'); $this->configuration->setCleanConditionally(false); + $this->attempts[] = ['articleContent' => $result, 'textLength' => $length]; } else { - $this->logger->emergency('[Parsing] Could not parse text, giving up :('); + $this->logger->debug('[Parsing] Threshold not met, searching across attempts for some content.'); + $this->attempts[] = ['articleContent' => $result, 'textLength' => $length]; + + // No luck after removing flags, just return the longest text we found during the different loops + usort($this->attempts, function ($a, $b) { + return $a['textLength'] < $b['textLength']; + }); + + // But first check if we actually have something + if (!$this->attempts[0]['textLength']) { + $this->logger->emergency('[Parsing] Could not parse text, giving up :('); - throw new ParseException('Could not parse text.'); + throw new ParseException('Could not parse text.'); + } + + $this->logger->debug('[Parsing] Threshold not met, but found some content in previous attempts.'); + + $result = $this->attempts[0]['articleContent']; + $parseSuccessful = true; + break; } } else { break; } } - $result = $this->postProcessContent($result); - - // If we haven't found an excerpt in the article's metadata, use the article's - // first paragraph as the excerpt. This can be used for displaying a preview of - // the article's content. - if (!$this->getExcerpt()) { - $this->logger->debug('[Parsing] No excerpt text found on metadata, extracting first p node and using it as excerpt.'); - $paragraphs = $result->getElementsByTagName('p'); - if ($paragraphs->length > 0) { - $this->setExcerpt(trim($paragraphs->item(0)->textContent)); + if ($parseSuccessful) { + $result = $this->postProcessContent($result); + + // If we haven't found an excerpt in the article's metadata, use the article's + // first paragraph as the excerpt. This can be used for displaying a preview of + // the article's content. + if (!$this->getExcerpt()) { + $this->logger->debug('[Parsing] No excerpt text found on metadata, extracting first p node and using it as excerpt.'); + $paragraphs = $result->getElementsByTagName('p'); + if ($paragraphs->length > 0) { + $this->setExcerpt(trim($paragraphs->item(0)->textContent)); + } } - } - $this->setContent($result); + $this->setContent($result); - $this->logger->info('*** Parse successful :)'); + $this->logger->info('*** Parse successful :)'); - return true; + return true; + } } /** @@ -468,6 +497,10 @@ class Readability if (count(preg_split('/\s+/', $curTitle)) < 3) { $curTitle = substr($originalTitle, strpos($originalTitle, ':') + 1); $this->logger->info(sprintf('[Metadata] Title too short, using the first part of the title instead: \'%s\'', $curTitle)); + } elseif (count(preg_split('/\s+/', substr($curTitle, 0, strpos($curTitle, ':')))) > 5) { + // But if we have too many words before the colon there's something weird + // with the titles and the H tags so let's just use the original title instead + $curTitle = $originalTitle; } } } elseif (mb_strlen($curTitle) > 150 || mb_strlen($curTitle) < 15) { @@ -549,7 +582,19 @@ class Readability */ public function getPathInfo($url) { - $pathBase = parse_url($url, PHP_URL_SCHEME) . '://' . parse_url($url, PHP_URL_HOST) . dirname(parse_url($url, PHP_URL_PATH)) . '/'; + // Check for base URLs + if ($this->dom->baseURI !== null) { + if (substr($this->dom->baseURI, 0, 1) === '/') { + // URLs starting with '/' override completely the URL defined in the link + $pathBase = parse_url($url, PHP_URL_SCHEME) . '://' . parse_url($url, PHP_URL_HOST) . $this->dom->baseURI; + } else { + // Otherwise just prepend the base to the actual path + $pathBase = parse_url($url, PHP_URL_SCHEME) . '://' . parse_url($url, PHP_URL_HOST) . dirname(parse_url($url, PHP_URL_PATH)) . '/' . rtrim($this->dom->baseURI, '/') . '/'; + } + } else { + $pathBase = parse_url($url, PHP_URL_SCHEME) . '://' . parse_url($url, PHP_URL_HOST) . dirname(parse_url($url, PHP_URL_PATH)) . '/'; + } + $scheme = parse_url($pathBase, PHP_URL_SCHEME); $prePath = $scheme . '://' . parse_url($pathBase, PHP_URL_HOST); @@ -1129,6 +1174,7 @@ class Readability $this->_clean($article, 'embed'); $this->_clean($article, 'h1'); $this->_clean($article, 'footer'); + $this->_clean($article, 'link'); // Clean out elements have "share" in their id/class combinations from final top candidates, // which means we don't remove the top candidates even they have "share". @@ -1480,6 +1526,28 @@ class Readability } /** + * Removes the class="" attribute from every element in the given + * subtree. + * + * Readability.js has a special filter to avoid cleaning the classes that the algorithm adds. We don't add classes + * here so no need to filter those. + * + * @param DOMDocument|DOMNode $node + * + * @return void + **/ + public function _cleanClasses($node) + { + if ($node->getAttribute('class') !== '') { + $node->removeAttribute('class'); + } + + for ($node = $node->firstChild; $node !== null; $node = $node->nextSibling) { + $this->_cleanClasses($node); + } + } + + /** * @param DOMDocument $article * * @return DOMDocument @@ -1532,6 +1600,8 @@ class Readability } } + $this->_cleanClasses($article); + return $article; } @@ -1564,7 +1634,7 @@ class Readability */ public function getContent() { - return $this->content->C14N(); + return ($this->content instanceof DOMDocument) ? $this->content->C14N() : null; } /** |