diff options
Diffstat (limited to 'classes/rssutils.php')
-rwxr-xr-x | classes/rssutils.php | 88 |
1 files changed, 58 insertions, 30 deletions
diff --git a/classes/rssutils.php b/classes/rssutils.php index 831ac1baf..dede50ba0 100755 --- a/classes/rssutils.php +++ b/classes/rssutils.php @@ -3,7 +3,12 @@ class RSSUtils { static function calculate_article_hash($article, $pluginhost) { $tmp = ""; + $ignored_fields = [ "feed", "guid", "guid_hashed", "owner_uid", "force_catchup" ]; + foreach ($article as $k => $v) { + if (in_array($k, $ignored_fields)) + continue; + if ($k != "feed" && isset($v)) { $x = strip_tags(is_array($v) ? implode(",", $v) : $v); @@ -469,7 +474,7 @@ class RSSUtils { foreach ($pluginhost->get_hooks(PluginHost::HOOK_FEED_PARSED) as $plugin) { Debug::log("... " . get_class($plugin), Debug::$LOG_VERBOSE); $start = microtime(true); - $plugin->hook_feed_parsed($rss); + $plugin->hook_feed_parsed($rss, $feed); Debug::log(sprintf("=== %.4f (sec)", microtime(true) - $start), Debug::$LOG_VERBOSE); } @@ -586,11 +591,11 @@ class RSSUtils { continue; } + $entry_guid_hashed_compat = 'SHA1:' . sha1("$owner_uid,$entry_guid"); + $entry_guid_hashed = json_encode(["ver" => 2, "uid" => $owner_uid, "hash" => 'SHA1:' . sha1($entry_guid)]); $entry_guid = "$owner_uid,$entry_guid"; - $entry_guid_hashed = 'SHA1:' . sha1($entry_guid); - - Debug::log("guid $entry_guid / $entry_guid_hashed", Debug::$LOG_VERBOSE); + Debug::log("guid $entry_guid (hash: $entry_guid_hashed compat: $entry_guid_hashed_compat)", Debug::$LOG_VERBOSE); $entry_timestamp = (int)$item->get_date(); @@ -632,8 +637,8 @@ class RSSUtils { Debug::log("done collecting data.", Debug::$LOG_VERBOSE); $sth = $pdo->prepare("SELECT id, content_hash, lang FROM ttrss_entries - WHERE guid = ? OR guid = ?"); - $sth->execute([$entry_guid, $entry_guid_hashed]); + WHERE guid IN (?, ?, ?)"); + $sth->execute([$entry_guid, $entry_guid_hashed, $entry_guid_hashed_compat]); if ($row = $sth->fetch()) { $base_entry_id = $row["id"]; @@ -828,8 +833,8 @@ class RSSUtils { RSSUtils::cache_media($entry_content, $site_url); $csth = $pdo->prepare("SELECT id FROM ttrss_entries - WHERE guid = ? OR guid = ?"); - $csth->execute([$entry_guid, $entry_guid_hashed]); + WHERE guid IN (?, ?, ?)"); + $csth->execute([$entry_guid, $entry_guid_hashed, $entry_guid_hashed_compat]); if (!$row = $csth->fetch()) { @@ -874,7 +879,7 @@ class RSSUtils { } - $csth->execute([$entry_guid, $entry_guid_hashed]); + $csth->execute([$entry_guid, $entry_guid_hashed, $entry_guid_hashed_compat]); $entry_ref_id = 0; $entry_int_id = 0; @@ -1032,6 +1037,11 @@ class RSSUtils { if (is_array($encs)) { foreach ($encs as $e) { + + foreach ($pluginhost->get_hooks(PluginHost::HOOK_ENCLOSURE_IMPORTED) as $plugin) { + $e = $plugin->hook_enclosure_imported($e, $feed); + } + $e_item = array( rewrite_relative_url($site_url, $e->link), $e->type, $e->length, $e->title, $e->width, $e->height); @@ -1221,6 +1231,32 @@ class RSSUtils { } } + static function cache_media_url($cache, $url, $site_url) { + $url = rewrite_relative_url($site_url, $url); + $local_filename = sha1($url); + + Debug::log("cache_media: checking $url", Debug::$LOG_VERBOSE); + + if (!$cache->exists($local_filename)) { + Debug::log("cache_media: downloading: $url to $local_filename", Debug::$LOG_VERBOSE); + + global $fetch_last_error_code; + global $fetch_last_error; + + $file_content = fetch_file_contents(array("url" => $url, + "http_referrer" => $url, + "max_size" => MAX_CACHE_FILE_SIZE)); + + if ($file_content) { + $cache->put($local_filename, $file_content); + } else { + Debug::log("cache_media: failed with $fetch_last_error_code: $fetch_last_error"); + } + } else if ($cache->isWritable($local_filename)) { + $cache->touch($local_filename); + } + } + static function cache_media($html, $site_url) { $cache = new DiskCache("images"); @@ -1229,33 +1265,24 @@ class RSSUtils { if ($doc->loadHTML($html)) { $xpath = new DOMXPath($doc); - $entries = $xpath->query('(//img[@src])|(//video/source[@src])|(//audio/source[@src])'); + $entries = $xpath->query('(//img[@src]|//source[@src|@srcset]|//video[@poster|@src])'); foreach ($entries as $entry) { - if ($entry->hasAttribute('src') && strpos($entry->getAttribute('src'), "data:") !== 0) { - $src = rewrite_relative_url($site_url, $entry->getAttribute('src')); - - $local_filename = sha1($src); - - Debug::log("cache_media: checking $src", Debug::$LOG_VERBOSE); + foreach (array('src', 'poster') as $attr) { + if ($entry->hasAttribute($attr) && strpos($entry->getAttribute($attr), "data:") !== 0) { + RSSUtils::cache_media_url($cache, $entry->getAttribute($attr), $site_url); + } + } - if (!$cache->exists($local_filename)) { - Debug::log("cache_media: downloading: $src to $local_filename", Debug::$LOG_VERBOSE); + if ($entry->hasAttribute("srcset")) { + $tokens = explode(",", $entry->getAttribute('srcset')); - global $fetch_last_error_code; - global $fetch_last_error; + for ($i = 0; $i < count($tokens); $i++) { + $token = trim($tokens[$i]); - $file_content = fetch_file_contents(array("url" => $src, - "http_referrer" => $src, - "max_size" => MAX_CACHE_FILE_SIZE)); + list ($url, $width) = explode(" ", $token, 2); - if ($file_content) { - $cache->put($local_filename, $file_content); - } else { - Debug::log("cache_media: failed with $fetch_last_error_code: $fetch_last_error"); - } - } else if ($cache->isWritable($local_filename)) { - $cache->touch($local_filename); + RSSUtils::cache_media_url($cache, $url, $site_url); } } } @@ -1343,6 +1370,7 @@ class RSSUtils { foreach ($filter["rules"] as $rule) { $match = false; $reg_exp = str_replace('/', '\/', $rule["reg_exp"]); + $reg_exp = str_replace("\n", "", $reg_exp); // reg_exp may be formatted with CRs now because of textarea, we need to strip those $rule_inverse = $rule["inverse"]; if (!$reg_exp) |