summaryrefslogtreecommitdiff
path: root/classes/rssutils.php
diff options
context:
space:
mode:
Diffstat (limited to 'classes/rssutils.php')
-rwxr-xr-xclasses/rssutils.php88
1 files changed, 58 insertions, 30 deletions
diff --git a/classes/rssutils.php b/classes/rssutils.php
index 831ac1baf..dede50ba0 100755
--- a/classes/rssutils.php
+++ b/classes/rssutils.php
@@ -3,7 +3,12 @@ class RSSUtils {
static function calculate_article_hash($article, $pluginhost) {
$tmp = "";
+ $ignored_fields = [ "feed", "guid", "guid_hashed", "owner_uid", "force_catchup" ];
+
foreach ($article as $k => $v) {
+ if (in_array($k, $ignored_fields))
+ continue;
+
if ($k != "feed" && isset($v)) {
$x = strip_tags(is_array($v) ? implode(",", $v) : $v);
@@ -469,7 +474,7 @@ class RSSUtils {
foreach ($pluginhost->get_hooks(PluginHost::HOOK_FEED_PARSED) as $plugin) {
Debug::log("... " . get_class($plugin), Debug::$LOG_VERBOSE);
$start = microtime(true);
- $plugin->hook_feed_parsed($rss);
+ $plugin->hook_feed_parsed($rss, $feed);
Debug::log(sprintf("=== %.4f (sec)", microtime(true) - $start), Debug::$LOG_VERBOSE);
}
@@ -586,11 +591,11 @@ class RSSUtils {
continue;
}
+ $entry_guid_hashed_compat = 'SHA1:' . sha1("$owner_uid,$entry_guid");
+ $entry_guid_hashed = json_encode(["ver" => 2, "uid" => $owner_uid, "hash" => 'SHA1:' . sha1($entry_guid)]);
$entry_guid = "$owner_uid,$entry_guid";
- $entry_guid_hashed = 'SHA1:' . sha1($entry_guid);
-
- Debug::log("guid $entry_guid / $entry_guid_hashed", Debug::$LOG_VERBOSE);
+ Debug::log("guid $entry_guid (hash: $entry_guid_hashed compat: $entry_guid_hashed_compat)", Debug::$LOG_VERBOSE);
$entry_timestamp = (int)$item->get_date();
@@ -632,8 +637,8 @@ class RSSUtils {
Debug::log("done collecting data.", Debug::$LOG_VERBOSE);
$sth = $pdo->prepare("SELECT id, content_hash, lang FROM ttrss_entries
- WHERE guid = ? OR guid = ?");
- $sth->execute([$entry_guid, $entry_guid_hashed]);
+ WHERE guid IN (?, ?, ?)");
+ $sth->execute([$entry_guid, $entry_guid_hashed, $entry_guid_hashed_compat]);
if ($row = $sth->fetch()) {
$base_entry_id = $row["id"];
@@ -828,8 +833,8 @@ class RSSUtils {
RSSUtils::cache_media($entry_content, $site_url);
$csth = $pdo->prepare("SELECT id FROM ttrss_entries
- WHERE guid = ? OR guid = ?");
- $csth->execute([$entry_guid, $entry_guid_hashed]);
+ WHERE guid IN (?, ?, ?)");
+ $csth->execute([$entry_guid, $entry_guid_hashed, $entry_guid_hashed_compat]);
if (!$row = $csth->fetch()) {
@@ -874,7 +879,7 @@ class RSSUtils {
}
- $csth->execute([$entry_guid, $entry_guid_hashed]);
+ $csth->execute([$entry_guid, $entry_guid_hashed, $entry_guid_hashed_compat]);
$entry_ref_id = 0;
$entry_int_id = 0;
@@ -1032,6 +1037,11 @@ class RSSUtils {
if (is_array($encs)) {
foreach ($encs as $e) {
+
+ foreach ($pluginhost->get_hooks(PluginHost::HOOK_ENCLOSURE_IMPORTED) as $plugin) {
+ $e = $plugin->hook_enclosure_imported($e, $feed);
+ }
+
$e_item = array(
rewrite_relative_url($site_url, $e->link),
$e->type, $e->length, $e->title, $e->width, $e->height);
@@ -1221,6 +1231,32 @@ class RSSUtils {
}
}
+ static function cache_media_url($cache, $url, $site_url) {
+ $url = rewrite_relative_url($site_url, $url);
+ $local_filename = sha1($url);
+
+ Debug::log("cache_media: checking $url", Debug::$LOG_VERBOSE);
+
+ if (!$cache->exists($local_filename)) {
+ Debug::log("cache_media: downloading: $url to $local_filename", Debug::$LOG_VERBOSE);
+
+ global $fetch_last_error_code;
+ global $fetch_last_error;
+
+ $file_content = fetch_file_contents(array("url" => $url,
+ "http_referrer" => $url,
+ "max_size" => MAX_CACHE_FILE_SIZE));
+
+ if ($file_content) {
+ $cache->put($local_filename, $file_content);
+ } else {
+ Debug::log("cache_media: failed with $fetch_last_error_code: $fetch_last_error");
+ }
+ } else if ($cache->isWritable($local_filename)) {
+ $cache->touch($local_filename);
+ }
+ }
+
static function cache_media($html, $site_url) {
$cache = new DiskCache("images");
@@ -1229,33 +1265,24 @@ class RSSUtils {
if ($doc->loadHTML($html)) {
$xpath = new DOMXPath($doc);
- $entries = $xpath->query('(//img[@src])|(//video/source[@src])|(//audio/source[@src])');
+ $entries = $xpath->query('(//img[@src]|//source[@src|@srcset]|//video[@poster|@src])');
foreach ($entries as $entry) {
- if ($entry->hasAttribute('src') && strpos($entry->getAttribute('src'), "data:") !== 0) {
- $src = rewrite_relative_url($site_url, $entry->getAttribute('src'));
-
- $local_filename = sha1($src);
-
- Debug::log("cache_media: checking $src", Debug::$LOG_VERBOSE);
+ foreach (array('src', 'poster') as $attr) {
+ if ($entry->hasAttribute($attr) && strpos($entry->getAttribute($attr), "data:") !== 0) {
+ RSSUtils::cache_media_url($cache, $entry->getAttribute($attr), $site_url);
+ }
+ }
- if (!$cache->exists($local_filename)) {
- Debug::log("cache_media: downloading: $src to $local_filename", Debug::$LOG_VERBOSE);
+ if ($entry->hasAttribute("srcset")) {
+ $tokens = explode(",", $entry->getAttribute('srcset'));
- global $fetch_last_error_code;
- global $fetch_last_error;
+ for ($i = 0; $i < count($tokens); $i++) {
+ $token = trim($tokens[$i]);
- $file_content = fetch_file_contents(array("url" => $src,
- "http_referrer" => $src,
- "max_size" => MAX_CACHE_FILE_SIZE));
+ list ($url, $width) = explode(" ", $token, 2);
- if ($file_content) {
- $cache->put($local_filename, $file_content);
- } else {
- Debug::log("cache_media: failed with $fetch_last_error_code: $fetch_last_error");
- }
- } else if ($cache->isWritable($local_filename)) {
- $cache->touch($local_filename);
+ RSSUtils::cache_media_url($cache, $url, $site_url);
}
}
}
@@ -1343,6 +1370,7 @@ class RSSUtils {
foreach ($filter["rules"] as $rule) {
$match = false;
$reg_exp = str_replace('/', '\/', $rule["reg_exp"]);
+ $reg_exp = str_replace("\n", "", $reg_exp); // reg_exp may be formatted with CRs now because of textarea, we need to strip those
$rule_inverse = $rule["inverse"];
if (!$reg_exp)