diff options
Diffstat (limited to 'classes/rssutils.php')
-rwxr-xr-x | classes/rssutils.php | 164 |
1 files changed, 108 insertions, 56 deletions
diff --git a/classes/rssutils.php b/classes/rssutils.php index aec17d538..561171d09 100755 --- a/classes/rssutils.php +++ b/classes/rssutils.php @@ -37,20 +37,29 @@ class RSSUtils { $pdo = Db::pdo(); $sth = $pdo->prepare("SELECT id FROM ttrss_feeds WHERE id = ?"); - // check icon files once every Config::get(Config::CACHE_MAX_DAYS) days - $icon_files = array_filter(glob(Config::get(Config::ICONS_DIR) . "/*.ico"), - function($f) { return filemtime($f) < time() - 86400 * Config::get(Config::CACHE_MAX_DAYS); }); + $cache = DiskCache::instance('feed-icons'); - foreach ($icon_files as $icon) { - $feed_id = basename($icon, ".ico"); + if ($cache->is_writable()) { + $dh = opendir($cache->get_full_path("")); - $sth->execute([$feed_id]); + if ($dh) { + while (($icon = readdir($dh)) !== false) { + if (preg_match('/^[0-9]{1,}$/', $icon) && $cache->get_mtime($icon) < time() - 86400 * Config::get(Config::CACHE_MAX_DAYS)) { - if ($sth->fetch()) { - @touch($icon); - } else { - Debug::log("Removing orphaned feed icon: $icon"); - unlink($icon); + $sth->execute([(int)$icon]); + + if ($sth->fetch()) { + $cache->put($icon, $cache->get($icon)); + } else { + $icon_path = $cache->get_full_path($icon); + + Debug::log("Removing orphaned feed icon: $icon_path"); + unlink($icon_path); + } + } + } + + closedir($dh); } } } @@ -338,6 +347,9 @@ class RSSUtils { $pdo = Db::pdo(); + /** @var DiskCache $cache */ + $cache = DiskCache::instance('feeds'); + if (Config::get(Config::DB_TYPE) == "pgsql") { $favicon_interval_qpart = "favicon_last_checked < NOW() - INTERVAL '12 hour'"; } else { @@ -387,7 +399,7 @@ class RSSUtils { $date_feed_processed = date('Y-m-d H:i'); - $cache_filename = Config::get(Config::CACHE_DIR) . "/feeds/" . sha1($feed_obj->feed_url) . ".xml"; + $cache_filename = sha1($feed_obj->feed_url) . ".xml"; $pluginhost = new PluginHost(); $user_plugins = get_pref(Prefs::_ENABLED_PLUGINS, $feed_obj->owner_uid); @@ -423,13 +435,13 @@ class RSSUtils { // try cache if (!$feed_data && - is_readable($cache_filename) && + $cache->exists($cache_filename) && !$feed_obj->auth_login && !$feed_obj->auth_pass && - filemtime($cache_filename) > time() - 30) { + $cache->get_mtime($cache_filename) > time() - 30) { Debug::log("using local cache: {$cache_filename}.", Debug::LOG_VERBOSE); - $feed_data = file_get_contents($cache_filename); + $feed_data = $cache->get($cache_filename); if ($feed_data) { $rss_hash = sha1($feed_data); @@ -447,7 +459,7 @@ class RSSUtils { Debug::log("not using CURL due to open_basedir restrictions", Debug::LOG_VERBOSE); } - if (time() - strtotime($feed_obj->last_unconditional) > Config::get(Config::MAX_CONDITIONAL_INTERVAL)) { + if (time() - strtotime($feed_obj->last_unconditional ?? "") > Config::get(Config::MAX_CONDITIONAL_INTERVAL)) { Debug::log("maximum allowed interval for conditional requests exceeded, forcing refetch", Debug::LOG_VERBOSE); $force_refetch = true; @@ -477,12 +489,12 @@ class RSSUtils { } // cache vanilla feed data for re-use - if ($feed_data && !$feed_obj->auth_pass && !$feed_obj->auth_login && is_writable(Config::get(Config::CACHE_DIR) . "/feeds")) { + if ($feed_data && !$feed_obj->auth_pass && !$feed_obj->auth_login && $cache->is_writable()) { $new_rss_hash = sha1($feed_data); if ($new_rss_hash != $rss_hash) { Debug::log("saving to local cache: $cache_filename", Debug::LOG_VERBOSE); - file_put_contents($cache_filename, $feed_data); + $cache->put($cache_filename, $feed_data); } } } @@ -492,7 +504,7 @@ class RSSUtils { // If-Modified-Since if (UrlHelper::$fetch_last_error_code == 304) { - Debug::log("source claims data not modified, nothing to do.", Debug::LOG_VERBOSE); + Debug::log("source claims data not modified (304), nothing to do.", Debug::LOG_VERBOSE); $error_message = ""; $feed_obj->set([ @@ -503,6 +515,24 @@ class RSSUtils { $feed_obj->save(); + } else if (UrlHelper::$fetch_last_error_code == 429) { + + // randomize interval using Config::HTTP_429_THROTTLE_INTERVAL as a base value (1-2x) + $http_429_throttle_interval = rand(Config::get(Config::HTTP_429_THROTTLE_INTERVAL), + Config::get(Config::HTTP_429_THROTTLE_INTERVAL)*2); + + $error_message = UrlHelper::$fetch_last_error; + + Debug::log("source claims we're requesting too often (429), throttling updates for $http_429_throttle_interval seconds.", + Debug::LOG_VERBOSE); + + $feed_obj->set([ + 'last_error' => $error_message . " (updates throttled for $http_429_throttle_interval seconds.)", + 'last_successful_update' => Db::NOW($http_429_throttle_interval), + 'last_updated' => Db::NOW($http_429_throttle_interval), + ]); + + $feed_obj->save(); } else { $error_message = UrlHelper::$fetch_last_error; @@ -572,21 +602,28 @@ class RSSUtils { if ($feed_obj->favicon_needs_check || $force_refetch) { - /* terrible hack: if we crash on floicon shit here, we won't check - * the icon avgcolor again (unless the icon got updated) */ + // restrict update attempts to once per 12h + $feed_obj->favicon_last_checked = Db::NOW(); + $feed_obj->save(); + + $favicon_cache = DiskCache::instance('feed-icons'); - $favicon_file = Config::get(Config::ICONS_DIR) . "/$feed.ico"; - $favicon_modified = file_exists($favicon_file) ? filemtime($favicon_file) : -1; + $favicon_modified = $favicon_cache->exists($feed) ? $favicon_cache->get_mtime($feed) : -1; + // don't try to redownload custom favicons if (!$feed_obj->favicon_is_custom) { Debug::log("favicon: trying to update favicon...", Debug::LOG_VERBOSE); self::update_favicon($feed_obj->site_url, $feed); - if ((file_exists($favicon_file) ? filemtime($favicon_file) : -1) > $favicon_modified) + if (!$favicon_cache->exists($feed) || $favicon_cache->get_mtime($feed) > $favicon_modified) { $feed_obj->favicon_avg_color = null; + $feed_obj->save(); + } } - if (is_readable($favicon_file) && function_exists("imagecreatefromstring") && empty($feed_obj->favicon_avg_color)) { + /* terrible hack: if we crash on floicon shit here, we won't check + * the icon avgcolor again (unless icon got updated) */ + if (file_exists($favicon_cache->get_full_path($feed)) && function_exists("imagecreatefromstring") && empty($feed_obj->favicon_avg_color)) { require_once "colors.php"; Debug::log("favicon: trying to calculate average color...", Debug::LOG_VERBOSE); @@ -594,13 +631,13 @@ class RSSUtils { $feed_obj->favicon_avg_color = 'fail'; $feed_obj->save(); - $feed_obj->favicon_avg_color = \Colors\calculate_avg_color($favicon_file); + $feed_obj->favicon_avg_color = \Colors\calculate_avg_color($favicon_cache->get_full_path($feed)); $feed_obj->save(); Debug::log("favicon: avg color: {$feed_obj->favicon_avg_color}", Debug::LOG_VERBOSE); } else if ($feed_obj->favicon_avg_color == 'fail') { - Debug::log("floicon failed $favicon_file, not trying to recalculate avg color", Debug::LOG_VERBOSE); + Debug::log("floicon failed on $feed, not trying to recalculate avg color", Debug::LOG_VERBOSE); } } @@ -644,7 +681,7 @@ class RSSUtils { print_r($item); } - if (ini_get("max_execution_time") > 0 && time() - $tstart >= ini_get("max_execution_time") * 0.7) { + if (ini_get("max_execution_time") > 0 && time() - $tstart >= ((float)ini_get("max_execution_time") * 0.7)) { Debug::log("looks like there's too many articles to process at once, breaking out.", Debug::LOG_VERBOSE); $pdo->commit(); break; @@ -715,7 +752,7 @@ class RSSUtils { $article_labels = Article::_get_labels($base_entry_id, $feed_obj->owner_uid); $existing_tags = Article::_get_tags($base_entry_id, $feed_obj->owner_uid); - $entry_tags = array_unique(array_merge($entry_tags, $existing_tags)); + $entry_tags = array_unique([...$entry_tags, ...$existing_tags]); } else { $base_entry_id = false; $entry_stored_hash = ""; @@ -847,7 +884,7 @@ class RSSUtils { $pluginhost->run_hooks(PluginHost::HOOK_FILTER_TRIGGERED, $feed, $feed_obj->owner_uid, $article, $matched_filters, $matched_rules, $article_filters); - $matched_filter_ids = array_map(function($f) { return $f['id']; }, $matched_filters); + $matched_filter_ids = array_map(fn(array $f) => $f['id'], $matched_filters); if (count($matched_filter_ids) > 0) { $filter_objs = ORM::for_table('ttrss_filters2') @@ -1172,8 +1209,7 @@ class RSSUtils { // check for manual tags (we have to do it here since they're loaded from filters) foreach ($article_filters as $f) { if ($f["type"] == "tag") { - $entry_tags = array_merge($entry_tags, - FeedItem_Common::normalize_categories(explode(",", $f["param"]))); + $entry_tags = [...$entry_tags, ...FeedItem_Common::normalize_categories(explode(",", $f["param"]))]; } } @@ -1284,7 +1320,7 @@ class RSSUtils { * @see FeedEnclosure */ static function cache_enclosures(array $enclosures, string $site_url): void { - $cache = new DiskCache("images"); + $cache = DiskCache::instance("images"); if ($cache->is_writable()) { foreach ($enclosures as $enc) { @@ -1306,8 +1342,6 @@ class RSSUtils { } else { Debug::log("cache_enclosures: failed with ".UrlHelper::$fetch_last_error_code.": ".UrlHelper::$fetch_last_error); } - } else if (is_writable($local_filename)) { - $cache->touch($local_filename); } } } @@ -1333,14 +1367,12 @@ class RSSUtils { } else { Debug::log("cache_media: failed with ".UrlHelper::$fetch_last_error_code.": ".UrlHelper::$fetch_last_error); } - } else if ($cache->is_writable($local_filename)) { - $cache->touch($local_filename); } } /* TODO: move to DiskCache? */ static function cache_media(string $html, string $site_url): void { - $cache = new DiskCache("images"); + $cache = DiskCache::instance("images"); if ($html && $cache->is_writable()) { $doc = new DOMDocument(); @@ -1656,9 +1688,36 @@ class RSSUtils { $tmph->run_hooks(PluginHost::HOOK_HOUSE_KEEPING); } + /** migrates favicons from legacy storage in feed-icons/ to cache/feed-icons/using new naming (sans .ico suffix) */ + static function migrate_feed_icons() : void { + $old_dir = Config::get(Config::ICONS_DIR); + $new_dir = Config::get(Config::CACHE_DIR) . '/feed-icons'; + + $dh = opendir($old_dir); + + $cache = DiskCache::instance('feed-icons'); + + if ($dh) { + while (($old_filename = readdir($dh)) !== false) { + if (strpos($old_filename, ".ico") !== false) { + $new_filename = str_replace(".ico", "", $old_filename); + $old_full_path = "$old_dir/$old_filename"; + + if (is_file($old_full_path) && $cache->put($new_filename, file_get_contents($old_full_path))) { + unlink($old_full_path); + } + } + } + + closedir($dh); + } + } + static function housekeeping_common(): void { - DiskCache::expire(); + $cache = DiskCache::instance(""); + $cache->expire_all(); + self::migrate_feed_icons(); self::expire_lock_files(); self::expire_error_log(); self::expire_feed_archive(); @@ -1676,8 +1735,6 @@ class RSSUtils { * @return false|string */ static function update_favicon(string $site_url, int $feed) { - $icon_file = Config::get(Config::ICONS_DIR) . "/$feed.ico"; - $favicon_urls = self::get_favicon_urls($site_url); if (count($favicon_urls) == 0) { @@ -1732,21 +1789,18 @@ class RSSUtils { break; } - Debug::log("favicon: $favicon_url looks valid, saving to $icon_file", Debug::LOG_VERBOSE); - - $fp = @fopen($icon_file, "w"); - - if ($fp) { + $favicon_cache = DiskCache::instance('feed-icons'); - fwrite($fp, $contents); - fclose($fp); - chmod($icon_file, 0644); - clearstatcache(); + if ($favicon_cache->is_writable()) { + Debug::log("favicon: $favicon_url looks valid, saving to cache", Debug::LOG_VERBOSE); - return $icon_file; + // we deal with this manually + if (!$favicon_cache->exists(".no-auto-expiry")) + $favicon_cache->put(".no-auto-expiry", ""); + return $favicon_cache->put((string)$feed, $contents); } else { - Debug::log("favicon: failed to open $icon_file for writing", Debug::LOG_VERBOSE); + Debug::log("favicon: $favicon_url skipping, local cache is not writable", Debug::LOG_VERBOSE); } } @@ -1778,12 +1832,10 @@ class RSSUtils { owner_uid = ? AND enabled = true ORDER BY order_id, title"); $sth->execute([$owner_uid]); - $check_cats = array_merge( - Feeds::_get_parent_cats($cat_id, $owner_uid), - [$cat_id]); + $check_cats = [...Feeds::_get_parent_cats($cat_id, $owner_uid), $cat_id]; $check_cats_str = join(",", $check_cats); - $check_cats_fullids = array_map(function($a) { return "CAT:$a"; }, $check_cats); + $check_cats_fullids = array_map(fn(int $a) => "CAT:$a", $check_cats); while ($line = $sth->fetch()) { $filter_id = $line["id"]; |