1 files changed, 108 insertions, 56 deletions

diff --git a/classes/rssutils.php b/classes/rssutils.php
index aec17d538..561171d09 100755
--- a/classes/rssutils.php
+++ b/classes/rssutils.php
@@ -37,20 +37,29 @@ class RSSUtils {
 		$pdo = Db::pdo();
 		$sth = $pdo->prepare("SELECT id FROM ttrss_feeds WHERE id = ?");
 
-		// check icon files once every Config::get(Config::CACHE_MAX_DAYS) days
-		$icon_files = array_filter(glob(Config::get(Config::ICONS_DIR) . "/*.ico"),
-			function($f) { return filemtime($f) < time() - 86400 * Config::get(Config::CACHE_MAX_DAYS); });
+		$cache = DiskCache::instance('feed-icons');
 
-		foreach ($icon_files as $icon) {
-			$feed_id = basename($icon, ".ico");
+		if ($cache->is_writable()) {
+			$dh = opendir($cache->get_full_path(""));
 
-			$sth->execute([$feed_id]);
+			if ($dh) {
+				while (($icon = readdir($dh)) !== false) {
+					if (preg_match('/^[0-9]{1,}$/', $icon) && $cache->get_mtime($icon) < time() - 86400 * Config::get(Config::CACHE_MAX_DAYS)) {
 
-			if ($sth->fetch()) {
-				@touch($icon);
-			} else {
-				Debug::log("Removing orphaned feed icon: $icon");
-				unlink($icon);
+						$sth->execute([(int)$icon]);
+
+						if ($sth->fetch()) {
+							$cache->put($icon, $cache->get($icon));
+						} else {
+							$icon_path = $cache->get_full_path($icon);
+
+							Debug::log("Removing orphaned feed icon: $icon_path");
+							unlink($icon_path);
+						}
+					}
+				}
+
+				closedir($dh);
 			}
 		}
 	}
@@ -338,6 +347,9 @@ class RSSUtils {
 
 		$pdo = Db::pdo();
 
+		/** @var DiskCache $cache */
+		$cache = DiskCache::instance('feeds');
+
 		if (Config::get(Config::DB_TYPE) == "pgsql") {
 			$favicon_interval_qpart = "favicon_last_checked < NOW() - INTERVAL '12 hour'";
 		} else {
@@ -387,7 +399,7 @@ class RSSUtils {
 
 		$date_feed_processed = date('Y-m-d H:i');
 
-		$cache_filename = Config::get(Config::CACHE_DIR) . "/feeds/" . sha1($feed_obj->feed_url) . ".xml";
+		$cache_filename = sha1($feed_obj->feed_url) . ".xml";
 
 		$pluginhost = new PluginHost();
 		$user_plugins = get_pref(Prefs::_ENABLED_PLUGINS, $feed_obj->owner_uid);
@@ -423,13 +435,13 @@ class RSSUtils {
 
 		// try cache
 		if (!$feed_data &&
-			is_readable($cache_filename) &&
+			$cache->exists($cache_filename) &&
 			!$feed_obj->auth_login && !$feed_obj->auth_pass &&
-			filemtime($cache_filename) > time() - 30) {
+			$cache->get_mtime($cache_filename) > time() - 30) {
 
 			Debug::log("using local cache: {$cache_filename}.", Debug::LOG_VERBOSE);
 
-			$feed_data = file_get_contents($cache_filename);
+			$feed_data = $cache->get($cache_filename);
 
 			if ($feed_data) {
 				$rss_hash = sha1($feed_data);
@@ -447,7 +459,7 @@ class RSSUtils {
 				Debug::log("not using CURL due to open_basedir restrictions", Debug::LOG_VERBOSE);
 			}
 
-			if (time() - strtotime($feed_obj->last_unconditional) > Config::get(Config::MAX_CONDITIONAL_INTERVAL)) {
+			if (time() - strtotime($feed_obj->last_unconditional ?? "") > Config::get(Config::MAX_CONDITIONAL_INTERVAL)) {
 				Debug::log("maximum allowed interval for conditional requests exceeded, forcing refetch", Debug::LOG_VERBOSE);
 
 				$force_refetch = true;
@@ -477,12 +489,12 @@ class RSSUtils {
 			}
 
 			// cache vanilla feed data for re-use
-			if ($feed_data && !$feed_obj->auth_pass && !$feed_obj->auth_login && is_writable(Config::get(Config::CACHE_DIR) . "/feeds")) {
+			if ($feed_data && !$feed_obj->auth_pass && !$feed_obj->auth_login && $cache->is_writable()) {
 				$new_rss_hash = sha1($feed_data);
 
 				if ($new_rss_hash != $rss_hash) {
 					Debug::log("saving to local cache: $cache_filename", Debug::LOG_VERBOSE);
-					file_put_contents($cache_filename, $feed_data);
+					$cache->put($cache_filename, $feed_data);
 				}
 			}
 		}
@@ -492,7 +504,7 @@ class RSSUtils {
 
 			// If-Modified-Since
 			if (UrlHelper::$fetch_last_error_code == 304) {
-				Debug::log("source claims data not modified, nothing to do.", Debug::LOG_VERBOSE);
+				Debug::log("source claims data not modified (304), nothing to do.", Debug::LOG_VERBOSE);
 				$error_message = "";
 
 				$feed_obj->set([
@@ -503,6 +515,24 @@ class RSSUtils {
 
 				$feed_obj->save();
 
+			} else if (UrlHelper::$fetch_last_error_code == 429) {
+
+				// randomize interval using Config::HTTP_429_THROTTLE_INTERVAL as a base value (1-2x)
+				$http_429_throttle_interval = rand(Config::get(Config::HTTP_429_THROTTLE_INTERVAL),
+					Config::get(Config::HTTP_429_THROTTLE_INTERVAL)*2);
+
+				$error_message = UrlHelper::$fetch_last_error;
+
+				Debug::log("source claims we're requesting too often (429), throttling updates for $http_429_throttle_interval seconds.",
+					Debug::LOG_VERBOSE);
+
+				$feed_obj->set([
+					'last_error' => $error_message . " (updates throttled for $http_429_throttle_interval seconds.)",
+					'last_successful_update' => Db::NOW($http_429_throttle_interval),
+					'last_updated' => Db::NOW($http_429_throttle_interval),
+				]);
+
+				$feed_obj->save();
 			} else {
 				$error_message = UrlHelper::$fetch_last_error;
 
@@ -572,21 +602,28 @@ class RSSUtils {
 
 			if ($feed_obj->favicon_needs_check || $force_refetch) {
 
-				/* terrible hack: if we crash on floicon shit here, we won't check
-				 * the icon avgcolor again (unless the icon got updated) */
+				// restrict update attempts to once per 12h
+				$feed_obj->favicon_last_checked = Db::NOW();
+				$feed_obj->save();
+
+				$favicon_cache = DiskCache::instance('feed-icons');
 
-				$favicon_file = Config::get(Config::ICONS_DIR) . "/$feed.ico";
-				$favicon_modified = file_exists($favicon_file) ? filemtime($favicon_file) : -1;
+				$favicon_modified = $favicon_cache->exists($feed) ? $favicon_cache->get_mtime($feed) : -1;
 
+				// don't try to redownload custom favicons
 				if (!$feed_obj->favicon_is_custom) {
 					Debug::log("favicon: trying to update favicon...", Debug::LOG_VERBOSE);
 					self::update_favicon($feed_obj->site_url, $feed);
 
-					if ((file_exists($favicon_file) ? filemtime($favicon_file) : -1) > $favicon_modified)
+					if (!$favicon_cache->exists($feed) || $favicon_cache->get_mtime($feed) > $favicon_modified) {
 						$feed_obj->favicon_avg_color = null;
+						$feed_obj->save();
+					}
 				}
 
-				if (is_readable($favicon_file) && function_exists("imagecreatefromstring") && empty($feed_obj->favicon_avg_color)) {
+				/* terrible hack: if we crash on floicon shit here, we won't check
+				 * the icon avgcolor again (unless icon got updated) */
+				if (file_exists($favicon_cache->get_full_path($feed)) && function_exists("imagecreatefromstring") && empty($feed_obj->favicon_avg_color)) {
 					require_once "colors.php";
 
 					Debug::log("favicon: trying to calculate average color...", Debug::LOG_VERBOSE);
@@ -594,13 +631,13 @@ class RSSUtils {
 					$feed_obj->favicon_avg_color = 'fail';
 					$feed_obj->save();
 
-					$feed_obj->favicon_avg_color = \Colors\calculate_avg_color($favicon_file);
+					$feed_obj->favicon_avg_color = \Colors\calculate_avg_color($favicon_cache->get_full_path($feed));
 					$feed_obj->save();
 
 					Debug::log("favicon: avg color: {$feed_obj->favicon_avg_color}", Debug::LOG_VERBOSE);
 
 				} else if ($feed_obj->favicon_avg_color == 'fail') {
-					Debug::log("floicon failed $favicon_file, not trying to recalculate avg color", Debug::LOG_VERBOSE);
+					Debug::log("floicon failed on $feed, not trying to recalculate avg color", Debug::LOG_VERBOSE);
 				}
 			}
 
@@ -644,7 +681,7 @@ class RSSUtils {
 					print_r($item);
 				}
 
-				if (ini_get("max_execution_time") > 0 && time() - $tstart >= ini_get("max_execution_time") * 0.7) {
+				if (ini_get("max_execution_time") > 0 && time() - $tstart >= ((float)ini_get("max_execution_time") * 0.7)) {
 					Debug::log("looks like there's too many articles to process at once, breaking out.", Debug::LOG_VERBOSE);
 					$pdo->commit();
 					break;
@@ -715,7 +752,7 @@ class RSSUtils {
 					$article_labels = Article::_get_labels($base_entry_id, $feed_obj->owner_uid);
 
 					$existing_tags = Article::_get_tags($base_entry_id, $feed_obj->owner_uid);
-					$entry_tags = array_unique(array_merge($entry_tags, $existing_tags));
+					$entry_tags = array_unique([...$entry_tags, ...$existing_tags]);
 				} else {
 					$base_entry_id = false;
 					$entry_stored_hash = "";
@@ -847,7 +884,7 @@ class RSSUtils {
 				$pluginhost->run_hooks(PluginHost::HOOK_FILTER_TRIGGERED,
 					$feed, $feed_obj->owner_uid, $article, $matched_filters, $matched_rules, $article_filters);
 
-				$matched_filter_ids = array_map(function($f) { return $f['id']; }, $matched_filters);
+				$matched_filter_ids = array_map(fn(array $f) => $f['id'], $matched_filters);
 
 				if (count($matched_filter_ids) > 0) {
 					$filter_objs = ORM::for_table('ttrss_filters2')
@@ -1172,8 +1209,7 @@ class RSSUtils {
 				// check for manual tags (we have to do it here since they're loaded from filters)
 				foreach ($article_filters as $f) {
 					if ($f["type"] == "tag") {
-						$entry_tags = array_merge($entry_tags,
-							FeedItem_Common::normalize_categories(explode(",", $f["param"])));
+						$entry_tags = [...$entry_tags, ...FeedItem_Common::normalize_categories(explode(",", $f["param"]))];
 					}
 				}
 
@@ -1284,7 +1320,7 @@ class RSSUtils {
 	 * @see FeedEnclosure
 	 */
 	static function cache_enclosures(array $enclosures, string $site_url): void {
-		$cache = new DiskCache("images");
+		$cache = DiskCache::instance("images");
 
 		if ($cache->is_writable()) {
 			foreach ($enclosures as $enc) {
@@ -1306,8 +1342,6 @@ class RSSUtils {
 						} else {
 							Debug::log("cache_enclosures: failed with ".UrlHelper::$fetch_last_error_code.": ".UrlHelper::$fetch_last_error);
 						}
-					} else if (is_writable($local_filename)) {
-						$cache->touch($local_filename);
 					}
 				}
 			}
@@ -1333,14 +1367,12 @@ class RSSUtils {
 			} else {
 				Debug::log("cache_media: failed with ".UrlHelper::$fetch_last_error_code.": ".UrlHelper::$fetch_last_error);
 			}
-		} else if ($cache->is_writable($local_filename)) {
-			$cache->touch($local_filename);
 		}
 	}
 
 	/* TODO: move to DiskCache? */
 	static function cache_media(string $html, string $site_url): void {
-		$cache = new DiskCache("images");
+		$cache = DiskCache::instance("images");
 
 		if ($html && $cache->is_writable()) {
 			$doc = new DOMDocument();
@@ -1656,9 +1688,36 @@ class RSSUtils {
 		$tmph->run_hooks(PluginHost::HOOK_HOUSE_KEEPING);
 	}
 
+	/** migrates favicons from legacy storage in feed-icons/ to cache/feed-icons/using new naming (sans .ico suffix) */
+	static function migrate_feed_icons() : void {
+		$old_dir = Config::get(Config::ICONS_DIR);
+		$new_dir = Config::get(Config::CACHE_DIR) . '/feed-icons';
+
+		$dh = opendir($old_dir);
+
+		$cache = DiskCache::instance('feed-icons');
+
+		if ($dh) {
+			while (($old_filename = readdir($dh)) !== false) {
+				if (strpos($old_filename, ".ico") !== false) {
+					$new_filename = str_replace(".ico", "", $old_filename);
+					$old_full_path = "$old_dir/$old_filename";
+
+					if (is_file($old_full_path) && $cache->put($new_filename, file_get_contents($old_full_path))) {
+						unlink($old_full_path);
+					}
+				}
+			}
+
+			closedir($dh);
+		}
+	}
+
 	static function housekeeping_common(): void {
-		DiskCache::expire();
+		$cache = DiskCache::instance("");
+		$cache->expire_all();
 
+		self::migrate_feed_icons();
 		self::expire_lock_files();
 		self::expire_error_log();
 		self::expire_feed_archive();
@@ -1676,8 +1735,6 @@ class RSSUtils {
 	 * @return false|string
 	 */
 	static function update_favicon(string $site_url, int $feed) {
-		$icon_file = Config::get(Config::ICONS_DIR) . "/$feed.ico";
-
 		$favicon_urls = self::get_favicon_urls($site_url);
 
 		if (count($favicon_urls) == 0) {
@@ -1732,21 +1789,18 @@ class RSSUtils {
 				break;
 			}
 
-			Debug::log("favicon: $favicon_url looks valid, saving to $icon_file", Debug::LOG_VERBOSE);
-
-			$fp = @fopen($icon_file, "w");
-
-			if ($fp) {
+			$favicon_cache = DiskCache::instance('feed-icons');
 
-				fwrite($fp, $contents);
-				fclose($fp);
-				chmod($icon_file, 0644);
-				clearstatcache();
+			if ($favicon_cache->is_writable()) {
+				Debug::log("favicon: $favicon_url looks valid, saving to cache", Debug::LOG_VERBOSE);
 
-				return $icon_file;
+				// we deal with this manually
+				if (!$favicon_cache->exists(".no-auto-expiry"))
+					$favicon_cache->put(".no-auto-expiry", "");
 
+				return $favicon_cache->put((string)$feed, $contents);
 			} else {
-				Debug::log("favicon: failed to open $icon_file for writing", Debug::LOG_VERBOSE);
+				Debug::log("favicon: $favicon_url skipping, local cache is not writable", Debug::LOG_VERBOSE);
 			}
 		}
 
@@ -1778,12 +1832,10 @@ class RSSUtils {
 				owner_uid = ? AND enabled = true ORDER BY order_id, title");
 		$sth->execute([$owner_uid]);
 
-		$check_cats = array_merge(
-			Feeds::_get_parent_cats($cat_id, $owner_uid),
-			[$cat_id]);
+		$check_cats = [...Feeds::_get_parent_cats($cat_id, $owner_uid), $cat_id];
 
 		$check_cats_str = join(",", $check_cats);
-		$check_cats_fullids = array_map(function($a) { return "CAT:$a"; }, $check_cats);
+		$check_cats_fullids = array_map(fn(int $a) => "CAT:$a", $check_cats);
 
 		while ($line = $sth->fetch()) {
 			$filter_id = $line["id"];