From a4f5fb34857233fe0c7f4f5fe41414d515ed1b1e Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Sun, 27 Aug 2023 19:08:14 +0300 Subject: generic version --- README.md | 8 ++--- init.php | 93 ++++++++++++++++++---------------------------------- sql/mysql/schema.sql | 11 ++++--- sql/pgsql/schema.sql | 11 ++++--- 4 files changed, 47 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index fd7dc4e..f259a9d 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ -## This plugin delays posts in Reddit feeds by a configurable delay (in hours) +## This plugin delays posts in RSS feeds by a configurable delay (in hours) -Optionally, it can filter out posts that were deleted (i.e. by Reddit moderators), while delayed. +Optionally, it can filter out posts that were deleted (either article or comment link goes nowhere) +while delayed. https://community.tt-rss.org/t/suggestions-for-how-to-delay-a-feed/4425 ### Installation -- Git clone to `plugins.local/reddit_delay` +- Git clone to `plugins.local/post_delay` - Set delay amount in Preferences → Feeds → Plugins ### Notes - Posts are stored in the backlog no longer than `Config::CACHE_MAX_DAYS`. -- If Reddit JSON API returns 404 for the post, it is considered deleted. diff --git a/init.php b/init.php index fef273b..b7d4840 100755 --- a/init.php +++ b/init.php @@ -1,12 +1,12 @@ where('feed_id', $feed_id) ->where('link', $link) ->find_one(); @@ -40,11 +40,12 @@ class Reddit_Delay extends Plugin { } private function cache_push(int $feed_id, FeedItem $item, DOMNode $node) : bool { - $entry = ORM::for_table('ttrss_plugin_reddit_delay_cache')->create(); + $entry = ORM::for_table('ttrss_plugin_post_delay_cache')->create(); $entry->set([ 'feed_id' => $feed_id, 'link' => $item->get_link(), + 'comments' => $item->get_comments_url(), 'item' => $node->ownerDocument->saveXML($node), 'orig_ts' => date("Y-m-d H:i:s", $item->get_date()) ]); @@ -62,7 +63,7 @@ class Reddit_Delay extends Plugin { $interval_query = "orig_ts < DATE_SUB(NOW(), INTERVAL $max_days DAY)"; } - $sth = $this->pdo->prepare("DELETE FROM ttrss_plugin_reddit_delay_cache + $sth = $this->pdo->prepare("DELETE FROM ttrss_plugin_post_delay_cache WHERE $interval_query"); $sth->execute([]); } @@ -84,7 +85,7 @@ class Reddit_Delay extends Plugin { $interval_query = "(orig_ts < DATE_SUB(NOW(), INTERVAL $delay HOUR))"; } - $entries = ORM::for_table('ttrss_plugin_reddit_delay_cache') + $entries = ORM::for_table('ttrss_plugin_post_delay_cache') ->where('feed_id', $feed_id) ->where_raw($interval_query) ->find_many(); @@ -99,58 +100,25 @@ class Reddit_Delay extends Plugin { $skip_post = false; $delete_post = false; - Debug::log(sprintf("[delay] pulling from cache: %s [%s]", - $entry->link, $entry->orig_ts), Debug::LOG_EXTENDED); - - if ($skip_removed && strpos($entry->link, "reddit.com") !== false) { - $matches = []; - - if (preg_match("/\/comments\/([^\/]+)\//", $entry->link, $matches)) { - $post_id = $matches[1]; - $post_api_url = "https://api.reddit.com/api/info/?id=t3_${post_id}"; - - Debug::log("[delay] API url: ${post_api_url}", Debug::LOG_EXTENDED); - - $json_data = UrlHelper::fetch(["url" => $post_api_url]); - - if ($json_data) { - $json = json_decode($json_data, true); - - if ($json) { - if (count($json["data"]["children"]) == 0) { - $skip_post = "[json:no-children]"; - } else { - foreach ($json["data"]["children"] as $child) { - if (empty($child["data"]["is_robot_indexable"])) { - $skip_post = "[removed]"; - $delete_post = true; - break; - } else if (empty($child["data"]["author"])) { - $skip_post = "[deleted]"; - $delete_post = true; - break; - } - } - } - } else { - $skip_post = "[json:parse-failed]"; - } - } else if (UrlHelper::$fetch_last_error_code == 404) { - $skip_post = "[json:404]"; - $delete_post = true; - } else { - $skip_post = "[json:no-data]"; - } - } - } else if ($skip_removed) { - // i guess we can check if the link leads anywhere + Debug::log(sprintf("[delay] pulling from cache: %s [c:%s] [%s]", + $entry->link, $entry->comments, $entry->orig_ts), Debug::LOG_EXTENDED); - $data = UrlHelper::fetch(["url" => $entry->item]); + if ($skip_removed) { + UrlHelper::fetch(["url" => $entry->item]); - if (UrlHelper::$fetch_last_error_code == 404) { - $skip_post = "[link:404]"; + if (UrlHelper::$fetch_last_error_code >= 400) { + $skip_post = "[link:4xx-or-5xx]"; $delete_post = true; } + + if (strlen($entry->comments) > 0) { + UrlHelper::fetch(["url" => $entry->comments]); + + if (UrlHelper::$fetch_last_error_code >= 400) { + $skip_post = "[comments:4xx-or-5xx]"; + $delete_post = true; + } + } } if (!$skip_post) { @@ -190,7 +158,7 @@ class Reddit_Delay extends Plugin { $delay = (int) $this->host->get($this, "delay"); $enabled_feeds = $this->host->get_array($this, "enabled_feeds"); - if (in_array($feed_id, $enabled_feeds) || preg_match("/\.?reddit\.com/", $fetch_url) === 1 && $delay > 0) { + if (in_array($feed_id, $enabled_feeds) && $delay > 0) { $doc = new DOMDocument(); @@ -227,8 +195,9 @@ class Reddit_Delay extends Plugin { if (!$item_timestamp) $item_timestamp = time(); if ($item_timestamp > $cutoff_timestamp) { - Debug::log(sprintf("[delay] %s [%s vs %s]", + Debug::log(sprintf("[delay] %s [c: %s] [%s vs %s]", $item->get_link(), + $item->get_comments_url(), date("Y-m-d H:i:s", $item->get_date()), date("Y-m-d H:i:s", $cutoff_timestamp)), Debug::LOG_EXTENDED); @@ -267,7 +236,7 @@ class Reddit_Delay extends Plugin { ?>
+ title="extension ">
@@ -305,7 +274,7 @@ class Reddit_Delay extends Plugin { pdo->prepare("SELECT COUNT(c.id) AS count - FROM ttrss_plugin_reddit_delay_cache c, ttrss_feeds f + FROM ttrss_plugin_post_delay_cache c, ttrss_feeds f WHERE f.id = c.feed_id AND f.owner_uid = ?"); $sth->execute([$_SESSION["uid"]]); @@ -317,7 +286,7 @@ class Reddit_Delay extends Plugin { pdo->prepare("SELECT COUNT(c.id) AS count, f.title, f.id AS feed_id - FROM ttrss_plugin_reddit_delay_cache c, ttrss_feeds f + FROM ttrss_plugin_post_delay_cache c, ttrss_feeds f WHERE f.id = c.feed_id AND f.owner_uid = ? GROUP BY f.title, f.id ORDER BY count DESC, f.title"); @@ -341,7 +310,7 @@ class Reddit_Delay extends Plugin { $this->host->set($this, "enabled_feeds", $enabled_feeds); if (count($enabled_feeds) > 0) { ?> -

+

    @@ -372,7 +341,7 @@ class Reddit_Delay extends Plugin {
    @@ -382,7 +351,7 @@ class Reddit_Delay extends Plugin { function hook_prefs_save_feed($feed_id) { $enabled_feeds = $this->host->get_array($this, "enabled_feeds"); - $enable = checkbox_to_sql_bool($_POST["reddit_delay_posts_enabled"] ?? ""); + $enable = checkbox_to_sql_bool($_POST["delay_posts_enabled"] ?? ""); $key = array_search($feed_id, $enabled_feeds); if ($enable) { diff --git a/sql/mysql/schema.sql b/sql/mysql/schema.sql index eda9f52..809ebc2 100644 --- a/sql/mysql/schema.sql +++ b/sql/mysql/schema.sql @@ -1,12 +1,13 @@ -drop table if exists ttrss_plugin_reddit_delay_cache; +drop table if exists ttrss_plugin_post_delay_cache; -create table ttrss_plugin_reddit_delay_cache (id integer not null primary key auto_increment, +create table ttrss_plugin_post_delay_cache (id integer not null primary key auto_increment, feed_id integer not null REFERENCES ttrss_feeds(id) on DELETE cascade, link text not null, + comments text not null, item text not NULL, orig_ts datetime not null) ENGINE=InnoDB DEFAULT CHARSET=UTF8; -create index ttrss_plugin_reddit_delay_cache_link_idx on ttrss_plugin_reddit_delay_cache(link(768)); -create index ttrss_plugin_reddit_delay_cache_feed_id_idx on ttrss_plugin_reddit_delay_cache(feed_id); +create index ttrss_plugin_post_delay_cache_link_idx on ttrss_plugin_post_delay_cache(link(768)); +create index ttrss_plugin_post_delay_cache_feed_id_idx on ttrss_plugin_post_delay_cache(feed_id); -create unique index ttrss_plugin_reddit_delay_cache_idx on ttrss_plugin_reddit_delay_cache(feed_id, link(768)); +create unique index ttrss_plugin_post_delay_cache_idx on ttrss_plugin_post_delay_cache(feed_id, link(768)); diff --git a/sql/pgsql/schema.sql b/sql/pgsql/schema.sql index f93f596..b70edff 100644 --- a/sql/pgsql/schema.sql +++ b/sql/pgsql/schema.sql @@ -1,11 +1,12 @@ -drop table if exists ttrss_plugin_reddit_delay_cache; +drop table if exists ttrss_plugin_post_delay_cache; -create table ttrss_plugin_reddit_delay_cache (id serial not null primary key, +create table ttrss_plugin_post_delay_cache (id serial not null primary key, feed_id integer not null REFERENCES ttrss_feeds(id) on DELETE cascade, link text not null, + comments text not null, item text not NULL, orig_ts timestamp not null); -create unique index ttrss_plugin_reddit_delay_cache_idx on ttrss_plugin_reddit_delay_cache(feed_id, link); -create index ttrss_plugin_reddit_delay_cache_link_idx on ttrss_plugin_reddit_delay_cache(link); -create index ttrss_plugin_reddit_delay_cache_feed_id_idx on ttrss_plugin_reddit_delay_cache(feed_id); +create unique index ttrss_plugin_post_delay_cache_idx on ttrss_plugin_post_delay_cache(feed_id, link); +create index ttrss_plugin_post_delay_cache_link_idx on ttrss_plugin_post_delay_cache(link); +create index ttrss_plugin_post_delay_cache_feed_id_idx on ttrss_plugin_post_delay_cache(feed_id); -- cgit v1.2.3