From fdb6066bf67751ebb388a3de082d80f5444e3681 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Tue, 13 Aug 2019 16:40:21 +0300 Subject: * HOOK_ENCLOSURE_ENTRY: pass article_id to handler * DiskCache: multiple fixes; support isWritable() for cache entries, set content-disposition for send() * public/cached_url: allow selecting files from sub-caches other than images * plugins/Cache_Starred_Images: rework to use DiskCache, can be enabled per-user, properly handles article enclosures, etc --- plugins/cache_starred_images/init.php | 230 +++++++++++++++++----------------- 1 file changed, 116 insertions(+), 114 deletions(-) (limited to 'plugins/cache_starred_images') diff --git a/plugins/cache_starred_images/init.php b/plugins/cache_starred_images/init.php index 9c64ac3d0..916cedd53 100755 --- a/plugins/cache_starred_images/init.php +++ b/plugins/cache_starred_images/init.php @@ -1,90 +1,80 @@ host = $host; + $this->cache = new DiskCache("starred-images"); - $this->cache_dir = CACHE_DIR . "/starred-images/"; - - if (!is_dir($this->cache_dir)) { - mkdir($this->cache_dir); - } + if ($this->cache->makeDir()) + chmod($this->cache->getDir(), 0777); - if (is_dir($this->cache_dir)) { - - if (!is_writable($this->cache_dir)) - chmod($this->cache_dir, 0777); - - if (is_writable($this->cache_dir)) { - $host->add_hook($host::HOOK_UPDATE_TASK, $this); - $host->add_hook($host::HOOK_HOUSE_KEEPING, $this); - $host->add_hook($host::HOOK_SANITIZE, $this); - $host->add_handler("public", "cache_starred_images_getimage", $this); - - } else { - user_error("Starred cache directory is not writable.", E_USER_WARNING); - } + if (!$this->cache->exists(".no-auto-expiry")) + $this->cache->touch(".no-auto-expiry"); + if ($this->cache->isWritable()) { + $host->add_hook($host::HOOK_HOUSE_KEEPING, $this); + $host->add_hook($host::HOOK_ENCLOSURE_ENTRY, $this); + $host->add_hook($host::HOOK_SANITIZE, $this); } else { - user_error("Unable to create starred cache directory.", E_USER_WARNING); + user_error("Starred cache directory ".$this->cache->getDir()." is not writable.", E_USER_WARNING); } } - function cache_starred_images_getimage() { - ob_end_clean(); + /** + * @SuppressWarnings(PHPMD.UnusedLocalVariable) + */ + function hook_house_keeping() { + /* since HOOK_UPDATE_TASK is not available to user plugins, this hook is a next best thing */ - $hash = basename($_REQUEST["hash"]); + Debug::log("caching media of starred articles for user " . $this->host->get_owner_uid() . "..."); - if ($hash) { + $sth = $this->pdo->prepare("SELECT content, ttrss_entries.title, + ttrss_user_entries.owner_uid, link, site_url, ttrss_entries.id, plugin_data + FROM ttrss_entries, ttrss_user_entries LEFT JOIN ttrss_feeds ON + (ttrss_user_entries.feed_id = ttrss_feeds.id) + WHERE ref_id = ttrss_entries.id AND + marked = true AND + site_url != '' AND + ttrss_user_entries.owner_uid = ? AND + plugin_data NOT LIKE '%starred_cache_images%' + ORDER BY ".sql_random_function()." LIMIT 100"); - $filename = $this->cache_dir . "/" . basename($hash); + if ($sth->execute([$this->host->get_owner_uid()])) { - if (file_exists($filename)) { - header("Content-Disposition: attachment; filename=\"$hash\""); + $usth = $this->pdo->prepare("UPDATE ttrss_entries SET plugin_data = ? WHERE id = ?"); - send_local_file($filename); - } else { - header($_SERVER["SERVER_PROTOCOL"]." 404 Not Found"); - echo "File not found."; + while ($line = $sth->fetch()) { + Debug::log("processing article " . $line["title"], Debug::$LOG_VERBOSE); + + if ($line["site_url"]) { + $success = $this->cache_article_images($line["content"], $line["site_url"], $line["owner_uid"], $line["id"]); + + if ($success) { + $plugin_data = "starred_cache_images,${line['owner_uid']}:" . $line["plugin_data"]; + + $usth->execute([$plugin_data, $line['id']]); + } + } } } - } - /** - * @SuppressWarnings(PHPMD.UnusedLocalVariable) - */ - function hook_house_keeping() { - $files = glob($this->cache_dir . "/*.{png,mp4,status}", GLOB_BRACE); + /* actual housekeeping */ + + Debug::log("expiring " . $this->cache->getDir() . "..."); + + $files = glob($this->cache->getDir() . "/*.{png,mp4,status}", GLOB_BRACE); $last_article_id = 0; $article_exists = 1; @@ -107,6 +97,16 @@ class Cache_Starred_Images extends Plugin implements IHandler { } } + function hook_enclosure_entry($enc, $article_id) { + $local_filename = $article_id . "-" . sha1($enc["content_url"]); + + if ($this->cache->exists($local_filename)) { + $enc["content_url"] = DiskCache::getUrl("starred-images/" . $local_filename); + } + + return $enc; + } + /** * @SuppressWarnings(PHPMD.UnusedFormalParameter) */ @@ -120,15 +120,12 @@ class Cache_Starred_Images extends Plugin implements IHandler { if ($entry->hasAttribute('src')) { $src = rewrite_relative_url($site_url, $entry->getAttribute('src')); - $extension = $entry->tagName == 'source' ? '.mp4' : '.png'; - $local_filename = $this->cache_dir . $article_id . "-" . sha1($src) . $extension; + $local_filename = $article_id . "-" . sha1($src); - if (file_exists($local_filename)) { - $entry->setAttribute("src", get_self_url_prefix() . - "/public.php?op=cache_starred_images_getimage&method=image&hash=" . - $article_id . "-" . sha1($src) . $extension); + if ($this->cache->exists($local_filename)) { + $entry->setAttribute("src", DiskCache::getUrl("starred-images/" . $local_filename)); + $entry->removeAttribute("srcset"); } - } } } @@ -136,42 +133,46 @@ class Cache_Starred_Images extends Plugin implements IHandler { return $doc; } - function hook_update_task() { - $res = $this->pdo->query("SELECT content, ttrss_user_entries.owner_uid, link, site_url, ttrss_entries.id, plugin_data - FROM ttrss_entries, ttrss_user_entries LEFT JOIN ttrss_feeds ON - (ttrss_user_entries.feed_id = ttrss_feeds.id) - WHERE ref_id = ttrss_entries.id AND - marked = true AND - (UPPER(content) LIKE '%pdo->prepare("UPDATE ttrss_entries SET plugin_data = ? WHERE id = ?"); + if (!$this->cache->getSize($local_filename) >= 0) { + Debug::log("cache_images: downloading: $url to $local_filename", Debug::$LOG_VERBOSE); - while ($line = $res->fetch()) { - if ($line["site_url"]) { - $success = $this->cache_article_images($line["content"], $line["site_url"], $line["owner_uid"], $line["id"]); + $data = fetch_file_contents(["url" => $url, "max_size" => MAX_CACHE_FILE_SIZE]); - if ($success) { - $plugin_data = "starred_cache_images,${line['owner_uid']}:" . $line["plugin_data"]; - - $usth->execute([$plugin_data, $line['id']]); + if ($data) { + if (strlen($data) > MIN_CACHE_FILE_SIZE) { + $this->cache->put($local_filename, $data); } + + return true; } + } else { + //Debug::log("cache_images: local file exists for $url", Debug::$LOG_VERBOSE); + + return true; } + + return false; } /** * @SuppressWarnings(PHPMD.UnusedFormalParameter) */ - function cache_article_images($content, $site_url, $owner_uid, $article_id) { - $status_filename = $this->cache_dir . $article_id . "-" . sha1($site_url) . ".status"; + private function cache_article_images($content, $site_url, $owner_uid, $article_id) { + $status_filename = $article_id . "-" . sha1($site_url) . ".status"; - Debug::log("status: $status_filename", Debug::$LOG_EXTENDED); + /* housekeeping might run as a separate user, in this case status/media might not be writable */ + if (!$this->cache->isWritable($status_filename)) { + Debug::log("status not writable: $status_filename", Debug::$LOG_VERBOSE); + return false; + } - if (file_exists($status_filename)) - $status = json_decode(file_get_contents($status_filename), true); + Debug::log("status: $status_filename", Debug::$LOG_VERBOSE); + + if ($this->cache->exists($status_filename)) + $status = json_decode($this->cache->get($status_filename), true); else $status = []; @@ -180,47 +181,48 @@ class Cache_Starred_Images extends Plugin implements IHandler { // only allow several download attempts for article if ($status["attempt"] > $this->max_cache_attempts) { Debug::log("too many attempts for $site_url", Debug::$LOG_VERBOSE); - return; + return false; } - if (!file_put_contents($status_filename, json_encode($status))) { + if (!$this->cache->put($status_filename, json_encode($status))) { user_error("unable to write status file: $status_filename", E_USER_WARNING); - return; + return false; } $doc = new DOMDocument(); - $doc->loadHTML('' . $content); - $xpath = new DOMXPath($doc); - - $entries = $xpath->query('(//img[@src])|(//video/source[@src])'); - $success = false; $has_images = false; + $success = false; + + if ($doc->loadHTML('' . $content)) { + $xpath = new DOMXPath($doc); + $entries = $xpath->query('(//img[@src])|(//video/source[@src])'); - foreach ($entries as $entry) { + foreach ($entries as $entry) { - if ($entry->hasAttribute('src') && strpos($entry->getAttribute('src'), "data:") !== 0) { + if ($entry->hasAttribute('src') && strpos($entry->getAttribute('src'), "data:") !== 0) { - $has_images = true; - $src = rewrite_relative_url($site_url, $entry->getAttribute('src')); + $has_images = true; - $extension = $entry->tagName == 'source' ? '.mp4' : '.png'; + $src = rewrite_relative_url($site_url, $entry->getAttribute('src')); - $local_filename = $this->cache_dir . $article_id . "-" . sha1($src) . $extension; + if ($this->cache_url($article_id, $src)) { + $success = true; + } + } + } + } - Debug::log("cache_images: downloading: $src to $local_filename", Debug::$LOG_VERBOSE); + $esth = $this->pdo->prepare("SELECT content_url FROM ttrss_enclosures WHERE post_id = ? AND + (content_type LIKE '%image%' OR content_type LIKE '%video%')"); - if (!file_exists($local_filename)) { - $file_content = fetch_file_contents(["url" => $src, "max_size" => MAX_CACHE_FILE_SIZE]); + if ($esth->execute([$article_id])) { + while ($enc = $esth->fetch()) { - if ($file_content) { - if (strlen($file_content) > MIN_CACHE_FILE_SIZE) { - file_put_contents($local_filename, $file_content); - } + $has_images = true; + $url = rewrite_relative_url($site_url, $enc["content_url"]); - $success = true; - } - } else { + if ($this->cache_url($article_id, $url)) { $success = true; } } -- cgit v1.2.3