diff options
author | Andrew Dolgov <[email protected]> | 2008-01-23 10:19:36 +0100 |
---|---|---|
committer | Andrew Dolgov <[email protected]> | 2008-01-23 10:19:36 +0100 |
commit | 02008cb19ac99727889cca05c7eddfbeef30d684 (patch) | |
tree | b1ff053ad8f846ea4171460edf0ba5220c2cb2e2 | |
parent | d4761137527576a249964eabaee66d36e62eb74f (diff) |
add multiprocess update daemon
-rw-r--r-- | functions.php | 13 | ||||
-rw-r--r-- | update_daemon2.php | 55 | ||||
-rw-r--r-- | update_daemon2_client.php | 161 |
3 files changed, 226 insertions, 3 deletions
diff --git a/functions.php b/functions.php index 919cd926c..9bd98795b 100644 --- a/functions.php +++ b/functions.php @@ -468,12 +468,19 @@ _debug("update_rss_feed: start"); } - $result = db_query($link, "SELECT id,update_interval,auth_login,auth_pass,cache_images - FROM ttrss_feeds WHERE id = '$feed'"); + if (DB_TYPE == "pgsql") { + $updstart_thresh_qpart = "(ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started >= NOW() - INTERVAL '120 seconds')"; + } else { + $updstart_thresh_qpart = "(ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started >= DATE_SUB(NOW(), INTERVAL 120 SECOND))"; + } + + $result = db_query($link, "SELECT id,update_interval,auth_login, + auth_pass,cache_images + FROM ttrss_feeds WHERE id = '$feed' AND $updstart_thresh_qpart"); if (db_num_rows($result) == 0) { if (defined('DAEMON_EXTENDED_DEBUG') || $_GET['xdebug']) { - _debug("update_rss_feed: feed $feed [$feed_url] NOT FOUND"); + _debug("update_rss_feed: feed $feed [$feed_url] NOT FOUND/SKIPPED"); } return; } diff --git a/update_daemon2.php b/update_daemon2.php new file mode 100644 index 000000000..3721a0979 --- /dev/null +++ b/update_daemon2.php @@ -0,0 +1,55 @@ +#!/usr/bin/php +<?php + // This is an experimental multiprocess update daemon + // It consists of the master server (this file) and + // client batch script (update_daemon2_client.php) which + // should only be run by the server process + + declare(ticks = 1); + + require "config.php"; + + define('MAX_JOBS', 2); + define('CLIENT_PROCESS', './update_daemon2_client.php SRV_RUN_OK'); + define('SPAWN_INTERVAL', DAEMON_SLEEP_INTERVAL); + + $running_jobs = 0; + $last_checkpoint = -1; + + function sigchld_handler($signal) { + global $running_jobs; + if ($running_jobs > 0) $running_jobs--; + print posix_getpid() . ": SIGCHLD received, jobs left: $running_jobs\n"; + pcntl_waitpid(-1, $status, WNOHANG); + } + + pcntl_signal(SIGCHLD, 'sigchld_handler'); + + while (true) { + + $next_spawn = $last_checkpoint + SPAWN_INTERVAL - time(); + + print "[MASTER] active jobs: $running_jobs, next spawn at $next_spawn sec\n"; + + if ($last_checkpoint + SPAWN_INTERVAL < time()) { + + for ($j = $running_jobs; $j < MAX_JOBS; $j++) { + print "[MASTER] spawning client $j..."; + $pid = pcntl_fork(); + if ($pid == -1) { + die("fork failed!\n"); + } else if ($pid) { + $running_jobs++; + print "OK [$running_jobs]\n"; + } else { + pcntl_signal(SIGCHLD, SIG_IGN); + passthru(CLIENT_PROCESS); + exit(0); + } + } + $last_checkpoint = time(); + } + sleep(1); + } + +?> diff --git a/update_daemon2_client.php b/update_daemon2_client.php new file mode 100644 index 000000000..2d069c37d --- /dev/null +++ b/update_daemon2_client.php @@ -0,0 +1,161 @@ +#!/usr/bin/php +<?php + // define('DEFAULT_ERROR_LEVEL', E_ALL); + define('DEFAULT_ERROR_LEVEL', E_ERROR | E_WARNING | E_PARSE); + + if ($argv[1] != "SRV_RUN_OK") { + die("This script should be started by update_daemon2.php.\n"); + } + + declare(ticks = 1); + + define('MAGPIE_CACHE_DIR', '/var/tmp/magpie-ttrss-cache-daemon'); + define('SIMPLEPIE_CACHE_DIR', '/var/tmp/simplepie-ttrss-cache-daemon'); + define('DISABLE_SESSIONS', true); + + require_once "version.php"; + + if (strpos(VERSION, ".99") !== false) { + define('DAEMON_EXTENDED_DEBUG', true); + } + + define('PURGE_INTERVAL', 3600); // seconds + + require_once "sanity_check.php"; + require_once "config.php"; + + if (!ENABLE_UPDATE_DAEMON) { + die("Please enable option ENABLE_UPDATE_DAEMON in config.php\n"); + } + + require_once "db.php"; + require_once "db-prefs.php"; + require_once "functions.php"; + require_once "magpierss/rss_fetch.inc"; + + error_reporting(DEFAULT_ERROR_LEVEL); + + function sigalrm_handler() { + die("received SIGALRM, hang in feed update?\n"); + } + + pcntl_signal(SIGALRM, sigalrm_handler); + + $link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME); + + if (!$link) { + if (DB_TYPE == "mysql") { + print mysql_error(); + } + // PG seems to display its own errors just fine by default. + return; + } + + if (DB_TYPE == "pgsql") { + pg_query("set client_encoding = 'utf-8'"); + pg_set_client_encoding("UNICODE"); + } else { + if (defined('MYSQL_CHARSET') && MYSQL_CHARSET) { + db_query($link, "SET NAMES " . MYSQL_CHARSET); +// db_query($link, "SET CHARACTER SET " . MYSQL_CHARSET); + } + } + + $last_purge = 0; + +// if (!make_stampfile('update_daemon.stamp')) { +// print "error: unable to create stampfile"; +// die; +// } + + if (time() - $last_purge > PURGE_INTERVAL) { + _debug("Purging old posts (random 30 feeds)..."); + global_purge_old_posts($link, true, 30); + $last_purge = time(); + } + + // Process all other feeds using last_updated and interval parameters + + $random_qpart = sql_random_function(); + + if (DAEMON_UPDATE_LOGIN_LIMIT > 0) { + if (DB_TYPE == "pgsql") { + $login_thresh_qpart = "AND ttrss_users.last_login >= NOW() - INTERVAL '".DAEMON_UPDATE_LOGIN_LIMIT." days'"; + } else { + $login_thresh_qpart = "AND ttrss_users.last_login >= DATE_SUB(NOW(), INTERVAL ".DAEMON_UPDATE_LOGIN_LIMIT." DAY)"; + } + } else { + $login_thresh_qpart = ""; + } + + if (DB_TYPE == "pgsql") { + $update_limit_qpart = "AND ttrss_feeds.last_updated < NOW() - INTERVAL '".(DAEMON_SLEEP_INTERVAL*2)." seconds'"; + } else { + $update_limit_qpart = "AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ".(DAEMON_SLEEP_INTERVAL*2)." SECOND)"; + } + + if (DB_TYPE == "pgsql") { + $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started >= NOW() - INTERVAL '120 seconds')"; + } else { + $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started >= DATE_SUB(NOW(), INTERVAL 120 SECOND))"; + } + + $result = db_query($link, "SELECT feed_url,ttrss_feeds.id,owner_uid, + SUBSTRING(last_updated,1,19) AS last_updated, + update_interval + FROM + ttrss_feeds,ttrss_users + WHERE + ttrss_users.id = owner_uid $login_thresh_qpart $update_limit_qpart + $updstart_thresh_qpart + ORDER BY $random_qpart DESC LIMIT " . DAEMON_FEED_LIMIT); + + $user_prefs_cache = array(); + + _debug(sprintf("Scheduled %d feeds to update...\n", db_num_rows($result))); + + while ($line = db_fetch_assoc($result)) { + + $upd_intl = $line["update_interval"]; + $user_id = $line["owner_uid"]; + + if (!$upd_intl || $upd_intl == 0) { + if (!$user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL']) { + $upd_intl = get_pref($link, 'DEFAULT_UPDATE_INTERVAL', $user_id); + $user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL'] = $upd_intl; + } else { + $upd_intl = $user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL']; + } + } + + if ($upd_intl < 0) { +# print "Updates disabled.\n"; + continue; + } + + _debug("Feed: " . $line["feed_url"] . ", " . $line["last_updated"]); + +// _debug(sprintf("\tLU: %d, INTL: %d, UID: %d) ", +// time() - strtotime($line["last_updated"]), $upd_intl*60, $user_id)); + + if (!$line["last_updated"] || + time() - strtotime($line["last_updated"]) > ($upd_intl * 60)) { + + _debug("Updating..."); + + pcntl_alarm(300); + + update_rss_feed($link, $line["feed_url"], $line["id"], true); + + pcntl_alarm(0); + + sleep(1); // prevent flood (FIXME make this an option?) + } else { + _debug("Update not needed."); + } + } + + if (DAEMON_SENDS_DIGESTS) send_headlines_digests($link); + + db_close($link); +?> |