From 2a649c447cfd7fd4e4d2345c73718de6738306b3 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 17 Apr 2013 09:09:59 +0400 Subject: enable logger in updater scripts --- update_daemon2.php | 1 + 1 file changed, 1 insertion(+) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index c6e215a29..8e1917b6f 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -20,6 +20,7 @@ require_once "config.php"; require_once "db.php"; require_once "db-prefs.php"; + require_once "errorhandler.php"; // defaults define('PURGE_INTERVAL', 3600); // seconds -- cgit v1.2.3 From 857efe49e653787f5408fc014ae71efec68494d5 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 17 Apr 2013 12:10:35 +0400 Subject: add schema checking to feed updating and update script --- update_daemon2.php | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index 8e1917b6f..77d05be66 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -183,6 +183,10 @@ db_close($link); + if ($schema_version != SCHEMA_VERSION) { + die("Schema version is wrong, please upgrade the database.\n"); + } + while (true) { // Since sleep is interupted by SIGCHLD, we need another way to -- cgit v1.2.3 From ba68b6815ab31d17cda113e7990eeb07558b02a9 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 17 Apr 2013 14:23:35 +0400 Subject: db updates, remove init_connection() --- update_daemon2.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index 77d05be66..e8a56eec9 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -177,7 +177,7 @@ // It is unnecessary to start the fork loop if database is not ok. $link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME); - if (!init_connection($link)) die("Can't initialize db connection.\n"); + if (!init_plugins($link)) die("Can't initialize db connection.\n"); $schema_version = get_schema_version($link); @@ -203,7 +203,7 @@ /* Check if schema version changed */ $link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME); - if (!init_connection($link)) die("Can't initialize db connection.\n"); + if (!init_plugins($link)) die("Can't initialize db connection.\n"); $test_schema_version = get_schema_version($link); db_close($link); @@ -255,7 +255,7 @@ $link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME); - if (!init_connection($link)) return; + if (!init_plugins($link)) return; // We disable stamp file, since it is of no use in a multiprocess update. // not really, tho for the time being -fox -- cgit v1.2.3 From 404e2e3603c852a3f82a21c14b8888005e2b3f99 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 17 Apr 2013 15:36:34 +0400 Subject: more work on singleton-based DB --- update_daemon2.php | 1 + 1 file changed, 1 insertion(+) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index e8a56eec9..6d13add00 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -14,6 +14,7 @@ define('DAEMON_EXTENDED_DEBUG', true); } + require_once "autoload.php"; require_once "functions.php"; require_once "rssfuncs.php"; require_once "sanity_check.php"; -- cgit v1.2.3 From 6322ac79a020ab584d412d782d62b2ee77d7c6cf Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 17 Apr 2013 16:23:15 +0400 Subject: remove $link --- update_daemon2.php | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index 6d13add00..f5e031c9e 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -174,15 +174,11 @@ "Maybe another daemon is already running.\n"); } - // Testing database connection. - // It is unnecessary to start the fork loop if database is not ok. - $link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME); + init_plugins(); - if (!init_plugins($link)) die("Can't initialize db connection.\n"); + $schema_version = get_schema_version(); - $schema_version = get_schema_version($link); - - db_close($link); + db_close(); if ($schema_version != SCHEMA_VERSION) { die("Schema version is wrong, please upgrade the database.\n"); @@ -203,10 +199,8 @@ /* Check if schema version changed */ - $link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME); - if (!init_plugins($link)) die("Can't initialize db connection.\n"); - $test_schema_version = get_schema_version($link); - db_close($link); + init_plugins(); + $test_schema_version = get_schema_version(); if ($test_schema_version != $schema_version) { echo "Expected schema version: $schema_version, got: $test_schema_version\n"; @@ -254,9 +248,7 @@ $start_timestamp = time(); - $link = db_connect(DB_HOST, DB_USER, DB_PASS, DB_NAME); - - if (!init_plugins($link)) return; + if (!init_plugins()) return; // We disable stamp file, since it is of no use in a multiprocess update. // not really, tho for the time being -fox @@ -271,15 +263,15 @@ _debug("Waiting before update [$j].."); sleep($j*5); - $nf = update_daemon_common($link); + $nf = update_daemon_common(); if (rand(0,100) > 50) { - $count = update_feedbrowser_cache($link); + $count = update_feedbrowser_cache(); _debug("Feedbrowser updated, $count feeds processed."); - purge_orphans($link, true); + purge_orphans( true); - $rc = cleanup_tags($link, 14, 50000); + $rc = cleanup_tags( 14, 50000); _debug("Cleaned $rc cached tags."); @@ -297,7 +289,7 @@ } } - db_close($link); + db_close(); // We are in a fork. // We wait a little before exiting to avoid to be faster than our parent process. -- cgit v1.2.3 From eefaa2df381686f771396baae2d0ae71b345c2e7 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 17 Apr 2013 17:00:24 +0400 Subject: remove db_connect, db_close; CLI fixes --- update_daemon2.php | 5 ----- 1 file changed, 5 deletions(-) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index f5e031c9e..a1d7e7d66 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -178,8 +178,6 @@ $schema_version = get_schema_version(); - db_close(); - if ($schema_version != SCHEMA_VERSION) { die("Schema version is wrong, please upgrade the database.\n"); } @@ -199,7 +197,6 @@ /* Check if schema version changed */ - init_plugins(); $test_schema_version = get_schema_version(); if ($test_schema_version != $schema_version) { @@ -289,8 +286,6 @@ } } - db_close(); - // We are in a fork. // We wait a little before exiting to avoid to be faster than our parent process. sleep(1); -- cgit v1.2.3 From 7440a7fe3c43644cc3a90fe94e7c23795f04eeba Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 17 Apr 2013 18:19:26 +0400 Subject: daemon2: use passthrough to daemon-loop --- update_daemon2.php | 61 +++--------------------------------------------------- 1 file changed, 3 insertions(+), 58 deletions(-) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index a1d7e7d66..05ccb1356 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -21,7 +21,6 @@ require_once "config.php"; require_once "db.php"; require_once "db-prefs.php"; - require_once "errorhandler.php"; // defaults define('PURGE_INTERVAL', 3600); // seconds @@ -230,68 +229,14 @@ register_shutdown_function('task_shutdown'); - $my_pid = posix_getpid(); - $lock_filename = "update_daemon-$my_pid.lock"; - - $lock_handle = make_lockfile($lock_filename); - - if (!$lock_handle) { - die("error: Can't create lockfile ($lock_filename). ". - "Maybe another daemon is already running.\n"); - } - - // ****** Updating RSS code ******* - // Only run in fork process. - - $start_timestamp = time(); - - if (!init_plugins()) return; - - // We disable stamp file, since it is of no use in a multiprocess update. - // not really, tho for the time being -fox - if (!make_stampfile('update_daemon.stamp')) { - _debug("warning: unable to create stampfile\n"); - } - - // Call to the feed batch update function - // and maybe regenerate feedbrowser cache - - $nf = 0; + $quiet = (isset($options["quiet"])) ? "--quiet" : ""; - _debug("Waiting before update [$j].."); - sleep($j*5); - $nf = update_daemon_common(); - - if (rand(0,100) > 50) { - $count = update_feedbrowser_cache(); - _debug("Feedbrowser updated, $count feeds processed."); - - purge_orphans( true); - - $rc = cleanup_tags( 14, 50000); - - _debug("Cleaned $rc cached tags."); - - global $pluginhost; - $pluginhost->run_hooks($pluginhost::HOOK_UPDATE_TASK, "hook_update_task", $op); - } - - _debug("Elapsed time: " . (time() - $start_timestamp) . " second(s)"); + $my_pid = posix_getpid(); - if ($nf > 0) { - _debug("Feeds processed: $nf"); + passthru(PHP_EXECUTABLE . " update.php --daemon-loop $quiet --task $j --pidlock $my_pid"); - if (time() - $start_timestamp > 0) { - _debug("Feeds/minute: " . sprintf("%.2d", $nf/((time()-$start_timestamp)/60))); - } - } - - // We are in a fork. - // We wait a little before exiting to avoid to be faster than our parent process. sleep(1); - unlink(LOCK_DIRECTORY . "/$lock_filename"); - // We exit in order to avoid fork bombing. exit(0); } -- cgit v1.2.3 From 0e6bdaeff04a9749c4a964b042459a8d2b124c9d Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 24 Apr 2013 13:52:09 +0400 Subject: daemon: use define_default --- update_daemon2.php | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index 05ccb1356..193a9c155 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -9,24 +9,22 @@ define('DISABLE_SESSIONS', true); require_once "version.php"; - - if (strpos(VERSION, ".99") !== false || getenv('DAEMON_XDEBUG')) { - define('DAEMON_EXTENDED_DEBUG', true); - } - + require_once "config.php"; require_once "autoload.php"; require_once "functions.php"; + + // defaults + define_default('PURGE_INTERVAL', 3600); // seconds + define_default('MAX_CHILD_RUNTIME', 600); // seconds + define_default('MAX_JOBS', 2); + define_default('SPAWN_INTERVAL', DAEMON_SLEEP_INTERVAL); // seconds + define_default('DAEMON_FEED_LIMIT', 250); + require_once "rssfuncs.php"; require_once "sanity_check.php"; - require_once "config.php"; require_once "db.php"; require_once "db-prefs.php"; - // defaults - define('PURGE_INTERVAL', 3600); // seconds - define('MAX_CHILD_RUNTIME', 600); // seconds - define('MAX_JOBS', 2); - define('SPAWN_INTERVAL', DAEMON_SLEEP_INTERVAL); // seconds if (!function_exists('pcntl_fork')) { die("error: This script requires PHP compiled with PCNTL module.\n"); -- cgit v1.2.3 From 867fb07c39bbabf0a9f9d56fca62ee6a23fbfe7e Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 24 Apr 2013 13:54:04 +0400 Subject: daemon: remove feed limit (unused) --- update_daemon2.php | 1 - 1 file changed, 1 deletion(-) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index 193a9c155..f841954cb 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -18,7 +18,6 @@ define_default('MAX_CHILD_RUNTIME', 600); // seconds define_default('MAX_JOBS', 2); define_default('SPAWN_INTERVAL', DAEMON_SLEEP_INTERVAL); // seconds - define_default('DAEMON_FEED_LIMIT', 250); require_once "rssfuncs.php"; require_once "sanity_check.php"; -- cgit v1.2.3 From f90728cde2d0a7c03f9046cfa94185ed845a03bb Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 24 Apr 2013 14:03:57 +0400 Subject: increase MAX_CHILD_RUNTIME --- update_daemon2.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index 193a9c155..51fe8cfec 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -15,7 +15,7 @@ // defaults define_default('PURGE_INTERVAL', 3600); // seconds - define_default('MAX_CHILD_RUNTIME', 600); // seconds + define_default('MAX_CHILD_RUNTIME', 1800); // seconds define_default('MAX_JOBS', 2); define_default('SPAWN_INTERVAL', DAEMON_SLEEP_INTERVAL); // seconds define_default('DAEMON_FEED_LIMIT', 250); -- cgit v1.2.3 From 8cabc200d5f50327641c6c3ee615095f632afd87 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 24 Apr 2013 14:15:46 +0400 Subject: increase updstart locking interval to prevent concurrent updates; decrease batch size for the daemon --- update_daemon2.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index 30787e6ab..7692eb930 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -12,6 +12,7 @@ require_once "config.php"; require_once "autoload.php"; require_once "functions.php"; + require_once "rssfuncs.php"; // defaults define_default('PURGE_INTERVAL', 3600); // seconds @@ -19,7 +20,6 @@ define_default('MAX_JOBS', 2); define_default('SPAWN_INTERVAL', DAEMON_SLEEP_INTERVAL); // seconds - require_once "rssfuncs.php"; require_once "sanity_check.php"; require_once "db.php"; require_once "db-prefs.php"; -- cgit v1.2.3 From e2cf81e21406f0b507446cf369f7da172ce9da14 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 24 Apr 2013 14:54:59 +0400 Subject: unify houskeeping stuff, increase spawn interval --- update_daemon2.php | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) (limited to 'update_daemon2.php') diff --git a/update_daemon2.php b/update_daemon2.php index 7692eb930..d5b6a45e3 100755 --- a/update_daemon2.php +++ b/update_daemon2.php @@ -170,14 +170,15 @@ "Maybe another daemon is already running.\n"); } - init_plugins(); - $schema_version = get_schema_version(); if ($schema_version != SCHEMA_VERSION) { die("Schema version is wrong, please upgrade the database.\n"); } + // Protip: children close shared database handle when terminating, it's a bad idea to + // do database stuff on main process from now on. + while (true) { // Since sleep is interupted by SIGCHLD, we need another way to @@ -190,17 +191,6 @@ } if ($last_checkpoint + $spawn_interval < time()) { - - /* Check if schema version changed */ - - $test_schema_version = get_schema_version(); - - if ($test_schema_version != $schema_version) { - echo "Expected schema version: $schema_version, got: $test_schema_version\n"; - echo "Schema version changed while we were running, bailing out\n"; - exit(100); - } - check_ctimes(); reap_children(); -- cgit v1.2.3