<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see
* <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlDaemon;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\models\Model;
/**
* CrawlDaemonTest checks how CrawlDaemon builds the command line that
* launches a background daemon, how it reads a lock file, and what a
* daemon does on its first process check. The point guarded is that, on
* systems that launch through a shell, the command closes the file
* descriptors the daemon inherited from its launcher before the program
* starts. A web server launches its background daemons; if a daemon kept
* the server's listening sockets open, those ports would stay in use
* until the daemon exited, and a server restart could not rebind them.
* The construction is a plain string built from its inputs, so it is
* checked directly with no process launched.
*
* @author Chris Pollett
*/
class CrawlDaemonTest extends UnitTest
{
/**
* $lock_path holds the path of a temporary lock file a lock case
* writes, and is empty when no such file is outstanding.
* @var string
*/
public $lock_path = "";
/**
* setUp points CrawlDaemon at SCHEDULES, a folder of this test's own
* in the tests folder, before each case, so the lock, origin and
* message files the cases write are not those of the site's daemons.
*/
public function setUp()
{
L\makePath(self::SCHEDULES);
CrawlDaemon::$schedules_folder = self::SCHEDULES;
}
/**
* tearDown removes any temporary lock file a lock case left behind,
* deletes SCHEDULES and points CrawlDaemon back at the work
* directory's schedules folder.
*/
public function tearDown()
{
if ($this->lock_path !== "" && is_file($this->lock_path)) {
unlink($this->lock_path);
}
(new Model())->db->unlinkRecursive(self::SCHEDULES);
CrawlDaemon::$schedules_folder = C\SCHEDULES_DIR;
}
/**
* SCHEDULES is the folder, under tests/test_files, that the cases have
* CrawlDaemon keep its daemons' files in.
*/
const SCHEDULES = C\TEST_DIR . "/test_files/crawl_daemon_schedules";
/**
* posixJobIsPlainBackgroundLaunchTestCase checks that on a non-Windows
* host the launch command runs the program in the background, taking
* no input and sending output where asked.
* It must not wrap the program in a descriptor-closing shell
* loop: that loop also closed the launching shell's own working
* descriptors and killed the daemon before it could start.
*/
public function posixJobIsPlainBackgroundLaunchTestCase()
{
$job = CrawlDaemon::composeBackgroundJob(
"php /tmp/Daemon.php child", "/tmp/out.log", false);
$this->assertEqual(
"php /tmp/Daemon.php child < /dev/null >> '/tmp/out.log' 2>&1 &",
$job, 'the program runs in the background, reading '.
'nothing and adding what it writes to the given file');
$this->assertTrue(
!str_contains($job, '>&-'),
'the launch does not close inherited descriptors in a loop');
}
/**
* windowsJobUsesPlainStartTestCase checks that on Windows the child is
* started without the launcher's socket handles, so the command is the
* plain background-start line with no descriptor closing.
*/
public function windowsJobUsesPlainStartTestCase()
{
$job = CrawlDaemon::composeBackgroundJob(
"php /tmp/Daemon.php child", "/tmp/out.log", true);
$this->assertEqual("start /B php /tmp/Daemon.php child ",
$job, 'Windows uses a plain background start line');
$this->assertTrue(
!str_contains($job, '>&-'),
'Windows does not close descriptors, none are inherited');
}
/**
* startedProgramsWriteToIndexLogTestCase checks that a start sends a
* program's output to index.log in the log folder, rather than to
* /dev/null, so a program that fails to load says why there.
*/
public function startedProgramsWriteToIndexLogTestCase()
{
$this->assertEqual(CrawlDaemon::startOutput(),
C\LOG_DIR . "/index.log", "a start sends a program's output " .
"to index.log");
}
/**
* indexServerUpWaitsForAnAnswerTestCase checks how a start of the web
* server tells that the server came up, with no file kept for it: it
* connects to the address the server answers on. An address nothing
* answers on is not up after the wait; one a listener answers on is.
* Before, it read IndexServerPort.txt, a file the server wrote at each
* start, and with no such file it never saw the server come up. A
* site that ran that version may still have the file, so the case
* checks that the wait does not write it, not that it is absent.
*/
public function indexServerUpWaitsForAnAnswerTestCase()
{
$ports_file = C\DATA_DIR . "/IndexServerPort.txt";
$ports_before = file_exists($ports_file) ?
filemtime($ports_file) : false;
$listener = stream_socket_server("tcp://127.0.0.1:0");
$address = stream_socket_get_name($listener, false);
$this->assertTrue(
CrawlDaemon::indexServerUp($address, 0.05, 10000),
"an address a listener answers on is up");
fclose($listener);
$this->assertTrue(
!CrawlDaemon::indexServerUp($address, 0.05, 10000),
"an address nothing answers on is not up");
clearstatcache();
$this->assertEqual(file_exists($ports_file) ?
filemtime($ports_file) : false, $ports_before,
"no file of ports is written");
}
/**
* onlyListedDaemonsMayBeInterruptedTestCase checks that only daemons
* listed as safe to cut off may be interrupted with a signal on stop;
* a daemon that writes long-lived index data is never signalled
* mid-job, and the listed one may be signalled only where the
* signal-sending function is present.
*/
public function onlyListedDaemonsMayBeInterruptedTestCase()
{
$this->assertTrue(
!CrawlDaemon::mayInterruptOnStop("QueueServer"),
'a queue server is never interrupted mid-job');
$this->assertTrue(
!CrawlDaemon::mayInterruptOnStop("Fetcher"),
'a fetcher is never interrupted mid-job');
$this->assertEqual(
function_exists("posix_kill"),
CrawlDaemon::mayInterruptOnStop("MediaUpdater"),
'MediaUpdater may be interrupted where signalling exists');
}
/**
* absentLockNotHeldTestCase checks that a lock file that does not
* exist is held by no one, so a fresh start is never blocked by a
* missing lock.
*/
public function absentLockNotHeldTestCase()
{
$this->lock_path = C\TEST_DIR . "/test_files/cd_lock_" .
uniqid("", true) . ".txt";
$this->assertTrue(
!CrawlDaemon::lockHeldByLiveProcess($this->lock_path),
'a lock file that is not there is not held');
}
/**
* livePidLockHeldTestCase checks that a lock recording the id of a
* process that is running is reported as held, so a running server
* still blocks a second start.
*/
public function livePidLockHeldTestCase()
{
$this->lock_path = C\TEST_DIR . "/test_files/cd_lock_" .
uniqid("", true) . ".txt";
file_put_contents($this->lock_path, time() . " " . getmypid());
$this->assertTrue(
CrawlDaemon::lockHeldByLiveProcess($this->lock_path),
'a lock with a live process id is held');
}
/**
* deadPidLockNotHeldTestCase checks that a lock recording the id of a
* process that has ended is reported as not held even when its
* timestamp is recent, so a server that crashed or was stopped from
* the keyboard does not block the next start.
*/
public function deadPidLockNotHeldTestCase()
{
$this->lock_path = C\TEST_DIR . "/test_files/cd_lock_" .
uniqid("", true) . ".txt";
$dead_process_id = 2147483646;
file_put_contents($this->lock_path,
time() . " " . $dead_process_id);
$this->assertTrue(
!CrawlDaemon::lockHeldByLiveProcess($this->lock_path),
'a fresh lock whose process has ended is not held');
}
/**
* windowsProcessIdsReadTasklistRowsTestCase checks how
* windowsProcessIds reads the rows tasklist writes on Windows with
* /FO CSV /NH, which isProcessAlive uses there in place of
* posix_kill. Each row is quoted and comma-separated, with the
* program's name and then its id, and a memory column that holds a
* comma of its own. A row that is not a process, such as the line
* tasklist writes when it lists nothing, gives no id.
*/
public function windowsProcessIdsReadTasklistRowsTestCase()
{
$rows = ['"System Idle Process","0","Services","0","8 K"',
'"php.exe","4312","Console","1","25,140 K"'];
$process_ids = CrawlDaemon::windowsProcessIds($rows);
$this->assertEqual($process_ids, [0 => true, 4312 => true],
"each row's second field is read as a process id");
$this->assertTrue(!isset($process_ids[25]),
"the memory column's comma does not make an id");
$this->assertEqual(CrawlDaemon::windowsProcessIds(
["INFO: No tasks are running which match the " .
"specified criteria."]), null,
"a list with no process rows gives no list");
}
/**
* noProcessIdFreshLockHeldTestCase checks that a lock with no recorded
* process id, as an older release wrote, falls back to its timestamp:
* a recent one is treated as still held.
*/
public function noProcessIdFreshLockHeldTestCase()
{
$this->lock_path = C\TEST_DIR . "/test_files/cd_lock_" .
uniqid("", true) . ".txt";
file_put_contents($this->lock_path, (string)time());
$this->assertTrue(
CrawlDaemon::lockHeldByLiveProcess($this->lock_path),
'a recent lock with no process id is treated as held');
}
/**
* noProcessIdStaleLockNotHeldTestCase checks that a lock with no
* recorded process id and an old timestamp is treated as stale, so the
* next start is not blocked.
*/
public function noProcessIdStaleLockNotHeldTestCase()
{
$this->lock_path = C\TEST_DIR . "/test_files/cd_lock_" .
uniqid("", true) . ".txt";
file_put_contents($this->lock_path, (string)(time() - 100000));
$this->assertTrue(
!CrawlDaemon::lockHeldByLiveProcess($this->lock_path),
'an old lock with no process id is treated as stale');
}
/**
* relayIsAmongTheDaemonsBroughtBackTestCase checks that the turn
* server sits in the list the own web server stops and relaunches on
* a restart, beside the media updater and the mail server, since a
* restart that left it out orphaned the running relay rather than
* bringing it back.
*/
public function relayIsAmongTheDaemonsBroughtBackTestCase()
{
$this->assertTrue(
in_array("TurnServer", CrawlDaemon::RESTART_DAEMONS, true),
'the turn server is brought back on a restart');
$this->assertTrue(
in_array("MediaUpdater", CrawlDaemon::RESTART_DAEMONS, true),
'the media updater is still brought back');
$this->assertTrue(
in_array("MailServer", CrawlDaemon::RESTART_DAEMONS, true),
'the mail server is still brought back');
}
/**
* freshDaemonWritesItsOwnLockTestCase checks that a daemon whose first
* process check finds no lock file writes one and carries on rather
* than stopping. The check runs in this process, as daemon DaemonTest
* with subname Half; a check that stopped the daemon would end the
* test run here.
*/
public function freshDaemonWritesItsOwnLockTestCase()
{
$lock_file = CrawlDaemon::getLockFileName("DaemonTest", "Half");
if (file_exists($lock_file)) {
unlink($lock_file);
}
$daemon = [CrawlDaemon::$name, CrawlDaemon::$subname,
CrawlDaemon::$mode];
CrawlDaemon::$name = "DaemonTest";
CrawlDaemon::$subname = "Half";
CrawlDaemon::$mode = "daemon";
CrawlDaemon::processHandler();
[CrawlDaemon::$name, CrawlDaemon::$subname, CrawlDaemon::$mode] =
$daemon;
$this->assertTrue(true,
"a first check with no lock does not stop the daemon");
$this->assertTrue(file_exists($lock_file),
"and the daemon wrote the lock itself");
if (file_exists($lock_file)) {
unlink($lock_file);
}
$origin = CrawlDaemon::getOriginFileName("DaemonTest", "Half");
if (file_exists($origin)) {
unlink($origin);
}
}
/**
* stopLeavesAYoungerDaemonAloneTestCase checks that a stop does not
* end a daemon started after the stop was asked for. Stopping a
* whole site takes long enough that another part of it can start
* the same daemon again meanwhile, and the daemon that then goes
* missing is the new one, which nobody asked to stop.
*/
public function stopLeavesAYoungerDaemonAloneTestCase()
{
$lock_file = CrawlDaemon::getLockFileName("MailServer", "");
$asked_at = time() - 10;
CrawlDaemon::writeLockFile($lock_file, time());
CrawlDaemon::stop("MailServer", "", false, $asked_at);
$this->assertTrue(file_exists($lock_file),
"a lock written after the stop was asked for is left");
unlink($lock_file);
}
/**
* stopEndsADaemonOlderThanItselfTestCase checks that a stop still
* ends a daemon that was there when the stop was asked for, which
* is every ordinary stop.
*/
public function stopEndsADaemonOlderThanItselfTestCase()
{
$lock_file = CrawlDaemon::getLockFileName("MailServer", "");
CrawlDaemon::writeLockFile($lock_file, time() - 30);
CrawlDaemon::stop("MailServer", "", false, time());
$this->assertFalse(file_exists($lock_file),
"a lock older than the stop is taken away");
@unlink(CrawlDaemon::$schedules_folder . "/MailServer" .
CrawlDaemon::STOP_ASKED_NAME);
}
/**
* daemonStartingIsNotCalledStoppedTestCase checks that a daemon
* just started, whose lock holds a time and no process number yet,
* is called starting rather than stopped. Calling it stopped told a
* reader to go looking for a fault in a service that was coming up
* perfectly well.
*/
public function daemonStartingIsNotCalledStoppedTestCase()
{
$lock_file = CrawlDaemon::getLockFileName("MailServer", "");
CrawlDaemon::writeLockFile($lock_file, time(), false);
$this->assertEqual(CrawlDaemon::daemonState("MailServer"),
CrawlDaemon::DAEMON_STARTING,
"a lock with a fresh time and no number is starting");
unlink($lock_file);
$this->assertEqual(CrawlDaemon::daemonState("MailServer"),
CrawlDaemon::DAEMON_STOPPED,
"no lock at all is stopped");
}
/**
* portHoldersNamesThisProcessTestCase opens a listening socket on a
* free loopback port in this process, then checks that portHolders
* names this process as holding it, and names no one once it is
* closed. portHolders reads /proc, which only Linux has, so elsewhere
* the case checks only that it finds no one.
*/
public function portHoldersNamesThisProcessTestCase()
{
$listener = stream_socket_server("tcp://127.0.0.1:0");
$port = intval(substr(strrchr(stream_socket_get_name($listener,
false), ":"), 1));
$holders = CrawlDaemon::portHolders($port);
if (is_readable("/proc/net/tcp")) {
$this->assertTrue(isset($holders[getmypid()]),
"the process holding the port is named");
} else {
$this->assertEqual($holders, [],
"without /proc no one is named");
}
fclose($listener);
$this->assertEqual(CrawlDaemon::portHolders($port), [],
"a closed port is held by no one");
}
}
- Yioop Repo - Seekquarry