<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
/* runWithTimeLimit's tick function runs only in code compiled with this,
so the slow work in the test below can be stopped. */
declare(ticks=1000);
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\UnitTest;
/**
* UtilityTest checks the parts of Utility.php that stand on their own.
* What is here so far is the counting of log messages that would
* otherwise be written thousands of times a second. A reader of a broken
* index meets the same fault on every posting, and writing an entry for
* each of them made the log the slow part of the machine. What is
* checked here is that many occurrences cost one entry, that the entry
* says how many there were and over what range of values, and that what
* a process still holds is written before it stops.
*
* @author Chris Pollett
*/
class UtilityTest extends UnitTest
{
/**
* COUNTER_LOG is the whole path, without its .log ending, of the log
* the crawlLogCounter cases write, in the tests folder.
*/
const COUNTER_LOG = C\TEST_DIR . "/test_files/counter_test";
/**
* $held is what the logging settings were before a case changed
* them, so tearDown can put them back.
* @var array
*/
public $held = [];
/**
* $log is the file the cases read back what was written from.
* @var string
*/
public $log = "";
/**
* setUp keeps the logging settings a case is about to change, sends
* what is written to a log of its own, and empties that log.
*/
public function setUp()
{
$this->held = ["NO_LOGGING" => $_SERVER["NO_LOGGING"] ?? null,
"LOG_TO_FILES" => $_SERVER["LOG_TO_FILES"] ?? null,
"NO_ROTATE_LOGS" => $_SERVER["NO_ROTATE_LOGS"] ?? null];
unset($_SERVER["NO_LOGGING"]);
$_SERVER["LOG_TO_FILES"] = true;
$_SERVER["NO_ROTATE_LOGS"] = true;
$this->log = self::COUNTER_LOG . ".log";
file_put_contents($this->log, "");
}
/**
* tearDown puts the logging settings back and takes the log away.
*/
public function tearDown()
{
@unlink($this->log);
foreach ($this->held as $name => $value) {
if ($value === null) {
unset($_SERVER[$name]);
} else {
$_SERVER[$name] = $value;
}
}
}
/**
* said gives back everything written to this case's log so far.
*
* @return string what the log holds
*/
public function said()
{
return file_exists($this->log) ? file_get_contents($this->log) : "";
}
/**
* seenSiteUnpacksAsPackedTestCase checks that a site packSeenSite
* packs with its page apart unpacks to the same fields, with the page
* as the gzip bytes the archive stores, and that a site a fetcher not
* yet updated sends, compressed whole, unpacks as before. A site
* without a page is compressed whole.
*/
public function seenSiteUnpacksAsPackedTestCase()
{
$page = "<html><body>" . str_repeat("mary pollett ", 40) .
"</body></html>";
$site = [L\CrawlConstants::URL => "https://www.example.com/",
L\CrawlConstants::TITLE => "Example",
L\CrawlConstants::PAGE => $page];
$packed = L\packSeenSite($site, 6, 1);
$this->assertEqual($packed[0], L\CrawlConstants::PAGE_APART_MARK,
"a site with a page is packed with the page apart");
$unpacked = L\unpackSeenSite($packed);
$sent_page = $unpacked[L\CrawlConstants::PAGE][
L\CrawlConstants::PRECOMPRESSED_PAGE] ?? "";
$this->assertEqual(unserialize(gzdecode($sent_page)), $page,
"the page comes back as gzip bytes of the page");
unset($unpacked[L\CrawlConstants::PAGE], $site[L\CrawlConstants::PAGE]);
$this->assertEqual($unpacked, $site,
"the rest of the site comes back as it was");
$old_packed = gzcompress(serialize($site + [L\CrawlConstants::PAGE =>
$page]), 1);
$this->assertEqual(L\unpackSeenSite($old_packed)[
L\CrawlConstants::PAGE] ?? "", $page,
"a site compressed whole unpacks as before");
$this->assertEqual(L\unpackSeenSite(L\packSeenSite($site, 6, 1)),
$site, "a site without a page is compressed whole");
}
/**
* precompressedPageGivesSameStringTestCase checks that a site whose
* page the fetcher has already gzip encoded, with the bytes under
* PRECOMPRESSED_PAGE, packs to the same string as the site with only
* its page, so the queue server receives what it received before.
*/
public function precompressedPageGivesSameStringTestCase()
{
$page = "<html><body>" . str_repeat("sled dogs of alaska ", 60) .
"</body></html>";
$site = [L\CrawlConstants::URL => "https://www.example.com/dogs",
L\CrawlConstants::TITLE => "Dogs",
L\CrawlConstants::PAGE => $page];
$encoded_site = $site + [L\CrawlConstants::PRECOMPRESSED_PAGE =>
gzencode(serialize($page), 6)];
$this->assertEqual(L\packSeenSite($encoded_site, 6, 1),
L\packSeenSite($site, 6, 1),
"an already encoded page packs to the same string");
}
/**
* manyOccurrencesCostOneEntryTestCase checks the whole point of
* counting a message: a thousand occurrences inside the waiting time
* write nothing at all.
*/
public function manyOccurrencesCostOneEntryTestCase()
{
for ($i = 0; $i < 1000; $i++) {
L\crawlLogCounter("a fault on every posting", self::COUNTER_LOG);
}
$this->assertEqual("", $this->said(),
"a thousand occurrences inside the waiting time write nothing");
}
/**
* waitOfNothingWritesASummaryTestCase checks the other end of the
* waiting time: a message told to wait no seconds writes a summary
* saying how many times it happened since the one before.
*/
public function waitOfNothingWritesASummaryTestCase()
{
L\crawlLogCounter("a fault worth an entry", self::COUNTER_LOG, 0);
L\crawlLogCounter("a fault worth an entry", self::COUNTER_LOG, 0);
$lines = array_filter(explode("\n", trim($this->said())));
$this->assertEqual(2, count($lines),
"each occurrence writes one line and no more");
$this->assertTrue(strpos($lines[1], "[1 times in 0 s]") !== false,
"and a summary counts only what came after the one before");
}
/**
* summaryReadsAsTheMessageDidTestCase checks that the message a
* caller wrote is still what the log says, with the count and the
* stretch added at the end.
*/
public function summaryReadsAsTheMessageDidTestCase()
{
L\crawlLogCounter("Posting decode error", self::COUNTER_LOG, 0);
$this->assertTrue(strpos($this->said(), "Posting decode error") !==
false, "the message a caller wrote is what the log says");
}
/**
* messagesAreCountedApartTestCase checks that two messages counted
* at once do not run into each other, since a reader of a broken
* index meets several faults and each has its own count.
*/
public function messagesAreCountedApartTestCase()
{
L\crawlLogCounter("the first fault", self::COUNTER_LOG, 0);
L\crawlLogCounter("the first fault", self::COUNTER_LOG, 0);
L\crawlLogCounter("the second fault", self::COUNTER_LOG, 0);
$said = $this->said();
$this->assertEqual(2, substr_count($said, "the first fault ["),
"the first message counts its own occurrences");
$this->assertTrue(strpos($said, "the second fault [1 times") !==
false, "and so does the second");
}
/**
* nothingIsWrittenWhenLoggingIsOffTestCase checks that a counted
* message obeys the same setting every other log message does.
*/
public function nothingIsWrittenWhenLoggingIsOffTestCase()
{
$_SERVER["NO_LOGGING"] = true;
L\crawlLogCounter("a fault nobody asked to hear about",
self::COUNTER_LOG, 0);
unset($_SERVER["NO_LOGGING"]);
$this->assertEqual("", $this->said(),
"a run that asked for no logging is given none");
}
/**
* fileIsHandedToTheWebUserTestCase checks that a file written
* before the server drops from root to the user it runs as is
* given to that user. Without it the file belongs to root, and
* every later run as that user finds a file it cannot write: the
* tests run from the screen printed a warning for each attempt.
*/
public function fileIsHandedToTheWebUserTestCase()
{
$file_name = C\TEST_DIR . "/test_files/handed_over_test.txt";
file_put_contents($file_name, "a file");
$this->assertTrue(!L\giveFileToWebUser(""),
"a file that is not there is left alone");
$done = L\giveFileToWebUser($file_name);
$named = !empty(C\WWW_USER);
$rooted = function_exists("posix_getuid") &&
posix_getuid() === 0;
if (!$named || !$rooted) {
$this->assertTrue(!$done, "nothing is handed over where " .
"no user is named or this process may not give it");
} else {
clearstatcache();
$owner = posix_getpwuid(fileowner($file_name));
$this->assertEqual(C\WWW_USER, $owner["name"],
"the file belongs to the user the server runs as");
}
unlink($file_name);
}
/**
* logMadeByRootIsWrittenAgainTestCase checks that a log file made
* while running as root can be appended to by the user the site
* runs as. Without the handing over the append fails and, being
* unchecked, says nothing, so a program started from the machines
* screen wrote no log at all while the same program run with sudo
* wrote one.
*/
public function logMadeByRootIsWrittenAgainTestCase()
{
$file_name = C\TEST_DIR . "/test_files/handover_log_test.log";
file_put_contents($file_name, "made first\n");
L\giveFileToWebUser($file_name);
clearstatcache();
$named = !empty(C\WWW_USER) && function_exists("posix_getuid") &&
posix_getuid() === 0;
if ($named) {
$owner = posix_getpwuid(fileowner($file_name));
$this->assertEqual(C\WWW_USER, $owner["name"],
"the log belongs to the user the site runs as");
}
$added = file_put_contents($file_name, "and again\n",
FILE_APPEND);
$this->assertTrue($added > 0,
"a later write to the log lands rather than failing");
unlink($file_name);
}
/**
* workPastItsLimitIsStoppedTestCase checks runWithTimeLimit. Work that
* finishes inside its limit gives back true and its result. A loop in
* a file that declares ticks, running past a limit of a twentieth of
* a second, is stopped near that limit and gives back false, which is
* how the indexer skips a document that takes too long.
*/
public function workPastItsLimitIsStoppedTestCase()
{
$quick = L\runWithTimeLimit(0.05, fn() => 42);
$this->assertEqual($quick, [true, 42], "work inside its limit " .
"gives back its result");
$start = microtime(true);
$slow = L\runWithTimeLimit(0.05, function () {
$count = 0;
$end = microtime(true) + 5;
while (microtime(true) < $end) {
$count++;
}
return $count;
});
$took = microtime(true) - $start;
$this->assertEqual($slow, [false, null], "work past its limit " .
"is stopped");
$this->assertTrue($took >= 0.05 && $took < 0.5,
"work past its limit stops near " .
"the limit");
}
/**
* emptySizeIsZeroTestCase checks that metricToInt reads an empty
* setting as 0 without a warning. A profile whose
* GIT_MAX_PUSH_SIZE, a setting since removed, was saved empty made
* metricToInt read the last
* letter of an empty string. The warning it printed was output, so
* index.php could no longer set session.gc_maxlifetime. Sizes with a
* letter, and with spaces around them, still convert in powers of
* 1000.
*/
public function emptySizeIsZeroTestCase()
{
$warnings = [];
set_error_handler(function ($number, $message) use (&$warnings) {
$warnings[] = $message;
return true;
});
$empty = L\metricToInt("");
restore_error_handler();
$this->assertEqual([$empty, $warnings], [0, []],
"an empty size is 0 with no warning");
$this->assertEqual([L\metricToInt("8M"), L\metricToInt(" 512K "),
L\metricToInt("2G"), L\metricToInt("100")],
[8000000, 512000, 2000000000, 100],
"sizes with a letter convert in powers of 1000");
}
/**
* metricCountsRoundToTheirLetterTestCase checks the short form
* intToMetric gives the counts on badges: a number under 1000 as it
* is, then thousands, millions and billions rounded to the nearest
* one with K, M or G, and a count that rounds up to 1000 of one letter
* written with the next letter, as 1M rather than 1000K.
*/
public function metricCountsRoundToTheirLetterTestCase()
{
$this->assertEqual(array_map("seekquarry\\yioop\\library\\intToMetric",
[0, 999, 1000, 1499, 1500, 999499, 999500, 2400000,
999999999]), ["0", "999", "1K", "1K", "2K", "999K", "1M", "2M",
"1G"], "each count with its letter");
$this->assertEqual(L\intToMetric("12"), "12",
"a value that is not a whole number is kept");
}
/**
* unlinkRecursiveTakesLinksTestCase checks that deleting a folder
* removes a symbolic link in it as a link. A link whose file is gone
* was left behind before, since deleteFileOrDir asked file_exists,
* which follows the link, so the folder stayed. A link to a folder
* outside was followed, and that folder's files were deleted. Where
* the system makes no symbolic links the case has nothing to check.
*/
public function unlinkRecursiveTakesLinksTestCase()
{
$folder = C\TEST_DIR . "/test_files/unlink_links_test";
$outside = C\TEST_DIR . "/test_files/unlink_links_outside";
$manager = new \seekquarry\yioop\models\datasources\Sqlite3Manager();
$manager->unlinkRecursive($folder);
$manager->unlinkRecursive($outside);
L\makePath($folder);
L\makePath($outside);
file_put_contents("$outside/kept.txt", "kept");
if (!@symlink("$folder/gone.css", "$folder/dangling.css") ||
!@symlink($outside, "$folder/elsewhere")) {
$manager->unlinkRecursive($folder);
$manager->unlinkRecursive($outside);
return;
}
$manager->unlinkRecursive($folder);
$this->assertFalse(file_exists($folder) || is_link($folder),
"the folder with its links is gone");
$this->assertTrue(file_exists("$outside/kept.txt"),
"a folder a link points to keeps its files");
$manager->unlinkRecursive($folder);
$manager->unlinkRecursive($outside);
}
}
- Yioop Repo - Seekquarry