<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\storage_formats\PartitionDocumentBundle;
use seekquarry\yioop\library\UnitTest;
/**
* PartitionDocumentBundleTest checks how PartitionDocumentBundle fills
* its partitions and reads a blob out of a partition file, where the
* length comes from the bundle's own index. A read asks the memory manager for
* the whole length before it reads anything, so a length that is wrong
* asks for that much memory whatever the file holds, and a long running
* server near its memory limit dies of one such read.
*
* @author Chris Pollett
*/
class PartitionDocumentBundleTest extends UnitTest
{
/**
* $folder stores where a case writes the partition file it reads.
* @var string
*/
public $folder = "";
/**
* $bundle stores the bundle a case reads through.
* @var object
*/
public $bundle = null;
/**
* setUp makes a partition file holding a hundred bytes, which is the
* file every case here reads from.
*/
public function setUp()
{
$this->folder = C\TEST_DIR . "/test_files/blobread" . getmypid();
if (!file_exists($this->folder)) {
mkdir($this->folder, 0777, true);
@chmod($this->folder, 0777);
}
$this->bundle = new PartitionDocumentBundle($this->folder,
["PRIMARY KEY" => "KEY", "VALUE" => "BLOB"]);
file_put_contents($this->folder . "/partition0.txt",
str_repeat("small", 20));
@chmod($this->folder . "/partition0.txt", 0777);
}
/**
* tearDown takes away the partition file and the folder a case made.
*/
public function tearDown()
{
if (!file_exists($this->folder)) {
return;
}
$entries = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($this->folder,
\FilesystemIterator::SKIP_DOTS),
\RecursiveIteratorIterator::CHILD_FIRST);
foreach ($entries as $entry) {
if ($entry->isDir()) {
rmdir($entry->getPathname());
} else {
unlink($entry->getPathname());
}
}
rmdir($this->folder);
}
/**
* trueLengthIsReadWholeTestCase checks that a length matching
* what the file holds is read as it always was, so the guard on the
* other lengths does not cost an ordinary read anything.
*/
public function trueLengthIsReadWholeTestCase()
{
$file = $this->folder . "/partition0.txt";
$said = PartitionDocumentBundle::lengthWorthReading($file, 0, 100);
$this->assertEqual(100, $said,
"a length the file can hold is left as it is");
$said = PartitionDocumentBundle::lengthWorthReading($file, 50, 50);
$this->assertEqual(50, $said,
"and so is one that reaches exactly the end");
}
/**
* lengthPastTheFileIsCutToWhatIsThereTestCase checks that a length
* running past the end of the file is cut to what is left, so the
* read asks for that much and no more.
*/
public function lengthPastTheFileIsCutToWhatIsThereTestCase()
{
$file = $this->folder . "/partition0.txt";
$said = PartitionDocumentBundle::lengthWorthReading($file, 0, 400);
$this->assertEqual(100, $said,
"a length past the end is cut to what the file holds");
$said = PartitionDocumentBundle::lengthWorthReading($file, 200, 50);
$this->assertEqual(0, $said,
"and an offset past the end asks for nothing at all");
}
/**
* lengthPastWhatABlobMayBeIsRefusedTestCase checks that a length
* larger than any blob is refused outright. Such a length asks the
* memory manager for that much before the file is touched, which is
* how a server near its limit is ended by one corrupt entry.
*/
public function lengthPastWhatABlobMayBeIsRefusedTestCase()
{
$file = $this->folder . "/partition0.txt";
$said = PartitionDocumentBundle::lengthWorthReading($file, 0,
C\MAX_BLOB_READ_LEN + 1);
$this->assertEqual(0, $said,
"a length past the most a blob may be asks for nothing");
$said = PartitionDocumentBundle::lengthWorthReading($file, 0,
291504352);
$this->assertEqual(0, $said,
"and so does the length that ended the live server");
}
/**
* blobIsReadWithoutAskingForMoreThanTheFileHoldsTestCase reads
* through the bundle itself and checks how much memory the read
* asks for. A read of two hundred and seventy-eight megabytes from
* a file of a hundred bytes used to ask for all of it.
*/
public function blobIsReadWithoutAskingForMoreThanTheFileHoldsTestCase()
{
$file = $this->folder . "/partition0.txt";
$before = memory_get_peak_usage(true);
$said = $this->bundle->getArchive($file, 0, 291504352);
$grew = memory_get_peak_usage(true) - $before;
$this->assertTrue($grew < C\MAX_BLOB_READ_LEN,
"the read asks for less than the most a blob may be, and " .
"asked for " . $grew . " bytes");
$this->assertTrue($said === false || strlen($said) <= 100,
"and hands back nothing more than the file holds");
}
/**
* pageSentAgainIsNotCountedAsNewTestCase checks that the count of new
* keys a put reports counts each key once, whether the repeat comes
* in the same put or in a later one against the same partition. The
* visited-pages tally on the crawl status screen rests on this
* count, so a page the fetchers send twice must add one, not two.
*/
public function pageSentAgainIsNotCountedAsNewTestCase()
{
$folder = C\TEST_DIR . "/test_files/newkeys" . getmypid();
if (!file_exists($folder)) {
mkdir($folder, 0777, true);
}
$bundle = new PartitionDocumentBundle($folder,
["PRIMARY KEY" => ["KEY", 8], "VALUE" => "BLOB"]);
$rows = [];
for ($i = 0; $i < 20; $i++) {
$rows[] = ["KEY" => sprintf("key%05d", $i), "VALUE" => "v$i"];
}
$rows[] = ["KEY" => "key00003", "VALUE" => "again"];
$bundle->put($rows);
$this->assertEqual($bundle->new_keys_put, 20,
"twenty distinct keys count as twenty, a repeat within the " .
"put as none");
$bundle->put([["KEY" => "key00007", "VALUE" => "later"],
["KEY" => "key99999", "VALUE" => "fresh"]]);
$this->assertEqual($bundle->new_keys_put, 1,
"a key already in the partition counts as none in a later put");
$second = new PartitionDocumentBundle($folder,
["PRIMARY KEY" => ["KEY", 8], "VALUE" => "BLOB"]);
$second->put([["KEY" => "key00001", "VALUE" => "x"],
["KEY" => "key99998", "VALUE" => "y"]]);
$this->assertEqual($second->new_keys_put, 1,
"a fresh object rebuilds the set from the partition on disk");
$this->removeFolder($folder);
}
/**
* deferredIndexIsSavedOnceTestCase checks that with
* $defer_index_save set, as the indexer sets it for the slices of one
* fetcher's file, put leaves the partition index on disk as it was and
* keeps adding to the copy in memory, and that saveIndex then writes
* every slice's records, which a fresh bundle reads back. Before, put
* read the index from disk and saved it again for each slice.
*/
public function deferredIndexIsSavedOnceTestCase()
{
$folder = C\TEST_DIR . "/test_files/deferred" . getmypid();
if (!file_exists($folder)) {
mkdir($folder, 0777, true);
}
$format = ["PRIMARY KEY" => ["KEY", 8], "VALUE" => "BLOB"];
$bundle = new PartitionDocumentBundle($folder, $format);
$bundle->put([["KEY" => "key00000", "VALUE" => "first"]]);
$index_file = $bundle->getPartitionIndex(0);
$before = file_get_contents($index_file);
$bundle->defer_index_save = true;
$bundle->put([["KEY" => "key00001", "VALUE" => "second"]]);
$bundle->put([["KEY" => "key00002", "VALUE" => "third"]]);
$this->assertEqual(file_get_contents($index_file), $before,
"the index on disk is unchanged while saving is put off");
$bundle->saveIndex();
$bundle->defer_index_save = false;
$reader = new PartitionDocumentBundle($folder, $format);
$values = [];
foreach (["key00000", "key00001", "key00002"] as $key) {
$row = $reader->get($key, 0, ["VALUE"]);
$values[] = $row["VALUE"] ?? "";
}
$this->assertEqual($values, ["first", "second", "third"],
"every slice's record is in the saved index");
$this->removeFolder($folder);
}
/**
* pageCompressedByFetcherReadsBackAsSentTestCase checks that a page
* a fetcher sends already serialized and gzip compressed reads back
* as the page itself, the same as a page sent plain. A bundle that
* gzip compresses its pages, as an index bundle does, stores the
* bytes as sent, so its partition file grows by exactly their
* length. A bundle that stores pages another way unpacks the page
* first.
*/
public function pageCompressedByFetcherReadsBackAsSentTestCase()
{
$page = "<html><body>" . str_repeat("mary pollett ", 50) .
"</body></html>";
$sent_bytes = gzencode(serialize($page),
C\ARCHIVE_COMPRESSION_LEVEL);
$compressors = ["gzip" => C\NS_COMPRESSORS . "GzipCompressor",
"plain" => C\NS_COMPRESSORS . "NonCompressor"];
foreach ($compressors as $name => $compressor) {
$folder = C\TEST_DIR . "/test_files/precompressed" .
getmypid() . $name;
if (!file_exists($folder)) {
mkdir($folder, 0777, true);
}
$bundle = new PartitionDocumentBundle($folder,
["PRIMARY KEY" => ["KEY", 8], "PAGE" => "SERIAL"],
PartitionDocumentBundle::MAX_ITEMS_PER_FILE,
PartitionDocumentBundle::PARTITION_SIZE_THRESHOLD,
PartitionDocumentBundle::DEFAULT_COMPRESSOR, $compressor);
$bundle->put([["KEY" => "packed01", "PAGE" =>
[CrawlConstants::PRECOMPRESSED_PAGE => $sent_bytes]],
["KEY" => "plain001", "PAGE" => $page]]);
$packed_row = $bundle->get("packed01", 0);
$plain_row = $bundle->get("plain001", 0);
$this->assertEqual($packed_row["PAGE"] ?? null, $page,
"$name bundle: a page compressed by the fetcher reads " .
"back as the page");
$this->assertEqual($plain_row["PAGE"] ?? null, $page,
"$name bundle: a page sent plain reads back as the page");
if ($name == "gzip") {
$archive = $bundle->getPartition(0);
clearstatcache();
$this->assertEqual(substr(file_get_contents($archive), 0,
strlen($sent_bytes)), $sent_bytes,
"gzip bundle: the bytes the fetcher sent are stored " .
"as sent");
}
$this->removeFolder($folder);
}
}
/**
* keyHoldingEscapedBytesIsFoundAgainTestCase checks that a key whose
* bytes are the two the packed index escapes is still recognized
* when it comes again. Those bytes take two bytes each in the index
* string, so reading a key back from an entry has to undo the
* escaping before comparing.
*/
public function keyHoldingEscapedBytesIsFoundAgainTestCase()
{
$folder = C\TEST_DIR . "/test_files/esckeys" . getmypid();
if (!file_exists($folder)) {
mkdir($folder, 0777, true);
}
$bundle = new PartitionDocumentBundle($folder,
["PRIMARY KEY" => ["KEY", 8], "VALUE" => "BLOB"]);
$odd_key = "\xFF\xFE\xFF\xFEabcd";
$bundle->put([["KEY" => $odd_key, "VALUE" => "one"],
["KEY" => "plainkey", "VALUE" => "two"]]);
$this->assertEqual($bundle->new_keys_put, 2,
"two distinct keys, one with escaped bytes, count as two");
$second = new PartitionDocumentBundle($folder,
["PRIMARY KEY" => ["KEY", 8], "VALUE" => "BLOB"]);
$second->put([["KEY" => $odd_key, "VALUE" => "three"]]);
$this->assertEqual($second->new_keys_put, 0,
"the escaped key read back from disk matches the one sent");
$this->removeFolder($folder);
}
/**
* removeFolder takes away a folder a case made and everything in it.
*
* @param string $folder path of the folder to remove
*/
public function removeFolder($folder)
{
$paths = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($folder,
\FilesystemIterator::SKIP_DOTS),
\RecursiveIteratorIterator::CHILD_FIRST);
foreach ($paths as $path) {
if ($path->isDir()) {
rmdir($path->getPathname());
} else {
unlink($path->getPathname());
}
}
rmdir($folder);
}
/**
* partitionHoldsAtMostItsLimitTestCase checks that a partition holds
* at most MAX_ITEMS_PER_FILE items, the limit WordIterator reads a
* partition's postings up to. It puts seven rows into a bundle whose
* limit is three, so the rows fill partitions 0 and 1 with three each
* and leave one in partition 2. When a partition was started only once
* the active one held more than the limit, partition 0 took four rows.
*/
public function partitionHoldsAtMostItsLimitTestCase()
{
$folder = $this->folder . "/limit";
$bundle = new PartitionDocumentBundle($folder,
["PRIMARY KEY" => "KEY", "VALUE" => "BLOB"], 3);
$rows = [];
for ($i = 0; $i < 7; $i++) {
$rows[] = ["KEY" => str_pad("$i", 8, "0", STR_PAD_LEFT),
"VALUE" => "row $i"];
}
$bundle->put($rows);
$this->assertEqual($bundle->parameters["SAVE_PARTITION"], 2,
"seven rows at three a partition end in partition 2");
$this->assertEqual($bundle->parameters["ACTIVE_COUNT"], 1,
"partition 2 holds the seventh row alone");
}
}
- Yioop Repo - Seekquarry