<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* This file contains unit tests of the PackedTableTools class
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\models\Model;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\library\data_structures as LDS;
/**
* Used to test the PackedTableTools class. PackedTableTools are used for
* reading and storing rows with respect to some signature
*/
class PackedTableToolsTest extends UnitTest
{
/**
* Prefix of folders for testing PackedTable's
*/
const TEST_DIR = C\TEST_DIR . '/test_files/packed_test';
/**
* Creates test directory for the tests
*/
public function setUp()
{
if (!file_exists(self::TEST_DIR)) {
mkdir(self::TEST_DIR);
}
}
/**
* After the tests delete test directory
*/
public function tearDown()
{
$model = new Model();
$model->db->unlinkRecursive(self::TEST_DIR);
}
/**
* Auxiliary function used during test that takes a PackedTableTools
* object instantiated according to some row signature and an array
* rows in that signature and checks that for all rows packing the row
* followed by unpacking it returns the original row. IF a row
* fails a testCase message is output
*
* @param PackedTableTools $table_factory used for pack and unpacking rows
* @param array $rows rows according to $table_factory's signature
*/
public function checkPackUnpackTableRows($table_factory, $rows)
{
foreach($rows as $row) {
$packed_row = $table_factory->pack($row);
$pack_unpack_row = $table_factory->unpack($packed_row);
$id = $row['ID'];
unset($row['ID']);
$this->assertEqual($row, $pack_unpack_row[0],
"Row $id pack unpack produces same.");
}
}
/**
* This tests packing and unpacking rows according to a PackedTableTools
* signature in the case that there is only one record associated with the
* primary key (no-clustering).
*/
public function packUnpackSingleRowTestCase()
{
$table_factory = new LDS\PackedTableTools(
["PRIMARY KEY" => "ID", "A" => "INT", "B" => "TEXT",
"C" => "INT"]);
$rows = [
["ID" => 1, "A" => 5, "B" => "Hello World", "C"=> 256],
["ID" => 2, "A" => 65000, "B" => "yo", "C"=> 2000000]
];
$this->checkPackUnpackTableRows($table_factory, $rows);
$table_factory = new LDS\PackedTableTools(
["PRIMARY KEY" => "ID", "A" => "DOUBLE", "B" => "TEXT",
"C" => "TEXT", "D" => "BOOL", "E" => "INT"]);
$rows = [
["ID" => 1, "A" => 1.59, "B" => "Bong", "C" => "yockledoo",
"D" => true, "E" => 1000],
["ID" => 2, "A" => 100.4, "B" => "Blah", "C" => "doodoo",
"D" => false, "E" => 9990],
];
$this->checkPackUnpackTableRows($table_factory, $rows);
}
/**
* This tests packing and unpacking rows according to a PackedTableTools
* signature in the case that where more than one record can be associated
* with the primary key (clustering).
*/
public function packUnpackMultiRowTestCase()
{
$table_factory = new LDS\PackedTableTools(
["PRIMARY KEY" => "ID", "A" => "INT", "B" => "TEXT",
"C" => "INT"]);
$rows = [
["A" => 5, "B" => "Hello World", "C"=> 256],
["A" => 20000, "B" => "laladida", "C"=> 5600]
];
$packed_rows = $table_factory->pack($rows);
$pack_unpack_rows = $table_factory->unpack($packed_rows);
$this->assertEqual($rows, $pack_unpack_rows,
"Two row pack unpack produces same.");
}
/**
* This tests if a set of rows packed according to a PackedTableTools record
* is saved that it can be reloaded and unpacked and still get the same
* results
*/
public function saveAddLoadTestCase()
{
$table_factory = new LDS\PackedTableTools(
["PRIMARY KEY" => "ID", "A" => "INT", "B" => "TEXT",
"C" => "INT"]);
$table_factory2 = new LDS\PackedTableTools(
["PRIMARY KEY" => "ID", "A" => "INT", "B" => "TEXT",
"C" => "INT"], C\NS_COMPRESSORS . "GzipCompressor");
$table = [];
$hash_key1 = md5("1", true);
$table_factory->add($table, $hash_key1,
$table_factory->pack(["A" => 5, "B" => "Hello World", "C"=> 256]),
mode: LDS\PackedTableTools::APPEND_MODE);
$table_factory->add($table, $hash_key1,
$table_factory->pack(["A" => 6, "B" => "Hello World2", "C"=> 257]),
mode: LDS\PackedTableTools::APPEND_MODE);
$hash_key2 = md5("2", true);
$table_factory->add($table, $hash_key2,
$table_factory->pack(["A" => 20000, "B" => "laladida",
"C"=> 5600]));
$table_factory->save(self::TEST_DIR . "/save.txt", $table);
$loaded_table = $table_factory->load(self::TEST_DIR . "/save.txt");
$this->assertEqual($table, $loaded_table,
"Add two rows, save, load table gives same result.");
$table_factory2->save(self::TEST_DIR . "/save2.txt", $table);
$loaded_table2 = $table_factory2->load(self::TEST_DIR . "/save2.txt");
$this->assertEqual($table, $loaded_table2,
"Add two rows, save, load compressed table gives same result.");
$entry1 = $loaded_table[$hash_key1];
$start = 0;
$num_entries = L\vByteDecode($entry1, $start);
$this->assertEqual($num_entries, 2,
"Row one has correct number of sub-entries.");
}
/**
* Tests that the in-memory table cache stays within its byte budget,
* drops the tables used longest ago first, spares a table that was just
* used, and drops a table's entry markers in step with the table. This
* guards the fix for the slow web-server memory growth that came from
* the cache holding every partition it was ever asked for.
*/
public function tableCacheBoundTestCase()
{
$signature = ["PRIMARY KEY" => "ID", "A" => "INT", "B" => "TEXT"];
$tools = new LDS\PackedTableTools($signature);
$paths = [];
for ($i = 1; $i <= 4; $i++) {
$one_table = [];
$tools->add($one_table, md5((string)$i, true),
$tools->pack(["A" => $i, "B" => "row for cache table $i"]));
$paths[$i] = self::TEST_DIR . "/cache_$i.txt";
$tools->save($paths[$i], $one_table);
}
$sample = $tools->load($paths[1],
LDS\PackedTableTools::AS_STRING_MODE, true);
$entry_size = strlen($sample);
$tools = new LDS\PackedTableTools($signature);
$tools->max_table_cache_bytes = 2 * $entry_size + 1;
foreach ([1, 2, 3] as $number) {
$tools->load($paths[$number],
LDS\PackedTableTools::AS_STRING_MODE, true);
}
$this->assertTrue(
$tools->table_cache_bytes <= $tools->max_table_cache_bytes,
"Cache bytes stay within the budget after loading three tables.");
$this->assertEqual(count($tools->table_cache), 2,
"Only the two most recent tables stay cached.");
$this->assertTrue(!isset($tools->table_cache[L\crawlHash($paths[1])]),
"The least recently used table was evicted.");
$tools->load($paths[2],
LDS\PackedTableTools::AS_STRING_MODE, true);
$tools->load($paths[4],
LDS\PackedTableTools::AS_STRING_MODE, true);
$this->assertTrue(!isset($tools->table_cache[L\crawlHash($paths[3])]),
"Using a table spares it and the next oldest goes instead.");
$this->assertTrue(isset($tools->table_cache[L\crawlHash($paths[2])]),
"The recently used table is kept.");
$tight = new LDS\PackedTableTools($signature);
$tight->max_table_cache_bytes = $entry_size + 1;
$tight->getEntryMarkers($paths[1]);
$this->assertTrue(
isset($tight->table_entry_markers[L\crawlHash($paths[1])]),
"Markers are cached when a table is read.");
$tight->getEntryMarkers($paths[2]);
$this->assertTrue(
!isset($tight->table_entry_markers[L\crawlHash($paths[1])]),
"Evicting a table drops its markers too.");
}
/**
* emptyTableIsSavedTestCase checks that save writes an empty table as
* an empty file. A partition stopped before any document was saved has
* empty tables, and its build is whole only once all four of its files
* exist; when save left them out, a stopped crawl waited for them.
*/
public function emptyTableIsSavedTestCase()
{
$tools = new LDS\PackedTableTools(["PRIMARY KEY" => ["TERM", 8],
"COUNT" => "INT"]);
$path = C\TEST_DIR . "/test_files/empty_table_test";
@unlink($path);
$this->assertTrue($tools->save($path, []), "saving an empty table " .
"succeeds");
$this->assertTrue(file_exists($path), "and leaves its file");
@unlink($path);
}
/**
* firstRowWithEscapedKeyByteIsFoundTestCase checks that
* findRowFromKeyTableString finds a table's first row when its key holds
* a \xFE byte. save writes such a byte as two bytes, and the lookup
* compared the first row with the key as given, so the row was missed.
* A partition's index of documents is read this way, and a document id
* is a hash that may hold such a byte.
*/
public function firstRowWithEscapedKeyByteIsFoundTestCase()
{
$tools = new LDS\PackedTableTools(["PRIMARY KEY" => ["KEY", 16],
"VALUE" => "INT"]);
$path = self::TEST_DIR . "/escaped_first_key.txt";
$first_key = "\xFE" . str_repeat("a", 15);
$tools->save($path, [$first_key => $tools->pack(["VALUE" => 7]),
str_repeat("b", 16) => $tools->pack(["VALUE" => 8])]);
$table_string = $tools->load($path,
LDS\PackedTableTools::AS_STRING_MODE);
$row = $tools->findRowFromKeyTableString($table_string, $first_key);
$this->assertEqual($tools->unpack($row ?? "")[0]["VALUE"] ?? null, 7,
"The first row is found though its key holds a \\xFE byte");
}
/**
* rowWithKeyOfAnyLengthIsFoundTestCase checks that
* findRowFromKeyTableString finds rows in a table whose keys vary in
* length. Such a row holds its key's length before the key, and the
* lookup searched for the key without it, so it found no row.
*/
public function rowWithKeyOfAnyLengthIsFoundTestCase()
{
$tools = new LDS\PackedTableTools(["PRIMARY KEY" => ["KEY", -1],
"VALUE" => "INT"]);
$path = self::TEST_DIR . "/varied_keys.txt";
$keys = ["apple" => 1, "b" => 2, "cherry_pie" => 3];
$table = [];
foreach ($keys as $key => $value) {
$table[$key] = $tools->pack(["VALUE" => $value]);
}
$tools->save($path, $table);
$table_string = $tools->load($path,
LDS\PackedTableTools::AS_STRING_MODE);
$found = [];
foreach ($keys as $key => $value) {
$row = $tools->findRowFromKeyTableString($table_string, $key);
$found[$key] = $tools->unpack($row ?? "")[0]["VALUE"] ?? null;
}
$this->assertEqual($found, $keys, "Every key is found with its value");
}
/**
* rowIsAppendedToFileAtPathTestCase checks that add with ADD_FILE_PATH
* writes rows to the file at the path given, so the file loads back as
* the table. add called fsize, which PHP does not have, so the first
* such call stopped with an error.
*/
public function rowIsAppendedToFileAtPathTestCase()
{
$tools = new LDS\PackedTableTools(["PRIMARY KEY" => ["KEY", 16],
"VALUE" => "INT"]);
$path = self::TEST_DIR . "/added_by_path.txt";
$table = [str_repeat("a", 16) => $tools->pack(["VALUE" => 1]),
str_repeat("b", 16) => $tools->pack(["VALUE" => 2])];
try {
foreach ($table as $key => $row) {
$tools->add($path, $key, $row,
LDS\PackedTableTools::ADD_FILE_PATH);
}
$loaded = $tools->load($path);
} catch (\Error $e) {
$loaded = $e->getMessage();
}
$this->assertEqual($loaded, $table, "The two rows load back");
}
/**
* emptyTableLoadsWithNoRowsTestCase checks that a table saved with no
* rows loads as an empty array. load split the empty string into one
* row whose key and records were empty, so a partition with no
* documents gave back a row for an empty key.
*/
public function emptyTableLoadsWithNoRowsTestCase()
{
$tools = new LDS\PackedTableTools(["PRIMARY KEY" => ["KEY", 16],
"VALUE" => "INT"]);
$path = self::TEST_DIR . "/no_rows.txt";
$tools->save($path, []);
$this->assertEqual($tools->load($path), [], "No rows are loaded");
}
/**
* negativeIntKeepsItsSignTestCase checks that an INT column below -127,
* which pack writes in two or more bytes, unpacks as the same negative
* number. pack left out the sign bit for such numbers, so they came back
* positive.
*/
public function negativeIntKeepsItsSignTestCase()
{
$tools = new LDS\PackedTableTools(["PRIMARY KEY" => ["KEY", 16],
"SMALL" => "INT", "MIDDLE" => "INT", "LARGE" => "INT"]);
$record = ["SMALL" => -5, "MIDDLE" => -200, "LARGE" => -70000];
$this->assertEqual($tools->unpack($tools->pack($record))[0], $record,
"Each negative number keeps its sign");
}
/**
* eightByteIntBesideOneByteIntsTestCase checks a record of three INT
* columns whose second needs eight bytes and whose others need one.
* Their size codes then make the byte 0x30, the character "0", which
* PHP's empty takes as empty, and unpack read every column as 0.
*/
public function eightByteIntBesideOneByteIntsTestCase()
{
$tools = new LDS\PackedTableTools(["PRIMARY KEY" => ["KEY", 16],
"FIRST" => "INT", "SECOND" => "INT", "THIRD" => "INT"]);
$record = ["FIRST" => 1, "SECOND" => 5000000000, "THIRD" => 2];
$this->assertEqual($tools->unpack($tools->pack($record))[0], $record,
"Each column unpacks as packed");
}
/**
* rowPastTheLastIsNotFoundTestCase checks that findEntryAtIndexTableName
* gives the empty string for a row number past a table's last row. For
* such a number it gave the whole table as if it were one row, so a
* document map read with a bad number gave a page other than the one
* asked for.
*/
public function rowPastTheLastIsNotFoundTestCase()
{
$tools = new LDS\PackedTableTools(["PRIMARY KEY" => ["KEY", 16],
"VALUE" => "INT"]);
$path = self::TEST_DIR . "/three_rows.txt";
$table = [];
foreach (["a", "b", "c"] as $i => $letter) {
$table[str_repeat($letter, 16)] = $tools->pack(["VALUE" => $i]);
}
$tools->save($path, $table);
$this->assertEqual(substr($tools->findEntryAtIndexTableName($path, 2),
0, 16), str_repeat("c", 16), "The last row is found");
$this->assertEqual($tools->findEntryAtIndexTableName($path, 3), "",
"Row 3 of rows 0 to 2 is not found");
}
}
- Yioop Repo - Seekquarry