<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\library as L;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library\UnitTest;
/**
* Used to test the various methods in utility, in particular, those
* related to posting lists and time.
*
* @author Chris Pollett
*/
class UtilityTest extends UnitTest
{
/**
* No set up being done for the time being
*/
public function setUp()
{
}
/**
* No tear down being done for the time being
*/
public function tearDown()
{
}
/**
* Determines if the checkTimeInterval method can correctly determine
* if a time of day is between the times of day of two timestamps
*/
public function checkTimeIntervalTestCase()
{
$three_oh_five = 1592172350;
$one_hour = 3600;
$this->assertEqual(-1, L\checkTimeInterval("14:00", -1, $three_oh_five),
"(a) no sleep duration (-1) does not contain 3:05pm");
$this->assertEqual(-1, L\checkTimeInterval("16:00", -1, $three_oh_five),
"(b) no sleep duration (-1) does not contain 3:05pm");
$this->assertEqual(-1, L\checkTimeInterval("14:00", $one_hour,
$three_oh_five), "2pm +1hr does not contain 3:05pm");
$this->assertEqual(1592175600, L\checkTimeInterval("14:00",
2 * $one_hour, $three_oh_five),
"2pm +2hr interval contains 3:05pm and ends at 4pm");
}
/**
* Checks passwordPolicyViolations against explicit policies rather
* than the ambient config, so the result does not depend on the
* machine's profile: a length-only policy accepts an eight character
* password and flags short, over-long, and forbidden-character ones,
* and each require-a-class policy flags a password lacking that class
* and accepts one that has it.
*/
public function passwordPolicyViolationsTestCase()
{
$length_only = ["min_length" => 8, "max_length" => 64,
"lowercase" => false, "uppercase" => false,
"digit" => false, "symbol" => false, "forbidden" => "'\""];
$this->assertTrue(empty(L\passwordPolicyViolations("abcdefgh",
$length_only)), "eight chars pass a length-only policy");
$this->assertTrue(in_array("too_short",
L\passwordPolicyViolations("abc", $length_only)),
"a short password is flagged too short");
$this->assertTrue(in_array("too_long",
L\passwordPolicyViolations(str_repeat("a", 65),
$length_only)), "an over-long password is flagged too long");
$this->assertTrue(in_array("forbidden",
L\passwordPolicyViolations("abcdefg'h", $length_only)),
"a single quote is a forbidden character");
$this->assertTrue(in_array("forbidden",
L\passwordPolicyViolations('abcdefg"h', $length_only)),
"a double quote is a forbidden character");
$require_upper = array_merge($length_only,
["uppercase" => true]);
$this->assertTrue(in_array("uppercase",
L\passwordPolicyViolations("abcdefgh", $require_upper)),
"requiring uppercase flags an all-lowercase password");
$this->assertTrue(empty(L\passwordPolicyViolations("Abcdefgh",
$require_upper)), "a password with uppercase then passes");
$require_digit = array_merge($length_only, ["digit" => true]);
$this->assertTrue(in_array("digit",
L\passwordPolicyViolations("abcdefgh", $require_digit)),
"requiring a digit flags a letters-only password");
$require_symbol = array_merge($length_only, ["symbol" => true]);
$this->assertTrue(in_array("symbol",
L\passwordPolicyViolations("abcdefgh", $require_symbol)),
"requiring a symbol flags an alphanumeric password");
$require_lower = array_merge($length_only,
["lowercase" => true]);
$this->assertTrue(in_array("lowercase",
L\passwordPolicyViolations("ABCDEFGH", $require_lower)),
"requiring lowercase flags an all-uppercase password");
}
/**
* Used to check Encoding decoding using unary coding
*/
public function unaryCodeTestCase()
{
$start = 0;
$current_string = "";
for($i = 1; $i <= 20; $i++) {
$current_string = L\appendUnary($i, $current_string, $start);
}
for($j = 20; $j >= 1; $j--) {
$current_string = L\appendUnary($j, $current_string, $start);
}
$start = 0;
for($i = 1; $i <= 20; $i++) {
$decoded = L\decodeUnary($current_string, $start);
$this->assertEqual($i, $decoded, "(a) Decode Encode $i");
}
for($j = 20; $j >= 1; $j--) {
$decoded = L\decodeUnary($current_string, $start);
$this->assertEqual($j, $decoded, "(b) Decode Encode $j");
}
$start = 0;
for($i = 0; $i <= 15; $i++) {
$decoded = L\decodeUnary("\xFF\xFF", $start);
$this->assertEqual(1, $decoded, "$i th encoded 1 decodes to 1");
}
}
/**
* Used to check Encoding decoding using unary coding
*/
public function encodeDecodeBitsCodeTestCase()
{
$to_encodes = [1, 257, 4, 9, 65535, 93];
$bit_lens = [1, 9, 3, 4, 16, 7];
$start = 0;
$encoded = "";
foreach ($to_encodes as $to_encode) {
$encoded = L\appendBits($to_encode, $encoded, $start);
}
$i = 0;
$start = 0;
foreach ($bit_lens as $bit_len) {
$decode = L\decodeBits($encoded, $start, $bit_len);
$this->assertEqual($to_encodes[$i], $decode, "Encode ".
$to_encodes[$i] ." decodes as $decode");
$i++;
}
}
/**
* Used to check Encoding decoding gamma codes
*/
public function encodeDecodeGammaTestCase()
{
$to_encodes = [1, 257, 4, 9, 65535, 93];
$start = 0;
$encoded = "";
foreach ($to_encodes as $to_encode) {
$encoded = L\appendGamma($to_encode, $encoded, $start);
}
$start = 0;
$num_encoded = count($to_encodes);
$decodes = L\decodeGammaList($encoded, $start, $num_encoded);
for ($i = 0; $i < $num_encoded; $i++) {
$this->assertEqual($to_encodes[$i], $decodes[$i], "Encode ".
"{$to_encodes[$i]} decodes as {$decodes[$i]}");
}
}
/**
* Check that encoding and decoding integers using the vByte scheme works
*/
public function encodeDecodeVByteTestCase()
{
for ($i = 0; $i < 1000000; $i += 500) {
$enc = L\vByteEncode($i);
$start = 0;
$decode = L\vByteDecode($enc, $start);
$this->assertEqual($i, $decode,
"Encoding and decoding $i give $i");
}
}
/**
* The encode255 / decode255 pair must round trip exactly and, just
* as importantly, the encoded form must never contain a 0xFF byte.
* The postings file uses 0xFF as its record separator, so a single
* stray 0xFF inside an encoded record would split it in the wrong
* place and hand the index reader a corrupt offset and length, the
* fault behind the posting-decode errors seen in production. This
* walks inputs built from the bytes the codec rewrites, plus a
* string of every byte value, and checks both properties.
*/
public function encodeDecode255RoundTripTestCase()
{
$samples = ["", "\xFE", "\xFF", "\xFE\xFE", "\xFE\xFD",
"\xFF\xFF", "a\xFFb", "\xFE\xFF\xFE", "plain text",
"\x00\x01\xFE\xFF\xFD\x80"];
$all_bytes = "";
for ($i = 0; $i < 256; $i++) {
$all_bytes .= chr($i);
}
$samples[] = $all_bytes . $all_bytes;
foreach ($samples as $sample) {
$encoded = L\encode255($sample);
$this->assertTrue(strpos($encoded, "\xFF") === false,
"the encoded form holds no 0xFF separator byte");
$this->assertEqual($sample, L\decode255($encoded),
"encode255 then decode255 returns the original bytes");
}
}
/**
* Used to check Encoding decoding using unary coding
*/
public function encodeDecodeRiceTestCase()
{
$position_list = [90, 101, 570, 581, 737, 950, 1100, 1119, 1127,
1147, 1175, 1185, 1930, 1969, 2020, 2040, 2068, 2083, 2090, 2102,
2126, 2170, 2182, 2191, 2217, 2228, 2250, 2260, 2370, 2392, 2403,
2447, 2456, 2467, 2476, 2486, 2503, 2508, 2610, 2628, 2629, 2641,
2674, 2693, 2710, 2753, 2761, 2770, 2847, 2885, 2899, 2920, 2934,
3000, 3019, 3039, 3058, 3070, 3133, 3168, 3227, 3240, 3249, 3266,
3277, 3296, 3309, 3327, 3348, 3366, 3368, 3375, 3424, 3456, 3458,
3463, 3478, 3487, 3511, 3513, 3523, 3557, 3614, 3828, 3880, 3896,
3910, 3999, 4039, 4056, 4165, 4226, 4248, 4269, 4308, 4324, 4338,
4444, 4484, 4560, 4577, 4597, 4622, 4695, 4710, 4801, 4824, 4859,
4876, 4981, 5071, 5109, 5131, 5199, 5232, 5270, 5287, 5317, 5330,
5373, 5409, 5426, 5490, 5500, 5501, 5533, 5544, 5722, 5765, 5799,
5821, 5854, 5938, 5967, 6004, 6036, 6195, 6262, 6319, 6337, 6345,
6346, 6391, 6430, 6452, 6460, 6514, 6580, 6736, 6758, 6794, 6820,
6976];
$num_positions = count($position_list);
$average_gap = ($position_list[$num_positions - 1] -
$position_list[0])/$num_positions;
$modulus = max(ceil(log($average_gap + 1, 2)), 2);
$start = 0;
$encoded = L\appendRiceSequence($position_list, $modulus, "",
$start, 0);
$start = 0;
$decodes = L\decodeRiceSequence($encoded, $start, $num_positions, 0);
for ($i = 0; $i < $num_positions; $i++) {
$this->assertEqual($position_list[$i], $decodes[$i], "Encode ".
"{$position_list[$i]} decodes as {$decodes[$i]}");
}
}
/**
* Used to check Encoding decoding using unary coding
*/
public function encodeDecodePositionListTestCase()
{
$position_list = [90, 101, 570, 581, 737, 950, 1100, 1119, 1127,
1147, 1175, 1185, 1930, 1969, 2020, 2040, 2068, 2083, 2090, 2102,
2126, 2170, 2182, 2191, 2217, 2228, 2250, 2260, 2370, 2392, 2403,
2447, 2456, 2467, 2476, 2486, 2503, 2508, 2610, 2628, 2629, 2641,
2674, 2693, 2710, 2753, 2761, 2770, 2847, 2885, 2899, 2920, 2934,
3000, 3019, 3039, 3058, 3070, 3133, 3168, 3227, 3240, 3249, 3266,
3277, 3296, 3309, 3327, 3348, 3366, 3368, 3375, 3424, 3456, 3458,
3463, 3478, 3487, 3511, 3513, 3523, 3557, 3614, 3828, 3880, 3896,
3910, 3999, 4039, 4056, 4165, 4226, 4248, 4269, 4308, 4324, 4338,
4444, 4484, 4560, 4577, 4597, 4622, 4695, 4710, 4801, 4824, 4859,
4876, 4981, 5071, 5109, 5131, 5199, 5232, 5270, 5287, 5317, 5330,
5373, 5409, 5426, 5490, 5500, 5501, 5533, 5544, 5722, 5765, 5799,
5821, 5854, 5938, 5967, 6004, 6036, 6195, 6262, 6319, 6337, 6345,
6346, 6391, 6430, 6452, 6460, 6514, 6580, 6736, 6758, 6794, 6820,
6976];
$num_positions = count($position_list);
$encoded = L\encodePositionList($position_list);
$decodes = L\decodePositionList($encoded, $num_positions);
for ($i = 0; $i < $num_positions; $i++) {
$this->assertEqual($position_list[$i], $decodes[$i], "Encode ".
"{$position_list[$i]} decodes as {$decodes[$i]}");
}
}
/**
* Checks that short position lists, the ones with fewer than three
* entries that take the plain gamma path rather than the Rice path,
* survive a round trip. The larger position-list test only exercises
* the Rice path, so without this a break in the short path would go
* unnoticed.
*/
public function encodeDecodeSmallPositionListTestCase()
{
$small_lists = [[5], [5, 9], [1, 2], [90, 101]];
foreach ($small_lists as $position_list) {
$encoded = L\encodePositionList($position_list);
$decodes = L\decodePositionList($encoded, count($position_list));
$this->assertEqual($position_list, $decodes,
"Small list " . json_encode($position_list) .
" decodes as " . json_encode($decodes));
}
}
/**
* Checks that decoding a Rice sequence does not break when asked for
* more numbers than the input actually holds. Reading past the end of
* the input used to apply a decrement to a boolean, which PHP warns
* about; this confirms the real numbers at the front still decode and
* the call simply returns the count requested without that warning.
*/
public function decodeRiceSequenceShortInputTestCase()
{
$positions = [3, 10, 17, 18];
$encoded = L\encodePositionList($positions);
$start_bit_offset = 0;
$first = L\decodeGammaList($encoded, $start_bit_offset, 1)[0];
$warned = false;
set_error_handler(function ($number, $text) use (&$warned) {
if (strpos($text, "Decrement on type bool") !== false) {
$warned = true;
}
return true;
});
$over_count = count($positions) + 8;
$decodes = L\decodeRiceSequence($encoded, $start_bit_offset,
$over_count, $first);
restore_error_handler();
$this->assertEqual([10, 17, 18], array_slice($decodes, 0, 3),
"the real positions at the front decode correctly");
$this->assertEqual($over_count, count($decodes),
"the decoder returns the number of values requested");
$this->assertTrue(!$warned,
"no decrement-on-boolean warning past the end of input");
}
/**
* Used to check Encoding decoding using Modified9 coding
*/
public function modified9TestCase()
{
$encode_list = [151466751, 11746, 11746];
$encoded = L\encodeModified9($encode_list);
$offset = 0;
$decode_list = L\decodeModified9($encoded, $offset);
$this->assertEqual($encode_list, $decode_list,
"Encoding and decoding an array with Modified9 gives same result");
}
/**
* Used to check if posting lists can be properly encoded/decoded
*/
public function packUnpackPostingTestCase()
{
$posting_list = [90, 101, 570, 581, 737, 950, 1100, 1119, 1127,
1147, 1175, 1185, 1930, 1969, 2020, 2040, 2068, 2083, 2090, 2102,
2126, 2170, 2182, 2191, 2217, 2228, 2250, 2260, 2370, 2392, 2403,
2447, 2456, 2467, 2476, 2486, 2503, 2508, 2610, 2628, 2629, 2641,
2674, 2693, 2710, 2753, 2761, 2770, 2847, 2885, 2899, 2920, 2934,
3000, 3019, 3039, 3058, 3070, 3133, 3168, 3227, 3240, 3249, 3266,
3277, 3296, 3309, 3327, 3348, 3366, 3368, 3375, 3424, 3456, 3458,
3463, 3478, 3487, 3511, 3513, 3523, 3557, 3614, 3828, 3880, 3896,
3910, 3999, 4039, 4056, 4165, 4226, 4248, 4269, 4308, 4324, 4338,
4444, 4484, 4560, 4577, 4597, 4622, 4695, 4710, 4801, 4824, 4859,
4876, 4981, 5071, 5109, 5131, 5199, 5232, 5270, 5287, 5317, 5330,
5373, 5409, 5426, 5490, 5500, 5501, 5533, 5544, 5722, 5765, 5799,
5821, 5854, 5938, 5967, 6004, 6036, 6195, 6262, 6319, 6337, 6345,
6346, 6391, 6430, 6452, 6460, 6514, 6580, 6736, 6758, 6794, 6820,
6976];
$packed = L\packPosting(10, $posting_list);
$offset = 0;
$out_doc_list = L\unpackPosting($packed, $offset, true);
$this->assertEqual($out_doc_list[0], 10,
"Doc index from unpack of long packed posting equal");
$this->assertEqual($out_doc_list[1], $posting_list,
"Unpack of long packed posting equal");
$offset = 0;
$posting_list = [254, 12000, 24000];
$packed = L\packPosting(33689, $posting_list);
$out_doc_list = L\unpackPosting($packed, $offset, true);
$this->assertEqual($out_doc_list[0], 33689,
"Doc index from unpack of first word has delta[0] case");
$this->assertEqual($out_doc_list[1], $posting_list,
"Unpack of delta[0] case");
$offset = 0;
$posting_list = [511, 12000, 24000];
$packed = L\packPosting(33689, $posting_list);
$out_doc_list = L\unpackPosting($packed, $offset, true);
$this->assertEqual($out_doc_list[0], 33689,
"Doc index from unpack of first word has delta[0] case 2");
$this->assertEqual($out_doc_list[1], $posting_list,
"Unpack of delta[0] case 2");
$posting_list = [6000, 12000, 24000];
$packed = L\packPosting(100000, $posting_list);
$offset = 0;
$out_doc_list = L\unpackPosting($packed, $offset, true);
$this->assertEqual($out_doc_list[0], 100000,
"Bigger Doc index from unpack of long packed posting equal");
$this->assertEqual($out_doc_list[1], $posting_list,
"Bigger Delta unpack of posting equal");
$posting_list = [1, 4, 7, 174];
$packed = L\packPosting(0, $posting_list);
$unpack_int = unpack("N*", $packed);
$offset = 0;
$out_doc_list = L\unpackPosting($packed, $offset, true);
$this->assertEqual($out_doc_list[0], 0,
"Doc index from unpack of doc index 0 case");
$this->assertEqual($out_doc_list[1], $posting_list,
"Unpack of doc index 0 case");
}
/**
* Checks webencode/webdecode to see inverses. Checks base64Hash/
* unbase64Hash to see inverses
*/
public function webencodeWebdecodeTestCase()
{
$expected = "=+~-@hi ya everyone!!@~+-=";
$encode_decoded = L\webdecode(L\webencode($expected));
$this->assertEqual($expected, $encode_decoded,
"Webencode/Webdecode works correctly");
$encode_decoded = L\unbase64Hash(L\base64Hash($expected));
$this->assertEqual($expected, $encode_decoded,
"base64Hash/unbase64Hash works correctly");
$expected = "\xFE\xFD\xFF\xFE\xFD";
$encode_decoded = L\decode255(L\encode255("\xFE\xFD\xFF\xFE\xFD"));
$this->assertEqual($expected, $encode_decoded,
"encode255/decode255 works correctly");
}
/**
* Tests crawlAuthHash, the keyed HMAC-SHA256 sibling of
* crawlHash that the CSRF token machinery uses. Covers
* determinism, output shape (32 raw / 43 URL-safe base64
* chars, no padding or unsafe chars), input-sensitivity,
* key-sensitivity by way of AUTH_KEY mixing through the
* underlying hash_hmac call, hash_equals acceptance of
* matching pairs, hash_equals rejection of single-byte
* tampered pairs, and a roundtrip mirroring how
* Controller::generateCSRFToken / checkCSRFToken use the
* primitive (hash($user . $time) plus "*$time" suffix,
* length-54 token, hash_equals verification).
*/
public function crawlAuthHashTestCase()
{
// Determinism: same input → same output
$a = L\crawlAuthHash("hello");
$b = L\crawlAuthHash("hello");
$this->assertEqual($a, $b,
"crawlAuthHash is deterministic for the same input");
// Output length: 32 raw bytes / 43 base64 chars
$raw = L\crawlAuthHash("hello", true);
$this->assertEqual(32, strlen($raw),
"crawlAuthHash raw output is 32 bytes");
$this->assertEqual(43, strlen($a),
"crawlAuthHash base64 output is 43 chars");
// URL-safe base64: no /, +, or = chars
$this->assertTrue(strpbrk($a, "/+=") === false,
"crawlAuthHash output is URL-safe (no /, +, or =)");
// Input-sensitivity: one-byte change flips many bits
$c = L\crawlAuthHash("hellp");
$this->assertNotEqual($a, $c,
"crawlAuthHash differs when the input differs by one byte");
// hash_equals accepts matching pairs
$this->assertTrue(hash_equals(
L\crawlAuthHash("token-input"),
L\crawlAuthHash("token-input")),
"hash_equals returns true for two equal crawlAuthHash outputs");
// hash_equals rejects a single-byte tamper
$good = L\crawlAuthHash("token-input");
$bad = substr($good, 0, -1) .
(($good[-1] === "x") ? "y" : "x");
$this->assertFalse(hash_equals($good, $bad),
"hash_equals returns false when one byte differs");
// CSRF-style roundtrip: token = hash($user.$time) . "*$time"
$user = "alice";
$time = (string) time();
$token = L\crawlAuthHash($user . $time) . "*$time";
$this->assertEqual(54, strlen($token),
"CSRF-shaped token is 54 chars (43 hash + 1 star + 10 time)");
$parts = explode("*", $token);
$this->assertEqual(2, count($parts),
"CSRF-shaped token splits into hash and time on '*'");
$this->assertTrue(hash_equals(
L\crawlAuthHash($user . $parts[1]), $parts[0]),
"CSRF-shaped token verifies with hash_equals on the same user");
// A different user with the same timestamp should not verify
$this->assertFalse(hash_equals(
L\crawlAuthHash("bob" . $parts[1]), $parts[0]),
"CSRF-shaped token does not verify for a different user");
}
/**
* crawlCrypt computes a bcrypt hash with crypt(). It behaves the
* same whether or not it is called inside a fiber: there is no
* helper process and no suspend, so a fiber that calls it runs to
* completion in a single step and returns the very same hash a
* direct call gives. A low bcrypt cost keeps the test quick.
*/
public function crawlCryptCooperativeTestCase()
{
$salt = '$2y$04$' .
strtr(base64_encode(random_bytes(16)), '+', '.');
$inline = L\crawlCrypt("hunter2", $salt);
$this->assertEqual(crypt("hunter2", $salt), $inline,
"outside a fiber crawlCrypt is a plain crypt");
$suspended = false;
$fiber = new \Fiber(function () use ($salt) {
return L\crawlCrypt("hunter2", $salt);
});
$fiber->start();
while ($fiber->isSuspended()) {
$suspended = true;
$fiber->resume();
}
$in_fiber = $fiber->getReturn();
$this->assertFalse($suspended,
"in a fiber crawlCrypt runs inline without suspending");
$this->assertEqual($inline, $in_fiber,
"in a fiber crawlCrypt produces the same hash as inline");
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software