<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* This file contains unit tests of the IndexDocumentBundleTable class
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\models\Model;
use seekquarry\yioop\library\CrawlConstants as CC;
use seekquarry\yioop\library\storage_formats\IndexDocumentBundle;
use seekquarry\yioop\library\data_structures\UndoLog;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\library\language_processing as LLP;
/**
* IndexDocumentBundleTest holds the unit tests for IndexDocumentBundle.
* The cases check that pages added to a bundle can be read back, that
* prepareIndexMap drops duplicate pages before a partition is inverted,
* that a partition's inverted index is built, and that its terms are added
* to the bundle's dictionary and can be looked up.
*/
class IndexDocumentBundleTest extends UnitTest
{
/**
* Prefix of folders for index document test
*/
const TEST_DIR = __DIR__ . '/test_files/index_document_test';
/**
* Test value used as the on-wire type byte for index records
* during link-document tests; 0x10 is the link record type.
*/
const TEST_DOC_TYPE = "\x10"; //link type;
/**
* $index_archive holds the IndexDocumentBundle each case works on
* @var IndexDocumentBundle
*/
public $index_archive;
/**
* setUp makes the bundle a case works on in TEST_DIR. The cases that
* need several partitions get partitions of four documents.
*/
public function setUp()
{
IndexDocumentBundle::$temp_folder = C\TEST_DIR . "/test_files";
$small_partition_methods = [
"addPartitionPostingsDictionaryTestCase",
"postingEntriesStayWithinPartitionFileTestCase",
"batchTotalCountsEveryPartitionTestCase",
"activePartitionRowTestCase",
"mergeNextPartitionTestCase",
"partitionBuiltWhileFilledIsBuiltAgainTestCase",
"partitionNotAddedStaysNextToMergeTestCase",
"signalSharesStartFromSavedTotalsTestCase"];
$num_docs_per_partition = (in_array($this->current_method,
$small_partition_methods)) ? 4 : C\NUM_DOCS_PER_PARTITION;
$this->index_archive = new IndexDocumentBundle(self::TEST_DIR,
false, "TestBundle", $num_docs_per_partition, 5);
}
/**
* tearDown deletes TEST_DIR and the bundle in it
*/
public function tearDown()
{
array_map("unlink", glob(C\TEST_DIR .
"/test_files/signal_totals_*") ?: []);
IndexDocumentBundle::$temp_folder = C\TEMP_DIR;
$model = new Model();
$model->db->unlinkRecursive(self::TEST_DIR);
}
/**
* saveDescriptionTestCase checks that making a bundle saves the
* description it was given in the bundle's archive information
*/
public function saveDescriptionTestCase()
{
$archive_info = IndexDocumentBundle::getArchiveInfo(self::TEST_DIR);
$this->assertEqual($archive_info["DESCRIPTION"], "TestBundle");
}
/**
* addGetPagesTestCase checks that a page added to an
* IndexDocumentBundle, and its summary, can be read back.
*/
public function addGetPagesTestCase()
{
$docs = [];
for ($i = 0; $i < 10; $i++)
{
$docs[] = [CC::DOC_ID => $this->docidFromInt($i),
CC::SUMMARY =>[CC::TITLE => "title $i",
CC::DESCRIPTION => "desc $i"],
CC::PAGE => "page $i"];
}
$this->index_archive->addPages($docs, 10);
for ($i = 0; $i < 10; $i++)
{
$summary =
$this->index_archive->getSummary($this->docidFromInt($i), 0);
$this->assertEqual($summary, $docs[$i][CC::SUMMARY],
"Summary Test $i");
$page =
$this->index_archive->getCachePage($this->docidFromInt($i), 0);
$this->assertEqual($page, $docs[$i][CC::PAGE],
"Page Test $i");
}
}
/**
* summaryStoredAtSummaryLevelTestCase checks that a summary goes into
* the archive gzip encoded at SUMMARY_COMPRESSION_LEVEL rather than
* ARCHIVE_COMPRESSION_LEVEL, since the indexer spent half its time on
* a file compressing summaries at the slower level.
*/
public function summaryStoredAtSummaryLevelTestCase()
{
$links = [];
for ($i = 0; $i < 60; $i++) {
$links["https://www.example.com/section/page-$i-" . md5($i)] =
"Link text for item $i about something";
}
$summary = [CC::TITLE => str_repeat("Title words ", 6),
CC::DESCRIPTION => str_repeat("Some description sentence " .
"with words. ", 30), CC::LINKS => $links];
$this->index_archive->addPages([[CC::DOC_ID =>
$this->docidFromInt(1), CC::SUMMARY => $summary,
CC::PAGE => "page"]], 1);
$stored = file_get_contents($this->index_archive->documents->
getPartition(0));
$at_summary_level = gzencode(serialize($summary),
C\SUMMARY_COMPRESSION_LEVEL);
$at_archive_level = gzencode(serialize($summary),
C\ARCHIVE_COMPRESSION_LEVEL);
$this->assertNotEqual($at_summary_level, $at_archive_level,
"the two levels give different bytes for this summary");
$this->assertTrue(str_contains($stored, $at_summary_level),
"the summary is stored at the summary level");
$this->assertEqual($this->index_archive->getSummary(
$this->docidFromInt(1), 0), $summary, "the summary reads back");
}
/**
* prepareIndexTestCase checks prepareIndexMap, which drops duplicate
* pages before a partition's inverted index is made. Pages with the
* same doc_id are added, and the case checks that they are grouped
* together and that the grouping scores them as expected.
*/
public function prepareIndexTestCase()
{
$docs = [];
for ($i = 260; $i > 250; $i--)
{
$docs[] = [CC::DOC_ID => $this->docidFromInt($i),
CC::SUMMARY =>[CC::TITLE => "title $i",
CC::DESCRIPTION => "desc $i"],
CC::PAGE => "page $i"];
}
$this->index_archive->addPages($docs, 10);
$index = $this->index_archive->documents->loadPartitionIndex(0, true);
$i = 0;
foreach ($index as $test_doc_id => $row)
{
$this->assertEqual($test_doc_id, $docs[$i][CC::DOC_ID],
"Key Partition Decode Test ". $i);
$i++;
}
$grouped_urls = $this->index_archive->prepareIndexMap(0);
/* hash_url component of doc_key is first eight bytes
in above should all be same, so will be grouped into one group
*/
$this->assertEqual(count($grouped_urls), 1,
"Grouping by hash url works count test");
/* none of the hash_page components of doc_ids above begin with d
will all be considered aux_keys (not main doc)
*/
$this->assertEqual(substr_count($grouped_urls['00000000'][CC::AUX_DOCS],
"\xFF"), 10, "Grouping by hash url works count aux_docs test");
/* the score of a doc_id is initial its position in key_partition.
groups have scores equal to the sum of the grouped doc_key's
scores. In this case for 10 urls have 1+2+ ... + 10 = 55
*/
$this->assertEqual($grouped_urls['00000000'][CC::SCORE], 55,
"Grouping by hash url works count score test");
$test_index = [];
$i = 1;
while ($i <= 100) {
/* every tenth $i should share a hash for both a doc and a link
given out scoring system the first 10 of these should have
the highest scores and be chosen as the "official" docs for
the grouping.
*/
for ($j = 1; $j <= 10; $j++) {
$test_index[$this->docidFromIntKeys($i, $j,
$j, false)] = "lnk";
$test_index[$this->docidFromIntKeys($i, $j,
$j, true)] = "doc";
$i++;
}
}
$grouped_urls = $this->index_archive->prepareIndexMap(0,
$test_index);
foreach ($grouped_urls as $hash_url => $group) {
$url_number = intval($hash_url);
if ($url_number <= 10) {
$this->assertTrue(substr_count(
$group[CC::AUX_DOCS], "\xFF") > 0,
"Nonempty group case multiple same hash_page case $i");
} else {
$this->assertEqual($group[CC::AUX_DOCS], "metas_only",
"Nonempty group case multiple same hash_page case $i");
}
}
}
/**
* shortestDuplicateKeepsItsWordsTestCase checks that, of two pages with
* the same content, the one with the shorter url keeps its words when
* a partition is built, however the two were ordered or linked. On a
* laptop crawl, https://www.google.com/webhp?tab=ww, added first, kept
* the words, and https://www.google.com/ was left with meta words only,
* so the home page answered no query by its words.
*/
public function shortestDuplicateKeepsItsWordsTestCase()
{
$longer = $this->docidFromIntKeys(1, 7, 1);
$shorter = $this->docidFromIntKeys(2, 7, 1);
$this->index_archive->addPages([
[CC::DOC_ID => $longer, CC::SUMMARY => [
CC::URL => "https://www.google.com/webhp?tab=ww",
CC::TITLE => "Google", CC::DESCRIPTION => "search"],
CC::PAGE => "same page"],
[CC::DOC_ID => $shorter, CC::SUMMARY => [
CC::URL => "https://www.google.com/",
CC::TITLE => "Google", CC::DESCRIPTION => "search"],
CC::PAGE => "same page"]], 10);
$grouped_urls = $this->index_archive->prepareIndexMap(0);
$this->assertEqual($grouped_urls["00000001"][CC::AUX_DOCS],
"metas_only", "the longer url keeps meta words only");
$this->assertTrue($grouped_urls["00000002"][CC::AUX_DOCS] !==
"metas_only", "the shorter url keeps its words");
$chosen = IndexDocumentBundle::chooseAmongDuplicates([
"a" => ["https://example.com/page?x=1", 9],
"b" => ["https://example.com/", 2],
"c" => ["", 3]]);
$this->assertEqual($chosen, "b", "the shortest known url wins");
$chosen = IndexDocumentBundle::chooseAmongDuplicates([
"a" => ["", 3], "b" => ["", 5]]);
$this->assertEqual($chosen, "b", "with no urls the score wins");
}
/**
* buildInvertedIndexPartitionTestCase adds two pages to a bundle and
* builds the inverted index of their partition. It then looks up
* terms known to be on the pages and checks the posting lists found.
*/
public function buildInvertedIndexPartitionTestCase()
{
$keys = [$this->docidFromIntKeys(1, 1, 1),
$this->docidFromIntKeys(2, 2, 2)];
$docs = [
[
CC::DOC_ID => $keys[0],
CC::SUMMARY =>
[
CC::DESCRIPTION => "To be or not to be...",
CC::HASH => str_pad("1", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Some Shakespeare Play",
CC::URL => "https://www.somewhere1.com/"
],
CC::PAGE => "Page 1",
],
[
CC::DOC_ID => $keys[1],
CC::SUMMARY =>
[
CC::DESCRIPTION => "Take me out to the ball game...",
CC::HASH => str_pad("2", 8, "0", STR_PAD_LEFT),
CC::TITLE => "A Dialog on Baseball for people now",
CC::URL => "https://www.somewhere2.com/"
],
CC::PAGE => "Page 2",
]
];
$this->index_archive->addPages($docs, 2);
$this->index_archive->buildInvertedIndexPartition();
$base_folder = $this->index_archive->getPartitionBaseFolder(0);
$doc_map_filename = $base_folder . "/" .
IndexDocumentBundle::DOC_MAP_FILENAME;
$doc_map_tools = $this->index_archive->doc_map_tools;
$doc_map = $doc_map_tools->load($doc_map_filename);
$i = 0;
foreach ($keys as $key) {
$row = $doc_map_tools->find($doc_map, $key);
//get row after bloom filter of terms
$row = substr($row, IndexDocumentBundle::TERMSFILTER_LEN + 1);
$entry = $doc_map_tools->unpack($row);
$preface_length = str_word_count(UrlParser::getWordsInHostUrl(
$docs[$i][CC::SUMMARY][CC::URL]) . " " .
$docs[$i][CC::SUMMARY][CC::TITLE]) + 3;
$entry_preface_length = ($entry[1]["POS"] & 255);
$this->assertEqual($preface_length, $entry_preface_length,
"Doc $i preface length matches calculated.");
$i++;
}
$posting_tools = $this->index_archive->postings_tools;
$postings_filename = $base_folder . "/" .
IndexDocumentBundle::POSTINGS_FILENAME;
$postings = $posting_tools->load($postings_filename);
$stemmed_ball = LLP\PhraseParser::stemTermsK(
"baseball", 'en-US', false)[0];
$baseball_id = L\canonicalTerm($stemmed_ball, true);
$row = $posting_tools->find($postings, $baseball_id);
$entry = $posting_tools->unpack($row);
$this->assertEqual($entry[0]['FREQUENCY'], 1,
"Test retrieve posting frequency");
$positions_filename = $base_folder . "/" .
IndexDocumentBundle::POSITIONS_FILENAME;
$encoded_positions = file_get_contents($positions_filename,
false, null, $entry[0]['POSITIONS_OFFSET'],
$entry[0]['POSITIONS_LEN']);
$position_list = L\decodePositionList($encoded_positions,
$entry[0]['FREQUENCY']);
$host_word_count = str_word_count(UrlParser::getWordsInHostUrl(
$docs[1][CC::SUMMARY][CC::URL]));
$this->assertEqual($position_list[0], 1 + $host_word_count + 1 + 3,
"Test Position List Decode");
}
/**
* fieldKeyHoldsPagesNamingTheWordInTitleTestCase checks the first tier
* of the tiered index. Both pages hold baseball, so both are filed
* under the word's combined key for English and safe search. Only the
* page whose title holds baseball is also filed under the word's field
* key, which a query reads as a disjunct of its own.
*/
public function fieldKeyHoldsPagesNamingTheWordInTitleTestCase()
{
$docs = [];
foreach ([1 => ["Baseball Tonight", "Scores from every game."],
2 => ["Sports Tonight", "Scores from baseball and more."]] as
$i => [$title, $description]) {
$docs[] = [CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [CC::DESCRIPTION => $description,
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => $title, CC::LANG => "en-US",
CC::URL => "https://www.somewhere$i.com/"],
CC::PAGE => "Page $i"];
}
$this->index_archive->addPages($docs, 2);
$this->index_archive->buildInvertedIndexPartition();
$base_folder = $this->index_archive->getPartitionBaseFolder(0);
$posting_tools = $this->index_archive->postings_tools;
$postings = $posting_tools->load($base_folder . "/" .
IndexDocumentBundle::POSTINGS_FILENAME);
$stem = LLP\PhraseParser::stemTermsK("baseball", 'en-US', false)[0];
$counts = [];
foreach (["true", "true" . IndexDocumentBundle::FIELD_TERM_MARK] as
$safe) {
$row = $posting_tools->find($postings,
L\combinedTerm($stem, "en", $safe));
$counts[] = ($row === false || $row === null) ? 0 :
count($posting_tools->unpack($row));
}
$this->assertEqual($counts, [2, 1],
"Both pages under the combined key, one under the field key");
}
/**
* redirectLinkCarriesHostWordsTestCase checks the words a redirect
* gives. www.coke.com redirects to
* https://www.coca-cola.com/?&redirect=true, and the Fetcher leaves a
* link from www.coke.com to that page and to https://www.coca-cola.com/.
* Both pages are filed under coke, the word of the host that
* redirected. The home page is also filed under coke's field key, as
* coke counts as one of its host words and gets the host bonus.
*/
public function redirectLinkCarriesHostWordsTestCase()
{
$from = "https://www.coke.com/";
$link_text = "location:$from";
$docs = [];
foreach ([7 => "https://www.coca-cola.com/",
5 => "https://www.coca-cola.com/?&redirect=true"] as $i => $to) {
$link_id = "url|$to|text|" . urlencode($link_text) .
"|eref|$from";
$docs[] = [CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [CC::DESCRIPTION => "Refresh the world.",
CC::HASH => "0000000$i", CC::LANG => "en-US",
CC::TITLE => "The Coca-Cola Company", CC::URL => $to],
CC::PAGE => "Page $i"];
$docs[] = [CC::DOC_ID => $this->docidFromIntKeys($i, $i + 1, 9,
false), CC::SUMMARY => [CC::TYPE => "link",
CC::DESCRIPTION => $link_text, CC::HASH => $link_id,
CC::LANG => "en-US", CC::TITLE => $to, CC::URL => $link_id],
CC::PAGE => ""];
}
$this->index_archive->addPages($docs, 4);
$this->index_archive->buildInvertedIndexPartition();
$base_folder = $this->index_archive->getPartitionBaseFolder(0);
$posting_tools = $this->index_archive->postings_tools;
$postings = $posting_tools->load($base_folder . "/" .
IndexDocumentBundle::POSTINGS_FILENAME);
$stem = LLP\PhraseParser::stemTermsK("coke", 'en-US', false)[0];
$counts = [];
foreach ([L\canonicalTerm($stem), L\combinedTerm($stem, "en",
"true" . IndexDocumentBundle::FIELD_TERM_MARK)] as $key) {
$row = $posting_tools->find($postings, $key);
$counts[] = ($row === false || $row === null) ? 0 :
count($posting_tools->unpack($row));
}
$this->assertEqual($counts, [2, 1], "both pages are filed under " .
"coke, and the home page under its field key");
}
/**
* addPartitionPostingsDictionaryTestCase runs the whole build: pages
* are added, duplicates dropped, partitions inverted and their terms
* added to the bundle's dictionary. It then looks up a term known to
* be on every page and checks the posting lists found.
*/
public function addPartitionPostingsDictionaryTestCase()
{
$index_archive = $this->index_archive;
$dictionary = $index_archive->dictionary;
$keys = [];
for ($i = 0; $i < 10; $i++) {
$keys[$i] = $this->docidFromIntKeys($i, $i, $i);
$docs[] = [
CC::DOC_ID => $keys[$i],
CC::SUMMARY =>
[
CC::DESCRIPTION => "to$i be or$i not$i to$i be...",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Some$i Shakespeare$i Play$i",
CC::URL => "https://www.somewhere$i.com/"
],
CC::PAGE => "Page $i",
];
}
$num_docs = count($docs);
$index_archive->addPages($docs, $num_docs);
$index_archive->updateDictionary();
$index_archive->forceSave();
$be_term_id = L\canonicalTerm("be");
$term_row = $dictionary->get($be_term_id);
$this->assertEqual(count($term_row), 2, "Two completed partitions");
$active_folder = $this->index_archive->getPartitionBaseFolder(
$this->index_archive->documents->parameters["SAVE_PARTITION"]);
$active_postings_filename = $active_folder . "/postings";
$posting_tools = $this->index_archive->postings_tools;
$this->assertTrue(file_exists($active_postings_filename),
"Active postings file exists");
$active_postings = [];
if (file_exists($active_postings_filename)) {
$active_dictionary = $posting_tools->load(
$active_postings_filename);
$active_term_row = $posting_tools->find($active_dictionary,
$be_term_id);
$active_postings = $posting_tools->unpack($active_term_row);
}
$sum = 0;
for ($i = 0; $i < 2; $i++) {
$sum += ($term_row[$i]['NUM_DOCS'] ?? 0);
}
$this->assertEqual($sum + count($active_postings), $num_docs,
"Term 'be' occurs in correct number of documents");
$key_len = $posting_tools->key_len;
for ($i = 0; $i < 2; $i++) {
$partition = $term_row[$i]['PARTITION'];
$partition_folder = $this->index_archive->getPartitionBaseFolder(
$partition);
$postings_filename = $partition_folder . "/" .
IndexDocumentBundle::POSTINGS_FILENAME;
$postings_string = file_get_contents($postings_filename,
false, null, $term_row[$i]['POSTINGS_OFFSET'],
$term_row[$i]['POSTINGS_LEN']);
$postings = $posting_tools->unpack($postings_string);
$base_folder = $this->index_archive->getPartitionBaseFolder(
$term_row[$i]['PARTITION']);
$positions_filename = $base_folder . "/" .
IndexDocumentBundle::POSITIONS_FILENAME;
$last = 0;
for ($j = 0; $j < 4; $j++) {
$encoded_positions = file_get_contents($positions_filename,
false, null, $last + $postings[$j]['POSITIONS_OFFSET'],
$postings[$j]['POSITIONS_LEN']);
$last += $postings[$j]['POSITIONS_OFFSET'];
$position_list = L\decodePositionList($encoded_positions,
$postings[$j]['FREQUENCY']);
$this->assertEqual($position_list[0], 9,
"Test Position List Decode");
}
}
}
/**
* postingEntriesStayWithinPartitionFileTestCase checks that every
* dictionary entry points at a posting list that lies wholly inside
* its partition's postings file: a non-negative
* offset, a positive length, and offset plus length no greater than
* the file size. An entry that breaks this is the corruption that
* made the live server try to read a posting list far larger than
* the file and run out of memory. This builds a small index and
* checks the invariant for every term the documents contain.
*/
public function postingEntriesStayWithinPartitionFileTestCase()
{
$index_archive = $this->index_archive;
$dictionary = $index_archive->dictionary;
$docs = [];
for ($i = 0; $i < 10; $i++) {
$docs[] = [
CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [
CC::DESCRIPTION => "to$i be or$i not$i to$i be...",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Some$i Shakespeare$i Play$i",
CC::URL => "https://www.somewhere$i.com/"
],
CC::PAGE => "Page $i",
];
}
$index_archive->addPages($docs, count($docs));
$index_archive->updateDictionary();
$index_archive->forceSave();
$terms = ["be", "page"];
$checked = 0;
foreach ($terms as $term) {
$term_rows = $dictionary->get(L\canonicalTerm($term));
if (empty($term_rows)) {
continue;
}
foreach ($term_rows as $row) {
$folder = $index_archive->getPartitionBaseFolder(
$row['PARTITION']);
$postings_filename = $folder . "/" .
IndexDocumentBundle::POSTINGS_FILENAME;
$file_size = filesize($postings_filename);
$offset = $row['POSTINGS_OFFSET'];
$len = $row['POSTINGS_LEN'];
$this->assertTrue($offset >= 0 && $len > 0 &&
$offset + $len <= $file_size,
"term '$term' posting entry lies within its file");
$checked++;
}
}
$this->assertTrue($checked > 0,
"at least one posting entry was checked");
}
/**
* getPostingsStringClampsOverlongLengthTestCase checks that
* getPostingsString never reads more than its partition file holds,
* however large a length it is handed. A corrupt dictionary
* entry asking for far more than the file contains comes back
* clamped to the file rather than pre-allocating a huge buffer, and
* an offset past the end comes back empty. This writes a tiny
* postings file and checks both.
*/
public function getPostingsStringClampsOverlongLengthTestCase()
{
$partition = 9091;
$folder = $this->index_archive->getPartitionBaseFolder(
$partition);
if (!is_dir($folder)) {
mkdir($folder, 0777, true);
}
$postings_filename = $folder . "/" .
IndexDocumentBundle::POSTINGS_FILENAME;
$bytes = L\encode255("a small set of posting bytes");
file_put_contents($postings_filename, $bytes);
$file_size = filesize($postings_filename);
$exact = $this->index_archive->getPostingsString($partition, 0,
$file_size);
$overlong = $this->index_archive->getPostingsString($partition,
0, $file_size + 100000000);
$this->assertEqual($exact, $overlong,
"an over-long length is clamped to the file contents");
$past_end = $this->index_archive->getPostingsString($partition,
$file_size + 50, 100000000);
$this->assertEqual("", $past_end,
"an offset past the end of the file returns empty");
}
/**
* docidFromInt makes a 24 byte docId by padding an int to the left
* with 0's
*
* @param int $i integer to make docId from
* @param string $type default type of docid
* @return string docid made by padding
*/
protected function docidFromInt($i, $type = self::TEST_DOC_TYPE)
{
$pre_key = str_pad("$i", 24, "0", STR_PAD_LEFT);
$pre_key[IndexDocumentBundle::DOCID_PART_LEN << 1] = $type;
return $pre_key;
}
/**
* docidFromIntKeys makes a 24 byte docid for a test page. A docid is
* made from three 8 byte strings, so this function left pads each of
* three ints with '0' (\x30) and joins them. The first byte of the
* third string says whether the id is for a document, 'd', or a link,
* 'l', and the $is_doc flag picks which one is written there.
*
* @param int $i_hash_url an int for first 8 bytes (in non-artificial docids
* would be for the crawlHash of url document from)
* @param int $j_hash_page an int for first 8 bytes (in non-artificial
* docids would be for the crawlHash of document)
* @param int $k_hash_host an int for first 8 bytes (in non-artificial
* docids would be for the crawlHash of hostname of site document from)
* @param bool $is_doc whether the hash is for a document or a link
* @return string 24 byte docid.
*/
protected function docidFromIntKeys($i_hash_url, $j_hash_page,
$k_hash_host, $is_doc = true)
{
$doc_or_link = ($is_doc) ? "d" : "l";
return str_pad("$i_hash_url", 8, "0", STR_PAD_LEFT) .
str_pad("$j_hash_page", 8, "0", STR_PAD_LEFT) .
$doc_or_link . str_pad("$k_hash_host", 7, "0", STR_PAD_LEFT);
}
/**
* postingCountBeyondStringIsRefusedTestCase checks that a posting
* string that says it holds more items than its bytes could encode is
* refused rather than walked. Each item
* takes at least two bytes, one naming its format and one of
* payload, so a count past half the string's length can only come
* from a corrupt index. Walking such a list builds an array of
* garbage items, which on a busy server is enough to exhaust its
* memory.
*/
public function postingCountBeyondStringIsRefusedTestCase()
{
/* vByteDecode reads 200 from a single byte with the high bit
set, then eight bytes of payload follow: far too few for the
two hundred items claimed. */
$postings_string = chr(200) . str_repeat("\x01", 8);
list($items, $sum_frequencies) =
$this->index_archive->unpackPostings($postings_string);
$this->assertEqual(0, count($items),
"a claim of two hundred items in nine bytes decodes to none");
$this->assertEqual(0, $sum_frequencies,
"and the frequencies it would have summed come to zero");
}
/**
* droppingAPostingHandleKeepsTheOthersPartitionsTestCase checks the
* list of postings files a query has open. It is keyed by the
* partition each file belongs to, and the list is trimmed when the
* process nears its memory limit. A trim must keep the keys of the
* files left, or a lookup by partition hands back another partition's
* file. This case trims a list whose keys are partition numbers and
* asks that each handle still answers to the partition it belongs
* to.
*/
public function droppingAPostingHandleKeepsTheOthersPartitionsTestCase()
{
$file_handles = [4 => "postings of 4", 9 => "postings of 9",
13 => "postings of 13"];
IndexDocumentBundle::dropOldestPostingHandle($file_handles);
$this->assertEqual([9, 13], array_keys($file_handles),
"The oldest handle goes and the others keep their partitions");
$this->assertEqual("postings of 9", $file_handles[9] ?? "",
"Partition 9 still answers with its own postings file");
$this->assertEqual("postings of 13", $file_handles[13] ?? "",
"Partition 13 still answers with its own postings file");
}
/**
* batchTotalCountsEveryPartitionTestCase checks that getWordInfo, asked
* for one partition's rows with the remaining total, still counts the
* word over every partition. Ten pages go into partitions of three, so
* the word be has rows in three completed partitions and one active
* one. A query reads a word's rows a batch at a time and reports the
* total of its first batch, which before this counted only that
* batch. Last it lowers the exact limit to one row, so the total past
* it is estimated from the share of partitions read.
*/
public function batchTotalCountsEveryPartitionTestCase()
{
$index_archive = $this->index_archive;
$docs = [];
for ($i = 0; $i < 10; $i++) {
$docs[] = [
CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [
CC::DESCRIPTION => "to$i be or$i not$i to$i be...",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Some$i Shakespeare$i Play$i",
CC::URL => "https://www.somewhere$i.com/"],
CC::PAGE => "Page $i"];
}
$index_archive->addPages($docs, count($docs));
$index_archive->updateDictionary();
$index_archive->forceSave();
$be_term_id = L\canonicalTerm("be");
$batch = $index_archive->getWordInfo($be_term_id, -1, 0, 1, true);
$this->assertEqual(count($batch['ROWS'] ?? []), 1,
"the batch holds its one row, and the active partition's row " .
"waits for the word's last batch");
$this->assertEqual($batch['TOTAL_COUNT'] ?? 0, 10,
"the total counts all ten pages");
$whole = $index_archive->getWordInfo($be_term_id);
$this->assertEqual($whole['TOTAL_COUNT'] ?? 0, 10,
"reading every row gives the same total");
/* With one row read exactly, the first completed partition's
count is scaled by the two completed partitions over the one
read. The two hold the same count here, so the estimate is
exact, and the active partition is added as read. */
$index_archive->exact_count_partitions = 1;
$estimate = $index_archive->getWordInfo($be_term_id, -1, 0, 1,
true);
$this->assertEqual($estimate['TOTAL_COUNT'] ?? 0, 10,
"an estimate past the exact rows scales to the whole count");
$this->assertEqual(count($estimate['ROWS'] ?? []), 1,
"the estimate returns the same rows");
}
/**
* activePartitionRowTestCase checks
* IndexDocumentBundle::activePartitionRow, which
* IndexManager::lookupLatestVersionPage asks instead of getWordInfo.
* Ten pages go into partitions of four, so pages 8 and 9 are in the
* active partition. For the word be, in every page, the row must be
* the last row getWordInfo gives, the only one holding postings. For
* or0, only in page 0 of the first partition, there is no row.
*/
public function activePartitionRowTestCase()
{
$index_archive = $this->index_archive;
$docs = [];
for ($i = 0; $i < 10; $i++) {
$docs[] = [
CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [
CC::DESCRIPTION => "to$i be or$i not$i to$i be...",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Some$i Shakespeare$i Play$i",
CC::URL => "https://www.somewhere$i.com/"],
CC::PAGE => "Page $i"];
}
$index_archive->addPages($docs, count($docs));
$index_archive->updateDictionary();
$index_archive->forceSave();
$be_term_id = L\canonicalTerm("be");
$row = $index_archive->activePartitionRow($be_term_id);
$whole = $index_archive->getWordInfo($be_term_id);
$this->assertEqual($row, end($whole['ROWS']),
"the active row is the last row of the whole word");
$this->assertEqual($row['PARTITION'] ?? -1,
$index_archive->documents->parameters["SAVE_PARTITION"],
"the row is the active partition's");
$this->assertEqual(count($row['POSTINGS'] ?? []), 2,
"the active partition holds pages 8 and 9");
$this->assertEqual($index_archive->activePartitionRow(
L\canonicalTerm("or0")), [],
"a word only in a completed partition has no active row");
}
/**
* termsFilterFindsItsTermsTestCase checks that
* IndexDocumentBundle::checkTermExists finds each word that
* IndexDocumentBundle::storeTerms put in a terms filter. storeTerms
* set bit $hash & 7 of a byte while checkTermExists tested bit
* $hash % 7, so a word put in the filter was reported absent, and a
* search dropped a page whose newer copy holds the word.
*/
public function termsFilterFindsItsTermsTestCase()
{
$words = ["mary", "pollett", "be", "university", "search"];
$terms_filter = substr(IndexDocumentBundle::storeTerms($words), 1);
foreach ($words as $word) {
$this->assertTrue(IndexDocumentBundle::checkTermExists(
L\canonicalTerm($word), $terms_filter),
"the filter holds $word");
}
}
/**
* docMapEntryMatchesWholeMapReadTestCase checks that
* IndexDocumentBundle::docMapEntry, which reads one document map entry
* through the partition's doc_map_marks file, gives the same entry as
* PackedTableTools::findEntryAtIndexTableName, which reads the whole
* map. A hundred pages make one partition, so entries lie past
* several marks. It checks every entry and the count, then again with
* the marks file removed, as for a partition built before such files
* were written, and with marks saved for a map of another size.
*/
public function docMapEntryMatchesWholeMapReadTestCase()
{
$index_archive = $this->index_archive;
$docs = [];
for ($i = 0; $i < 100; $i++) {
$docs[] = [
CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [
CC::DESCRIPTION => "to$i be or$i not$i to$i be...",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Some$i Shakespeare$i Play$i",
CC::URL => "https://www.somewhere$i.com/"],
CC::PAGE => "Page $i"];
}
$index_archive->addPages($docs, count($docs));
$index_archive->forceSave();
$doc_map = $index_archive->getPartitionBaseFolder(0) . "/" .
IndexDocumentBundle::DOC_MAP_FILENAME;
$marks_file = dirname($doc_map) . "/" .
IndexDocumentBundle::DOC_MAP_MARKS_FILENAME;
$this->assertTrue(file_exists($marks_file),
"the build writes the marks file");
$tools = $index_archive->doc_map_tools;
$count = count($tools->getEntryMarkers($doc_map)) + 1;
foreach (["as built", "with no marks file",
"with marks for another size"] as $case) {
if ($case == "with no marks file") {
unlink($marks_file);
} else if ($case == "with marks for another size") {
file_put_contents($marks_file, pack("N*", 1, 1, 0));
}
IndexDocumentBundle::$doc_map_marks_cache = [];
$this->assertEqual($index_archive->docMapMarks($doc_map)[1],
$count, "the marks count every entry, $case");
$differ = 0;
for ($i = 0; $i <= $count; $i++) {
if ($index_archive->docMapEntry(0, $i) !==
$tools->findEntryAtIndexTableName($doc_map, $i)) {
$differ++;
}
}
$this->assertEqual($differ, 0,
"every entry matches the whole map read, $case");
}
}
/**
* docMapsKeptOpenAreFewTestCase checks that reading document map
* entries from more partitions than DOC_MAP_HANDLES_KEPT leaves at
* most that many maps open, and that each entry still matches the
* whole map read. A search of a large index reads entries from
* hundreds of partitions, and a process that kept every map open ran
* out of the files it may open.
*/
public function docMapsKeptOpenAreFewTestCase()
{
$index_archive = $this->index_archive;
$docs = [];
for ($i = 0; $i < 40; $i++) {
$docs[] = [
CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [
CC::DESCRIPTION => "open$i files$i",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Map$i",
CC::URL => "https://www.somewhere$i.com/"],
CC::PAGE => "Page $i"];
}
$index_archive->addPages($docs, count($docs));
$index_archive->forceSave();
$first_folder = $index_archive->getPartitionBaseFolder(0);
$doc_map = $first_folder . "/" .
IndexDocumentBundle::DOC_MAP_FILENAME;
$num_partitions = C\DOC_MAP_HANDLES_KEPT + 8;
for ($partition = 1; $partition < $num_partitions; $partition++) {
$folder = $index_archive->getPartitionBaseFolder($partition);
if (!file_exists($folder)) {
mkdir($folder, 0777, true);
}
copy($doc_map, $folder . "/" .
IndexDocumentBundle::DOC_MAP_FILENAME);
}
$tools = $index_archive->doc_map_tools;
$differ = 0;
for ($partition = 0; $partition < $num_partitions; $partition++) {
$entry_number = $partition % 40;
if ($index_archive->docMapEntry($partition, $entry_number) !==
$tools->findEntryAtIndexTableName($doc_map,
$entry_number)) {
$differ++;
}
}
$this->assertEqual($differ, 0,
"each entry matches the whole map read");
$this->assertTrue(count(IndexDocumentBundle::$doc_map_handles) <=
C\DOC_MAP_HANDLES_KEPT, "at most DOC_MAP_HANDLES_KEPT maps " .
"stay open after reading from $num_partitions partitions");
}
/**
* signalSharesStartFromSavedTotalsTestCase checks that signalShares
* saves the totals of the partitions' signal counts under tests/test_files,
* that a later process's first look starts from them, and that totals
* saved before the last counted partition's signals file was written
* again are not used. Partitions here hold four documents, so ten
* documents close two partitions, which are merged.
*/
public function signalSharesStartFromSavedTotalsTestCase()
{
$docs = [];
for ($i = 0; $i < 10; $i++) {
$docs[] = [CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [CC::DESCRIPTION => "to$i be or$i not$i",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Some$i Play$i",
CC::URL => "https://www.somewhere$i.com/"],
CC::PAGE => "Page $i"];
}
$this->index_archive->addPages($docs, count($docs));
$this->index_archive->updateDictionary();
$bundle = $this->index_archive;
$saved_path = C\TEST_DIR . "/test_files/signal_totals_" .
md5($bundle->dir_name) . ".txt";
@unlink($saved_path);
IndexDocumentBundle::$signal_shares_cache = [];
$shares = $bundle->signalShares();
$this->assertTrue(file_exists($saved_path),
"the first look saves the totals of the two merged partitions");
$saved = json_decode(file_get_contents($saved_path), true);
$saved["TOTALS"] = ["DOCS" => 10, "WIKI_BONUS" => 5];
file_put_contents($saved_path, json_encode($saved));
IndexDocumentBundle::$signal_shares_cache = [];
$this->assertEqual($bundle->signalShares(), ["WIKI_BONUS" => 0.5],
"a later process's first look starts from the saved totals");
touch($bundle->getPartitionBaseFolder(1) . "/" .
IndexDocumentBundle::SIGNALS_FILENAME, time() + 5);
IndexDocumentBundle::$signal_shares_cache = [];
$this->assertEqual($bundle->signalShares(), $shares,
"totals saved before a partition's counts changed are not used");
@unlink($saved_path);
}
/**
* mergeNextPartitionTestCase checks IndexDocumentBundle::
* mergeNextPartition on a bundle whose partitions hold four documents,
* so ten documents close two partitions. It merges nothing while the
* next partition's inverted index is not built, nor while a fresh
* merge_claim says another process is merging. Once the partition is
* built it merges partition 0, moves the next partition on to 1, and
* the dictionary then finds a word from it. A merge_claim older than
* BUILD_CLAIM_TIMEOUT counts as abandoned and is taken over.
*/
public function mergeNextPartitionTestCase()
{
$docs = [];
for ($i = 0; $i < 10; $i++) {
$docs[] = [
CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [
CC::DESCRIPTION => "to$i be or$i not$i to$i be...",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Some$i Shakespeare$i Play$i",
CC::URL => "https://www.somewhere$i.com/"],
CC::PAGE => "Page $i"];
}
$this->index_archive->addPages($docs, count($docs));
$this->index_archive->forceSave();
$this->assertEqual(IndexDocumentBundle::mergeNextPartition(
self::TEST_DIR), -1, "nothing is merged before a build");
IndexDocumentBundle::buildClosedPartition(self::TEST_DIR, 0);
$claim = self::TEST_DIR . "/merge_claim";
touch($claim);
$this->assertEqual(IndexDocumentBundle::mergeNextPartition(
self::TEST_DIR), -1, "a fresh merge claim keeps others out");
touch($claim, time() - IndexDocumentBundle::BUILD_CLAIM_TIMEOUT - 5);
$this->assertEqual(IndexDocumentBundle::mergeNextPartition(
self::TEST_DIR), 0, "an abandoned claim is taken over and " .
"partition 0 is merged");
$this->assertFalse(file_exists($claim),
"the merge removes its claim when done");
$this->assertEqual(intval(file_get_contents(self::TEST_DIR . "/" .
IndexDocumentBundle::NEXT_PARTITION_FILE)), 1,
"the next partition to merge is now 1");
$reader = new IndexDocumentBundle(self::TEST_DIR, true);
$rows = $reader->dictionary->get(L\canonicalTerm("be"));
$this->assertTrue(!empty($rows),
"the dictionary finds a word from the merged partition");
$checkpoint = (string)@file_get_contents(self::TEST_DIR . "/" .
UndoLog::CHECKPOINT_NAME);
$this->assertFalse(str_contains($checkpoint, "/" .
IndexDocumentBundle::DICTIONARY_FOLDER . "/"),
"the bundle's checkpoint names no dictionary tier file");
$this->assertFalse(str_contains($checkpoint, $this->index_archive->
getPartitionBaseFolder(0)),
"the bundle's checkpoint names no file of the built partition");
}
/**
* partitionNotAddedStaysNextToMergeTestCase checks that a partition
* whose postings could not be added to the dictionary is not recorded
* as merged. Partitions here hold four documents, so ten documents
* close two partitions. A bundle whose addPartitionPostingsDictionary
* returns false, as it does when the postings file is missing, merges
* partition 0. The next partition to merge must stay 0, so that
* partition 0's words still get into the dictionary. A
* merge by an ordinary bundle then adds partition 0, and the
* dictionary finds the info: word of one of its pages.
*/
public function partitionNotAddedStaysNextToMergeTestCase()
{
$docs = [];
for ($i = 0; $i < 10; $i++) {
$docs[] = [
CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [
CC::DESCRIPTION => "to$i be or$i not$i to$i be...",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Some$i Shakespeare$i Play$i",
CC::URL => "https://www.somewhere$i.com/"],
CC::PAGE => "Page $i"];
}
$this->index_archive->addPages($docs, count($docs));
$this->index_archive->forceSave();
IndexDocumentBundle::buildClosedPartition(self::TEST_DIR, 0);
$failing = new class(self::TEST_DIR, false, "TestBundle", 4, 5)
extends IndexDocumentBundle {
/**
* addPartitionPostingsDictionary adds nothing and returns
* false, as IndexDocumentBundle's does when the partition's
* postings file is missing.
*
* @param int $partition the partition to add
* @param string $taking_too_long_touch file to touch while busy
* @return bool false, for a partition that was not added
*/
public function addPartitionPostingsDictionary($partition = -1,
$taking_too_long_touch = null)
{
return false;
}
};
$next_file = self::TEST_DIR . "/" .
IndexDocumentBundle::NEXT_PARTITION_FILE;
$failing->updateDictionary(null, false, "merge_0");
$next = (file_exists($next_file)) ?
intval(file_get_contents($next_file)) : 0;
$this->assertEqual($next, 0,
"a partition not added stays the next partition to merge");
$merger = new IndexDocumentBundle(self::TEST_DIR, false,
"TestBundle", 4, 5);
$merger->updateDictionary(null, false, "merge_0");
$reader = new IndexDocumentBundle(self::TEST_DIR, true);
$rows = $reader->dictionary->get(L\canonicalTerm(
"info:https://www.somewhere0.com/"));
$this->assertTrue(!empty($rows),
"a later merge adds the partition's info: words");
}
/**
* partitionBuiltWhileFilledIsBuiltAgainTestCase checks that inverted
* index files written while a partition was still being filled are not
* merged as they are once the partition is closed. Partitions here hold
* four documents. Partition 0 is built holding two, as the Indexer's
* periodic save does, and partitionIsBuilt says it is built. Two more
* are then added to it. partitionIsBuilt must then say no,
* mergeNextPartition must merge nothing, buildClosedPartition must
* build partition 0 again, and once merged the dictionary must find the
* words of the documents added after the first build.
*/
public function partitionBuiltWhileFilledIsBuiltAgainTestCase()
{
$make_doc = function ($i) {
return [CC::DOC_ID => $this->docidFromIntKeys($i, $i, $i),
CC::SUMMARY => [CC::DESCRIPTION => "word$i shared words",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Title$i", CC::URL => "https://site$i.com/"],
CC::PAGE => "Page $i"];
};
$index_archive = $this->index_archive;
$index_archive->addPages([$make_doc(0), $make_doc(1)], 2);
$index_archive->forceSave();
$this->assertTrue($index_archive->partitionIsBuilt(0),
"files built from the documents as they are count as built");
$later_docs = [];
for ($i = 2; $i < 6; $i++) {
$later_docs[] = $make_doc($i);
}
$index_archive->addPages($later_docs, count($later_docs));
$this->assertFalse($index_archive->partitionIsBuilt(0),
"files built before documents were added are out of date");
$this->assertEqual(IndexDocumentBundle::mergeNextPartition(
self::TEST_DIR), -1, "out of date files are not merged");
$this->assertEqual(IndexDocumentBundle::buildClosedPartition(
self::TEST_DIR), 0, "partition 0 is built again");
$this->assertTrue($index_archive->partitionIsBuilt(0),
"the new build counts as built");
$this->assertEqual(IndexDocumentBundle::mergeNextPartition(
self::TEST_DIR), 0, "partition 0 is then merged");
$reader = new IndexDocumentBundle(self::TEST_DIR, true);
foreach ([0, 3] as $i) {
$rows = $reader->dictionary->get(L\canonicalTerm("word$i"));
$this->assertEqual($rows[0]["NUM_DOCS"] ?? 0, 1,
"the dictionary finds word$i of partition 0");
}
}
/**
* claimIsAbandonedTestCase checks that a claim stands for as long as
* the process that made it runs, however old the claim file is, and is
* abandoned at once when that process is gone. A claim holding no
* process id still counts as abandoned only after BUILD_CLAIM_TIMEOUT.
*/
public function claimIsAbandonedTestCase()
{
$claim = self::TEST_DIR . "/build_claim_test";
$old = time() - IndexDocumentBundle::BUILD_CLAIM_TIMEOUT - 60;
file_put_contents($claim, (string)getmypid());
touch($claim, $old);
$this->assertFalse(IndexDocumentBundle::claimIsAbandoned($claim),
"an old claim whose holder still runs stands");
$finished_id = self::finishedProcessId();
file_put_contents($claim, (string)$finished_id);
$this->assertTrue(IndexDocumentBundle::claimIsAbandoned($claim),
"a fresh claim whose holder has ended is abandoned");
file_put_contents($claim, "");
$this->assertFalse(IndexDocumentBundle::claimIsAbandoned($claim),
"a fresh claim with no holder stands");
touch($claim, $old);
$this->assertTrue(IndexDocumentBundle::claimIsAbandoned($claim),
"a claim with no holder is abandoned after the timeout");
unlink($claim);
}
/**
* signalCountsTestCase checks the url signals read from a document id,
* each counted as occurrences of a pseudo word: a company-level root
* page and a host root page each count once in their own signal, and
* the path depth counts 1/(number of slashes + 1).
*/
public function signalCountsTestCase()
{
$part = str_repeat("a", IndexDocumentBundle::DOCID_PART_LEN);
$host = str_repeat("h", IndexDocumentBundle::DOCID_PART_LEN);
foreach ([[chr(128 | 1), "CLD_URL_BONUS", 1 / 2],
[chr(2), "HOST_URL_BONUS", 1 / 3]] as [$type, $root, $depth]) {
$counts = IndexDocumentBundle::signalCounts($host . $part .
$type . substr($host, 1));
$this->assertEqual($counts[$root] ?? 0, 1, "$root counts once");
$this->assertEqual($counts["NUM_SLASHES_BONUS"], $depth,
"the path depth counts 1/(slashes + 1)");
}
}
/**
* signalSharesReadNewPartitionsTestCase writes signal counts for two
* built partitions of the test bundle and checks the shares
* signalShares works out from them. It then marks a third partition
* built, with counts of its own, and checks that the next look, once
* SIGNAL_SHARES_CHECK_SECONDS has passed, adds that partition's counts
* to the totals rather than starting over. Last, it writes counts in a
* fourth partition's folder without marking it built, and checks they
* are not read: signalShares finds partitions from next_partition.txt,
* not by listing the bundle's folders.
*/
public function signalSharesReadNewPartitionsTestCase()
{
$bundle = $this->index_archive;
$write = function ($partition, $docs, $wiki) use ($bundle) {
$folder = $bundle->getPartitionBaseFolder($partition);
if (!file_exists($folder)) {
mkdir($folder, 0777, true);
}
file_put_contents($folder . "/" .
IndexDocumentBundle::SIGNALS_FILENAME,
json_encode(["DOCS" => $docs, "WIKI_BONUS" => $wiki]));
};
$write(0, 100, 10);
$write(1, 100, 30);
file_put_contents(self::TEST_DIR . "/" .
IndexDocumentBundle::NEXT_PARTITION_FILE, 2);
IndexDocumentBundle::$signal_shares_cache = [];
$shares = $bundle->signalShares();
$this->assertEqual($shares["WIKI_BONUS"] ?? -1, 0.2,
"two built partitions give 40 wiki pages in 200");
$write(2, 200, 160);
file_put_contents(self::TEST_DIR . "/" .
IndexDocumentBundle::NEXT_PARTITION_FILE, 3);
IndexDocumentBundle::$signal_shares_cache[$bundle->dir_name][2] = 0;
$shares = $bundle->signalShares();
$this->assertEqual($shares["WIKI_BONUS"] ?? -1, 0.5,
"a third built partition adds its 160 in 200 to make 200 in 400");
$write(3, 100, 100);
IndexDocumentBundle::$signal_shares_cache[$bundle->dir_name][2] = 0;
$shares = $bundle->signalShares();
$this->assertEqual($shares["WIKI_BONUS"] ?? -1, 0.5,
"a partition folder not yet counted as built is not read");
}
}
- Yioop Repo - Seekquarry