<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\data_structures\BloomFilterFile;
use seekquarry\yioop\library\data_structures\Tier;
use seekquarry\yioop\library\storage_formats\FeedDocumentBundle;
use seekquarry\yioop\library\UnitTest;
/**
* FeedDocumentBundleTest checks the feed bundle, the folder of news feed
* items that FeedsUpdateJob adds to, as MediaUpdater uses it: every
* item added is indexed, every term merged into the dictionary is found
* there, and a bundle with both of its seen-item filters saved opens
* again.
*
* @author Chris Pollett
*/
class FeedDocumentBundleTest extends UnitTest
{
/**
* TEST_DIR is the folder the test feed bundle is made in and removed
* from around each case.
*/
const TEST_DIR = __DIR__ . '/test_files/feed_document_test';
/**
* NUM_DOCS_PER_PARTITION is how many documents a partition of the test
* bundle holds, more than any case adds, so every case builds one
* partition.
*/
const NUM_DOCS_PER_PARTITION = 1000;
/**
* TEST_FILTER_SIZE is how many keys the test bundle's seen filter is
* made for. A bundle made without a filter on disk makes one sized
* for URL_FILTER_SIZE keys, about 90 MB, more than a test run with a
* 128 MB memory limit can hold.
*/
const TEST_FILTER_SIZE = 10000;
/**
* setUp removes any test bundle a case before left, so each case
* starts from no bundle.
*/
public function setUp()
{
$this->removeTestBundle();
}
/**
* tearDown removes the test bundle the case made.
*/
public function tearDown()
{
$this->removeTestBundle();
}
/**
* removeTestBundle deletes the test bundle folder and everything in
* it, if it is there. setUp and tearDown call it.
*/
public function removeTestBundle()
{
if (!is_dir(self::TEST_DIR)) {
return;
}
$entries = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator(self::TEST_DIR,
\FilesystemIterator::SKIP_DOTS),
\RecursiveIteratorIterator::CHILD_FIRST);
foreach ($entries as $entry) {
if ($entry->isDir()) {
rmdir($entry->getPathname());
} else {
unlink($entry->getPathname());
}
}
rmdir(self::TEST_DIR);
}
/**
* openTestBundle opens the test feed bundle for writing, making it if
* it is not there, the way FeedsUpdateJob::getFeedBundle does.
*
* @return FeedDocumentBundle the opened test bundle
*/
public function openTestBundle()
{
$this->makeSmallFilter(self::TEST_DIR);
return new FeedDocumentBundle(self::TEST_DIR, $this->makeTestDatabase(),
false, serialize(['DESCRIPTION' => "feed"]),
self::NUM_DOCS_PER_PARTITION);
}
/**
* everyItemIsIndexedTestCase adds six feed items and builds the
* partition, then checks that the partition's document map holds all
* six of them.
*/
public function everyItemIsIndexedTestCase()
{
$bundle = $this->openTestBundle();
$bundle->addPagesAndSeenKeys($this->makeTestPages(6), 6);
$bundle->forceSave();
$this->assertEqual($bundle->doc_map_counter, 6,
"all six items are in the partition's document map");
}
/**
* savedSecondFilterIsReopenedTestCase saves a second seen-item filter
* in the bundle, as the bundle does once its first filter is half full,
* then checks that the bundle opens again with that filter as its
* second filter and its first filter left as it was.
*/
public function savedSecondFilterIsReopenedTestCase()
{
$bundle = $this->openTestBundle();
$bundle->addPagesAndSeenKeys($this->makeTestPages(2), 2);
$bundle->forceSave();
$second = new BloomFilterFile(self::TEST_DIR . "/filter_b.ftr",
self::TEST_FILTER_SIZE);
$second->add("second filter key");
$second->save();
$reopened = $this->openTestBundle();
$this->assertTrue($reopened->filter_b !== null &&
$reopened->filter_b->contains("second filter key"),
"the saved second filter is opened as the second filter");
$first_page = $this->makeTestPages(1)[0];
$this->assertTrue($reopened->filter_a->contains(
$first_page[FeedDocumentBundle::SUMMARY][
FeedDocumentBundle::HASH]),
"the first filter still holds the items added before");
}
/**
* dictionaryFindsEveryTermTestCase makes a feed bundle of four items a
* partition, adds eight items so that partition 0 is closed, and has
* updateDictionary build partition 0 and merge it into the dictionary.
* It then reads every row of the dictionary's first tier in turn and
* checks that looking up each row's term finds it, which needs the
* terms to have been saved in order.
*/
public function dictionaryFindsEveryTermTestCase()
{
$this->makeSmallFilter(self::TEST_DIR);
$bundle = new FeedDocumentBundle(self::TEST_DIR,
$this->makeTestDatabase(), false,
serialize(['DESCRIPTION' => "feed"]), 4);
$bundle->addPagesAndSeenKeys($this->makeTestPages(8), 8);
$bundle->updateDictionary();
$dictionary = $bundle->dictionary;
$first_tier = new Tier($dictionary->getTierFolder(0) . "/A",
$dictionary->table_tools);
$key_len = $dictionary->table_tools->key_len;
$terms = 0;
$not_found = [];
while ($row = $first_tier->next()) {
$terms++;
$term = substr($row, 0, $key_len);
if (empty($dictionary->get($term))) {
$not_found[] = $term;
}
}
$this->assertTrue($terms > 20, "the dictionary holds the terms " .
"of partition 0, $terms of them");
$this->assertEqual($not_found, [], "every term in the dictionary " .
"is found by looking it up");
}
/**
* makeSmallFilter saves a seen filter for TEST_FILTER_SIZE keys in
* $folder when none is there yet, so the feed bundle opened on
* $folder loads it rather than making a full-size one.
* openTestBundle calls it.
*
* @param string $folder the folder of the feed bundle
*/
public function makeSmallFilter($folder)
{
if (file_exists("$folder/filter_a.ftr")) {
return;
}
if (!file_exists($folder)) {
mkdir($folder, 0777, true);
}
$filter = new BloomFilterFile("$folder/filter_a.ftr",
self::TEST_FILTER_SIZE);
$filter->save();
}
/**
* makeTestDatabase makes an object that stands in for the database a
* feed bundle writes its trending term counts to, so a partition can
* be built without a real database. Each of its methods answers as an
* empty database would: a statement finds no rows.
*
* @return object the stand-in database
*/
public function makeTestDatabase()
{
return new class {
/**
* execute stands in for running a statement, and answers that
* the statement found nothing.
*
* @param string $sql the statement that would be run
* @param array $parameters the values it would be given
* @return bool false, as for a statement with no rows
*/
public function execute($sql, $parameters = [])
{
return false;
}
/**
* fetchArray stands in for reading the next row of a result,
* and answers that there are no more rows.
*
* @param mixed $result the result rows would be read from
* @return bool false, as at the end of a result
*/
public function fetchArray($result)
{
return false;
}
/**
* limitOffset stands in for the clause that limits how many
* rows a statement returns, and gives that clause as SQL text.
*
* @param int $limit the most rows to return
* @return string the clause that limits the rows
*/
public function limitOffset($limit)
{
return " LIMIT $limit";
}
};
}
/**
* makeTestPages makes the pages FeedsUpdateJob would add to a feed
* bundle for $count feed items, each from the same test source with a
* title and a description of a few words.
*
* @param int $count how many feed items to make
* @return array pages, each with a document id and a summary
*/
public function makeTestPages($count)
{
$pages = [];
for ($i = 0; $i < $count; $i++) {
$link = "https://news.example.com/story$i";
$guid = L\crawlHash($link);
$summary = [CrawlConstants::TITLE => "Story number $i",
CrawlConstants::DESCRIPTION => "A short story about item $i",
CrawlConstants::URL => $link, CrawlConstants::HASH => $guid,
CrawlConstants::SOURCE_NAME => "Test News",
CrawlConstants::IMAGE_LINK => "",
CrawlConstants::PUBDATE => time() - $i];
$pages[] = [CrawlConstants::DOC_ID =>
FeedDocumentBundle::computeDocId($summary),
CrawlConstants::SUMMARY => $summary,
CrawlConstants::PAGE => ""];
}
return $pages;
}
/**
* findTestFiles finds every file named $name in $folder or in a folder
* within it, such as the document maps of a feed bundle's partitions.
*
* @param string $folder the folder to search
* @param string $name the file name to find
* @return array paths of the files found
*/
public function findTestFiles($folder, $name)
{
$found = [];
if (!is_dir($folder)) {
return $found;
}
$entries = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($folder,
\FilesystemIterator::SKIP_DOTS));
foreach ($entries as $entry) {
if ($entry->getFilename() === $name) {
$found[] = $entry->getPathname();
}
}
return $found;
}
}
- Yioop Repo - Seekquarry