- Yioop Repo - Seekquarry

<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * This file contains unit tests of DocIterator and NegationIterator
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\tests;

use seekquarry\yioop\library as L;
use seekquarry\yioop\models\Model;
use seekquarry\yioop\library\CrawlConstants as CC;
use seekquarry\yioop\library\storage_formats\IndexDocumentBundle;
use seekquarry\yioop\library\IndexManager;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\library\index_bundle_iterators\DocIterator;
use seekquarry\yioop\library\index_bundle_iterators\NegationIterator;
use seekquarry\yioop\library\index_bundle_iterators\WordIterator;

/**
 * DocIteratorTest checks DocIterator, which walks every document of an
 * IndexDocumentBundle, and NegationIterator, which walks the documents
 * without a word.
 */
class DocIteratorTest extends UnitTest
{
    /**
     * Name of the IndexDocumentBundle the cases store pages in
     */
    const TEST_BUNDLE = "IndexData200000000";
    /**
     * Folder the test bundle is made in
     */
    const TEST_DIR = __DIR__ . '/test_files/doc_iterator_test';
    /**
     * setUp makes the test folder and a bundle in it of ten pages, in
     * partitions of four pages. Every page holds the words common and
     * words, and page 2 also holds the word rare.
     */
    public function setUp()
    {
        if (!file_exists(self::TEST_DIR)) {
            mkdir(self::TEST_DIR);
        }
        $index_archive = new IndexDocumentBundle(self::TEST_DIR . "/" .
            self::TEST_BUNDLE, false, "TestBundle", 4, 5);
        $docs = [];
        for ($i = 0; $i < 10; $i++) {
            $summary = [CC::URL => "https://www.somewhere$i.com/",
                CC::TITLE => "Page$i title",
                CC::DESCRIPTION => "common words" . (($i == 2) ? " rare" :
                    ""), CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
                CC::TYPE => "text/html", CC::HTTP_CODE => 200];
            $docs[] = [CC::DOC_ID => IndexDocumentBundle::computeDocId(
                $summary), CC::SUMMARY => $summary, CC::PAGE => "Page $i"];
        }
        $index_archive->addPages($docs, count($docs));
        $index_archive->updateDictionary();
        $index_archive->forceSave();
        IndexManager::clearCache();
    }
    /**
     * tearDown deletes the test folder and the bundle in it.
     */
    public function tearDown()
    {
        IndexManager::clearCache();
        $model = new Model();
        $model->db->unlinkRecursive(self::TEST_DIR);
    }
    /**
     * walk reads the blocks of $iterator until it has none left, and gives
     * the URL of each page in them, from the page's summary.
     *
     * @param object $iterator an IndexBundleIterator to walk
     * @return array the URLs of the pages, in the order walked
     */
    public function walk($iterator)
    {
        $index = IndexManager::getIndex(self::TEST_DIR . "/" .
            self::TEST_BUNDLE);
        $urls = [];
        for ($i = 0; $i < 100; $i++) {
            $pages = $iterator->currentDocsWithWord();
            if (!is_array($pages)) {
                break;
            }
            foreach ($pages as $doc_key => $page) {
                $summary = $index->getSummary($doc_key,
                    $page[CC::GENERATION]);
                $urls[] = $summary[CC::URL] ?? "";
            }
            $iterator->advance();
        }
        return $urls;
    }
    /**
     * everyPageIsWalkedTestCase checks that a DocIterator gives each of the
     * ten pages once, walked ascending and descending. A walk stopped
     * before the last row of each partition's document map, so it gave
     * seven pages.
     */
    public function everyPageIsWalkedTestCase()
    {
        $expected = [];
        for ($i = 0; $i < 10; $i++) {
            $expected[] = "https://www.somewhere$i.com/";
        }
        foreach ([CC::ASCENDING => "ascending",
            CC::DESCENDING => "descending"] as $direction => $label) {
            $urls = $this->walk(new DocIterator(self::TEST_DIR . "/" .
                self::TEST_BUNDLE, null, 1, $direction));
            sort($urls);
            $this->assertEqual($urls, $expected,
                "A walk $label gives every page once");
        }
    }
    /**
     * negatedWordKeepsPagesPastItsLastTestCase checks that a
     * NegationIterator gives the pages without a word, including those
     * after the word's last page. It stopped once the word's walk had no
     * page left, so for rare, on page 2, it gave pages 0 and 1, and for a
     * word on no page it gave nothing.
     */
    public function negatedWordKeepsPagesPastItsLastTestCase()
    {
        $name = self::TEST_DIR . "/" . self::TEST_BUNDLE;
        $without_rare = $this->walk(new NegationIterator(new WordIterator(
            L\canonicalTerm("rare"), $name, true, null, 1)));
        sort($without_rare);
        $expected = [];
        for ($i = 0; $i < 10; $i++) {
            if ($i != 2) {
                $expected[] = "https://www.somewhere$i.com/";
            }
        }
        $this->assertEqual($without_rare, $expected,
            "The nine pages without rare are given");
        $without_missing = $this->walk(new NegationIterator(
            new WordIterator(L\canonicalTerm("missing"), $name, true, null,
            1)));
        $this->assertEqual(count($without_missing), 10,
            "All ten pages lack a word on no page");
    }
}