- Yioop Repo - Seekquarry

<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\tests;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\models\Model;
use seekquarry\yioop\library\storage_formats\MessagesBundle;

/**
 * MessagesBundleTest checks the send window that hands fetch batches to
 * fetchers: how long a batch may be out before it is offered again, and
 * that the window still reads batch files written before the batch size
 * was kept in the name.
 *
 * @author Chris Pollett
 */
class MessagesBundleTest extends UnitTest
{
    /**
     * The folder the test's messages bundle is made in
     * @var string
     */
    public $dir;
    /**
     * Makes an empty messages bundle folder for each case.
     */
    public function setUp()
    {
        $this->dir = C\TEST_DIR . "/test_files/MessagesBundleTest";
        $this->tearDown();
        mkdir($this->dir, 0777, true);
    }
    /**
     * Removes the test's messages bundle folder.
     */
    public function tearDown()
    {
        if (file_exists($this->dir)) {
            (new Model())->db->unlinkRecursive($this->dir, true);
        }
    }
    /**
     * smallBatchIsOfferedAgainSoonTestCase checks that a batch of one url
     * that has been out longer than the time one pass of urls needs is
     * offered to a fetcher again, while a full batch out for the same time
     * is not: a full batch is allowed the whole SCHEDULE_TIMEOUT and a
     * smaller one a share of it.
     */
    public function smallBatchIsOfferedAgainSoonTestCase()
    {
        $one_pass = MessagesBundle::scheduleTimeout(1);
        $full = MessagesBundle::scheduleTimeout(C\MAX_FETCH_SIZE);
        $this->assertTrue($one_pass < $full,
            "a one url batch is allowed less time than a full one");
        $this->assertEqual(C\SCHEDULE_TIMEOUT, $full,
            "a full batch is allowed the whole schedule timeout");
        $this->assertEqual(C\SCHEDULE_TIMEOUT,
            MessagesBundle::scheduleTimeout(0),
            "a batch of unknown size is allowed the whole timeout");
        $bundle = new MessagesBundle($this->dir);
        $bundle->window_size = 8;
        $bundle->initWindows();
        $small = $this->dir . "/small.txt";
        file_put_contents($small, "one url");
        $bundle->insertSendWindow($small, 1);
        $bundle->extractSendWindow();
        $this->assertTrue(count(glob($this->dir . "/send/*")) == 1,
            "the batch went out once and stays in the window");
        foreach ($bundle->send as $sequence => $item) {
            $old = $bundle->makeSendWindowFilename($sequence,
                time() - $one_pass - 1, $item['tries'], 1);
            rename($item['file'], $old);
        }
        $bundle->initWindows();
        $this->assertTrue($bundle->extractSendWindow() !== false,
            "a one url batch out longer than one pass is offered again");
    }
    /**
     * batchThatWaitedGoesOutOnceTestCase checks that the timeout of a full
     * batch made most of SCHEDULE_TIMEOUT ago, but not asked for by a
     * fetcher until now, starts when a fetcher gets it. Otherwise the
     * batch is sent to a second fetcher soon after the first gets it.
     */
    public function batchThatWaitedGoesOutOnceTestCase()
    {
        mkdir($this->dir . "/send", 0777, true);
        mkdir($this->dir . "/receive", 0777, true);
        $bundle = new MessagesBundle($this->dir);
        $now = time();
        file_put_contents($bundle->makeSendWindowFilename(1,
            $now - C\SCHEDULE_TIMEOUT + 60, 0, C\MAX_FETCH_SIZE), "batch");
        $bundle->initWindows();
        $this->assertEqual("batch", $bundle->extractSendWindow(),
            "the first fetcher to ask gets the batch");
        $bundle->initWindows();
        $this->assertTrue($bundle->send[1]['schedule_time'] >= $now,
            "its timeout starts when the fetcher got it");
    }
    /**
     * oldWindowNameStillReadsTestCase checks that a send window file named
     * the way batches were named before the size was kept, three parts
     * rather than four, is still read into the window and given the whole
     * schedule timeout.
     */
    public function oldWindowNameStillReadsTestCase()
    {
        mkdir($this->dir . "/send", 0777, true);
        mkdir($this->dir . "/receive", 0777, true);
        file_put_contents($this->dir . "/send/3-1000-0.txt", "old batch");
        $bundle = new MessagesBundle($this->dir);
        $bundle->window_size = 8;
        $bundle->initWindows();
        $this->assertTrue(isset($bundle->send[3]),
            "the three part name is read into the window");
        $this->assertEqual(0, $bundle->send[3]['num_urls'],
            "its size is not known");
    }
}