<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see
* <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @package seek_quarry\tests
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlDaemon;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\models\CrawlModel;
use seekquarry\yioop\models\Model;
/**
* Checks that a crawl whose stop has been sent shows as stopping before
* its queue server has read the stop. The cases write the name server's
* message file for an unused channel, as clicking stop does, and ask
* CrawlModel::markStopRequested what the status becomes.
*
* @author Chris Pollett
*/
class CrawlModelTest extends UnitTest
{
/**
* CHANNEL is the channel number of the crawl the cases make up.
*/
const CHANNEL = 97;
/**
* The crawl model under test.
* @var CrawlModel
*/
public $crawl_model;
/**
* setUp builds the crawl model and points CrawlDaemon at SCHEDULES, so
* the message files the cases write are in the tests folder.
*/
public function setUp()
{
$this->crawl_model = new CrawlModel();
L\makePath(self::SCHEDULES);
CrawlDaemon::$schedules_folder = self::SCHEDULES;
}
/**
* SCHEDULES is the folder, under tests/test_files, that CrawlDaemon
* keeps the name server's message file in for these cases.
*/
const SCHEDULES = C\TEST_DIR . "/test_files/crawl_model_schedules";
/**
* tearDown deletes SCHEDULES, with the channel's name server message
* file, and points CrawlDaemon back at the work directory's schedules
* folder.
*/
public function tearDown()
{
(new Model())->db->unlinkRecursive(self::SCHEDULES);
CrawlDaemon::$schedules_folder = C\SCHEDULES_DIR;
}
/**
* writeNameServerMessage writes the channel's message file as the
* crawl component does.
*
* @param string $status the STATUS the message carries
*/
public function writeNameServerMessage($status)
{
file_put_contents(CrawlDaemon::getMessageFileName("NameServer",
self::CHANNEL), serialize([CrawlModel::STATUS => $status]));
}
/**
* A running crawl whose stop has been sent shows SHUTDOWN_QUEUE, and a
* crawl on another channel is left as it was.
*/
public function sentStopShowsAtOnceTestCase()
{
$this->writeNameServerMessage("STOP_CRAWL");
$status = [self::CHANNEL => ['DESCRIPTION' => "Test Syn 3"],
self::CHANNEL + 1 => ['DESCRIPTION' => "Another crawl"]];
$this->crawl_model->markStopRequested($status);
$this->assertEqual($status[self::CHANNEL]['DESCRIPTION'],
"SHUTDOWN_QUEUE", "a sent stop shows as the first stopping step");
$this->assertEqual($status[self::CHANNEL + 1]['DESCRIPTION'],
"Another crawl", "a channel with no stop is unchanged");
}
/**
* Once the queue server writes its own shutdown step, that step shows
* rather than SHUTDOWN_QUEUE.
*/
public function queueServerStepIsKeptTestCase()
{
$this->writeNameServerMessage("STOP_CRAWL");
$status = [self::CHANNEL => ['DESCRIPTION' => "SHUTDOWN_DICTIONARY"]];
$this->crawl_model->markStopRequested($status);
$this->assertEqual($status[self::CHANNEL]['DESCRIPTION'],
"SHUTDOWN_DICTIONARY", "the queue server's step is kept");
}
/**
* A crawl whose message file holds a start, or which has no message
* file, shows its own description.
*/
public function noStopLeavesDescriptionTestCase()
{
$status = [self::CHANNEL => ['DESCRIPTION' => "Test Syn 3"]];
$this->crawl_model->markStopRequested($status);
$this->assertEqual($status[self::CHANNEL]['DESCRIPTION'],
"Test Syn 3", "no message file leaves the description");
$this->writeNameServerMessage("NEW_CRAWL");
$this->crawl_model->markStopRequested($status);
$this->assertEqual($status[self::CHANNEL]['DESCRIPTION'],
"Test Syn 3", "a start message leaves the description");
}
/**
* speedHistoryReadsBacklogAndFetcherRateTestCase checks that
* crawlSpeedHistory gives a line's indexer backlog and fetcher
* downloads per hour, and gives "" for both from a line logged before
* those columns were kept. Lines logged before 27 October hold the
* crawl's channel after its timestamp, and later lines do not; both
* give the same row. A line of another crawl is left out.
*/
public function speedHistoryReadsBacklogAndFetcherRateTestCase()
{
$speed_log = self::SCHEDULES . "/CrawlSpeed.log";
file_put_contents($speed_log,
"100\t7\t0\t1500.5\t20\t900\n" .
"200\t8\t0\t9.0\t1\t2\t3\t4\n" .
"300\t7\t0\t1600.25\t30\t950\t12\t170000\n" .
"400\t7\t1700.75\t40\t990\t3\t180000\n");
$this->assertEqual($this->crawl_model->crawlSpeedHistory(7,
$speed_log), [["100", "1500.5", "20", "900", "", ""],
["300", "1600.25", "30", "950", "12", "170000"],
["400", "1700.75", "40", "990", "3", "180000"]],
"the old line has no backlog or rate, the later ones have " .
"both, with or without the channel");
}
/**
* fetcherRatesCountEachFetcherOnceTestCase checks that aggregateStatuses
* keeps one rate for a fetcher in the FETCHER_RATES of two machines,
* the newer one, and that FETCHER_DOWNLOADS_PER_HOUR sums the rates
* of the different fetchers.
*/
public function fetcherRatesCountEachFetcherOnceTestCase()
{
$statuses = [];
foreach ([["0-97-Yioop" => [100, 17000], "1-97-Yioop" => [90, 9000]],
["0-97-Yioop" => [120, 18000]]] as $rates) {
$statuses[] = [CrawlModel::PAGE => L\webencode(serialize([
self::CHANNEL => ["FETCHER_RATES" => $rates,
"CRAWL_TIME" => 7]]))];
}
$status = $this->crawl_model->aggregateStatuses($statuses);
$this->assertEqual($status[self::CHANNEL]["FETCHER_RATES"],
["0-97-Yioop" => [120, 18000], "1-97-Yioop" => [90, 9000]],
"a fetcher on two machines keeps its newer rate");
$this->assertEqual($status[self::CHANNEL]
["FETCHER_DOWNLOADS_PER_HOUR"], 27000,
"the rates of different fetchers are summed");
}
/**
* backlogIsSummedAcrossMachinesTestCase checks that aggregateStatuses
* adds up the INDEXER_BACKLOG of each machine's crawl status for a
* channel, as it does the visited urls count.
*/
public function backlogIsSummedAcrossMachinesTestCase()
{
$statuses = [];
foreach ([[5, 100], [7, 50]] as [$backlog, $visited]) {
$statuses[] = [CrawlModel::PAGE => L\webencode(serialize([
self::CHANNEL => ["INDEXER_BACKLOG" => $backlog,
"VISITED_URLS_COUNT" => $visited, "CRAWL_TIME" => 7]]))];
}
$status = $this->crawl_model->aggregateStatuses($statuses);
$this->assertEqual([$status[self::CHANNEL]["INDEXER_BACKLOG"],
$status[self::CHANNEL]["VISITED_URLS_COUNT"]], [12, 150],
"the backlogs are summed like the visited counts");
}
}
- Yioop Repo - Seekquarry