<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\executables\QueueServer;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\storage_formats\CrawlQueueBundle;
use seekquarry\yioop\library\storage_formats\IndexDocumentBundle;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\models\Model;
/* Both FetcherTest and QueueServerTest load an executable that only
starts its process when this is not defined, and the suite loads both. */
if (!C\nsdefined("UNIT_TEST_MODE")) {
C\nsdefine("UNIT_TEST_MODE", true);
}
/**
* Used to test functions related to scheduling websites to crawl for
* a web crawl (the responsibility of a QueueServer)
*
* @author Chris Pollett
*/
class QueueServerTest extends UnitTest
{
/**
* Creates a QueueServer object with an initial set of indexed file types
*/
public function setUp()
{
$this->test_objects['Q_SERVER'] = new QueueServer();
$this->test_objects['Q_SERVER']->temp_folder = C\TEST_DIR .
"/test_files";
$this->test_objects['Q_SERVER']->indexed_file_types = ["html", "txt"];
}
/**
* Used to get rid of any object/files we created during a test case.
* that need to be disposed of.
*/
public function tearDown()
{
// get rid of the queue_server from previous test case
$this->test_objects['Q_SERVER'] = null;
}
/**
* allowedToCrawlSite check if a url is matches a list of url
* and domains stored in a QueueServer's allowed_sites and that it
* is of an allowed to crawl file type. This function tests these
* properties
*/
public function allowedToCrawlSiteTestCase()
{
$q_server = $this->test_objects['Q_SERVER'];
$q_server->allowed_sites = ["http://www.example.com/",
"domain:ca", "domain:somewhere.tv"];
$test_urls = [
["http://www.yoyo.com/", true,
"not restrict by url case", ["restrict_sites_by_url" => false]],
["http://www.yoyo.com/", false,
"simple not allowed case", ["restrict_sites_by_url" => true]],
["http://www.example.com/", true,
"simple allowed site", [ "restrict_sites_by_url" => true]],
["http://www.example.com/a.bad", false, "not allowed filetype"],
["http://www.example.com/a.txt", true, "allowed filetype"],
["http://www.uchicago.com/", false,
"domain disallowed", ["restrict_sites_by_url" => true]],
["http://www.findcan.ca/", true,
"domain disallowed", ["restrict_sites_by_url" => true]],
["http://somewhere.tv.com/", false,
"domain disallowed", ["restrict_sites_by_url" => true]],
["http://woohoo.somewhere.tv/", true,
"domain disallowed", ["restrict_sites_by_url" => true]],
];
if (!C\nsdefined("KEEP_SPAM_DOMAINS")) {
$test_urls[] =
["http://www.jhdsgjhd.com/", false,
"spam site case", ["restrict_sites_by_url" => false]];
$test_urls[] =
["http://www.jhds.com/", true,
"spam site case 2", ["restrict_sites_by_url" => false]];
}
foreach ($test_urls as $test_url) {
if (isset($test_url[3])) {
foreach ($test_url[3] as $field => $value) {
$q_server->$field = $value;
}
}
$result = $q_server->allowedToCrawlSite($test_url[0]);
$this->assertEqual($result, $test_url[1], $test_url[2]);
}
}
/**
* disallowedToCrawlSite check if a url is matches a list of url
* and domains stored in a QueueServer's disallowed_sites. This function
* tests this properties (The test cases are similar to those of
* urlMemberSiteArrayTestCase, but are using the disallowed_sites array)
*/
public function disallowedToCrawlSiteTestCase()
{
$q_server = $this->test_objects['Q_SERVER'];
$q_server->disallowed_sites = ["http://www.example.com/",
"http://www.cs.sjsu.edu/faculty/pollett/*/*/",
"http://www.bing.com/video/search?*&*&",
"http://*.cool.*/a/*/"];
$test_urls = [
["http://www.cs.sjsu.edu/faculty/pollett/", false,
"regex url negative 1"],
["http://www.bing.com/video/search?", false,"regex url negative 2"],
["http://www.cool.edu/a", false, "regex url negative 3"],
["http://www.cs.sjsu.edu/faculty/pollett/a/b/c", true,
"regex url positive 1"],
["http://www.bing.com/video/search?a&b&c", true,
"regex url positive 2"],
["http://www.cool.bob.edu/a/b/c", true, "regex url positive 3"],
];
foreach ($test_urls as $test_url) {
$result = $q_server->disallowedToCrawlSite($test_url[0]);
$this->assertEqual($result, $test_url[1], $test_url[2]);
}
}
/**
* builtPartitionsDoNotHoldBatchesBackTestCase makes a bundle of four
* documents a partition and closes MAX_UNMERGED_PARTITIONS + 2
* partitions, none of them merged into the dictionary. While none is
* built, unbuiltPartitions counts past MAX_UNMERGED_PARTITIONS, so
* produceFetchBatch holds crawl batches back. Once every closed
* partition is built, still with none merged, it counts none, so
* batches go out while the partitions wait only for the merge claim.
*/
public function builtPartitionsDoNotHoldBatchesBackTestCase()
{
$folder = C\TEST_DIR . "/test_files/unbuilt_partitions_test";
$model = new Model();
$model->db->unlinkRecursive($folder);
$bundle = new IndexDocumentBundle($folder, false, "Test", 4);
$closed = C\MAX_UNMERGED_PARTITIONS + 2;
$pages = [];
for ($i = 0; $i <= 4 * $closed; $i++) {
$pages[] = [CrawlConstants::DOC_ID =>
str_pad("$i", 8, "0", STR_PAD_LEFT) .
str_pad("$i", 8, "0", STR_PAD_LEFT) . "d" .
str_pad("$i", 7, "0", STR_PAD_LEFT),
CrawlConstants::SUMMARY => [
CrawlConstants::DESCRIPTION => "word$i shared words",
CrawlConstants::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CrawlConstants::TITLE => "Title$i",
CrawlConstants::URL => "https://site$i.com/"],
CrawlConstants::PAGE => "Page $i"];
}
$bundle->addPages($pages, count($pages));
$this->assertEqual($bundle->documents->parameters["SAVE_PARTITION"],
$closed, "$closed partitions are closed and none is merged");
$this->assertEqual(QueueServer::unbuiltPartitions($folder),
C\MAX_UNMERGED_PARTITIONS + 1,
"unbuilt partitions are counted past the limit");
do {
$built = IndexDocumentBundle::buildClosedPartition($folder);
} while ($built >= 0);
$this->assertEqual(QueueServer::unbuiltPartitions($folder), 0,
"built partitions waiting to be merged are not counted");
$model->db->unlinkRecursive($folder);
}
/**
* lastTryUrlsGoInScheduleTestCase checks that the Scheduler names, in
* a fetch schedule's first line, the urls it is sending out for the
* last time. A url whose $fetch_tries count is one less than
* FETCH_TRIES_MAX is on its last try; a url tried fewer times, a url
* never tried and the empty slot that spaces out crawl delays are not.
* A schedule with no such url has no list in its first line.
*/
public function lastTryUrlsGoInScheduleTestCase()
{
$queue_server = $this->test_objects['Q_SERVER'];
$last = "https://www.pepsi.com/";
$earlier = "https://www.coke.com/";
$queue_server->fetch_tries = [
L\crawlHash($last, true) => QueueServer::FETCH_TRIES_MAX - 1,
L\crawlHash($earlier, true) => 1];
$sites = [[$last, 1.0, 0, 0, ""], [$earlier, 1.0, 0, 0, ""],
["https://www.example.com/", 1.0, 0, 0, ""],
[QueueServer::DUMMY, 0.0, 0, 0, ""]];
$this->assertEqual($queue_server->lastTryUrls($sites), [$last],
"only the url on its last try is listed");
$queue_server->messages_bundle = new \stdClass();
$queue_server->messages_bundle->max_sequence_number = 0;
$meta = unserialize(base64_decode(
$queue_server->calculateScheduleMetaInfo(time(), [$last])));
$this->assertEqual($meta[QueueServer::LAST_TRY_URLS] ?? [], [$last],
"the schedule's first line names the url");
$meta = unserialize(base64_decode(
$queue_server->calculateScheduleMetaInfo(time())));
$this->assertTrue(!isset($meta[QueueServer::LAST_TRY_URLS]),
"a schedule with no url on its last try has no list");
}
/**
* urlGoesInBatchOnceTestCase queues a landing page five times among
* other urls, as the queue holds a host's landing page once for each
* link to the host seen before it is fetched, and has the Scheduler
* produce a fetch batch from the queue. The schedule it writes holds
* the landing page once and each of the other urls once; a fetcher
* downloaded and processed every copy, http://www.jyb.cn/ 39 times in
* two and a half minutes of the crawl of 23 October.
*/
public function urlGoesInBatchOnceTestCase()
{
$queue_server = $this->test_objects['Q_SERVER'];
$folder = C\TEST_DIR . "/test_files/QueueServerBatchTest";
$model = new Model();
$model->db->unlinkRecursive($folder);
$queue_server->crawl_time = 1;
$queue_server->crawl_type = CrawlConstants::WEB_CRAWL;
$queue_server->crawl_order = CrawlConstants::BREADTH_FIRST;
$queue_server->robots_txt = C\IGNORE_ROBOTS;
/* takes the schedule as the messages bundle would, leaving the
file where produceFetchBatch wrote it */
$queue_server->messages_bundle = new class {
/**
* Sequence number of the last schedule, read by the Scheduler
* @var int
*/
public $max_sequence_number = 0;
/**
* insertSendWindow takes the schedule produceFetchBatch wrote
* and leaves it where it is, for the test to read.
* @param string $file_name the schedule's file
* @param int $num_urls how many urls the schedule holds
* @return bool true, the schedule always being taken
*/
public function insertSendWindow($file_name, $num_urls)
{
return true;
}
};
$queue_server->crawl_queue = new CrawlQueueBundle($folder, 1000,
1000);
$landing = "https://www.landing.com/";
$others = [];
$tuples = [];
for ($i = 0; $i < 5; $i++) {
$others[] = "https://www.landing.com/page$i.html";
$tuples[] = [$landing, 1.0, 1, "", 1.0];
$tuples[] = [$others[$i], 1.0, 1, "", 1.0];
}
$queue_server->crawl_queue->addSendFetcherQueue($tuples,
CrawlConstants::BREADTH_FIRST);
$queue_server->produceFetchBatch();
$schedule_file = C\TEST_DIR . "/test_files/" .
QueueServer::schedule_name . "1.txt";
$lines = file($schedule_file, FILE_IGNORE_NEW_LINES);
unlink($schedule_file);
$urls = [];
foreach (array_slice($lines, 1) as $line) {
$url = explode("###", substr(base64_decode($line), 12))[0];
if ($url != QueueServer::DUMMY) {
$urls[] = $url;
}
}
$counts = array_count_values($urls);
$this->assertEqual($counts[$landing] ?? 0, 1,
"the landing page queued five times goes out once");
$this->assertEqual(count($urls), 6,
"each of the other urls goes out once");
$model->db->unlinkRecursive($folder);
}
/**
* sendQueuesUrlOnceTestCase hands the Scheduler a fetcher's send whose
* to-crawl list names a landing page five times, once for each page
* that linked to it, as a site's navigation does, and one other url.
* The url filter holds neither url yet, so every copy passes it. The
* queue then holds the landing page once and the other url once;
* before, it held every copy, and each went out in a fetch batch.
*/
public function sendQueuesUrlOnceTestCase()
{
$queue_server = $this->test_objects['Q_SERVER'];
$folder = C\TEST_DIR . "/test_files/QueueServerSendTest";
$model = new Model();
$model->db->unlinkRecursive($folder);
$queue_server->crawl_order = CrawlConstants::BREADTH_FIRST;
$queue_server->all_file_types = ["html", "txt"];
$queue_server->restrict_sites_by_url = false;
$queue_server->crawl_queue = new CrawlQueueBundle($folder, 1000,
1000);
$queue_server->crawl_queue->robot_table->put([[
"HOSTNAME" => "https://www.landing.com", "CAPTURE_TIME" => time(),
"CRAWL_DELAY" => 0, "ROBOT_PATHS" => [], "FLAGS" => 0]]);
$landing = "https://www.landing.com/";
$other = "https://www.landing.com/page.html";
$to_crawl = [];
for ($i = 0; $i < 5; $i++) {
$to_crawl[] = [$landing, 0, 1, "https://www.linker.com/$i", 0.0];
}
$to_crawl[] = [$other, 0, 1, "https://www.linker.com/0", 0.0];
$sites = [CrawlConstants::TO_CRAWL => $to_crawl];
$queue_server->processToCrawlArchive($sites);
$crawl_queue = $queue_server->crawl_queue;
$queued = [];
foreach (glob($folder . "/" . CrawlQueueBundle::URL_QUEUE_FOLDER .
"/*", GLOB_ONLYDIR) as $tier) {
foreach ($crawl_queue->getDayFolders($tier) as $day) {
foreach ($crawl_queue->getUrlsFiles($day) as $file) {
foreach ($crawl_queue->getUrlsFileContents($file) as
$tuple) {
$queued[] = $tuple[0];
}
}
}
}
$counts = array_count_values($queued);
$this->assertEqual($counts[$landing] ?? 0, 1,
"the landing page linked five times is queued once");
$this->assertEqual($counts[$other] ?? 0, 1,
"the other url is queued once");
$model->db->unlinkRecursive($folder);
}
/**
* fetcherRatesKeepRecentFetchersTestCase checks that
* updateMostRecentFetcher keeps, from a channel's robot table, the time
* and rate of each fetcher heard from in the last RECENT_FETCHER_TIME
* seconds, gives 0 to one that sent no rate, and leaves out one heard
* from earlier.
*/
public function fetcherRatesKeepRecentFetchersTestCase()
{
$q_server = $this->test_objects['Q_SERVER'];
$folder = C\TEST_DIR . "/test_files/fetcher_rates_test";
$model = new Model();
$model->db->unlinkRecursive($folder);
L\makePath($folder);
$q_server->channel = 97;
$now = 10000;
$old = $now - QueueServer::RECENT_FETCHER_TIME - 1;
file_put_contents("$folder/97-" . CrawlConstants::robot_table_name,
serialize([
"0-97-Yioop" => ["1.2.3.4", "/", $now - 30, 17000],
"1-97-Yioop" => ["1.2.3.4", "/", $now - 100],
"2-97-Yioop" => ["1.2.3.5", "/", $old, 50000]]));
$q_server->updateMostRecentFetcher($folder, $now);
$this->assertEqual($q_server->fetcher_rates, [
"0-97-Yioop" => [$now - 30, 17000],
"1-97-Yioop" => [$now - 100, 0]],
"the recent fetchers are kept with their rates");
$this->assertEqual($q_server->most_recent_fetcher, "0-97-Yioop",
"the most recent fetcher is still found");
$model->db->unlinkRecursive($folder);
}
}
- Yioop Repo - Seekquarry