<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\storage_formats\CrawlQueueBundle;
use seekquarry\yioop\library\UnitTest;
/**
* UnitTest for the CrawlQueueBundle class.
*
* @author Chris Pollett
*/
class CrawlQueueBundleTest extends UnitTest
{
/** our dbms manager handle so we can call unlinkRecursive
* @var object
*/
public $db;
/**
* Sets up a minimal DBMS manager class so that we will be able to use
* unlinkRecursive to tear down own CrawlQueueBundle
*/
public function __construct()
{
$db_class = C\NS_DATASOURCES . ucfirst(C\DBMS)."Manager";
$this->db = new $db_class();
}
/**
* Set up a web queue bundle that can store 1000 urls in ram, has bloom
* filter space for 1000 urls and which uses a maximum value returning
* priority queue.
*/
public function setUp()
{
$this->test_objects['FILE1'] =
new CrawlQueueBundle(C\TEST_DIR . "/test_files/QueueTest",
1000, 1000);
}
/**
* Delete the directory and files associated with the CrawlQueueBundle
*/
public function tearDown()
{
$this->db->unlinkRecursive(C\TEST_DIR . "/test_files/QueueTest");
}
/**
* Tests the methods addRobotPaths and checkRobotOkay
*/
public function addRobotPathsCheckRobotOkayTestCase()
{
$crawl_queue = $this->test_objects['FILE1'];
$paths = [
CrawlConstants::ALLOWED_SITES => ["/trapdoor"],
CrawlConstants::DISALLOWED_SITES => ["/trap","/*?",'/cgi-bin'],
];
$empty_paths = [CrawlConstants::ALLOWED_SITES => [],
CrawlConstants::DISALLOWED_SITES => []];
$robot_rows = [];
$robot_rows[] = ["HOSTNAME" => "http://www.test.com",
"CAPTURE_TIME" => time(), "CRAWL_DELAY" => 0,
"ROBOT_PATHS" => $paths,
"FLAGS" => 0];
$robot_rows[] = ["HOSTNAME" => "http://www.cs.sjsu.edu",
"CAPTURE_TIME" => time(), "CRAWL_DELAY" => 0,
"ROBOT_PATHS" => $empty_paths,
"FLAGS" => 0];
$crawl_queue->robot_table->put($robot_rows);
$test_urls = [
["http://www.cs.sjsu.edu/", true,
"url with no stored rules"],
["http://www.test.com/trapdoor", true,
"allowed url"],
["http://www.test.com/trapdoor?b", true,
"allowed overrides all disallows"],
["http://www.test.com/trap", false,
"forbidden url 1"],
["http://www.test.com/abc?", false,
"forbidden url 2"],
["http://www.test.com/a?b", false,
"forbidden url 3"],
["http://www.test.com/cgi-bin/psearch?-list", false,
"forbidden url 4"],
];
foreach ($test_urls as $test_url) {
$result = $crawl_queue->checkRobotOkay($test_url[0]);
$this->assertEqual($result, $test_url[1], $test_url[2]);
}
}
/**
* slowKindsGoFurtherBackTestCase gives the queue processing times for
* web pages, PDFs and pictures, then works out the tier of a url of
* each kind from a host with nothing seen. A PDF that takes 100 times
* as long as a web page goes two tiers back, a picture taking 5 times
* as long goes one back, and a web page none. Until times came in,
* every kind but a web page went back one tier.
*/
public function slowKindsGoFurtherBackTestCase()
{
$queue = $this->test_objects['FILE1'];
$queue->noteProcessTimes(["html" => [2, 40], "pdf" => [100, 20],
"jpg" => [5, 20]]);
$host = ["COMPANY_LEVEL_DOMAIN" => "example.com", "SEEN_URLS" => 0,
"WEIGHTED_SEEN_URLS" => 0.0, "WEIGHTED_INCOMING_URLS" => 0.0];
$tiers = [];
foreach (["page.html", "paper.pdf", "photo.jpg"] as $name) {
$tiers[] = $queue->computeTierUrl(["https://example.com/$name",
0, 1, "", 0], $host, CrawlConstants::HOST_BUDGETING);
}
$this->assertEqual($tiers, [0, 2, 1],
"a web page, a PDF and a picture go 0, 2 and 1 tiers back");
}
/**
* processTimesAreBoundedSavedAndReadBackTestCase gives the queue times
* for one more kind of document than it keeps, then makes a new queue
* on the same folder, as a restarted queue server does. Only the first
* MAX_TIMED_TYPES kinds are kept, and the new queue reads back the
* times and the tiers they give.
*/
public function processTimesAreBoundedSavedAndReadBackTestCase()
{
$queue = $this->test_objects['FILE1'];
$times = ["html" => [2, 40], "pdf" => [100, 20]];
for ($i = count($times); $i <= CrawlQueueBundle::MAX_TIMED_TYPES;
$i++) {
$times["kind$i"] = [1, 1];
}
$queue->noteProcessTimes($times);
$this->assertEqual(count($queue->process_times),
CrawlQueueBundle::MAX_TIMED_TYPES,
"only the first kinds seen are kept");
$again = new CrawlQueueBundle($queue->dir_name, 1000, 1000);
$this->assertEqual($again->process_times, $queue->process_times,
"a new queue on the folder reads the times back");
$this->assertEqual($again->type_tiers["pdf"] ?? -1, 2,
"and the tiers they give");
}
/**
* chooserSkipsTiersWithNoUrlsTestCase checks that under host
* budgeting the fetch batch chooser never hands back a tier holding
* no day of urls while some tier does. Six tiers are made, and only
* the two highest hold a day folder; on a fresh counter only the low
* tiers are opened, and all are empty, so the chooser must reach past
* them to a tier with urls rather than bail. It used to return the
* empty tier it landed on and the caller bailed on the whole pass.
*/
public function chooserSkipsTiersWithNoUrlsTestCase()
{
$queue = $this->test_objects['FILE1'];
$tiers = $queue->dir_name . "/" . CrawlQueueBundle::URL_QUEUE_FOLDER;
for ($rank = 0; $rank < 6; $rank++) {
$tier = $tiers . "/" . CrawlQueueBundle::TIER_PREFIX . $rank;
if (!file_exists($tier)) {
mkdir($tier, 0777, true);
}
if ($rank >= 4) {
mkdir($tier . "/20260909", 0777, true);
}
}
$chosen = $queue->chooseFetchBatchQueueFolder(
CrawlConstants::HOST_BUDGETING);
$this->assertTrue($chosen !== false,
"a tier is chosen while some tier holds urls");
$this->assertTrue(!empty($queue->getDayFolders($chosen)),
"and the chosen tier is one that holds a day of urls");
}
/**
* Checks a tier holding urls is offered to fill out a fetch batch,
* and one already drawn from is passed over. A batch whose tier ran
* dry used to go out with room left in it.
*/
public function anotherTierIsOfferedToFillABatchTestCase()
{
$queue = $this->test_objects['FILE1'];
$tiers = $queue->dir_name . "/" . CrawlQueueBundle::URL_QUEUE_FOLDER;
$with_urls = [];
for ($rank = 0; $rank < 4; $rank++) {
$tier = $tiers . "/" . CrawlQueueBundle::TIER_PREFIX . $rank;
if (!file_exists($tier)) {
mkdir($tier, 0777, true);
}
if ($rank == 1 || $rank == 3) {
mkdir($tier . "/20260909", 0777, true);
$with_urls[] = $tier;
}
}
$first = $queue->anotherFetchBatchQueueFolder([]);
$this->assertTrue(in_array($first, $with_urls),
"a tier holding urls is offered");
$second = $queue->anotherFetchBatchQueueFolder([$first]);
$this->assertTrue(in_array($second, $with_urls) &&
$second != $first,
"the other tier holding urls is offered next");
$this->assertFalse(
$queue->anotherFetchBatchQueueFolder($with_urls),
"nothing is offered once every tier with urls has been used");
}
/**
* Checks filling out a batch does not advance the tier rotation,
* which is chosen once per batch so the most interesting outstanding
* sites go out first.
*/
public function fillingABatchLeavesTheRotationAloneTestCase()
{
$queue = $this->test_objects['FILE1'];
$tiers = $queue->dir_name . "/" . CrawlQueueBundle::URL_QUEUE_FOLDER;
for ($rank = 0; $rank < 3; $rank++) {
$tier = $tiers . "/" . CrawlQueueBundle::TIER_PREFIX . $rank;
if (!file_exists($tier)) {
mkdir($tier, 0777, true);
}
mkdir($tier . "/20260909", 0777, true);
}
$before = $queue->chooseFetchBatchQueueFolder(
CrawlConstants::HOST_BUDGETING);
$queue->anotherFetchBatchQueueFolder([$before]);
$queue->anotherFetchBatchQueueFolder([$before]);
$after = $queue->chooseFetchBatchQueueFolder(
CrawlConstants::HOST_BUDGETING);
$this->assertTrue($after != $before,
"the rotation moved on by one choice, not by three");
}
}
- Yioop Repo - Seekquarry